diff --git a/.gitattributes b/.gitattributes index ceb7898e..3a257ea9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,2 @@ -languages/tla-plus/v1/grammar-source/tla+.jj -whitespace -languages/fhirpath/v2.0.0/grammar-source/fhirpath.g4 -text -whitespace +languages/tla-plus/grammar-source/tla+.jj -whitespace +languages/fhirpath/grammar-source/fhirpath.g4 -text -whitespace diff --git a/.github/actions/gerbil-setup/action.yml b/.github/actions/gerbil-setup/action.yml new file mode 100644 index 00000000..842faf53 --- /dev/null +++ b/.github/actions/gerbil-setup/action.yml @@ -0,0 +1,82 @@ +name: Immutable Gerbil SDK +description: Admit a cached SDK by full digest and platform, using GCC +inputs: + release-os: + required: true + description: release-os + release-arch: + required: true + description: release-arch + release-tag: + required: true + description: release-tag + release-sha256: + required: true + description: release-sha256 +runs: + using: composite + steps: + - name: Install Linux system dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + ca-certificates \ + curl \ + git \ + jq \ + libsqlite3-dev \ + libssl-dev \ + openjdk-17-jre-headless \ + openssl \ + pkg-config \ + sqlite3 \ + zlib1g-dev + shell: bash + - name: Install macOS system dependencies + if: runner.os == 'macOS' + env: + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 + run: | + brew list --formula ghostscript >/dev/null 2>&1 && brew unlink ghostscript || true + brew install gcc jq openssl pkg-config sqlite zlib + gcc_prefix="$(brew --prefix gcc)" + gcc_bin="$( + find "$gcc_prefix/bin" -maxdepth 1 -type f -name 'gcc-[0-9]*' \ + | awk -F- '{print $NF "\t" $0}' \ + | sort -n \ + | tail -n 1 \ + | cut -f2- + )" + test -x "$gcc_bin" + echo "GERBIL_GCC=$gcc_bin" >> "$GITHUB_ENV" + openssl_prefix="$(brew --prefix openssl@3 2>/dev/null || brew --prefix openssl)" + sqlite_prefix="$(brew --prefix sqlite)" + echo "LIBRARY_PATH=$openssl_prefix/lib:$sqlite_prefix/lib${LIBRARY_PATH:+:$LIBRARY_PATH}" >> "$GITHUB_ENV" + macos_major="$(sw_vers -productVersion | cut -d. -f1)" + echo "PKG_CONFIG_PATH=$openssl_prefix/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" >> "$GITHUB_ENV" + echo "MACOSX_DEPLOYMENT_TARGET=$macos_major.0" >> "$GITHUB_ENV" + shell: bash + - name: Install native task tool + uses: taiki-e/install-action@f7e5d7c961414b23f5b25b2da9294395d08513ad + with: + tool: just + - name: Cache immutable SDK archive + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/gerbil-sdk.tar.gz + key: parser-sdk-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.release-sha256 }} + - name: Extract immutable Gerbil host SDK + run: | + test -f "$RUNNER_TEMP/gerbil-sdk.tar.gz" || curl --fail --location --retry 3 --output "$RUNNER_TEMP/gerbil-sdk.tar.gz" \ + "https://github.com/tao3k/gerbil-bazel/releases/download/${{ inputs.release-tag }}/${{ inputs.release-tag }}.tar.gz" + echo '${{ inputs.release-sha256 }} '"$RUNNER_TEMP/gerbil-sdk.tar.gz" | shasum -a 256 --check + mkdir -p "$GERBIL_RELEASE_ROOT" + tar -xzf "$RUNNER_TEMP/gerbil-sdk.tar.gz" -C "$GERBIL_RELEASE_ROOT" --strip-components=1 + GERBIL_HOME="$GERBIL_RELEASE_ROOT/current" \ + GAMBOPT="~~=$GERBIL_RELEASE_ROOT/current,~~bin=$GERBIL_RELEASE_ROOT/current/bin,~~lib=$GERBIL_RELEASE_ROOT/current/lib" \ + "$GERBIL_RELEASE_ROOT/current/bin/gxi" tools/ci/bootstrap.ss \ + "$GERBIL_RELEASE_ROOT" '${{ inputs.release-os }}' '${{ inputs.release-arch }}' + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ef71f73..dd054e0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,226 +1,120 @@ name: CI - -on: +'on': push: branches: - main - pull_request: - + pull_request: null + workflow_dispatch: + inputs: + cpu-scope: + description: CPU qualification scope + type: choice + options: + - all + - capture + default: all permissions: contents: read - concurrency: - group: ci-${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + group: ci-${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true - jobs: - rust-rowan: - name: Rust/Rowan AOT runtime + rust-runtime: + name: Rust ${{ matrix.lane }} runs-on: ubuntu-latest timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + lane: + - contracts + - lints + - aot-performance steps: - name: Checkout gerbil-parser - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Verify parser semantics and operation costs + if: matrix.lane == 'contracts' + uses: leanprover/lean-action@f061402b660e0c34644504b324e830f2991d4865 # v1 + with: + lake-package-directory: formal + auto-config: false + build: true + test: false + lint: false + use-mathlib-cache: false + - name: Set up uv + if: matrix.lane == 'contracts' + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 + with: + version: '0.11.3' + - name: Verify publication model and evidence host controls + if: matrix.lane == 'contracts' + run: | + npm ci --ignore-scripts --prefix formal/quint + npm --prefix formal/quint test + uv run --project formal/evidence --locked python -m unittest discover -s formal/evidence -k HostControls -v + uv run --project formal/evidence --locked python -m unittest discover -s formal/evidence -p test_model_inputs.py -v + - name: Identify Rust toolchain + id: rust-version + run: echo "version=$(rustc -Vv | shasum -a 256 | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + - name: Restore Rust build cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: |- + ~/.cargo/registry + ~/.cargo/git + target + t/fixtures/runtime-record-assignments/target + t/fixtures/runtime-ascii-digit-run/target + key: parser-rust-v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.lane }}-${{ steps.rust-version.outputs.version }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }} - name: Verify Rust formatting run: cargo fmt --all -- --check - - - name: Verify ASP Rust source policy - run: cargo test -p gerbil-parser-build-support --test asp-rust-gate --locked -- --nocapture - + if: matrix.lane == 'contracts' - name: Verify Rust contracts - run: cargo test --workspace --locked - - - name: Verify downstream Cargo-only language consumer - run: cargo test --locked --manifest-path t/fixtures/rust-rowan-downstream/Cargo.toml - + run: cargo test --workspace --all-features --locked + if: matrix.lane == 'contracts' + - name: Verify record assignments Cargo consumer + run: cargo test --locked --manifest-path t/fixtures/runtime-record-assignments/Cargo.toml + if: matrix.lane == 'contracts' + - name: Verify external ASCII digit adapter + run: | + cargo fmt --manifest-path t/fixtures/runtime-ascii-digit-run/Cargo.toml -- --check + cargo test --locked --manifest-path t/fixtures/runtime-ascii-digit-run/Cargo.toml + if: matrix.lane == 'contracts' - name: Verify Rust lints - run: cargo clippy --workspace --all-targets --locked -- -D warnings - + run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + if: matrix.lane == 'lints' - name: Verify publishable Rust products - run: cargo package -p gerbil-parser-rowan --locked --allow-dirty - - - name: Verify generated-table Rowan engine red zones - run: cargo test --release -p gerbil-parser-rowan-scenarios --test performance_test -- --ignored --nocapture --test-threads=1 - - native-gerbil: - name: Native Gerbil parser and language packs (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 240 + run: cargo package -p gerbil-parser-artifact -p gerbil-parser-runtime --locked --allow-dirty + if: matrix.lane == 'lints' + - name: Verify generated-table AOT engine red zones + run: cargo test --release -p gerbil-parser-build-support --test performance_test -- --ignored --nocapture --test-threads=1 + if: matrix.lane == 'aot-performance' + scheme: + name: Scheme ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-latest gerbil-label: release-v0.19-2591dcd + release-os: linux + release-arch: x86_64 + release-tag: gerbil-v0.19-2591dcd9b7c6d2c4e9dd8611a17c5b1a5d82bbdb-linux-x86_64-portable-full-single-host-unlimited-patchf5cedd8168cb + release-sha256: 9844ba362fdf6f1c5e8a4411e462a466d52dee0d541f8e84f61091cec0e5e740 - os: macos-latest gerbil-label: release-v0.19-2591dcd - env: - GERBIL_RELEASE_ROOT: ${{ github.workspace }}/.ci/gerbil-release/${{ matrix.gerbil-label }} - steps: - - name: Checkout gerbil-parser - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Install Linux system dependencies - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y \ - build-essential \ - ca-certificates \ - git \ - jq \ - libsqlite3-dev \ - libssl-dev \ - openssl \ - pkg-config \ - sqlite3 \ - zlib1g-dev - - - name: Install macOS system dependencies - if: runner.os == 'macOS' - env: - HOMEBREW_NO_AUTO_UPDATE: 1 - HOMEBREW_NO_INSTALL_CLEANUP: 1 - run: | - brew list --formula ghostscript >/dev/null 2>&1 && brew unlink ghostscript || true - brew install gcc jq openssl pkg-config sqlite zlib - gcc_prefix="$(brew --prefix gcc)" - gcc_bin="$( - find "$gcc_prefix/bin" -maxdepth 1 -type f -name 'gcc-[0-9]*' \ - | awk -F- '{print $NF "\t" $0}' \ - | sort -n \ - | tail -n 1 \ - | cut -f2- - )" - test -x "$gcc_bin" - echo "GERBIL_GCC=$gcc_bin" >> "$GITHUB_ENV" - openssl_prefix="$(brew --prefix openssl@3 2>/dev/null || brew --prefix openssl)" - macos_major="$(sw_vers -productVersion | cut -d. -f1)" - echo "PKG_CONFIG_PATH=$openssl_prefix/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" >> "$GITHUB_ENV" - echo "MACOSX_DEPLOYMENT_TARGET=$macos_major.0" >> "$GITHUB_ENV" - - - name: Detect native build capacity - run: | - cores="$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)" - test "$cores" -gt 0 - echo "GERBIL_BUILD_CORES=$cores" >> "$GITHUB_ENV" - echo "Native Gerbil capacity: $cores cores" - - - name: Install immutable Gerbil v0.19 release - shell: bash - run: bash tools/ci/install-gerbil-v19-release.sh "$GERBIL_RELEASE_ROOT" - - - name: Restore Gerbil package cache - id: package-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - .gerbil/pkg - .gerbil/lib - .gerbil/bin - key: gerbil-parser-packages-v2-${{ runner.os }}-${{ runner.arch }}-${{ matrix.gerbil-label }}-${{ hashFiles('gerbil.pkg') }} - - - name: Install declared package dependencies - run: | - mkdir -p "$HOME/.gerbil/pkg" - gerbil pkg dir --add git.cons.io/mighty-gerbils/gerbil-directory - if [ "${{ steps.package-cache.outputs.cache-hit }}" = "true" ]; then - if ! gerbil env gerbil interactive -e '(begin (import :core/object-family/interface :asp-gerbil-scheme/building-api :asp-gerbil-scheme/src/benchmark/framework) (displayln "dependency-cache-ready"))'; then - gerbil deps --update - fi - else - gerbil deps --install - fi - gerbil env gerbil interactive -e '(begin (import :core/object-family/interface :asp-gerbil-scheme/building-api :asp-gerbil-scheme/src/benchmark/framework) (displayln "dependencies-ready"))' - - - name: Save Gerbil package cache - if: steps.package-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - .gerbil/pkg - .gerbil/lib - .gerbil/bin - key: gerbil-parser-packages-v2-${{ runner.os }}-${{ runner.arch }}-${{ matrix.gerbil-label }}-${{ hashFiles('gerbil.pkg') }} - - - name: Build parser and versioned language packs - run: | - gerbil clean - GERBIL_BUILD_VERBOSE=1 gerbil build - cc -std=c11 -Wall -Wextra -Werror \ - -Wno-unused-command-line-argument -fsyntax-only \ - -Iinclude t/native-ffi-header-smoke.c - - - name: Build optional native commands - run: | - gerbil env gxi build-gparse.ss compile - gerbil env gxi build-rust-rowan-aot.ss compile - - - name: Compile Scheme test support - run: | - gerbil env gxc t/line-structure-assertions.ss - gerbil env gxc \ - t/fixtures/rust-rowan-downstream/languages/records/v1/grammar.ss \ - t/fixtures/rust-rowan-downstream/languages/records/v1/parser.ss \ - t/fixtures/rust-rowan-downstream/languages/records/v1/fixtures.ss - - - name: Run native Gerbil tests - timeout-minutes: 15 - run: gerbil test -v 3 t/... languages/... - - - name: Verify Rust/Rowan AOT generation is current - run: | - arithmetic_generated="$(mktemp)" - gerbil env gxi generate-rust-rowan.ss "$arithmetic_generated" - rustfmt --edition 2024 "$arithmetic_generated" - cmp "$arithmetic_generated" rust/gerbil-parser-rowan-arithmetic/src/generated/mod.rs - - - name: Verify Scheme event strategy AOT fixture - run: | - event_generated="$(mktemp)" - grammar_generated="$(mktemp)" - fold_ir_generated="$(mktemp)" - outline_ir_generated="$(mktemp)" - gerbil env gxi t/generate-event-strategy-fixture.ss "$event_generated" - gerbil env gxi t/generate-event-strategy-grammar.ss "$grammar_generated" - gerbil env gxi t/generate-event-fold-ir.ss "$fold_ir_generated" - gerbil env gxi t/generate-outline-fold-ir.ss "$outline_ir_generated" - rustfmt --edition 2024 "$event_generated" "$grammar_generated" - cmp "$event_generated" rust/gerbil-parser-rowan/tests/unit/event_strategy_generated.rs - cmp "$grammar_generated" rust/gerbil-parser-rowan/tests/unit/event_strategy_grammar.rs - cmp "$fold_ir_generated" t/fixtures/event_fold.ir.json - cmp "$outline_ir_generated" t/fixtures/outline_fold.ir.json - - - name: Verify Scheme pure owned-string AOT fixture - run: | - owned_generated="$(mktemp)" - gerbil env gxi t/generate-owned-word-fixture.ss "$owned_generated" - cmp "$owned_generated" rust/gerbil-parser-rowan/tests/unit/owned_first_word_generated.rs - - - name: Verify downstream custom Rust/Rowan language pack - run: | - fixture="t/fixtures/rust-rowan-downstream" - generated="$(mktemp)" - clean_home="$RUNNER_TEMP/rowan-aot-home" - mkdir -p "$clean_home" - gerbil env gxc \ - "$fixture/languages/records/v1/grammar.ss" \ - "$fixture/languages/records/v1/parser.ss" \ - "$fixture/languages/records/v1/fixtures.ss" \ - "$fixture/languages/records/v1/parser-test.ss" - env -i \ - HOME="$clean_home" \ - PATH=/usr/bin:/bin \ - GERBIL_PATH="$PWD/.gerbil" \ - GERBIL_PARSER_ROWAN_AOT_LIB="$PWD/.gerbil/lib:$GERBIL_HOME/lib" \ - "$PWD/.gerbil/bin/gerbil-parser-rowan-aot" \ - "$fixture/languages/records/v1/grammar.ss" \ - "$generated" - rustfmt --edition 2024 "$generated" - cmp "$generated" "$fixture/src/generated/records_v1.rs" - - - name: Verify native command - run: | - gerbil env gparse help - gerbil env gparse check "1 + 2 * value" + release-os: darwin + release-arch: aarch64 + release-tag: gerbil-v0.19-2591dcd9b7c6d2c4e9dd8611a17c5b1a5d82bbdb-darwin-aarch64-gcc16-arm64-aot-tools-single-host-unlimited-patchf5cedd8168cb + release-sha256: 8d9c88434aed6301eaebb719bf52472f05c2368eafc630e8a9f1d67cc9895a21 + uses: ./.github/workflows/scheme.yml + with: + os: ${{ matrix.os }} + gerbil-label: ${{ matrix.gerbil-label }} + cpu-scope: ${{ inputs.cpu-scope || 'all' }} + release-os: ${{ matrix.release-os }} + release-arch: ${{ matrix.release-arch }} + release-tag: ${{ matrix.release-tag }} + release-sha256: ${{ matrix.release-sha256 }} diff --git a/.github/workflows/scheme.yml b/.github/workflows/scheme.yml new file mode 100644 index 00000000..f604823f --- /dev/null +++ b/.github/workflows/scheme.yml @@ -0,0 +1,851 @@ +name: Prepared Scheme qualification +'on': + workflow_call: + inputs: + cpu-scope: + required: false + type: string + default: all + os: + required: true + type: string + gerbil-label: + required: true + type: string + release-os: + required: true + type: string + release-arch: + required: true + type: string + release-tag: + required: true + type: string + release-sha256: + required: true + type: string +permissions: + contents: read +jobs: + prepare: + name: Prepare Scheme (${{ inputs.release-os }}) + runs-on: ${{ inputs.os }} + timeout-minutes: 120 + env: + GERBIL_RELEASE_ROOT: ${{ github.workspace }}/.ci/gerbil-release/${{ inputs.gerbil-label }} + GERBIL_SDK_SHA256: ${{ inputs.release-sha256 }} + GERBIL_PREPARED_HEAD: ${{ github.sha }} + steps: + - name: Checkout gerbil-parser + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Activate immutable Gerbil SDK + uses: ./.github/actions/gerbil-setup + with: + release-os: ${{ inputs.release-os }} + release-arch: ${{ inputs.release-arch }} + release-tag: ${{ inputs.release-tag }} + release-sha256: ${{ inputs.release-sha256 }} + - name: Restore Gerbil package cache + id: package-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + .gerbil/pkg + .gerbil/lib + .gerbil/bin + key: gerbil-parser-packages-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.release-sha256 }}-${{ hashFiles('gerbil.pkg') }} + - name: Install declared package dependencies + run: | + mkdir -p "$HOME/.gerbil/pkg" + gerbil pkg dir --add git.cons.io/mighty-gerbils/gerbil-directory + if [ "${{ steps.package-cache.outputs.cache-hit }}" = "true" ]; then + if ! gerbil env gerbil interactive -e '(begin (import :core/object-family/interface :asp-gerbil-scheme/building-api :asp-gerbil-scheme/src/benchmark/framework) (displayln "dependency-cache-ready"))'; then + gerbil deps --update + fi + else + gerbil deps --install + fi + gerbil env gerbil interactive -e '(begin (import :core/object-family/interface :asp-gerbil-scheme/building-api :asp-gerbil-scheme/src/benchmark/framework) (displayln "dependencies-ready"))' + - name: Save Gerbil package cache + if: steps.package-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + .gerbil/pkg + .gerbil/lib + .gerbil/bin + key: gerbil-parser-packages-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.release-sha256 }}-${{ hashFiles('gerbil.pkg') }} + - name: Build parser and language packs + run: | + gerbil clean + GERBIL_BUILD_VERBOSE=1 gerbil build + "${GERBIL_GCC:-gcc}" -std=c11 -Wall -Wextra -Werror \ + -fsyntax-only \ + -Iinclude t/ffi-header-smoke.c + - name: Build optional compiled commands + run: | + gerbil env gxi build-gparse.ss compile + gerbil env gxi build-rust-runtime-aot.ss compile + - name: Compile test exit support + run: gerbil env gxi build-test-driver.ss compile + - name: Compile qualification boundary + run: gerbil env gxi build-qualification.ss compile + - name: Compile reusable Scheme library objects + run: | + gerbil env gxi qualify.ss run --timeout 240 --idle-timeout 5 \ + --log .data/native-library-objects.log --require COMPILED-LIBRARY-OBJECTS-READY \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!)' \ + -e '(call-with-compiled-interface-trace (lambda () (load "build-language-conformance-link.ss") (eval (quote (main "library-objects"))))) (exit 0)' + - name: Restore conformance preparation + id: conformance-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .gerbil/conformance-store + key: parser-conformance-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.release-sha256 }}-${{ hashFiles('gerbil.pkg') }}-${{ github.sha }} + restore-keys: | + parser-conformance-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.release-sha256 }}-${{ hashFiles('gerbil.pkg') }}- + - name: Compile language conformance modules + run: | + GERBIL_BUILD_VERBOSE=1 GERBIL_PARSER_LR_TRACE=1 \ + gerbil env gxi qualify.ss run --timeout 240 --idle-timeout 5 \ + --log .data/language-conformance-build.log \ + -- gxi build-conformance.ss + - name: Save admitted conformance preparation + if: steps.conformance-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .gerbil/conformance-store + key: parser-conformance-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.release-sha256 }}-${{ hashFiles('gerbil.pkg') }}-${{ github.sha }} + - name: Compile Scheme test support + run: | + gerbil env gxc t/fixtures/tla-sany-differential/worker-control.ss \ + t/abi-datum-support.ss \ + t/line-structure-assertions.ss \ + t/benchmarks/lr-request-stack/derivation.ss \ + t/benchmarks/lr-request-stack/lookahead.ss \ + t/scenarios/performance/selective-glr/scenario.ss \ + t/benchmarks/lr-request-stack/benchmark.ss \ + t/benchmarks/gql/runtime/actors.ss \ + t/benchmarks/lr-request-stack/capture-reference.ss \ + t/benchmarks/capture-prefix/controls.ss \ + t/support/qualification/benchmark-qos.ss + if [ "$RUNNER_OS" = macOS ]; then + gerbil env gxc -ld-options "-Wl,-undefined,dynamic_lookup" \ + t/benchmarks/gql/runtime/compiled-timing.ss + else + gerbil env gxc t/benchmarks/gql/runtime/compiled-timing.ss + fi + gerbil env gxc t/header-delimiter-test.ss \ + t/structured-lexical-cases.ss \ + t/structured-lexical-test.ss \ + t/module-source-cases.ss \ + t/module-source-test.ss \ + t/fixtures/shared-scanner/bash.ss + - name: Prepare gxtest source owners + run: | + gerbil env gxi qualify.ss run --timeout 240 --idle-timeout 5 \ + --log .data/gxtest-source-prepare.log \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!)' \ + -e '(call-with-compiled-interface-trace (lambda () (load "build-language-conformance.ss") (eval (quote (prepare-gxtest!))))) (exit 0)' + - name: Compile FFI Scheme fixtures once + run: GERBIL_BUILD_VERBOSE=1 gerbil env gxi build-ffi-tests.ss compile + - name: Prepare standalone language library once + run: |- + library="$PWD/.gerbil/ci/libparsernative.so" + if [ "$RUNNER_OS" = macOS ]; then library="$PWD/.gerbil/ci/libparsernative.dylib"; fi + mkdir -p .gerbil/ci + gerbil env gxi qualify.ss shared-library --language-module gerbil-parser/t/fixtures/shared-scanner/records-engine-abi --output "$library" + - name: Pack exact-head Scheme products + run: bash tools/ci/prepared-products.sh pack "$RUNNER_TEMP/scheme-products.tar.gz" + - name: Publish prepared Scheme products + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: scheme-${{ inputs.release-os }}-${{ inputs.release-arch }}-${{ github.sha }} + path: ${{ runner.temp }}/scheme-products.tar.gz + retention-days: 2 + compression-level: 0 + - name: Upload preparation logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: logs-${{ inputs.release-os }}-prepare-${{ github.sha }} + path: .data/**/*.log + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + semantics: + name: semantics (${{ inputs.release-os }}) + needs: prepare + runs-on: ${{ inputs.os }} + timeout-minutes: 45 + env: + GERBIL_RELEASE_ROOT: ${{ github.workspace }}/.ci/gerbil-release/${{ inputs.gerbil-label }} + GERBIL_SDK_SHA256: ${{ inputs.release-sha256 }} + GERBIL_PREPARED_HEAD: ${{ github.sha }} + steps: + - name: Checkout gerbil-parser + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + - name: Activate immutable Gerbil SDK + uses: ./.github/actions/gerbil-setup + with: + release-os: ${{ inputs.release-os }} + release-arch: ${{ inputs.release-arch }} + release-tag: ${{ inputs.release-tag }} + release-sha256: ${{ inputs.release-sha256 }} + - name: Download exact-head Scheme products + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: scheme-${{ inputs.release-os }}-${{ inputs.release-arch }}-${{ github.sha }} + path: ${{ runner.temp }}/scheme-products + - name: Admit prepared Scheme products + run: bash tools/ci/prepared-products.sh restore "$RUNNER_TEMP/scheme-products/scheme-products.tar.gz" + - name: Verify native process contracts + run: just test-process-contracts + - name: Verify capture prefix field and tail parity + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/capture-prefix-controls.log --require '^CAPTURE-PREFIX-CONTROLS-OK productions=11$' \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/benchmarks/capture-prefix/controls") (preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") (eval (quote (import :gerbil-parser/t/benchmarks/capture-prefix/controls :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process))) (eval (quote (verify-capture-prefix!))) (eval (quote (test-child-process-exit! 0)))' + - name: Verify callback history commit admission + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/history-commit-controls.log --require HISTORY-COMMIT-CONTROLS-OK \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "scripts/tests/history-commit-test.ss")' scripts/tests/history-commit-test.ss + - name: Verify official module loading and source invalidation + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/module-loading-controls.log \ + --require '^MODULE-LOADING-CONTROLS-OK$' \ + -- gxi t/benchmarks/module-loading/controls.ss + - name: Verify generated arithmetic Scheme LR is current + run: gerbil env gxi t/benchmarks/arithmetic-scale/generate-fused-lr.ss check src/compiler/arithmetic-drive.ss + - name: Verify generic POO Scheme reductions are current + run: | + gerbil env gxi scripts/generate-reductions.ss check :gerbil-parser/languages/hcl/grammar src/compiler/hcl-reductions.ss + gerbil env gxi scripts/generate-reductions.ss check :gerbil-parser/languages/arithmetic/grammar t/fixtures/arithmetic-reductions.ss + - name: Verify generated HCL recursive parser is current + run: gerbil env gxi scripts/generate-source-parser.ss check :gerbil-parser/languages/hcl/grammar src/compiler/hcl-source.ss + - name: Verify Scheme preparation and test ownership controls + run: | + gerbil env gxi scripts/tests/process-qualification-test.ss + gerbil env gxi scripts/tests/compiler-process-test.ss + gerbil env gxi scripts/tests/preparation-test.ss + gerbil env gxi scripts/tests/test-plan-test.ss + gerbil env gxi test-processes.ss conformance + - name: Verify Scheme DSL and POO declaration closure + id: dsl-research + run: gerbil env gxi test-processes.ss research .data/dsl-research + - name: Diagnose failed Ubuntu semantic process + if: ${{ failure() && runner.os == 'Linux' && steps.dsl-research.outcome == 'failure' }} + run: | + if ! command -v gdb; then + sudo apt-get update + sudo apt-get install -y gdb + fi + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/dsl-research-gdb.log \ + -- gdb --nx --batch --return-child-result \ + -ex run -ex 'thread apply all bt' \ + --args .gerbil/bin/gerbil-parser-conformance semantic + - name: Verify Source caller survives natural collection + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/source-callback-gc.log --require SOURCE-CALLBACK-GC-OK \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (call-with-compiled-interface-trace (lambda () (eval (quote (import :gerbil-parser/t/benchmarks/source-edits/benchmark))) (eval (quote (verify-callback-gc)))))' + - name: Verify shared-runtime Worker controls + run: gerbil env gxi test-processes.ss workers + - name: Run Scheme tests + timeout-minutes: 15 + run: | + GERBIL_PATH="$PWD/.gerbil" GERBIL_LOADPATH="$PWD" \ + GAMBOPT=max-heap=3G,debug=q GERBIL_PARSER_LR_TRACE=1 \ + gerbil env gxi t/fixtures/tla-sany-differential/test-suite.ss --remaining-conformance t languages + - name: Verify native command + run: | + gerbil env gparse help + gerbil env gparse check "1 + 2 * value" + - name: Upload qualification logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: logs-${{ inputs.release-os }}-semantics-suite-${{ github.sha }} + path: .data/**/*.log + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + generation: + name: generation (${{ inputs.release-os }}) + needs: prepare + runs-on: ${{ inputs.os }} + timeout-minutes: 45 + env: + GERBIL_RELEASE_ROOT: ${{ github.workspace }}/.ci/gerbil-release/${{ inputs.gerbil-label }} + GERBIL_SDK_SHA256: ${{ inputs.release-sha256 }} + GERBIL_PREPARED_HEAD: ${{ github.sha }} + steps: + - name: Checkout gerbil-parser + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Activate immutable Gerbil SDK + uses: ./.github/actions/gerbil-setup + with: + release-os: ${{ inputs.release-os }} + release-arch: ${{ inputs.release-arch }} + release-tag: ${{ inputs.release-tag }} + release-sha256: ${{ inputs.release-sha256 }} + - name: Download exact-head Scheme products + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: scheme-${{ inputs.release-os }}-${{ inputs.release-arch }}-${{ github.sha }} + path: ${{ runner.temp }}/scheme-products + - name: Admit prepared Scheme products + run: bash tools/ci/prepared-products.sh restore "$RUNNER_TEMP/scheme-products/scheme-products.tar.gz" + - name: Verify header-derived delimiter Rust AOT regeneration + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/header-delimiter-generation.log \ + --require HEADER-DELIMITER-HL7-GENERATED --require HEADER-DELIMITER-RECORD-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "t/generate-header-delimiters.ss")' \ + t/generate-header-delimiters.ss /tmp/hl7-header.rs /tmp/header-record.rs + rustfmt --edition 2024 /tmp/hl7-header.rs /tmp/header-record.rs + cmp /tmp/hl7-header.rs rust/gerbil-parser-runtime/tests/fixtures/generated/hl7_header.rs + cmp /tmp/header-record.rs rust/gerbil-parser-runtime/tests/fixtures/generated/header_record.rs + - name: Verify strict quoted profile Rust AOT regeneration + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/structured-lexical-generation.log \ + --require STRUCTURED-LEXICAL-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/structured-lexical-test")' \ + t/generate-structured-lexical.ss /tmp/structured-lexical.rs + rustfmt --edition 2024 /tmp/structured-lexical.rs + cmp /tmp/structured-lexical.rs rust/gerbil-parser-runtime/tests/fixtures/generated/structured_lexical.rs + gerbil env gxi t/generate-scheme-alphabetic-class.ss /tmp/scheme-alphabetic.rs + rustfmt --edition 2024 /tmp/scheme-alphabetic.rs + cmp /tmp/scheme-alphabetic.rs rust/gerbil-parser-runtime/src/engine/unicode_alphabetic.rs + gerbil env gxi t/generate-scheme-numeric-class.ss /tmp/scheme-numeric.rs + rustfmt --edition 2024 /tmp/scheme-numeric.rs + cmp /tmp/scheme-numeric.rs rust/gerbil-parser-runtime/src/engine/unicode_numeric.rs + gerbil env gxi t/generate-quoted-string-profile.ss /tmp/quoted-profile.rs + rustfmt --edition 2024 /tmp/quoted-profile.rs + cmp /tmp/quoted-profile.rs rust/gerbil-parser-runtime/tests/fixtures/quoted_profile_generated.rs + gerbil env gxi t/generate-text-profile.ss /tmp/fhirpath-profile.rs + rustfmt --edition 2024 /tmp/fhirpath-profile.rs + cmp /tmp/fhirpath-profile.rs rust/gerbil-parser-runtime/tests/fixtures/generated/fhirpath_profile.rs + - name: Verify prepared module framing Rust AOT regeneration + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/module-source-generation.log --require MODULE-SOURCE-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/module-source-test")' \ + t/generate-module-source.ss /tmp/module-source.rs + rustfmt --edition 2024 /tmp/module-source.rs + cmp /tmp/module-source.rs rust/gerbil-parser-runtime/tests/fixtures/generated/module_source.rs + gerbil env gxi t/generate-scheme-whitespace-class.ss /tmp/scheme-whitespace.rs + rustfmt --edition 2024 /tmp/scheme-whitespace.rs + cmp /tmp/scheme-whitespace.rs rust/gerbil-parser-runtime/src/engine/unicode_whitespace.rs + - name: Verify shared result projection Rust AOT regeneration + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/result-projections-generation.log --require RESULT-PROJECTIONS-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "t/generate-result-projections.ss")' \ + t/generate-result-projections.ss /tmp/result-projections.rs + rustfmt --edition 2024 /tmp/result-projections.rs + cmp /tmp/result-projections.rs rust/gerbil-parser-runtime/tests/fixtures/generated/result_projections.rs + - name: Verify ordered result catalog Rust AOT regeneration + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/ordered-results-generation.log --require ORDERED-RESULTS-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "t/generate-ordered-results.ss")' \ + t/generate-ordered-results.ss /tmp/ordered-results.rs + rustfmt --edition 2024 /tmp/ordered-results.rs + cmp /tmp/ordered-results.rs rust/gerbil-parser-runtime/tests/fixtures/generated/ordered_results.rs + - name: Verify shared Command Source Rust AOT regeneration + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/command-source-generation.log --require COMMAND-SOURCE-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "t/generate-command-source.ss")' \ + t/generate-command-source.ss /tmp/command-source.rs /tmp/command-source.bin + rustfmt --edition 2024 /tmp/command-source.rs + cmp /tmp/command-source.rs rust/gerbil-parser-runtime/tests/fixtures/generated/command_source.rs + cmp /tmp/command-source.bin rust/gerbil-parser-runtime/tests/fixtures/generated/command_source.bin + - name: Verify shared Word and Binding Rust AOT regeneration + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/word-parts-generation.log --require WORD-PARTS-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "t/generate-word-parts.ss")' \ + t/generate-word-parts.ss /tmp/word-parts.rs + rustfmt --edition 2024 /tmp/word-parts.rs + cmp /tmp/word-parts.rs rust/gerbil-parser-runtime/tests/fixtures/generated/word_parts.rs + - name: Verify shared region recognition Rust AOT regeneration + run: | + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/region-boundaries-generation.log --require REGION-BOUNDARIES-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "t/generate-region-boundaries.ss")' \ + t/generate-region-boundaries.ss /tmp/region-boundaries.rs + rustfmt --edition 2024 /tmp/region-boundaries.rs + cmp /tmp/region-boundaries.rs rust/gerbil-parser-runtime/tests/fixtures/generated/region_boundaries.rs + - name: Verify shared scanner and contextual Rust AOT regeneration + run: | + gerbil env gxi t/fixtures/shared-scanner/generate.ss /tmp/shared-scanner.rs + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/bash-scanner-generation.log --require BASH-SCANNER-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/fixtures/shared-scanner/bash")' \ + t/fixtures/shared-scanner/generate-bash.ss /tmp/bash-scanner.rs + rustfmt --edition 2024 /tmp/bash-scanner.rs + cmp /tmp/bash-scanner.rs rust/gerbil-parser-runtime/tests/fixtures/generated/bash_scanner.rs + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log /tmp/binding-names-generation.log --require BINDING-NAMES-RUST-GENERATED \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "t/generate-binding-names.ss")' \ + t/generate-binding-names.ss /tmp/binding-names.rs + rustfmt --edition 2024 /tmp/binding-names.rs + cmp /tmp/binding-names.rs rust/gerbil-parser-runtime/tests/fixtures/generated/binding_names.rs + gerbil env gxi t/fixtures/shared-scanner/generate-contextual.ss /tmp/records-contextual.rs + gerbil env gxi t/fixtures/shared-scanner/generate-artifact.ss /tmp/artifact.rs + rustfmt --edition 2024 /tmp/artifact.rs + cmp /tmp/artifact.rs rust/gerbil-parser-runtime/tests/fixtures/artifact_generated.rs + rustfmt --edition 2024 /tmp/shared-scanner.rs /tmp/records-contextual.rs + cmp /tmp/shared-scanner.rs rust/gerbil-parser-runtime/tests/fixtures/shared_scanner_generated.rs + cmp /tmp/records-contextual.rs rust/gerbil-parser-runtime/tests/fixtures/records_contextual_generated.rs + gerbil env gxi t/fixtures/shared-scanner/generate-abi.ss /tmp/records-abi.ss /tmp/records-abi.h + cmp /tmp/records-abi.ss t/fixtures/shared-scanner/records-abi.ss + cmp /tmp/records-abi.h t/fixtures/shared-scanner/records-abi.h + gerbil env gxc t/fixtures/shared-scanner/records.ss + - name: Verify Rust AOT generation is current + run: | + arithmetic_generated="$(mktemp)" + gerbil env gxi generate-rust-runtime.ss "$arithmetic_generated" + rustfmt --edition 2024 "$arithmetic_generated" + cmp "$arithmetic_generated" build-support/gerbil-parser-build-support/tests/fixtures/generated/arithmetic.rs + - name: Verify Scheme event strategy AOT fixture + run: | + event_generated="$(mktemp)" + grammar_generated="$(mktemp)" + fold_ir_generated="$(mktemp)" + outline_ir_generated="$(mktemp)" + gerbil env gxi t/generate-event-strategy-fixture.ss "$event_generated" + gerbil env gxi t/generate-event-strategy-grammar.ss "$grammar_generated" + gerbil env gxi t/generate-event-fold-ir.ss "$fold_ir_generated" + gerbil env gxi t/generate-outline-fold-ir.ss "$outline_ir_generated" + rustfmt --edition 2024 "$event_generated" "$grammar_generated" + cmp "$event_generated" rust/gerbil-parser-runtime/tests/unit/event_strategy_generated.rs + cmp "$grammar_generated" rust/gerbil-parser-runtime/tests/unit/event_strategy_grammar.rs + cmp "$fold_ir_generated" t/fixtures/event_fold.ir.json + cmp "$outline_ir_generated" t/fixtures/outline_fold.ir.json + - name: Verify Scheme pure owned-string AOT fixture + run: | + owned_generated="$(mktemp)" + gerbil env gxi t/generate-owned-word-fixture.ss "$owned_generated" + cmp "$owned_generated" rust/gerbil-parser-runtime/tests/unit/owned_first_word_generated.rs + - name: Verify record assignments Rust language pack + run: | + fixture="t/fixtures/runtime-record-assignments" + generated="$(mktemp)" + clean_home="$RUNNER_TEMP/runtime-aot-home" + mkdir -p "$clean_home" + gerbil env gxc \ + "$fixture/languages/records/grammar.ss" \ + "$fixture/languages/records/fixtures.ss" \ + "$fixture/languages/records/parser.ss" \ + "$fixture/languages/records/parser-test.ss" + env -i \ + HOME="$clean_home" \ + PATH=/usr/bin:/bin \ + GERBIL_PATH="$PWD/.gerbil" \ + GERBIL_PARSER_RUNTIME_AOT_LIB="$PWD/.gerbil/lib:$GERBIL_HOME/lib" \ + "$PWD/.gerbil/bin/gerbil-parser-runtime-aot" \ + "$fixture/languages/records/grammar.ss" \ + "$generated" + rustfmt --edition 2024 "$generated" + cmp "$generated" "$fixture/src/generated/records.rs" + - name: Upload qualification logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: logs-${{ inputs.release-os }}-generation-suite-${{ github.sha }} + path: .data/**/*.log + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + ffi: + name: ffi (${{ inputs.release-os }}) + needs: prepare + runs-on: ${{ inputs.os }} + timeout-minutes: 45 + env: + GERBIL_RELEASE_ROOT: ${{ github.workspace }}/.ci/gerbil-release/${{ inputs.gerbil-label }} + GERBIL_SDK_SHA256: ${{ inputs.release-sha256 }} + GERBIL_PREPARED_HEAD: ${{ github.sha }} + GERBIL_PARSER_NATIVE_HOST_LIBRARY: ${{ github.workspace }}/.gerbil/ci/libparsernative.${{ inputs.release-os == 'darwin' && 'dylib' || 'so' }} + steps: + - name: Checkout gerbil-parser + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Activate immutable Gerbil SDK + uses: ./.github/actions/gerbil-setup + with: + release-os: ${{ inputs.release-os }} + release-arch: ${{ inputs.release-arch }} + release-tag: ${{ inputs.release-tag }} + release-sha256: ${{ inputs.release-sha256 }} + - name: Download exact-head Scheme products + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: scheme-${{ inputs.release-os }}-${{ inputs.release-arch }}-${{ github.sha }} + path: ${{ runner.temp }}/scheme-products + - name: Admit prepared Scheme products + run: bash tools/ci/prepared-products.sh restore "$RUNNER_TEMP/scheme-products/scheme-products.tar.gz" + - name: Identify Rust toolchain + id: rust-version + run: echo "version=$(rustc -Vv | shasum -a 256 | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + - name: Cache Rust FFI consumer + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: |- + ~/.cargo/registry + ~/.cargo/git + target + key: parser-ffi-v1-${{ runner.os }}-${{ runner.arch }}-ffi-${{ inputs.release-sha256 }}-${{ steps.rust-version.outputs.version }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }} + - name: Verify every language artifact ABI contract + env: + GERBIL_PARSER_LR_TRACE: '1' + run: | + GERBIL_LOADPATH="$PWD/.." gerbil env gerbil test -v 5 t/language-abi-alignment-test.ss + - name: Compile and call public C ABI + run: | + GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" \ + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/native-abi.log --require NATIVE-ABI-OK \ + --require 'ABI-RUNTIME-GENERATED-MATCH calls=3' \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/fixtures/ffi/abi-probe") (preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process")' \ + -e '(import :gerbil-parser/t/fixtures/ffi/abi-probe :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process) (main) (test-child-process-exit! 0)' + - name: Verify Rust ownership over the C ABI + run: | + CARGO_TARGET_DIR="$PWD/target" \ + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 30 \ + --log .data/rust-native-build.log \ + -- cargo build --locked --manifest-path t/fixtures/rust-language-abi/Cargo.toml + GERBIL_PARSER_RUST_NATIVE_ARCHIVE="$PWD/target/debug/librust_language_abi_fixture.a" \ + GERBIL_BUILD_VERBOSE=1 gerbil env gxi qualify.ss run \ + --timeout 90 --idle-timeout 30 --log .data/rust-native-link.log \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-test-imports "build-rust-ffi-tests.ss")' \ + -e '(call-with-compiled-interface-trace (lambda () (load "build-rust-ffi-tests.ss") (eval (quote (main "compile")))))' + GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" \ + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/rust-native-ownership.log --require RUST-NATIVE-OK \ + --require 'RUST-NATIVE-OWNERSHIP-OK handles=2 results=113' \ + --require RUST-NATIVE-100-CALLS \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/fixtures/ffi/rust-language-probe")' \ + -e '(import :gerbil-parser/t/fixtures/ffi/rust-language-probe) (main)' + - name: Verify language-independent C ABI and ASP performance + run: | + GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" \ + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/language-abi.log --require LANGUAGE-ABI-OK \ + --require LANGUAGE-ABI-100-CALLS \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/fixtures/ffi/language-probe")' \ + -e '(import :gerbil-parser/t/fixtures/ffi/language-probe) (main)' + - name: Verify independent C and Rust lifecycle + run: GERBIL_BUILD_CORES=4 GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" gerbil env gxi qualify.ss host + - name: Upload qualification logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: logs-${{ inputs.release-os }}-ffi-suite-${{ github.sha }} + path: .data/**/*.log + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + corpus: + name: corpus (${{ inputs.release-os }}) + needs: prepare + runs-on: ${{ inputs.os }} + timeout-minutes: 45 + env: + GERBIL_RELEASE_ROOT: ${{ github.workspace }}/.ci/gerbil-release/${{ inputs.gerbil-label }} + GERBIL_SDK_SHA256: ${{ inputs.release-sha256 }} + GERBIL_PREPARED_HEAD: ${{ github.sha }} + steps: + - name: Checkout gerbil-parser + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Activate immutable Gerbil SDK + uses: ./.github/actions/gerbil-setup + with: + release-os: ${{ inputs.release-os }} + release-arch: ${{ inputs.release-arch }} + release-tag: ${{ inputs.release-tag }} + release-sha256: ${{ inputs.release-sha256 }} + - name: Download exact-head Scheme products + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: scheme-${{ inputs.release-os }}-${{ inputs.release-arch }}-${{ github.sha }} + path: ${{ runner.temp }}/scheme-products + - name: Admit prepared Scheme products + run: bash tools/ci/prepared-products.sh restore "$RUNNER_TEMP/scheme-products/scheme-products.tar.gz" + - name: Compare TLA+ candidate structure with pinned SANY + run: | + jar="$RUNNER_TEMP/tla2tools-v1.7.4.jar" + curl --fail --location --retry 3 --silent --show-error \ + --output "$jar" \ + https://github.com/tlaplus/tlaplus/releases/download/v1.7.4/tla2tools.jar + GERBIL_PATH="$PWD/.gerbil" GERBIL_LOADPATH="$PWD" \ + GAMBOPT=max-heap=1G,debug=q GERBIL_PARSER_LR_TRACE=1 \ + gerbil env gxi qualify.ss run --timeout 600 --idle-timeout 5 --log .data/tla-source.log -- \ + gerbil env gxi \ + -e '(load "t/fixtures/tla-sany-differential/preload.ss") (preload-module "gerbil-parser/languages/tla-plus/parser")' \ + t/fixtures/tla-sany-differential/compare.ss "$jar" + - name: Verify complete pinned TLA+ Examples corpus + run: | + corpus="$RUNNER_TEMP/tlaplus-examples" + git init "$corpus" + git -C "$corpus" remote add origin https://github.com/tlaplus/Examples.git + git -C "$corpus" fetch --depth=1 origin ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10 + git -C "$corpus" checkout --detach FETCH_HEAD + GERBIL_PATH="$PWD/.gerbil" GERBIL_LOADPATH="$PWD" \ + GAMBOPT=max-heap=1G,debug=q GERBIL_PARSER_LR_TRACE=1 \ + gerbil env gxi qualify.ss run --timeout 600 --idle-timeout 5 --log .data/tla-source.log -- \ + gerbil env gxi \ + -e '(load "t/fixtures/tla-sany-differential/preload.ss") (preload-module "gerbil-parser/languages/tla-plus/parser")' \ + t/fixtures/tla-sany-differential/corpus.ss "$corpus" + - name: Upload qualification logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: logs-${{ inputs.release-os }}-corpus-suite-${{ github.sha }} + path: .data/**/*.log + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + if: inputs.release-os == 'linux' + diagnostics: + name: diagnostics (${{ inputs.release-os }}) + needs: prepare + runs-on: ${{ inputs.os }} + timeout-minutes: 45 + env: + GERBIL_RELEASE_ROOT: ${{ github.workspace }}/.ci/gerbil-release/${{ inputs.gerbil-label }} + GERBIL_SDK_SHA256: ${{ inputs.release-sha256 }} + GERBIL_PREPARED_HEAD: ${{ github.sha }} + steps: + - name: Checkout gerbil-parser + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Activate immutable Gerbil SDK + uses: ./.github/actions/gerbil-setup + with: + release-os: ${{ inputs.release-os }} + release-arch: ${{ inputs.release-arch }} + release-tag: ${{ inputs.release-tag }} + release-sha256: ${{ inputs.release-sha256 }} + - name: Download exact-head Scheme products + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: scheme-${{ inputs.release-os }}-${{ inputs.release-arch }}-${{ github.sha }} + path: ${{ runner.temp }}/scheme-products + - name: Admit prepared Scheme products + run: bash tools/ci/prepared-products.sh restore "$RUNNER_TEMP/scheme-products/scheme-products.tar.gz" + - name: Verify matched Source edit calls and phase observations + run: | + GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" \ + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/source-edit-cost.log --require SOURCE-EDIT-BENCHMARK-OK \ + -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (call-with-compiled-interface-trace (lambda () (eval (quote (import :gerbil-parser/t/benchmarks/source-edits/benchmark))) (eval (quote (main "8" "32")))))' + - name: Report GQL native component and actor costs + run: | + gerbil env gxi qualify.ss local \ + --prepared-diagnostics --suite gql-profile --suite gql-actors \ + --gerbil-path .gerbil --log-dir .data/gql-diagnostics + - name: Verify shared LR complete request matrix + run: | + GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" \ + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/lr-request-stack.log --require LR-REQUEST-STACK-OK \ + -- gxi -e ' + (load "t/fixtures/tla-sany-differential/preload.ss") + (preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") + (preload-module "gerbil-parser/t/benchmarks/lr-request-stack/benchmark") + (gerbil-parser/t/benchmarks/lr-request-stack/benchmark#benchmark-lr-request-stack 20 20) + (gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process#test-child-process-exit! 0)' + - name: Upload qualification logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: logs-${{ inputs.release-os }}-diagnostics-suite-${{ github.sha }} + path: .data/**/*.log + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + if: github.event_name == 'workflow_dispatch' + cpu: + name: CPU ${{ matrix.lane }} + if: github.event_name == 'workflow_dispatch' && inputs.release-os == 'linux' + needs: prepare + runs-on: ${{ inputs.os }} + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + lane: ${{ fromJSON(inputs.cpu-scope == 'capture' && '["capture"]' || '["scanner","events","engine","ranked","utf8","tokio","module-generation","module-preload","selector-input","capture"]') }} + env: + GERBIL_RELEASE_ROOT: ${{ github.workspace }}/.ci/gerbil-release/${{ inputs.gerbil-label }} + GERBIL_SDK_SHA256: ${{ inputs.release-sha256 }} + GERBIL_PREPARED_HEAD: ${{ github.sha }} + GAMBOPT: max-heap=3G,debug=q + steps: + - name: Checkout gerbil-parser + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Activate immutable Gerbil SDK + uses: ./.github/actions/gerbil-setup + with: + release-os: ${{ inputs.release-os }} + release-arch: ${{ inputs.release-arch }} + release-tag: ${{ inputs.release-tag }} + release-sha256: ${{ inputs.release-sha256 }} + - name: Download exact-head Scheme products + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: scheme-${{ inputs.release-os }}-${{ inputs.release-arch }}-${{ github.sha }} + path: ${{ runner.temp }}/scheme-products + - name: Admit prepared Scheme products + run: bash tools/ci/prepared-products.sh restore "$RUNNER_TEMP/scheme-products/scheme-products.tar.gz" + - name: Identify Rust toolchain + id: rust-version + run: echo "version=$(rustc -Vv | shasum -a 256 | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + if: matrix.lane == 'tokio' + - name: Cache Rust FFI consumer + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: |- + ~/.cargo/registry + ~/.cargo/git + target + key: parser-ffi-v1-${{ runner.os }}-${{ runner.arch }}-cpu-${{ inputs.release-sha256 }}-${{ steps.rust-version.outputs.version }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }} + if: matrix.lane == 'tokio' + - name: Select an allowed CPU + run: |- + allowed=$(awk '/^Cpus_allowed_list:/ {print $2}' /proc/self/status) + processor=${allowed%%[-,]*} + test -n "$processor" + echo "CPU_PROOF_PROCESSOR=$processor" >> "$GITHUB_ENV" + - name: Prove shared Scheme scanner CPU improvement + env: + GAMBOPT: max-heap=3G,debug=q + run: | + # Affinity is inherited by each Scheme child; GC and CPU remain charged. + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/scanner-cpu-proof.log --require SCANNER-CPU-PROOF-OK \ + -- gxi t/benchmarks/ranked-literal/cpu-proof.ss bounded 20 2000000 + if: matrix.lane == 'scanner' + - name: Prove complete 64-record event publication CPU improvement + env: + GAMBOPT: max-heap=3G,debug=q + run: | + # Affinity is inherited by each Scheme child; GC and CPU remain charged. + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + for unicode in false true; do + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log ".data/ffi-event-records-${unicode}-cpu.log" --require ABI-EVENT-CPU-PROOF-OK \ + -- gxi t/benchmarks/language-abi/utf8-proof.ss 20 256 64 records "$unicode" events + done + if: matrix.lane == 'events' + - name: Prove scanner and LR CPU improvement on complete 64-record requests + env: + GAMBOPT: max-heap=3G,debug=q + run: | + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + for unicode in false true; do + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log ".data/record-engine-${unicode}-cpu.log" --require ENGINE-CPU-PROOF-OK \ + -- gxi t/benchmarks/language-abi/engine-proof.ss combined "$unicode" + done + if: matrix.lane == 'engine' + - name: Prove complete ranked scanning and public parser CPU improvement + env: + GAMBOPT: max-heap=3G,debug=q + run: | + # Affinity is inherited by each Scheme child; GC and CPU remain charged. + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/ranked-parser-long-cpu-proof.log --require SCANNER-CPU-PROOF-OK \ + -- gxi t/benchmarks/ranked-literal/cpu-proof.ss parser 20 256 layout 48 + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/ranked-parser-cpu-proof.log --require SCANNER-CPU-PROOF-OK \ + -- gxi t/benchmarks/ranked-literal/cpu-proof.ss parser 20 1024 layout 12 + for scenario in keyword identifier; do + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log ".data/ranked-${scenario}-cpu-proof.log" --require SCANNER-CPU-PROOF-OK \ + -- gxi t/benchmarks/ranked-literal/cpu-proof.ss lexical 20 250000 "$scenario" + done + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/ranked-literal-prefix-cpu-proof.log --require SCANNER-CPU-PROOF-OK \ + -- gxi t/benchmarks/ranked-literal/cpu-proof.ss lexical 20 750000 literal-prefix + if: matrix.lane == 'ranked' + - name: Prove upstream FFI UTF-8 complete request CPU improvement + env: + GAMBOPT: max-heap=3G,debug=q + run: | + # Affinity is inherited by each Scheme child; GC and CPU remain charged. + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + for unicode in false true; do + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log ".data/ffi-utf8-${unicode}-cpu.log" --require ABI-UTF8-CPU-PROOF-OK \ + -- gxi t/benchmarks/language-abi/utf8-proof.ss 20 1024 128 identifier "$unicode" + done + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/ffi-utf8-records-cpu.log --require ABI-UTF8-CPU-REGRESSION-OK \ + -- gxi t/benchmarks/language-abi/utf8-proof.ss 20 256 64 records false regression + if: matrix.lane == 'utf8' + - name: Prepare exact-head selector input reference + if: matrix.lane == 'selector-input' + run: | + git fetch --depth=1 origin 7c184acd77b9fcdfec0537be11129b3a8df6616b + just prepare-capture-reference 7c184acd77b9fcdfec0537be11129b3a8df6616b + - name: Prove additional full request selector CPU improvement + if: matrix.lane == 'selector-input' + run: | + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + just benchmark-capture-prefix + - name: Compile matched current capture owners + if: matrix.lane == 'capture' + run: gerbil env gxi t/benchmarks/capture-prefix/build-reference.ss src/runtime/lr-parser/recognition.ss .gerbil/lib selector-input + - name: Prepare captured LR allocation reference + if: matrix.lane == 'capture' + run: | + git fetch --depth=1 origin 459ec36e1961f3d8d95ce4ebec13843d164a4bdf + just prepare-capture-reference 459ec36e1961f3d8d95ce4ebec13843d164a4bdf .data/capture/reference selector-input + - name: Qualify captured LR complete request CPU twice + if: matrix.lane == 'capture' + run: | + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + for trial in 1 2; do + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log ".data/capture-${trial}-cpu.log" --require '^CAPTURE-COMPLETE-CPU-PROOF-OK$' \ + -- gxi t/benchmarks/capture-prefix/benchmark.ss .data/capture/reference/lib selector-input + done + - name: Prove complete Rust generation CPU improvement + if: matrix.lane == 'module-generation' + run: | + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/module-generation-cpu.log --require '^MODULE-LOADING-CPU-PROOF-OK generation$' \ + -- gxi t/benchmarks/module-loading/benchmark.ss generation + - name: Prove warm runtime preparation CPU improvement + if: matrix.lane == 'module-preload' + run: | + taskset -pc "$CPU_PROOF_PROCESSOR" "$$" + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/module-preload-cpu.log --require '^MODULE-LOADING-CPU-PROOF-OK preload$' \ + -- gxi t/benchmarks/module-loading/benchmark.ss preload + - name: Prove Rust Tokio complete 64-record requests + if: matrix.lane == 'tokio' + env: + GERBIL_PARSER_RUST_CPU_PROOF: '1' + GERBIL_PARSER_NATIVE_HOST_LIBRARY: ${{ github.workspace }}/.gerbil/ci/libparsernative.so + run: GERBIL_BUILD_CORES=4 GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" gerbil env gxi qualify.ss host + - name: Upload qualification logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: logs-${{ inputs.release-os }}-cpu-${{ matrix.lane }}-${{ github.sha }} + path: .data/**/*.log + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index 56fee01c..2356e743 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ .data/ manifest.ss target/ +formal/.lake/ +formal/quint/node_modules/ +formal/evidence/__pycache__/ +formal/evidence/.venv/ diff --git a/Cargo.lock b/Cargo.lock index b84b49c8..030861da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,7 +20,7 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "asp-rust" version = "0.1.2" -source = "git+https://github.com/tao3k/asp-rust?rev=2419414ee5379bad89506496e3eaab928df68daf#2419414ee5379bad89506496e3eaab928df68daf" +source = "git+https://github.com/tao3k/asp-rust?rev=9b95fd6744a89f3f38e87f9bda33ea762f1f65e1#9b95fd6744a89f3f38e87f9bda33ea762f1f65e1" dependencies = [ "asp-rust-build-support", "blake3", @@ -41,7 +41,7 @@ dependencies = [ [[package]] name = "asp-rust-build-support" version = "0.1.0" -source = "git+https://github.com/tao3k/asp-rust?rev=2419414ee5379bad89506496e3eaab928df68daf#2419414ee5379bad89506496e3eaab928df68daf" +source = "git+https://github.com/tao3k/asp-rust?rev=9b95fd6744a89f3f38e87f9bda33ea762f1f65e1#9b95fd6744a89f3f38e87f9bda33ea762f1f65e1" dependencies = [ "blake3", ] @@ -126,12 +126,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" -[[package]] -name = "countme" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" - [[package]] name = "cpufeatures" version = "0.2.17" @@ -212,43 +206,53 @@ dependencies = [ "version_check", ] +[[package]] +name = "gerbil-parser-artifact" +version = "0.1.0" +dependencies = [ + "serde_json", + "sha2 0.10.9", +] + [[package]] name = "gerbil-parser-build-support" version = "0.1.0" dependencies = [ "asp-rust", "asp-rust-build-support", + "gerbil-parser-runtime", "gerbil-scheme-rust-ir", ] [[package]] -name = "gerbil-parser-rowan" +name = "gerbil-parser-ffi" version = "0.1.0" dependencies = [ - "rowan", - "sha2 0.10.9", + "gerbil-parser-artifact", + "tokio", ] [[package]] -name = "gerbil-parser-rowan-arithmetic" +name = "gerbil-parser-runtime" version = "0.1.0" dependencies = [ - "gerbil-parser-rowan", + "gerbil-parser-artifact", + "sha2 0.10.9", ] [[package]] -name = "gerbil-parser-rowan-scenarios" +name = "gerbil-parser-runtime-external" version = "0.1.0" dependencies = [ - "gerbil-parser-build-support", - "gerbil-parser-rowan", - "gerbil-parser-rowan-arithmetic", + "gerbil-parser-artifact", + "gerbil-parser-runtime", + "sha2 0.10.9", ] [[package]] name = "gerbil-scheme-rust-ir" version = "0.1.0" -source = "git+https://github.com/tao3k/gerbil-scheme-rust?rev=2aac6d5b24b9c4d1382b232035f6e242a31a5130#2aac6d5b24b9c4d1382b232035f6e242a31a5130" +source = "git+https://github.com/tao3k/gerbil-scheme-rust?rev=0c18941cc6b6b7ff0ef3943abdb2b5fac8e92ccb#0c18941cc6b6b7ff0ef3943abdb2b5fac8e92ccb" dependencies = [ "prettyplease", "proc-macro2", @@ -271,12 +275,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - [[package]] name = "hashbrown" version = "0.17.1" @@ -299,7 +297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.17.1", + "hashbrown", ] [[package]] @@ -326,6 +324,12 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "prettyplease" version = "0.3.0" @@ -383,24 +387,6 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" -[[package]] -name = "rowan" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "417a3a9f582e349834051b8a10c8d71ca88da4211e4093528e36b9845f6b5f21" -dependencies = [ - "countme", - "hashbrown 0.14.5", - "rustc-hash", - "text-size", -] - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "semver" version = "1.0.28" @@ -503,10 +489,25 @@ dependencies = [ ] [[package]] -name = "text-size" -version = "1.1.1" +name = "tokio" +version = "1.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "toml" diff --git a/Cargo.toml b/Cargo.toml index 10f4ba30..efd1f004 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,37 @@ [workspace] members = [ "build-support/gerbil-parser-build-support", - "rust/gerbil-parser-rowan", - "rust/gerbil-parser-rowan-arithmetic", - "rust/gerbil-parser-rowan-scenarios", + "rust/gerbil-parser-runtime", + "rust/gerbil-parser-ffi", + "rust/gerbil-parser-artifact", + "rust/gerbil-parser-runtime-external", ] resolver = "2" +# Dependency workspaces do not export profiles to this Cargo root. Match the +# native Gerbil AOT contract while retaining debugger and safety checks. +[profile.dev] +opt-level = 2 +debug = 2 +debug-assertions = true +overflow-checks = true + +# Source-policy tooling is a build/test dependency, not a parser hot path. +# Avoid optimizing its large syn visitors during a cold qualification build. +[profile.dev.package.asp-rust] +opt-level = 0 + +[profile.dev.build-override] +opt-level = 2 + +[profile.release] +opt-level = 3 +lto = "thin" +overflow-checks = true + +[profile.release.build-override] +opt-level = 3 + [workspace.package] version = "0.1.0" edition = "2024" @@ -15,13 +40,12 @@ repository = "https://github.com/tao3k/gerbil-parser" rust-version = "1.95" [workspace.dependencies] -asp-rust = { git = "https://github.com/tao3k/asp-rust", rev = "2419414ee5379bad89506496e3eaab928df68daf", version = "0.1.2", default-features = false } -asp-rust-build-support = { git = "https://github.com/tao3k/asp-rust", rev = "2419414ee5379bad89506496e3eaab928df68daf", version = "0.1.0" } +asp-rust = { git = "https://github.com/tao3k/asp-rust", rev = "9b95fd6744a89f3f38e87f9bda33ea762f1f65e1", version = "0.1.2", default-features = false } +asp-rust-build-support = { git = "https://github.com/tao3k/asp-rust", rev = "9b95fd6744a89f3f38e87f9bda33ea762f1f65e1", version = "0.1.0" } gerbil-parser-build-support = { path = "build-support/gerbil-parser-build-support", version = "0.1.0" } -gerbil-scheme-rust-ir = { git = "https://github.com/tao3k/gerbil-scheme-rust", rev = "2aac6d5b24b9c4d1382b232035f6e242a31a5130", version = "0.1.0" } -gerbil-parser-rowan = { path = "rust/gerbil-parser-rowan", version = "0.1.0" } -gerbil-parser-rowan-arithmetic = { path = "rust/gerbil-parser-rowan-arithmetic", version = "0.1.0" } -rowan = "0.16.1" +gerbil-scheme-rust-ir = { git = "https://github.com/tao3k/gerbil-scheme-rust", rev = "0c18941cc6b6b7ff0ef3943abdb2b5fac8e92ccb", version = "0.1.0" } +gerbil-parser-runtime = { path = "rust/gerbil-parser-runtime", version = "0.1.0" } +gerbil-parser-artifact = { path = "rust/gerbil-parser-artifact", version = "0.1.0" } sha2 = "0.10.9" [workspace.lints.rust] diff --git a/FOLD-TASK-RECEIPT.md b/FOLD-TASK-RECEIPT.md new file mode 100644 index 00000000..7854ff6e --- /dev/null +++ b/FOLD-TASK-RECEIPT.md @@ -0,0 +1,1680 @@ +# Real Fold cross-statement qualification + +Status: whole-engine preflight incomplete; retain local buffer/wire candidate, +reject slower scan/publication extensions. No promotion, push or full matrix. + +## Source checkpoint requested on 2026-10-13 + +The user requested committing the current task changes before continuing. +This checkpoint retains source, controls, distinct Lean/Quint obligations and +negative benchmark records; it is not throughput acceptance. Generated SDKs, +binaries and temporary logs remain local and are not staged. The existing +GitHub PR #13 head is 23b1b493135881b3e4c68b68a8e2ba376c32a721; +the checkpoint starts from 16f539901aeaf231ad65f0236a6c2b60283d5f4e. +Upstream module-admission work must be preserved during alignment. The +1000-request 10 ms gate remains open; no full matrix or Automation restart. + +Checkpoint 450c27e44845ad9fa477b491fff2bf936fcd1128 is rebased on that +GitHub head. The sole Justfile conflict retained both independent recipes. +Post-rebase checks passed: 13 run controls, 8 tape/buffer controls, 6 single-use +call controls, and 24 retained actual Org AOT controls. The latter reruns the +qualified retained binary; it does not qualify a newly regenerated upstream +module-loading closure. The fresh model collector passed Lean and Quint: +Lean faa7c2ec17f4844c986094f4f84a244115a857f14967a12d792a9f03cf4a1050; +Quint dd6abc197b27fa3bd83d3f1ad13551cef82c32556b9c6996334c6c94c2b44ba3. +Model-only collection leaves Scheme/performance joins Missing and execution +unauthorized, as expected. Initial cache permission and missing Ascent library +environment failures were setup failures, not passing model runs. + +Continuation closes the observed diagnostic-link workflow gap: attribution, +stall and allocation link recipes now use the existing static object-lock +precheck, as prepared preflight already does. The existing lock rejection and +evidence-preservation test passes. This is fail-fast detection, not an atomic +cross-process lock or a throughput improvement. Links must still run only +after their owned compilation has completed; no SDK locks are removed. + +## Whole-span recognition/state/consumer fusion trial + +An architectural span candidate replaces the generated per-byte Scheme loop +with one synchronous borrowed-buffer leaf. Each source byte is read once for +stop classification and all isolated reset/increment consumers. The leaf +produces trailing lengths; Scheme applies carry to the unbounded initial value, +then executes the original ordered last-byte residual and event consumption. +No callback, allocation or pointer retention occurs inside the leaf. The +compiler's existing effect/dependency admission is unchanged; this is NOT +permission to skip arbitrary triggers or cross consumer/helper/frame effects. + +First product `.real-span-leaf` used the checked general traversal entry and +one local scratch vector per loop, including empty counter sets. Unit controls +(13 cases), 24 actual complete tapes and source-bound qualification passed: +Lean fdd4e8de2eca6233347a1f42dffce665b56322280e395c520b99bf4f1f56caf8; +Quint d5e43e5f69f3defcadad16553e81f3fa9c44997c8b2329a31ff84f1b607dcd17; +Scheme/AOT 37e36e26535bb1250d58f9491262b34858e5e725f2c107f7c84fd093728950b3. +Lean proves the finite reset/carry summary; Quint separately models synchronous +borrow, epoch validity, commit and residual order. Neither proves C storage or +compiler eligibility; real AOT controls bind those implementation obligations. + +Actual SHA a421efa46b073d4bab4ba10accf1fd151ebfe62d437bfef62e9e251ec9e3ad77; +prepared SHA ba135fea6395b333e3fe81a6c2ce5baff025d2e7c2c35385f4f109891c421fc8. +The corpus and retained publication control remain byte-identical. +Four matched ABBA lanes/product, complete 1000 changing tapes each, gate exit70: +candidate wall37.815/67.181/37.052/36.544 ms; CPU37.711/37.216/36.916/36.533 ms; +control wall36.636/50.843/36.016/35.916 ms; CPU36.615/37.608/35.974/35.878 ms. +Candidate wall median37.4335 ms, CPU37.066 ms, RSS26664960 bytes; +control wall36.326 ms, CPU36.2945 ms, RSS26574848 bytes. +Candidate P95 52-56 us, P99 66-121 us, throughput14885.16-27364.27 requests/s; +control P95 53-55 us, P99 71-252 us. All lanes have four GCs. The candidate +regresses CPU/RSS and is NOT admitted. Its67.181 ms lane has29.965 ms beyond +process CPU; that difference alone cannot locate the pause or classify GC. + +`.span-leaf-stalls-1.log` is diagnostic only: wall40516 us, CPU40496 us, +complete comparisons1112 us. A captured447 us call has412.226 us GC wall; +the800 us first call has802 us CPU and no GC. This run does NOT reproduce +or retrospectively attribute the earlier46.798/67.181 ms slow batches. + +Initial test failure was an unbound library name in the isolated test eval, +fixed by explicit lexical injection (no production eval introduced). Initial +prepared link correctly rejected an active object lock; initial real-control +launch preceded binary completion and failed before parsing. All failures +remain in `.span-leaf-*.log`; their finished retries, not failures, qualify. + +Second product `.real-span-entry` removes repeated direct-call admission from +compiler-proven intervals and removes scratch allocation for zero-counter +spans. The checked general library entry remains. Installed std/ffi vector +body macros and Gambit's existing U64 storage macros are used; no kernel/SDK +changes. Thirteen unit and 24 complete actual controls pass. Source-bound +collector session80053 and prepared linker session41191 are active at this +receipt point; do not duplicate them or edit bound sources before completion. +No second-product performance has yet been admitted. No full20-batch matrix, +baseline promotion, commit, push, Automation, downstream or SMP admission. + +### Completion and withdrawal of both foreign span candidates + +All sessions listed above have ended. Second-product source-bound chain passed: +Lean89ac43249a0ef1d7b0c5ae3faba367544c500a34068480119b389468dd705d12; +Quinta128d4353df5aa5b1ed122bf052edd123cf292b574963440b97cd9f7d8791d35; +Scheme/AOTcf573f72b61fc8114947fd928dba778f56e272ac41768ad5ba8ab2f80dbf4b0c. +Actuald966ac143a841a6dfd7453cba462541df62710cfee91a50075bc180bfd4ab96d; +prepared0149b06a61627cde14cbaf86f98a4f2238cc5bec23a34375b2ff3a7145b25a48. +Four matched complete1000-tape ABBA lanes/product, all gate exit70: +candidate walls37.148/36.514/36.934/77.678 ms; CPU37.074/36.464/36.823/36.559 ms; +control walls36.987/35.617/35.986/36.252 ms; CPU36.118/35.555/35.899/36.103 ms. +Candidate medians wall37.041 ms, CPU36.691 ms, RSS26648576 bytes; +control wall36.119 ms, CPU36.001 ms, RSS26566656 bytes. Candidate P9552-56 us, +P9989-116 us, throughput12873.66-27386.76 requests/s. All eight lanes have4 GCs. +The77.678 ms lane has41.119 ms beyond process CPU; this is NOT a root-cause +classification or permission to discard the lane. No throughput admission. + +Both new scanner/compiler-runtime entries and their production imports were +removed with narrow reverse patches. Existing single-use call fusion and owned +tape publication remain; source byte-spans returns to its original implementation. +Retained scalar controls now include a counter above2^64 and nonzero carry. +The temporary duplicate Lean counter refinement was replaced by a genuine +representation obligation built on the existing Fold model. Its first empty +case proof failed and remains negative evidence in.span-encoding-models.log; +the corrected proof has no sorry/custom axioms and passes. + +Diagnostic-only instrumentation extends the existing request-owned count session, +not a new runtime/benchmark framework. It emits grouped source visits, isolated +counter byte visits, spans/buckets and fallback byte transitions; all hooks are +absent when diagnostics is disabled. Existing two cost tests include weighted +increments and rejection of negative increments. Compiler/zero controls pass. +Fresh.real-span-census actual24 controls and the independent1000 complete-tape +count census pass; output events1017619 is WIRE BYTES, not an event count. +Diagnostic CPU253.95 ms is instrumented and NOT a performance observation. +Observed source visits326949, summary byte visits323278, spans45996, +fallback byte transitions71640. Buckets1/2-4/5-16/>16 =6335/11332/25995/2334. +Mean7.108 bytes/span;38.410% spans<=4,5.074%>16. Visits can overlap across +owners and are not unique coverage of the2856770 raw input bytes. Static +.span-census-static-invariants.log has285 loop occurrences,33 admitted; +all reported required inline guard cells may be written within their loop. +Hoisting them unchanged would be unsound. The dependency-write boundaries, +not a blanket faster byte scanner, are the next execution-region obligation. +Installed SSAX look-for-str.scm was consulted for sequential recognition and +partial-match overlap handling; its port API was NOT substituted for byte Fold. + +The final retained-source model/AOT join passes: +Leanfaa7c2ec17f4844c986094f4f84a244115a857f14967a12d792a9f03cf4a1050; +Quintc1552051fa4f127d2df8054daf9403df42ed22bb3bc80dbb692f902ffdf1348a; +Scheme/AOT20e0922dee1af8ab72d6c4d9a35d13c990e33cf1bf8ec741b151bd97fe720882. +Matched-performance remains Missing; executionAuthorized=false. No current +artifact meets1000/10ms. No full3x20 matrix, promotion, commit, push, kernel/SDK +change, Automation restart or Org/FFI/Tokio/Python/SMP speed claim. +Diagnostic build/link sessions12400/34662, census52823 and collector20633 all +ended; no active matrix exists. An early census control launch failed before +binary completion, then the completed-binary retry passed. Preserve both logs. + +## 2026-10-13 reset/increment run summary rejection and live stall attribution + +### Generated call-region fusion: real implementation, preflight still open + +#### Follow-up: exclusive event-tape publication (work in progress) + +Production fold-tape-publish! now follows installed std/io/bio/api.ss +get-memory-output-u8vector: seal owner, clear its byte slot, shrink the privately +allocated u8vector to used length, and transfer it rather than subu8vector-copy +the complete tape. No input/header alias or foreign pointer escapes; all +sealed writers still reject before accessing the cleared slot. This removes +one publication allocation/copy, not the owner or capacity-growth allocations. +Shrinking logical length is NOT proof of immediate physical RSS reclamation. +Existing eight buffer/tape controls include post-growth, zero-size, explicit +GC, retained bytes and sealed reentry checks; they pass. Actual fresh AOT has +24 complete tape controls passing. An initial wrong controls pathname failed +before execution; .owned-publication-real-controls.log preserves that error, +and the valid retry is .owned-publication-real-controls-retry.log. + +Lean owned_publication_prefix models observable prefix equality, not allocation +or primitive correctness. Quint TapeOwner separately owns seal/transfer/reentry +lifecycle, with four tests, finite one-step closure and bounded histories. +First Lean simplification attempt failed; corrected List.take_take proof +builds successfully without sorry. No failed attempt is admitted as passing. +Source-bound qualification runs as session45192; sequential prepared/stalls/ +allocation diagnostic links run as session93341. Do not duplicate these jobs +or modify bound code during qualification. New product .real-owned-publication; +staged immutable evidence .owned-publication-real-evidence. No timing/large-load +or allocation improvement qualified yet; prior published baseline unchanged. + +Publication-stage completion: sessions45192/93341/11489/60824 ended. Model/AOT +join passed: Lean372f63ab693316b01db28f9cfb788288410a7c98c25b3367275331e7c6303b6c; +Quintda0863009441f32fa1920ba2c43c01dd85a60c14954c9a987d068a9e4186c357; +Scheme/AOT60124fda8e7ce187dc63ce8aa5ef45dd894ecd5962e4ae1c54c373df9973e557. +Actual org-fold SHA1b753e93b00f4f08214d5430b1a0d68a98418f4b5c6c101283bf3e86053479dd; +prepared953ae85f308f8471a4d9f7ee09c1a2ec185d9966c31ee0a446d9b3e0283325f0. +These identities qualify that stage, NOT subsequent single-read source edits. + +Allocation subset bytes/request by residue0/1/2 is14608/23456/28528 versus +15904/24976/31888; denominators332/332/331 versus332/331/332. Reduction is +1296/1520/3360 bytes per observed no-GC request, not whole-process allocation. +Logs .owned-publication-allocation.log/.owned-publication-control-allocation.log. +Eight interleaved1000-request lanes per product retain every result: walls +candidate38.524/35.534/35.642/36.220/35.549/36.259/35.755/35.268 ms; +control38.058/36.191/39.826/36.583/38.761/73.321/36.322/36.057 ms. +Median35.6985 versus37.3205 ms; CPU35.6645 versus36.279 ms. Candidate P95 +50-56 us; P9960-133 us; throughput25957.84-28354.32 requests/s. All complete +1000 tapes pass; GC4 versus5; all unchanged10ms gates fail. First eight lanes +have sysctl-denied RSS and time exit1, not complete resource qualification; +later eight escalated lanes have parser gate exit70 and RSS. RSS medians +26583040 versus26558464 bytes (candidate slightly higher), not an RSS saving. +External compiler activity makes peak throughput and tail admission conditional. +Retain candidate1 wall/P99 regression and control6 wall73.321 ms/P99411 us. +No full20, publication/push or claimed stable architecture-wide speed admission. + +Current AOT stalls diagnostic: wall133.837 ms versusCPU46.434 ms with probes. +Slowest call monotonic8157 us, CPU319 us, zeroGC and3 involuntary switches; +other multi-ms rows also show switches withoutGC. Concurrent external compiler +jobs were observed. This supports non-CPU waiting for those calls, NOT a +retrospective explanation of the historic46.798 ms batch. Profile sample +.owned-publication-stack.sample has top-frame counts part14=66, fold-tape=54, +part17=24, source-lines=11. Module samples are NOT exclusive function costs, +dynamic scan counts or parser-only percentages; complete tape comparison remains +in the profile loop. .owned-publication-stack-profile.log confirms complete tapes. + +Follow-up single-read production lowering now binds each admitted next byte +once for membership recognition and all counter consumers. No read at limit, +no cache and no intervening consumer/source writes. The existing counter control +also checks one generated next-byte source load for two summaries. Current +tests/AOT/prepared link run as session92630; no large matrix active. Do not +duplicate or promote that product before actual AOT/model and matched checks. +NUL/empty/exact-end scalar-reference cases now pass inside the13-case run +suite; all24actual single-read tapes pass. Immutable copied AOT is staged at +.single-read-real-evidence. The second source-bound chain is active separately; +do not confuse earlier publication identities with this source generation. + +Single-read final result: source-bound Leanb64a4528fe3f7b07ca91b892bf421911a41ae3b997136af1e64e81b633dc1d30, +Quintbcb115adc394796bcc7572785b0bb569041f6132ef2bcf6dc78b6b73fc68285e, +Scheme/AOT04c51ba78cbca8cd7bc254871d9da59ce67dbf791154f93923e2f50fee4b1ead +pass, but performance fails. Candidate walls37.200/36.401/36.751/36.410 ms; +publication-only controls38.752/36.636/35.166/35.982 ms. CPU medians36.5015 +versus36.2355 ms; P50 consistently30 versus29 us. Candidate P9551-53 us, +P9970-83 us, throughput26881.72-27471.77 requests/s, RSS26574848-26607616B; +control RSS26460160-26574848B. All8lanes full1000tapes pass, GC4, gateexit70. +Actual SHA b5a93216c1b16246d711aa7873606ec8d18fda704a7aebdb44228cc244bdc76b; +prepared70be3ec5f238bfab48d51ec0365592a8b574385710438a7f3c87dd159e410b6a. +The new let/guard/source-read lowering was withdrawn with a narrow reverse +patch; independent scalar NUL/empty/end tests are retained. No unrelated dirty +edits reverted. Publication-only runtime remains an unpromoted candidate. +Current final-source model/AOT requalification session27278 uses a fresh +.publication-retained-evidence-controls root and the unchanged real publication +artifact. Do not restart it or claim old source identities cover this rollback. +Next architectural candidate must collapse an admitted entire pure source span, +not introduce another per-byte binding/guard. Counter suffix/initial-state +semantics and ordered residual/event effects must remain explicit obligations. + +Final retained-source qualification session27278 completed exit0 under the +unchanged240s budget: Leana20290bd26c1a78d545c08eef2b13d7ae2a0dfef9f9bc6bb2043c5b027a5a2f9; +Quint0f934b1cbb01db6962f6220da494ef6bfe30fc9a45a35b04c4731e868a03608c; +Scheme/AOT1de91e7e76708a2b2702379a39cee39deffc2530baa0e86bbaf74a0570dca8b5. +Review still lacks matched-performance and executionAuthorized=false. All owned +compiler/model/benchmark/OS-sampling processes ended. No matrix active, no push, +baseline promotion or Automation restart. Live GitHub PR13 head verified still +16f539901aeaf231ad65f0236a6c2b60283d5f4e (open). Published prepared baseline +a94f3fb148469e3f3f04bffc89bec90be66d65efcdd32664a8787e844ffa17c0 and +immutable corpus e9ccee6c366bdc07f733c525634ed9efed2eb1d2932ef93b54f5ba4cd820df40 +remain unchanged; git diff --check passes. Runtime publication benefit remains +diagnostic/unpromoted and historical46.798ms root cause is NOT established. + + +Head remains 16f539901aeaf231ad65f0236a6c2b60283d5f4e; fixed installed SDK +1cfb032, unchanged corpus/input indices/independent full tape oracle. +New event-fold-call-plan.ss counts whole-program private references and fuses +only single-use expression calls with scalar-reference arguments and bounded +expanded bodies. Shared consumers, helper frames and effectful argument calls +retain their boundary. Alpha conversion preserves let/let*/named-loop scope; +final reachability protects children after rejected speculative expansions. +The pass is integrated before AOT module partitioning, on production tapes +only. It introduces no runtime cache, duplicate consumer or event format. + +An initial real-generation failure exposed Gerbil single-binding let syntax. +Retained .single-use-aot-build.log and incomplete .real-single-use; fixed the +converter and added this syntax to the scope control. Six directed controls +and 24 complete real AOT tapes now pass on fresh .real-single-use-v2. +Initial compile/Lean syntax failures remain recorded; none counted as passes. + +Current static census: 3310->1300 definitions; 7381->5371 private calls; +27210->18920 private argument slots; 70->25 event owner assignments. +Counts are syntactic, not dynamic scans, allocations or CPU shares. + +Source-bound complete qualification passed within unchanged 240 s budget: +Lean 140046f1d390704266b2a069740736f03b580015139290e3422a6fec04166696; +Quint c628def5c3b8c03d1ba0ca971957b34ab4e213f843f896061993795abf5271a3; +Scheme/AOT 822ef6711f124d8a2ce6e5ebc39a4801e74b464484433fb6c5bc0dc65825be14. +Lean splice_call_preserves_order proves ordered region composition under its +model, not the alpha converter. Quint fused-call history owns extent +invalidation. No custom axioms or speed-proof claim. Model review still has +matched-performance missing and executionAuthorized=false. + +Actual AOT SHA256 685fcbf700d04bf684b91c7f811501d003a0d1f7e0b7e122227ad220e4721687; +prepared e39d5ad4448d123f4047413649b2e41d0ffd75d1e548490803c76124d7893d29. + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| control1 | 44.609 | 64 | 80 | 22417.00 | 44.095 | 28803072 | +| candidate1 | 37.430 | 55 | 76 | 26716.54 | 37.369 | 26525696 | +| candidate2 | 46.798 | 51 | 156 | 21368.43 | 36.389 | 26558464 | +| control2 | 43.357 | 64 | 82 | 23064.33 | 43.322 | 28786688 | +| candidate3 | 36.032 | 54 | 63 | 27753.11 | 36.022 | 26525696 | +| control3 | 43.264 | 63 | 93 | 23113.91 | 43.222 | 28786688 | +| control4 | 44.095 | 65 | 116 | 22678.31 | 43.908 | 28803072 | +| candidate4 | 36.307 | 53 | 69 | 27542.90 | 36.289 | 26542080 | +| candidate5 | 37.025 | 55 | 101 | 27008.78 | 36.852 | 26558464 | +| control5 | 44.349 | 65 | 95 | 22548.42 | 44.182 | 28835840 | +| control6 | 43.352 | 61 | 96 | 23066.99 | 43.295 | 28803072 | +| candidate6 | 36.212 | 53 | 83 | 27615.16 | 36.145 | 26574848 | + +ABBA/BAAB followed by an additional BAAB diagnostic: six lanes per product, +1000 changing requests each, all full comparisons pass, all gate exits70. +These are preflight diagnostics, NOT twenty-batch acceptance or large loads. +All six candidate CPU times improve against controls; six-lane median CPU +36.339 vs43.615 ms. Wall medians36.666 vs43.726 ms retain the 46.798 ms +candidate2 spike (CPU36.389 ms; P99156 us). No retrospective attribution of +that ~10.4 ms non-CPU gap. Additional reverse controls do not replace it. +Five collections each; no GC-count improvement or allocation-total claim. +RSS medians26550272 vs28803072 bytes include retained inputs/oracles/code. +Call percentiles and comparison-inclusive batch wall remain distinct; this +is not public Org::parse, FFI graph projection, Rust queues, Python or SMP. +Another session's parser-quality evidence process was observed during earlier +qualification; owned compiler jobs had ended before timing. Do not claim an +isolated-host capacity limit from these diagnostics. + +Retain this as an unpromoted implementation candidate: broad CPU/RSS benefit +observed, but the hard10ms preflight fails and wall/tail regression is retained. +Published baseline hash unchanged; no push, commit or full20 matrix. Do not +label scan-count or allocation reduction established from call/RSS counts. +Serial allocation diagnostics against both existing AOT closures completed +with complete-tape agreement; all owned build/model/benchmark jobs have ended. +The no-GC-request subset has identical observed bytes per request: +fixture residues0/1/2:15904/24976/31888 bytes for candidate and control. +Candidate denominators332/331/332; control332/332/332. Collections by residue +are2/2/1 versus2/1/1; CPU sums8.171/11.158/17.941 ms versus +9.935/12.897/21.082 ms. These diagnostics include observation overhead and +exclude requests crossing GC; they do not establish total or parser-exclusive +allocation, lower GC pressure, or performance acceptance. Logs: +.single-use-allocation.log and .single-use-control-allocation.log. +The current benefit is call-structure/CPU/code-RSS, not demonstrated allocation +reduction; direct event allocation remains an explicit open cost. +No kernel/SDK change, acceptance relaxation or Automation restart. + +Next coherent step is lexical dominance of immutable recognition facts across +dependent consumers, now inside fused regions, plus direct-consumer allocation +cost. Do not reintroduce an all-idle guard or maintained request cache. + + +### Whole-engine model and rejected local-transition trial + +The optimization unit is now the complete parser engine, not Fold-local guards. +formal/ParserEngine/Execution.lean models ordered dependent consumers sharing +immutable recognition facts and invariant-preserving ordered transitions. +The existing FoldPlan.qnt owns lifecycle/invalidation, not a second semantics +proof. Lean build passes (40 jobs); FoldPlan has eleven passing cases and the +existing bounded Quint checks pass. Compiler refinement and whole-request +dynamic cost attribution remain open; these are not speed or GC proofs. + +Local completion-support trial: 13 run controls and 24 complete real AOT tapes +passed. Main mode guards and 33/285 admitted loops did not change. Generated +definitions remain 3310; private call sites 7374->7371; owner assignments +69->70. Static counts cannot establish execution cost reduction. + +Source-bound trial receipts (historical after withdrawal): +Lean 7551302d363bbdc86c8870059c405fa97673cd025dedcd4de392f7da5a25b50b; +Quint bf9a111d595868549c3f5e69a9e9d54d2dba844978c650f0dd7f41ead4483434; +Scheme/AOT e564c5d3ef7be0fcbc3fe17c9c6bd1ca7dac2a10a2e3ce31a4aec37c65b54c06. +Trial AOT 1ad8c00c117674f708eac86424e084daedca6024065a667491119dc092e66963; +prepared 71ba046e73d315b1d974a8bc9321cec631ef3360da56a780bfa38253cdbcd223. +All retained under .real-local-transition and .local-transition-*. +No timeout widening, baseline promotion, commit or push. + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| control1 | 43.803 | 59 | 121 | 22829.49 | 43.286 | 28819456 | +| candidate1 | 43.344 | 63 | 119 | 23071.24 | 43.179 | 28803072 | +| candidate2 | 46.836 | 67 | 129 | 21351.10 | 44.568 | 28868608 | +| control2 | 43.244 | 62 | 97 | 23124.60 | 43.189 | 28819456 | +| candidate3 | 42.116 | 63 | 84 | 23743.95 | 42.046 | 28803072 | +| control3 | 55.164 | 60 | 85 | 18127.76 | 42.611 | 28835840 | +| control4 | 43.562 | 64 | 95 | 22955.79 | 43.503 | 28819456 | +| candidate4 | 42.716 | 63 | 89 | 23410.43 | 42.642 | 28803072 | + +Serial ABBA/BAAB: all eight lanes completed 1000 full independent comparisons, +all exits 70, five GCs each. Wall medians 43.030 ms candidate vs 43.683 ms +control do not qualify a consistent gain: candidate2 regressed; control3 has +55.164 ms wall vs 42.611 ms CPU. The non-CPU gap is observed, not attributed +to a specific scheduler/GC cause. No 10 ms admission or full twenty-batch matrix. +Call P95/P99 and comparison-inclusive batch wall are distinct; RSS includes +retained input/oracle/code, not Rust capacity. No FFI/public Org/Python/SMP claim. + +Withdrawn only this turn's local-support admission/lowering policy. The prior +selective-table implementation and immutable baseline remain unchanged. The +source-only invariant checker and directed negative controls remain tooling. +Restoration session69320 completed exit0: fresh compilation, existing source +controls and all 13 run controls have final OK. All owned jobs ended. +Baseline prepared SHA256 remains +a94f3fb148469e3f3f04bffc89bec90be66d65efcdd32664a8787e844ffa17c0; +corpus SHA256 remains +e9ccee6c366bdc07f733c525634ed9efed2eb1d2932ef93b54f5ba4cd820df40. +No kernel/SDK/Automation changes. Trial source-bound receipts remain historical, +not qualification of the restored lowering. git diff --check passes. + +Next architecture target: jointly plan source recognition facts and consumer +dependency frontiers across frame/helper boundaries, preserving original +state/event effect order. Then reduce generated call/return boundaries and +publication work, not another all-idle guard or maintained runtime cache. + + +### Continuation: dependency-selected interval execution + +Final qualification on head 16f539901aeaf231ad65f0236a6c2b60283d5f4e: +the fresh aligned retry passed the unchanged 240 s budget. Eleven run controls +and 24 complete real AOT tape controls pass. Source-bound receipts are Lean +527fbd50fe56b58580c7588f9198029f6e16515b46db69c041e4acbc5e421f6f, +Quint bfbed6809deed73f6d12dc61aab981aa37b2e8590319097269fb60c096a84a70, +and Scheme/AOT a1b6649ca18b0347a653625dfa8a68e2673c672d5ad4f890c697a3bf1c70d29f. +The earlier timeout below remains a failure, not converted to a pass. + +The interval-selected narrow/wide candidate has AOT SHA256 +884f7c9f7ca657393c0667fa8978289af7b3c5f3a5d2db1ed481cd1d8a6b0edc +and prepared-binary SHA256 +98f8ca51ecb0fd7d31628b1340beb708b83212aed7a80cbfae06b0579981f9ec. +The unchanged baseline prepared binary is +a94f3fb148469e3f3f04bffc89bec90be66d65efcdd32664a8787e844ffa17c0; +input/oracle SHA256 is +e9ccee6c366bdc07f733c525634ed9efed2eb1d2932ef93b54f5ba4cd820df40. +All timing lanes use the same installed 1cfb032 SDK, not another session's +2591dcd build. No SDK rebuild, kernel change or Automation restart occurred. + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Control 1 | 57.987 | 154 | 275 | 17245.24 | 54.903 | 28770304 | +| Candidate 1 | 43.931 | 62 | 109 | 22762.97 | 43.781 | 28770304 | +| Candidate 2 | 43.262 | 63 | 89 | 23114.97 | 43.220 | 28770304 | +| Control 2 | 42.219 | 59 | 87 | 23686.02 | 42.112 | 28753920 | +| Candidate 3 | 41.011 | 58 | 63 | 24383.70 | 41.015 | 28803072 | +| Control 3 | 42.779 | 62 | 73 | 23375.96 | 42.733 | 28753920 | +| Control 4 | 42.382 | 59 | 65 | 23594.92 | 42.388 | 28803072 | +| Candidate 4 | 41.871 | 58 | 91 | 23882.88 | 41.856 | 28786688 | + +Sequential ABBA/BAAB lanes each compare all 1000 changing inputs against the +independent complete tape oracle. All comparisons pass; all preflight exits +are 70 and each lane has five collections. The first control spike is retained. +Four-lane wall medians are 42.5665 ms candidate and 42.5805 ms control: no stable +speed improvement is established. No promotion, commit, push or full twenty- +batch matrix. The 1000-request <=10 ms gate remains failed. All owned processes +have ended. This candidate replaces the slower unconditional wide policy but +remains unqualified; published baseline artifacts are unchanged. + +P95/P99 are compiled Fold-call durations; wall includes call timing/sample +recording and full semantic comparison. RSS is whole-process retained inputs, +oracles and native code, not Rust reserved capacity. This is not public +Org::parse, Python, Tokio or FFI projection acceptance. Counter deltas across GC +are not allocation totals. Unsupported SMP remains open/skipped, never passed. +The next algorithmic gap is dependent active consumer transitions: derive +local completion/transition support and safe zero-preserving reset witnesses, +instead of requiring whole-mode inactivity. Do not repeat the rejected wide +policy or infer speed from static guard counts; no maintained runtime cache. + +The new candidate admits ordinary-byte intervals from the complete consumer +body. A consumer selector is stable only when its read support is disjoint +from every interval write and exact counter target, it has no observable +effects, and it does not depend on the changing cursor/current byte. Leaf +actions are total state-independent overwrites; isolated unconditional exact +counters retain their existing summary. Source/order boundaries and event +actions reject interval admission. Conditional counters are not moved into +unconditional accumulators: their ordinary reset remains under its selector. + +For consumers that still require an idle premise, the planner removes one +support at a time only with a complete admitted transfer witness for the new +body. This is not equality with the all-idle body: independent active consumers +may execute their ordered overwrite once at the last ordinary byte. Required +premises must remain disjoint from all common writes. There is one scanner, +not a source pass per consumer, no maintained runtime mode/scan cache, and the +first triggering byte remains unconsumed for the original complete body. + +Lean selective_run_last owns conditional-transfer equivalence under untouched +context. Quint RunPlan adds activeConsumersCanCommitTest to the existing +commit/residual lifecycle; ConsumerScope retains dependency invalidation. +These models do not prove the Scheme extractor or physical performance. +Standalone Lean and Quint pass. Actual AOT and performance remain pending; +fresh closure .selective-mixed-controls is being validated. No matrix or +baseline promotion is authorized by these structural results. + +Mixed-state AOT 1b5cb5227e65fb936587590b7bf2c6cf490152b0b42619514ad73c9489d5f69b +and prepared binary 0622b17e8c2d30dd38e4c7371b69d995a709f39b11eed8b7476bcad32b679293 +pass the 24 real tape controls. Source-bound receipts: Lean +bb2c67a7794241927125a54534a6ff6cd255360e1417e337d77561f77043e3f5; +Quint c6cb7c5a31bf197a8aee3424a82d5992a636f791ff5ed28fa2c37ae2cc6b47b4; +Scheme/AOT 3de53a6e9f803f298a105512a82125b64f5aed36a29231a5dec962db016858b4. +These are historical after subsequent classification-selection edits. + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Control 1 | 45.051 | 60 | 109 | 22197.07 | 44.604 | 28770304 | +| Candidate 1 | 47.277 | 63 | 103 | 21151.93 | 47.161 | 28819456 | +| Candidate 2 | 48.004 | 65 | 123 | 20831.60 | 47.555 | 28868608 | +| Control 2 | 56.601 | 109 | 312 | 17667.53 | 54.177 | 28803072 | +| Candidate 3 | 49.267 | 63 | 211 | 20297.56 | 48.168 | 28819456 | +| Control 3 | 58.144 | 113 | 458 | 17198.68 | 53.361 | 28803072 | +| Control 4 | 44.004 | 60 | 80 | 22725.21 | 43.990 | 28770304 | +| Candidate 4 | 46.399 | 62 | 114 | 21552.19 | 46.256 | 28868608 | + +All 1000 comparisons per lane pass; all preflights fail (70), five collections +each. Mixed-state plans reduce guards to 10/8/7 but broaden trigger alphabets +from 17/16 to 29/21. Stable endpoints show regression despite removing guard +premises; noisy controls are retained, not used to claim improvement. This +unconditional wide-alphabet policy is not promoted. The follow-on candidate +retains narrow and wide precompiled tables, selecting once at interval entry +from disjoint unchanged consumer support. Both tables share one scanner and +the same admitted wide transfer; no text/mode cache or additional source pass. +The selection support must be disjoint from every common write/counter target; +the narrow alphabet must be a proper subset of the wide alphabet. It is an +unqualified candidate, pending fresh native controls, AOT and matched timing. + +GitHub PR #13 head advanced to 16f539901aeaf231ad65f0236a6c2b60283d5f4e. +The checkout fast-forwarded to that exact commit, not the local origin's later +unpublished head. Dirty edits were preserved; collect_models.py was backed up +as .pre-align-collect-models.py and its additional Fold inputs restored while +retaining upstream recognition/token inputs. The upstream diff does not alter +the Fold compiler, its directly imported runtime modules, or preflight code; +generated AOT imports do not use the changed LR recognition/token/reduction +path. Existing immutable Fold artifacts are therefore reused, but model/AOT +source binding is rerun fresh at .selective-tables-aligned-evidence-controls. +The pre-alignment source-bound receipts are not the new head's qualification. +The first aligned chain failed its unchanged 240 s conformance budget while +linking event-fold-controls, after all six source controls and eleven run +controls passed. Timeout receipt: +6d5fcbd7735835b5326dbd92c130449f57941a6a48a435dadd9ce8fbdccf1551. +Another session's parallel parser-quality build was active at timeout. This +supports possible compiler contention, not proof of a sole cause. The timeout +is retained as a failure; budget and acceptance are not relaxed. Owned timed-out +processes exited. A fresh complete retry must wait for compiler contention to +end; never benchmark under that build or count this receipt as passed. + +### Continuation: consumer-local selector support, not an all-idle guard + +The new private implementation adds fold-consumer-state-body to the existing +zero-state analysis and invokes it at actual for-line-bytes lowering. Its +initial fact set is empty. An executed unsigned selector's false-positive +branch, true zero-equality branch, or a literal zero write introduces a fact +for that consumer only. Unrelated writes preserve it; support writes, opaque +effects, helpers/bounds and lexical joins invalidate it conservatively. Branch +merges intersect facts. Each byte starts afresh; no runtime fact table/cache or +new all-machine guard is emitted. Ordinary-run admission remains separate. + +Seven native analysis controls pass, including independent consumers, dependent +writes and effect barriers. All 24 independent actual Org complete tapes pass +in .consumer-state-native-controls.log. Static (NOT executed) coverage reports +285 loops, 4 refined bodies, and 714 -> 672 selector occurrences. This limited +coverage is not claimed to remove the entire heavily exercised state chain. +The installed SDK, original IR/corpus and immutable baseline remain unchanged. + +Lean adds unsigned-selector and independent-support refinement; Quint's new +ConsumerScope checks support-local invalidation, barriers and next-byte scope +across five tests and 1000 seeded bounded traces. These are different duties, +not duplicated semantics or a full compiler/liveness proof. Both focused +checks pass. The full source-bound chain is currently running against staged +.consumer-state-real-evidence and the actual .real-consumer-state product; +no performance promotion, complete matrix, commit or push is authorized by +these model results alone. Logs/products have unique .consumer-state-* names. +The full chain then exits 0; Scheme/actual AOT receipt identity is +5b86d3642b33a6ba9150acc59c923d3f75b9ce5f8434df6e2756e4c4d256e15b. +That packet becomes historical upon subsequent source changes. + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| control 1 | 43.883 | 62 | 95 | 22787.87 | 43.289 | 28786688 | +| candidate 1 | 43.509 | 64 | 86 | 22983.75 | 43.418 | 28803072 | +| candidate 2 | 43.223 | 64 | 95 | 23135.83 | 43.168 | 28770304 | +| control 2 | 42.596 | 61 | 86 | 23476.38 | 42.571 | 28819456 | +| candidate 3 | 58.390 | 154 | 276 | 17126.22 | 57.711 | 28819456 | +| control 3 | 42.553 | 62 | 88 | 23500.11 | 42.512 | 28819456 | +| control 4 | 43.424 | 62 | 95 | 23028.74 | 43.333 | 28819456 | +| candidate 4 | 43.858 | 64 | 106 | 22800.86 | 43.772 | 28770304 | + +All eight strictly serial/awaited lanes compare 1000 complete tapes and fail +the unchanged gate (exit 70), with five GCs and equal raw allocation counters +(NOT allocated bytes). No major gain; one reverse lane regresses. Do not promote +this zero-only integration. Its actual product SHA is +acd8d77f47bfbf1bcc39b9c42c9e44eb6d959a7606b7eecb5eaef064e491689b; +prepared SHA 696f8a81db749a1a9504be888cddf418ce56a62137154ffea1e90972c09e854d. + +The follow-on consumer-selector implementation tracks executed pure selector +truth, including positive/Boolean/composite selectors, against each predicate's +declared read support. Writes invalidate only intersecting support; unknown +effects discard all knowledge. A branch merge keeps common facts and loop/frame +handoffs are not recursively specialized under stale facts. No runtime table, +new admission guard or source-input cache is emitted. New native controls cover +positive/Boolean facts, independent/dependent writes, barriers, composite event +order and lexical/iteration scope. Fresh closure .consumer-support-controls and +actual product .real-consumer-support are the next unqualified owners. +Five new consumer-support native controls pass with exit 0. All 24 actual Org +complete tapes also pass. Static (not executed) selector count is 714 -> 669 +over the same four refined bodies; generated helper definitions are 3308, +not an expansive per-byte specialization. Actual product SHA +da30c8c2c8be5b316b7885f71ab396e23f664a310864984ba5075b46bfbf0968. +The original IR remains d37b4b160c9d956a549e86486e0b590b0cbdbf2a273ccb16f683c100d3158c38. +Lean's disjoint-write theorem explicitly assumes sound extracted predicate +support; it does NOT prove the Scheme extractor from a source hash. Quint +checks eligibility scope/invalidation histories, not unsigned mathematics. +The source-bound chain is running at the unchanged 240-second limit; current +packet may not be reported passed before it exits. No production promotion. + +The full chain subsequently exits 0: Lean e149b7b1521236f473ea82bb38bd763ed3cfc30147da284b7a993b6a3d144dea, +Quint c7c1effc4c6c5a2e01e136c4f14790026d92815473adbc821edfc0be013e1044, +Scheme/AOT 85f8bb55417535afbd9299474d9888f00e713c29fcdae7db8a77309c0c679748. +That staged .consumer-support-real-evidence packet is now historical because +the following support-minimization source is different. Strict serial results: + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| control 1 | 42.220 | 59 | 72 | 23685.46 | 41.699 | 28786688 | +| candidate 1 | 42.123 | 59 | 106 | 23740.00 | 42.066 | 28868608 | +| candidate 2 | 41.967 | 60 | 87 | 23828.25 | 41.897 | 28852224 | +| control 2 | 41.218 | 59 | 86 | 24261.25 | 41.197 | 28835840 | +| candidate 3 | 41.664 | 58 | 65 | 24001.54 | 41.645 | 28835840 | +| control 3 | 42.205 | 61 | 93 | 23693.88 | 42.158 | 28803072 | +| control 4 | 41.906 | 61 | 93 | 23862.93 | 41.836 | 28786688 | +| candidate 4 | 40.876 | 58 | 72 | 24464.23 | 40.841 | 28868608 | + +All lanes compare 1000 complete independent tapes, have five GCs and fail the +10 ms gate (exit 70). These essentially tied results do not qualify major +state-chain removal or a baseline promotion. Candidate prepared product SHA +1de980282f1c661e129e22b3560dba688ff17b754d8fa2b031085b8cc37de368. + +The follow-on guard-support minimizer drops an external zero premise ONLY +when rerunning the conservative cofactor without it produces the identical +ordered IR body. Explicit guard witnesses are not minimized recursively. Eight +native zero/support controls pass, including elimination of a payload guard +inside an unreachable consumer. Unnecessary actual guard cells are: +timestamp-diary-opens, footnote-label-count and export-backend-count. Inline +ordinary admission drops from 16/12 to 13/10 state guards. This is a structural +guard-dependency finding, not a measured admission-hit or speed claim. + +The earlier exact reset/increment summary is now tested with this minimal +guard support, not reinstated unchanged. Seven run controls pass; summaries +cannot be read/written by other common actions or guards, conditionally updated +or multiply declared. Local accumulators preserve exact counter state and +publish before the unconsumed triggering residual; no event reorder or runtime +cache is introduced. Static trigger alphabets become 17/16, with the unrelated +payload guards removed. Fresh closure .consumer-supported-runs-controls and +actual .real-consumer-supported-runs have completed correctness qualification, +but failed performance preflight. Eight zero-state, seven run, five consumer +and twenty-four actual AOT controls pass. Source-bound model receipts are Lean +ac8aac991e0171e8741486c3384930cf89a75c7638d3c409c0816c677641612a, +Quint d424baf94d853dcb4941eb6d0d27e84b3e1502f0514ef8139774d8d81adf46a6, +Scheme/AOT ddf93974b19a44d1cddf0c2961f6fcaff056dd33ee077b4d3b2fc6f1ae1880df. +These receipts bind that candidate, not subsequent edits. + +Matched A/B/B/A and B/A/A/B, 1000 changing requests each: + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Control 1 | 44.423 | 61 | 92 | 22510.86 | 43.796 | 28852224 | +| Candidate 1 | 44.692 | 64 | 171 | 22375.37 | 44.490 | 28934144 | +| Candidate 2 | 41.504 | 60 | 75 | 24094.06 | 41.489 | 28934144 | +| Control 2 | 42.465 | 61 | 94 | 23548.80 | 42.413 | 28819456 | +| Candidate 3 | 41.620 | 60 | 99 | 24026.91 | 41.532 | 28884992 | +| Control 3 | 43.107 | 63 | 84 | 23198.09 | 43.058 | 28835840 | +| Control 4 | 42.048 | 61 | 76 | 23782.34 | 42.018 | 28803072 | +| Candidate 4 | 42.096 | 61 | 96 | 23755.23 | 42.022 | 28884992 | + +All lanes compare every independent OEV1 tape and exit 70: the 10 ms gate +fails. Five GC collections per lane; the raw allocation-counter difference +is not allocated-byte evidence. No full twenty-batch matrix or promotion. +Actual artifact SHA256: 368784c6dd09926baf3da40a2e654bb18ada288ae5686a6a32256d856cceaf24. +Prepared artifact SHA256: f8676ebd8c95e6f332129f96549f888b16a510fd6a467478c5a6f03fc5928a51. + + +PR 13 remains b5e1895c8bc9a79dcb3d4e845130cdc113b7d2be (verified on GitHub). +No SDK/kernel changes, baseline overwrite, push or automation restart. + +The next unqualified experiment is .real-consumer-scalar, compiled from +.consumer-scalar-controls. Scalar operator trees remain directly in their +consumer block instead of receiving an artificial helper boundary. Unknown +calls, scans, allocation and statement bodies remain outlined; evaluation +order is unchanged. Five calling-convention controls pass. Generated helper +definitions decrease from 3308 to 3256; this is a static call-boundary count, +not proof that primitive evaluations or end-to-end latency decreased. Actual +AOT linking, twenty-four complete-tape controls and the full source-bound +model chain completed. Lean: 758705c205df2f9d6fb8d4b237a584b7eedb1ac442bf4525cc7dc29c01d72492; +Quint: 099be0856ab57f4449fa89fe95c0809ed1fe3c59b9edde1701628d2522ee9541; +Scheme/AOT: 4855cc0e4c5a1fec3ab32387de00188a5d501cf4714081fefcd982895c78563d. +Actual binary: e6571cbfbbba3fa685faca9cb143434d23ed8e2e97ecc3dbe886afe475d7a376; +prepared: a79a198dc038973e18fd48110dad1d00a15e0945c7164413635bc242064c123d. + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Control 1 | 67.527 | 154 | 415 | 14808.89 | 55.885 | 28819456 | +| Candidate 1 | 90.921 | 140 | 376 | 10998.56 | 49.338 | 30916608 | +| Candidate 2 | 128.127 | 140 | 283 | 7804.76 | 52.344 | 30883840 | +| Control 2 | 43.036 | 63 | 77 | 23236.36 | 42.945 | 28753920 | +| Candidate 3 | 39.846 | 57 | 87 | 25096.62 | 39.780 | 30900224 | +| Control 3 | 43.475 | 61 | 101 | 23001.73 | 43.386 | 28786688 | +| Control 4 | 43.518 | 60 | 73 | 22979.00 | 43.528 | 28803072 | +| Candidate 4 | 41.378 | 57 | 99 | 24167.43 | 40.794 | 30900224 | + +Every lane completed all 1000 full semantic comparisons and failed the wall +gate (exit 70). All had five collections. Mixed CPU/wall behavior does not +attribute the 90/128 ms stalls to encoding, GC or scheduler internals. Nor is +the raw allocation-counter decrease an allocated-byte claim. No best-lane +promotion is justified. The scalar classifier, its test and case-count edit +were withdrawn after this negative matched result; existing dependency-derived +calling conventions remain unchanged. All artifacts/logs are retained. This +model packet is historical after rollback, not qualification of current files. +Post-withdrawal fresh closure .consumer-rollback-controls builds successfully; +eight zero-state and four private-call-environment controls pass with final OK. +All owned compilation, model and timing processes have ended. No matrix is +active. The baseline and immutable input corpus hashes remain unchanged. +Remaining open work is selective consumer execution across ordinary spans; +neither predicate cofactoring nor scalar helper elimination demonstrated the +required architecture-level throughput. Do not rerun these rejected candidates +unchanged or treat their static reductions as actual execution-count savings. + +The candidate removes unconditional exact current-byte reset/increment machines +from ordinary-run trigger classification, accumulates their counters locally, +and publishes them before the unconsumed trigger residual. Other reads/writes, +conditional execution, duplicate targets and guard dependencies reject admission. +It is a transition summary, not a request cache or foreign scan. Static inline +trigger alphabets shrink from 52/53 to 16/17; executed coverage is not inferred. +Seven native admission controls and 24 independent complete actual Org AOT tapes +pass. Lean proves the reset/increment run-summary algebra; existing Quint checks +retain epoch, owner and residual/publication responsibilities. The full existing +source-bound chain passes at its unchanged 240-second Scheme limit: + +- Lean: d7267559381ccb1991e8c82ef6ec2981954cf7adf9d8d682fe903461d77b10db +- Quint: 7af95f23d99a50f41cc8924fcd14eee3aa3aab7061c823e5d82b2b9eb43d92f2 +- Scheme/actual AOT: 0ca97e6fbbc27638480a7d9f4d213a7086fe903bb91cbc08bfdff19e41d7222a + +These observations describe the subsequently REJECTED source version, not the +restored current engine. Immutable sources and actual product are preserved in +.counter-summary-real-evidence; .counter-summary-full-evidence-retry.log records +matched-performance Missing and executionAuthorized=false. The first evidence +invocation failed before checks because an absolute receipt path was supplied; +the existing policy requires repository-relative receipt identifiers. + +The first .counter-summary-abba-* launch did not explicitly await one yielded +process before the next request. Treat its timing as diagnostic-only; retain +its 321.481 ms candidate batch and do not claim strict isolation. The subsequent +.counter-summary-serial-* run explicitly awaits EVERY child before the next: + +| Lane | 1000-request wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| control 1 | 49.831 | 70 | 157 | 20067.8 | 46.700 | 28819456 | +| candidate 1 | 44.380 | 61 | 89 | 22532.7 | 43.773 | 28852224 | +| candidate 2 | 43.538 | 62 | 98 | 22968.4 | 43.422 | 28852224 | +| control 2 | 46.177 | 63 | 185 | 21655.8 | 45.389 | 28786688 | +| candidate 3 | 43.169 | 62 | 87 | 23164.8 | 43.023 | 28868608 | +| control 3 | 43.416 | 63 | 101 | 23033.0 | 43.306 | 28819456 | +| control 4 | 44.172 | 64 | 95 | 22638.8 | 43.886 | 28819456 | +| candidate 4 | 43.881 | 63 | 110 | 22788.9 | 43.716 | 28852224 | + +Every lane compares all 1000 independent complete tapes, has five GCs, and fails +the unchanged 10 ms preflight (exit 70). Reverse CPU times are essentially tied; +the candidate offers no substantial qualified gain and retains slightly more +RSS. The earlier disturbed timing was worse. The candidate admission, scanner, +test-count and plan-diagnostic edits were withdrawn, preserving the prior +condition lowering. .rejected-counter-summary-source.diff archives the attempted +source (also contains pre-existing context changes; NOT a wholesale rollback). +The pure Lean summary lemma remains only an algebraic contract for future work. +Existing baseline binary SHA a94f3fb148469e3f3f04bffc89bec90be66d65efcdd32664a8787e844ffa17c0 +is unchanged. Rejected product SHA 891837346cadead2f3b5bd804aaefbc89ce36943e740bade904267b9b236c4be; +rejected prepared product SHA 842f37e58d5179f96fa89c497f78a7a5b1608dc59abf036384d85830086ca0bb. + +Live BASELINE diagnosis, not throughput acceptance, now distinguishes realtime +and monotonic clocks, parser probes, complete tape comparison, GC CPU/wall and +voluntary/involuntary switches. The installed official std/os/time.ss uses +CLOCK_REALTIME; the independent POSIX monotonic probe does not alter that clock +or the SDK. Four .call-environment-stalls-*.log runs compare all 1000 tapes: + +| Run | Batch wall ms | CPU ms | Parser monotonic ms | Tape comparison ms | +| --- | ---: | ---: | ---: | ---: | +| 1 | 71.491 | 62.702 | 68.028 | 1.268 | +| 2 | 47.720 | 47.386 | 44.909 | 1.039 | +| 3 | 111.849 | 71.842 | 108.193 | 1.401 | +| 4 | 48.670 | 47.593 | 45.910 | 1.014 | + +Run 3 input 440 takes 4.908 ms realtime / 4.917 ms monotonic, only 0.438 ms CPU, +zero GC and two involuntary switches; input 675 takes 4.339/4.351 ms with 0.100 ms +CPU, zero GC and two involuntary switches. This reproduced class includes OS +descheduling, not GC or an isolated realtime-clock jump. It does NOT identify +the historical 142-148 ms batches retroactively, the scheduling competitor, +or all of the increased CPU cost. Probe overhead is included; no RSS/throughput +acceptance is inferred from this diagnostic. Diagnostic binary SHA +a0cee10298a820dffdb4344a863cbfa91e125c2046ad376679d4ef5e13eb216e. + +The changing corpus remains e9ccee6c366bdc07f733c525634ed9efed2eb1d2932ef93b54f5ba4cd820df40. +All 20-batch matrices remain unopened: the 1000-request gate fails. SMP remains +OPEN/SKIP, not PASS. Rust queues, FFI and Python are outside these engine numbers. +Next: actual changing-input call census on the RESTORED baseline, rather than +inferring CPU dominance from the reduced static trigger alphabet. New count +products are diagnostic-only and may not replace the immutable timing baseline. + +### Restored-baseline changing-input native call census + +The .real-baseline-census diagnostic product retains exactly the original IR +file bytes (SHA d37b4b160c9d956a549e86486e0b590b0cbdbf2a273ccb16f683c100d3158c38). +Its 24 independent controls pass. The new prepared-counts attribution driver +then compares all 1000 changing inputs and 1017619 output tape bytes against the +existing independent oracle: .baseline-census-changing-counts-retry.log exits 0 +and ends DIAGNOSTIC-OK. No speed/allocated-bytes claim: 302.614 ms diagnostic CPU +includes every instrumentation table update and four GCs. Native census binary +SHA ac22a196537ba6b56acbd49ea88b3e854df8ba8a846c4311f2ac28a9815ed8ff. + +The first prepared-counts call attempted the production performance binder and +was correctly rejected (negative .baseline-census-changing-counts.log). That +protection is UNCHANGED. The corrected diagnostic-only interface binds the +corpus digest; generated IR and actual binary hashes are sealed separately in +this receipt. It does not obtain production admission or infer backend parity. + +Expressions 54 and 565 each execute 95001 times; 13 expressions (including 120, +130, 555, 558, 560 and intermediate dispatch nodes) execute 94668 times each. +The latter include an ordered active-state selection chain over slots 15, 32, +50, 73 etc. Expression 42 computes a clamped two-byte successor 89606 times. +These are dynamic CALL counts, not per-expression exclusive CPU attribution. +Source encoding and event publication each execute 1000 times, and inline-span +is entered 5331 times. Thus static ordinary-run alphabet reduction alone does +not remove the heavily exercised residual selection path. + +New guard-name diagnostics (.baseline-run-guard-cells.log) expose all 12/16 +ordinary guard cells rather than only their count. Counts include inactive mode +payload counters; whether they cause fast-path rejection still needs executed +guard-hit evidence, not inference from names. + +### Whole-residual byte-region candidate: rejected + +The barrier-aware byte-region cofactor was implemented over the complete +residual, retaining ordered writes, else arms and original positive conditions. +Six run controls and 24 independent actual Org tapes passed. The source-bound +Lean/Quint/Scheme-AOT chain passed at the unchanged 240-second limit, recorded +in .residual-region-full-evidence.log and .residual-region-real-evidence. +That packet describes the subsequently rejected version, NOT current sources. +All compiler/evidence jobs and an unrelated SDK source experiment had ended +before the following strictly serial, awaited timing lanes: + +| Lane | 1000-request wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| control 1 | 58.877 | 156 | 370 | 16984.56 | 56.124 | 28868608 | +| candidate 1 | 46.834 | 70 | 191 | 21352.01 | 45.378 | 30769152 | +| candidate 2 | 46.490 | 71 | 197 | 21510.00 | 45.636 | 30801920 | +| control 2 | 48.220 | 72 | 257 | 20738.28 | 46.625 | 28819456 | +| candidate 3 | 41.581 | 60 | 80 | 24049.45 | 41.558 | 30818304 | +| control 3 | 43.780 | 61 | 116 | 22841.48 | 43.714 | 28803072 | +| control 4 | 43.807 | 62 | 103 | 22827.40 | 43.678 | 28819456 | +| candidate 4 | 82.181 | 209 | 603 | 12168.26 | 62.133 | 30801920 | + +All lanes compare 1000 complete independent tapes and fail the 10 ms gate +(exit 70). Candidate GC counts are four, versus five for controls, but this +does NOT offset the unstable wall/CPU or roughly 2 MB higher process RSS. +The compiler grows from 52 to 74 generated parts and 4713 helper definitions. +The stable reverse lane improves CPU only about five percent; the last lane +regresses. Withdraw this integration, preserving pre-existing condition +lowering and the immutable baseline. Archive .rejected-residual-region-source.diff +includes pre-existing context: it is NOT a wholesale rollback patch. Rejected +actual product SHA 04c0722e05e0f9d2008d4d5328d3ebaecbf4e5548d4a97c1bfa6e839b5d209ee; +prepared SHA 4a8713405500c9df15ffa453ebc528b8f2439079bac844019c4ec6154111b5a4. + +### Idle trigger execution: rejected after actual AOT qualification + +The next implementation separates state admission from source-byte admission: +a trigger byte need not imply an active machine. Keep existing ordinary-run +scanning; on a residual byte, check the complete body's zero-state guards and +execute its existing barrier-aware cofactor when eligible, otherwise execute +the original ordered body. Facts are invalidated at writes/opaque barriers +inside the byte and checked anew at the next iteration; no maintained cache. +This replaces the rejected whole-byte case expansion rather than stacking it. +Six native run controls pass, including distinct idle trigger and active-state +paths. .idle-trigger-controls owns compilation; .real-idle-trigger passes all +24 independent actual AOT complete-tape controls. Generated parts/definitions +are 53/3377, versus the control's 52/~3319, not the rejected 74/4713 expansion. +The complete source-bound chain exits 0 at the unchanged 240-second limit: + +- Lean: c6a054e06c60245c6e79966f6e65ddaefd8f1ede70c9f572250a820844f4c8a8 +- Quint: 20e5bcaf3d31f32b8802837803c7805254043c244059ca92a0744a98c3f742f8 +- Scheme/actual AOT: 73c9e3d7421957adee17bd78ea8a840f67b1c125a98d14f32151d434ade0e7f3 + +.idle-trigger-full-evidence.log retains matched-performance Missing and +executionAuthorized=false. Staged source/product: .idle-trigger-real-evidence. +Actual product SHA 8b108a5aaab8b8f64537b46bd541c75df34a9966cadb2bad14cfb8215cca5aa5; +prepared SHA 0acd3c22f413771c029e29144576264736796930763df4b672345afcb33a5d6a. +Original IR and corpus hashes are unchanged. Before timing, another session's +new-SDK reference diagnostic was found consuming roughly 94 percent CPU; +wait for it to end without terminating or modifying its work. A subsequent +Ascent compiler batch also ended before timing. No performance qualification +follows from models. The strictly serial, explicitly awaited lanes are: + +| Lane | 1000-request wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| control 1 | 44.102 | 60 | 129 | 22674.71 | 43.509 | 28803072 | +| candidate 1 | 75.223 | 157 | 428 | 13293.81 | 68.337 | 28901376 | +| candidate 2 | 42.067 | 64 | 92 | 23771.60 | 41.814 | 28884992 | +| control 2 | 45.846 | 66 | 145 | 21812.15 | 45.561 | 28819456 | +| candidate 3 | 45.751 | 73 | 182 | 21857.45 | 45.126 | 28901376 | +| control 3 | 45.547 | 61 | 154 | 21955.34 | 45.243 | 28868608 | +| control 4 | 46.747 | 68 | 136 | 21391.75 | 46.381 | 28819456 | +| candidate 4 | 43.806 | 65 | 152 | 22827.92 | 43.197 | 28884992 | + +All lanes compare all 1000 complete independent tapes, report five GCs and +fail the unchanged preflight (exit 70). The raw allocation counter deltas are +identical, but are NOT allocated-byte totals under GC. No stable major gain: +candidate 1 regresses markedly, candidate 3 CPU is essentially tied, and +candidate RSS is slightly higher. A new external reference diagnostic appears +in the post-timing process snapshot; polling does not establish an isolated +host throughout the entire interval. Thus no architectural speedup or exact +cause of candidate 1's extra CPU is inferred. Reject the integration rather +than select its best singleton. Full matrices remain unopened. + +The added idle-residual lowering, experiment-only test and test-count change +are withdrawn; pre-existing condition lowering stays intact. Five restored +native run controls pass with exit 0 in .idle-trigger-rollback-controls. +.rejected-idle-trigger-source.diff archives the experiment (also contains +pre-existing condition lowering; NOT a wholesale rollback). The model/product +packet is now historical rather than current source qualification. Immutable +baseline hash remains a94f3fb148469e3f3f04bffc89bec90be66d65efcdd32664a8787e844ffa17c0. + +Next architectural obligation: establish per-consumer/state dependency support +and executed idle-admission coverage before another specializer. A monolithic +all-zero entry guard still couples independent payload counters and machines; +repeating that guard per byte does not prove removal of residual work. Do not +assume a counter is irrelevant from its name or introduce a maintained cache. +Lean's eligible-domain equivalence remains separate from Quint's invalidation, +ordered publication and ownership protocol. Real AOT and the 10 ms gate must +discharge the implementation/performance obligations, not just these models. + +## 2026-10-13 bounded run leaf rejection and dependency-derived calling convention + +GitHub PR 13 was verified at b5e1895c8bc9a79dcb3d4e845130cdc113b7d2be. +The installed pinned SDK and all existing baseline binaries remain unchanged. +No automation was restarted. Two engine candidates were implemented in order. + +The first moved the already-admitted ordinary-run endpoint search into a +synchronous borrowed-buffer C leaf, with a single-byte specialization. Four +native controls include exhaustive byte/suffix differential checks; 24 real +Org AOT complete-tape controls pass. The candidate's standalone Lean and Quint +checks pass, but the combined source-bound Scheme gate times out at its original +240 seconds; that failed observation is retained, not counted as conformance. +The original memchr version failed dynamic linking, and was replaced by a +bounded C loop without changing the installed SDK or its link configuration. + +Sequential A/B/B/A diagnostics under other active compiler workloads: + +| Lane | 1000-request wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Existing anchored control 1 | 76.117 | 97 | 499 | 13137.670 | 52.118 | 28983296 | +| Borrowed run leaf 1 | 56.158 | 96 | 337 | 17806.902 | 53.835 | 29048832 | +| Borrowed run leaf 2 | 93.750 | 229 | 818 | 10666.667 | 62.464 | 29016064 | +| Existing anchored control 2 | 75.350 | 200 | 693 | 13271.400 | 59.364 | 28950528 | + +All four compare all 1000 independent complete tapes, have five GCs and fail +the unchanged 10 ms gate (exit 70). CPU cost did not improve; RSS increased. +These disturbed diagnostics do not qualify throughput. The new runtime APIs, +compiler import/lowering and active scanner tests were rolled back. Negative +artifacts remain under .real-run-leaf, .run-leaf-*, and the test is archived as +.rejected-run-leaf-scan-test.ss. The pure first-trigger/partition lemmas remain +applicable to the restored Scheme scanner; they do not prove the rejected C. + +The second candidate derives each generated helper's fixed arguments from its +normalized body, including transitive calls. Unused source/bounds/slots/owner +arguments are removed without request caches, speculative effects, code +inlining, or changes to event publication. Four focused native controls pass +for state-only bodies, quoted names, lexical cursor binding and list ownership; +Lean's EnvironmentExpr.project_required proves read-dependent projection for +its explicit expression fragment. Quint retains lifecycle responsibility. +The real generated product has 3319 definitions versus 3499 in its anchored +predecessor; static private argument slots decrease approximately 28 percent. +Static syntax counts are not dynamic instruction or allocation measurements. +Its real AOT binary 71412dddaf0a3920844c8743120d52266aff886f82ddefed022e2697e15a3cb6 +passes 24 complete-tape controls. Full evidence-real-fold-controls passes within +the unchanged 240-second Scheme limit, with realAotControlsBound=true: +Lean d54d8d3b80bacc978181469c260854af00cc79d3c8d904fca0e21cc675e32be8; +Quint 7c9a6ae3b8eaf0e2fb8b2843081571d3230dbce5448dc23d8ad291f1efc22f35; +Scheme 8c0faa156f6020e026340dfe33eab4aa50643e9acd6d2cf890812290819a7ea1. +Native review 9993bfcece3a0f79cd0f16d160cf5957bd176ce404cfcbe58854ba79df1b8a91 +still has matched-performance missing and executionAuthorized=false. +The prepared binary hash is a94f3fb148469e3f3f04bffc89bec90be66d65efcdd32664a8787e844ffa17c0. + +After this task's compilation ended, matched A/B/B/A and reversed B/A/A/B +preflights retained the same corpus, full tapes, flags and five GCs: + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Anchored control 1 | 45.327 | 64 | 100 | 22061.906 | 44.865 | 28950528 | +| Projected environment 1 | 43.471 | 60 | 95 | 23003.842 | 43.063 | 28803072 | +| Projected environment 2 | 43.217 | 62 | 107 | 23139.043 | 43.136 | 28835840 | +| Anchored control 2 | 44.048 | 63 | 93 | 22702.506 | 43.994 | 28934144 | +| Projected environment 3 | 142.624 | 164 | 1226 | 7011.443 | 73.236 | 28819456 | +| Anchored control 3 | 212.749 | 350 | 3135 | 4700.375 | 84.107 | 28950528 | +| Anchored control 4 | 70.518 | 120 | 476 | 14180.777 | 65.431 | 28950528 | +| Projected environment 4 | 148.276 | 237 | 1509 | 6744.180 | 70.871 | 28868608 | + +Every lane passes 1000 complete semantic comparisons but exits 70 at the hard +10 ms gate. The first round has a modest wall/CPU decrease and lower peak RSS; +the reversed round is unstable for both products and does not reproduce a +qualified benefit. No concurrent compiler was found in the snapshot after +that round, so its pauses must NOT be attributed to compiler load by assumption. +Raw receipts remain .call-environment-abba-*.log; none are discarded. +The projected candidate's non-CPU remainder rises from 0.408/0.081 ms in the +first round to 69.388/77.405 ms in the second. CPU cost also rises: subtracting +CPU from wall alone cannot identify GC, scheduling or blocking mechanisms. +Keep this private candidate unpromoted; the installed/committed baseline stays +unchanged. No three-load twenty-batch matrix, performance acceptance or push. +Do not extrapolate these engine-only times to public Org::parse or FFI/Tokio. + +## 2026-10-12 run specialization and spike attribution + +Verified GitHub parser PR 13 still has head a7eb67a3c6c5398c7e591b88fafa2d2f4275fa94. +Attempted a contiguous foreign scan only for admitted overwrite-only ordinary +runs (87 generated real Org call sites), never arbitrary state/effect regions. +Nine focused controls and 24 complete actual AOT tapes passed. Sequential B/R/R/B: + +| Product | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Buffer 9 | 49.054 | 65 | 70 | 20385.697 | 48.541 | 29048832 | +| Run leaf 1 | 48.578 | 65 | 75 | 20585.450 | 48.563 | 29081600 | +| Run leaf 2 | 48.326 | 66 | 72 | 20692.795 | 48.325 | 29081600 | +| Buffer 10 | 47.879 | 64 | 71 | 20885.983 | 47.879 | 29065216 | + +No stable timing/RSS improvement; all failed the unchanged 10 ms gate. +Rolled back byte scanner API and compiler import/lowering; archive +.rejected-ordinary-leaf.patch. .real-buffer-runs is REJECTED, not a new baseline. +Restored eight active consumer controls; compiler closure rebuilt after rollback. +No full 20-batch matrix, README performance promotion or push. + +Lean standalone Fold check passed with two new universal theorems: run residual +retention and contiguous partition. Quint full existing protocol suite passed +(7 publication, 8 shared-scan, 4 idle, 4 run controls); each of the three lifecycle +invariants ran 1000 seeded traces of 20 steps. These are bounded traces, not +exhaustive verification. Model collection now binds actual admission, lowering, +runtime scanner/writer and control source files. Aggregate Lake/ASCENT evidence +was not rerun: private checkout has no materialized LeanPoo dependency. + +Three diagnostic-only complete-tape runs read installed header VM GC counters +and getrusage counters per call. Logs .stalls-runs-{1,2,3}.log retain top 20 rows. +Observed GC pauses 234-564 us account for most of the 314-696 us GC-bearing +requests; non-GC requests also show involuntary switches. Cold initial call and +extra probes mean these rows cannot replace matched performance receipts. +The historical 71.459 ms batch had 62.551 ms CPU, an 8.908 ms wall/CPU gap and +712 whole-process involuntary switches. Those process-wide counts include +initialization and do not identify timed-call switches. Its full root cause +remains UNPROVEN; do not attribute the whole spike to GC or scheduling. + +Kernel/SDK unchanged; unsupported SMP still skipped; Automation remains off. + +Source baseline: parser PR 13 a7eb67a3c6c5398c7e591b88fafa2d2f4275fa94; +Orgize PR 18 8f04aa246290b56b5605f4a38f25c55d0b416268. +Private checkout paths are experimental build locations, not dependency pins. +SDK: installed Gerbil 1cfb032 / Gambit ea114fc, unchanged. +ASP f5b7c009, POO 099b3815 and Core e85fd453 were privately rebuilt. +Complete parser source and current Org syntax declarations compiled successfully. + +The candidate emitted 13 lexical shared-scan bindings and 28 frame-byte +bindings. Both control and candidate passed all 24 complete OEV1 tape cases. +Five planner and five lowering safety cases passed before rollback. +Independent changing-input preparation contains 1000 inputs and complete tapes. + +Sequential unsandboxed paired preflights (identical binaries/options/input, +no task compiler active; order candidate/control/control/candidate): + +| Product | Wall ms | P95 us | P99 us | Requests/s | Peak RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | +| Candidate 2 | 60.211 | 86 | 99 | 16608.26 | 29507584 | +| Control 2 | 61.121 | 86 | 104 | 16360.99 | 29310976 | +| Control 3 | 58.712 | 83 | 95 | 17032.29 | 29278208 | +| Candidate 3 | 62.922 | 90 | 168 | 15892.69 | 29491200 | + +All four failed the unchanged 1000-request <=10 ms gate. No 20-batch matrix +was launched. P95/P99 are per compiled parser call; wall includes complete +semantic comparison. Peak RSS includes executable, retained inputs/oracles, +and process initialization, not a parser-only allocation measurement. +Each batch reported five GCs and the same raw allocation delta 23014968; +the runtime did not admit that counter as allocated bytes. CPU approximately +matched timed wall. These are Fold/OEV1 results, not public Org, Tokio, Python, +or FFI end-to-end acceptance. Earlier sandboxed runs could not report RSS and +are retained separately, not pooled into these controls. + +Rollback: removed candidate runtime lowering and the candidate generation +mode. Original runtime lowering restored; the rejected lowering patch and test +are retained as `.rejected-cross-lowering.patch` and +`.rejected-frame-guard-test.ss`. Existing `.real-cross` binaries are REJECTED +artifacts compiled before rollback, not the active source baseline. +Pure analysis, its division-error safety fix, compiler dependency separation +and private build/qualification support remain uncommitted and unpromoted. + +Previous architectural target: fuse the full-range inline trigger precheck with +the guarded ordinary-run traversal. Actual Org currently scans for triggers +before traversing the same source range again. Any fusion must preserve the +no-trigger branch's lack of state writes, entry guard cells, exceptional-byte +ordering, final ordinary-prefix state, and complete tapes. Do not simply +delete the precheck or execute skipped state updates. No cross-request cache. + +Receipt locations: `.control-preflight-{1,2,3}.log`, +`.candidate-preflight-{1,2,3}.log`, `.real-{control,cross}/org-fold-test.log`. +## Subsequent whole-engine experiments + +Parser PR 13 head was reverified through GitHub as a7eb67a3c6c5398c7e591b88fafa2d2f4275fa94. +Scope remains the upstream parser engine; no downstream FFI/runtime changes. + +Trigger/traversal cursor fusion reached six actual Org AOT sites and passed +five planner/lowering controls plus all 24 complete tape cases. Strict awaited +order control 6 / fused 3 / fused 4 / control 7 produced: + +| Product | Wall ms | P95 us | P99 us | Requests/s | Peak RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | +| Control 6 | 65.184 | 89 | 169 | 15341.188 | 29294592 | +| Fused 3 | 68.016 | 90 | 177 | 14702.423 | 30474240 | +| Fused 4 | 64.157 | 89 | 151 | 15586.764 | 30507008 | +| Control 7 | 62.530 | 86 | 137 | 15992.324 | 29278208 | + +Rejected and rolled back compiler lowering, context flag, generator mode, +and active tests. Archive: .rejected-trigger-fusion.patch and +.rejected-trigger-fusion-test.ss. Fused binaries are rejected artifacts. +Earlier overlapping/yielded preflights 1/2 and controls 4/5 are diagnostic +only, excluded from this matched sequence. + +Bounded one-use generated procedure contraction reduced private definitions +3588 -> 1830 and units 57 -> 29. Seven native controls covered argument-once, +sharing/escape, quotation, size budgets, lexical assignment, nested remaining +references and recursion. The actual AOT product passed all 24 complete tape +cases. No request cache, source snapshot, SDK or kernel changes. + +Two strictly sequential C/K/K/C preflight groups were run; source corpus, +SDK, linker, process options and hard gate were unchanged: + +| Product | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Control 8 | 72.001 | 93 | 354 | 13888.696 | 65.599 | 29310976 | +| Contracted 1 | 77.681 | 109 | 589 | 12873.161 | 59.773 | 27590656 | +| Contracted 2 | 60.711 | 107 | 211 | 16471.480 | 60.478 | 27590656 | +| Control 9 | 98.548 | 87 | 321 | 10147.339 | 65.739 | 29310976 | +| Control 10 | 62.873 | 89 | 132 | 15905.078 | 62.546 | 29310976 | +| Contracted 3 | 84.686 | 204 | 443 | 11808.327 | 75.512 | 27623424 | +| Contracted 4 | 54.205 | 79 | 114 | 18448.483 | 54.073 | 27557888 | +| Control 11 | 61.735 | 88 | 141 | 16198.267 | 61.608 | 29327360 | + +Wall/CPU divergences and CPU variation preclude a stable performance claim. +Candidate GC counts 4 vs control 5 and lower RSS do not qualify throughput. +Every run failed the original <=10 ms / >=100000 requests/s gate (exit 70). +Do not pool batch P95/P99: these remain individual per-call distributions. +Allocation-counter deltas were not admitted as bytes by the runtime. + +Contraction activation and lowering were rolled back. Archive: +.rejected-call-contraction.patch, .rejected-call-graph.ss and +.rejected-call-graph-test.ss. .real-contracted binaries are REJECTED artifacts, +not the accepted compiler. Baseline .real-control binary remains unchanged. +Restored compiler closure was rebuilt through build-fold-attribution (exit 0); +two cost controls and five state/effect controls passed with final OK. +Fresh tape-run generation in .real-restored matched every control +parse-org-compiled-events*.ss byte-for-byte (SOURCE-ROLLBACK-OK). +All task compilation/test/preflight processes have finished; no matrix active. + +No full 60-batch matrix, public Org/FFI acceptance, README speed claim or push +was performed. Unsupported SMP remains skipped. Automation stays off. +Do not repeat either experiment as a fresh candidate. Next work must address +whole-engine execution/scan ownership and actual state dependence, not just +function counts, GC counts, or downstream queue concurrency. + +## Continuous buffer / record consumer experiment + +Parser PR 13 exact head was again verified unchanged through GitHub. +The retained local candidate changes only engine source traversal and tape +consumer: one synchronous std/ffi borrowed-buffer leaf per line boundary, +and one leaf per complete OEV1 start/token record. No retained pointer, +Scheme callback from C, host-alignment cast, request cache or kernel change. +Unsigned bounds, all wire byte orders, source visit order and seal checks +are retained. Eight native controls and all 24 actual tape controls passed. +The grammar digest and independent 1000-input/oracle corpus are unchanged. +The original .real-control executable remains immutable and accepted source. +The local .real-buffer candidate is NOT promoted or pushed. + +First matched awaited C/B/B/C group: + +| Product | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Control 12 | 67.740 | 97 | 301 | 14762.327 | 65.687 | 29310976 | +| Buffer 1 | 50.733 | 69 | 102 | 19711.036 | 50.679 | 29114368 | +| Buffer 2 | 50.243 | 71 | 100 | 19903.270 | 50.152 | 29097984 | +| Control 13 | 62.013 | 89 | 115 | 16125.651 | 61.867 | 29310976 | + +Additional buffer controls (never suppress noisy/negative records): + +| Product | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Buffer 3 | 50.085 | 67 | 105 | 19966.058 | 49.643 | 29097984 | +| Buffer 4 | 49.580 | 67 | 78 | 20169.423 | 49.566 | 29130752 | +| Buffer 5 | 54.252 | 75 | 123 | 18432.500 | 53.140 | 29114368 | +| Buffer 6 | 50.270 | 69 | 111 | 19892.580 | 50.162 | 29130752 | +| Buffer 7 | 71.459 | 176 | 383 | 13994.039 | 62.551 | 29097984 | +| Buffer 8 | 54.555 | 74 | 183 | 18330.126 | 54.271 | 29163520 | +| Control 14 | 64.159 | 88 | 133 | 15586.278 | 62.211 | 29278208 | + +All preflights failed the unchanged <=10 ms / >=100000 requests/s gate. +No batch percentiles are pooled or called a per-request distribution. +Eight preflight processes are NOT a twenty-batch qualification matrix. + +The extension lowering full-range membership probes, paired masks and +ordinary-run seeks to checked foreign leaves was slower than .real-buffer: +51.683/52.367 ms vs bracketing 50.085/49.580 ms; RSS 29130752/29114368. +P95 69/72 us, P99 90/112 us, CPU 51.611/52.286 ms, five GCs throughout. +Rolled back byte-spans API and every compiler lowering site. Evidence: +.rejected-buffer-scans.patch, .rejected-buffer-consumer-scans-test.ss, +.buffer-scans-preflight-{1,2}.log. .real-buffer-scans is a rejected artifact. + +Official std/crypto/cipher.ss and std/misc/ports.ss were inspected for +private-buffer shrink publication. It removed the final copy in an experiment +but did not establish stable throughput/RSS improvement over .real-buffer: + +| Owned publication | Wall ms | P95 us | P99 us | Requests/s | CPU ms | RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 76.635 | 194 | 397 | 13048.868 | 70.494 | 29114368 | +| 2 | 49.250 | 68 | 97 | 20304.569 | 49.160 | 29147136 | +| 3 | 56.690 | 77 | 187 | 17639.795 | 54.085 | 29147136 | +| 4 | 49.776 | 66 | 76 | 20090.003 | 49.767 | 29130752 | + +GC fell 5 -> 4; raw allocation-counter delta 23037048 -> 21104040, but +neither delta is admitted allocated bytes because GC occurred. GC count is +not a wall/throughput claim. Conservative selection restored copy publication; +archive .rejected-owned-publication.patch and .real-buffer-owned artifacts. +Retain every negative sample; neither GC nor scheduler cause is proven by +wall/CPU differences alone. No unsupported SMP acceptance or formal-model +proof is inferred from these controls. + +Active private source now corresponds to .real-buffer, not scans/owned. +Final restored test-fold-buffer-consumer completed with eight CASE-OK and +HARNESS-OK/OK (exit 0). Static input comparisons are not byte-identical +because compile products include varying generated metadata; do not claim +artifact identity from source restoration alone. All task build +and preflight processes finished; no duplicated matrix/process is running. +No full matrix is active; no README qualified-performance claim or push. +Continue with actual generated state execution/dependence and remaining +allocation ownership. Do not repeat the rejected unconditional foreign scan +or shrink publication as newly discovered candidates. Automation stays off. + +## Architecture review: event representation versus recognition execution + +The user requested an architecture diagnosis before further optimization. +New implementation is paused. Condition-only Boolean lowering and lexical +horizontal-prefix sharing are experimental source edits: compilation is not +actual AOT conformance or performance acceptance. Do not promote these edits. +The immutable .real-buffer/prepared-tape-fold executable remains the control. + +The ready-gated .fold-profile-2.sample observes the actual Fold generated units, +state execution and event writer. A concurrent build was active; the sample is +not matched exclusive CPU attribution or qualified performance evidence. +.fold-profile-1.sample captured startup and is excluded from Fold attribution. + +The diagnostic .real-counts/counts.log records 4518 calls to expression-3471 +across 24 complete AOT controls. In .real-buffer/parse-org-compiled-events-part-40.ss, +3471 scans horizontal whitespace; 3473 invokes 3471 and 3476 invokes 3473. +Code interning therefore does not establish shared dynamic scan results. +Counts are not exclusive timing, and the diagnostic Just invocation did not +complete successfully at its external resource-reporting boundary. The printed +diagnostic completion does not turn that invocation into a passed recipe. + +Primary research distinguishes output representation from efficient recognition: +flap (https://arxiv.org/html/2304.05276) fuses deterministic lexer/parser work +and eliminates ephemeral tokens; its LL(1) assumptions are not established for +the whole Org strategy. A Typed, Algebraic Approach to Parsing +(https://www.cl.cam.ac.uk/~jdy22/papers/a-typed-algebraic-approach-to-parsing.pdf) +uses grammar restrictions and staging, not merely AOT compilation of every +rule. These papers do not prove this EventFold compiler correct or fast. + +Current judgment: retain the lossless ordered event boundary provisionally; +do not qualify the recognition/execution architecture as high-performance. +Audit frame-local scan dependencies, effect/ordering barriers, active-rule +dispatch, source progress and per-byte work before selecting a restructuring. +Independent language conformance remains distinct from agreement between an +interpreter and a compiler consuming the same strategy. Lean owns semantic +refinement/progress/work bounds; Quint owns finite ownership/publication and +activation histories, not duplicate algebra or a physical latency proof. +The 71.459 ms historical spike remains only partly attributed; wall-minus-CPU +alone cannot identify a cause. No full matrix, baseline promotion, PR push, +kernel/SDK change or Automation restart occurred. + +## Authorized architecture implementation follow-up + +Audit: docs/research/event-fold-recognition-architecture.org records research +assumptions, recognizer/output separation and unproved whole-program bounds. +The frame-fact/condition candidate .real-frame-plan passes five frame lowering +controls, four condition controls and 24 actual complete Org AOT controls. +Its native prepared executable is .real-frame-plan/sdk/prepared-tape-fold. +Same independent input/oracle corpus and grammar identity are retained. + +Sequential control/candidate/candidate/control preflight results: + +| Lane | Batch wall ms | P95 us | P99 us | Requests/s | CPU ms | Peak RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Buffer 11 | 83.344 | 150 | 445 | 11998.464 | 60.326 | 29097984 | +| Frame 1 | 47.997 | 67 | 114 | 20834.635 | 47.856 | 28803072 | +| Frame 2 | 47.991 | 65 | 108 | 20837.240 | 47.823 | 28819456 | +| Buffer 12 | 50.837 | 67 | 106 | 19670.712 | 50.802 | 29065216 | + +All four exit 70 at the unchanged physical gate. Buffer 11 includes a 13.505 ms +request spike: no large causal speedup is admitted from that disturbed lane. +Both candidates retain five GCs; raw counter deltas are not allocated bytes. +Static call graphs change 3590/7599 definitions/calls to 3494/7444. Counts are +not a runtime throughput proof. Do not start the twenty-batch matrix. + +Baseline executable SHA256: +0db434809bbc5af932bc587a76f1f6663f7f9550401548546924d4dbbd5cf73d +Independent .real-preflight.ss SHA256: +e9ccee6c366bdc07f733c525634ed9efed2eb1d2932ef93b54f5ba4cd820df40 + +The next opt-in candidate moves private state into lexical execution-region +owners. Four ownership lowering controls pass. Its generated .real-lexical-plan +has 1616 definitions / 6365 private call sites; these are static counts only. +Actual linking is owned by exec session 2108; do not duplicate that build. +Its complete AOT controls and physical preflight have NOT yet run. +Compiler flag current-fold-scheme-lexical-state defaults to false. Root/helper +layout and parameter overrides are changed only in the explicit candidate. + +Lean formal-fold, including the new conditional frame-representation transition +refinement theorem, passes. The theorem requires transition commutation; it is +not a proof of the concrete compiler. Quint formal-protocol passes publication, +scan, idle and run directed controls plus seeded bounded histories. Source +controls are wired into qualify-fold-plan and source identities bound by the +collector; the full Lake/ASCENT chain was not executed in this private checkout +because its pinned LeanPoo materialization is absent. Do not claim full closure. + +## Follow-up closure: rejected lexical owners and fresh restored qualification + +Supersedes the earlier missing-dependency state: exact LeanPoo manifest commit +0a111f6c5de43b261de5bba9287b48540d9f7c2e was materialized offline in the private +Lake package checkout; GitHub origin and manifests remain unchanged. Full +`just formal` passes all 38 jobs. Locked Quint dependencies are prepared offline. +The existing upstream .gerbil/native/libascent.dylib is loaded, not rebuilt. +The helper source/AOT bundle passed native evidence-real-fold-controls with +Lean, Quint and scheme-conformance established, matched-performance missing and +executionAuthorized=false. That source-bound packet is historical after rollback. + +Strict matched results (1000 changing requests, complete comparison in wall): + +| Lane | Wall ms | P95 us | P99 us | Requests/s | CPU ms | Peak RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Frame 3 | 46.292 | 61 | 76 | 21602.005 | 45.174 | 28819456 | +| Lexical 1 | 41.072 | 57 | 65 | 24347.487 | 41.066 | 28147712 | +| Lexical 2 | 104.958 | 154 | 1168 | 9527.621 | 57.008 | 28196864 | +| Frame 4 | 44.342 | 61 | 75 | 22551.982 | 44.317 | 28770304 | +| Frame 5 | 44.977 | 59 | 69 | 22233.586 | 44.193 | 28770304 | +| Helper 1 | 145.675 | 299 | 2130 | 6864.596 | 69.181 | 27426816 | +| Helper 2 | 44.563 | 62 | 103 | 22440.141 | 44.497 | 27361280 | +| Frame 6 | 48.229 | 65 | 132 | 20734.413 | 47.908 | 28803072 | + +GC: lexical six, helper seven, controls five. Raw deltas under GC remain +unadmitted allocated bytes. Slow-lane non-CPU gaps remain unattributed. +Both candidates passed 24 actual complete AOT controls but fail physical gates. +Their compiler flag/root/helper layouts and temporary lexical test were removed; +compiled artifacts and every negative timing receipt are retained. The source +snapshot .rejected-lexical-owner-source.diff includes other task changes versus +base and is not a standalone rollback patch. No user source was deleted. + +Active source retains frame-fact/condition lowering and explicit vector state. +`git diff --check` passes. Immutable baseline SHA256 remains +0db434809bbc5af932bc587a76f1f6663f7f9550401548546924d4dbbd5cf73d. +Restored current-source evidence is running as exec 80767 against fresh +.restored-evidence-controls/.restored-evidence-real. Do not duplicate this run; +the historical helper-source packet cannot stand in for the restored source. +No matrix, PR push, kernel/SDK change or Automation restart. + +Restored exec 80767 completed exit 0. Native ASCENT now establishes the current +source's Lean model f5ca890c3f848f63c7a19456ffd5051504cef8b1d63238b8e1d939f813e8d681, +Quint protocol 675f8d76e6213f6d8e37b78e073f03bb831c0bd06276c0b5eb64e64083b3039b, +and Scheme/AOT conformance fd9fe9fedf9bd499a1f16effd613878eddee1668fc81d393c686a257060851da. +Review bundle 8fbb6f1450f89e5b7ab89c6b2c5a8560bbadd88e2c94760b509faf02e1d7186a +retains matched-performance missing and executionAuthorized=false. Its durable +packet is .data/parser-evidence/fold-controls.json. Do not mislabel it full +compiler proof, independent Org grammar completeness, FFI/Tokio/Python admission +or a physical speed claim. Fresh restored compiler controls are in +.restored-evidence-controls/lib; prefer them over the older .aligned-1cfb +compiler products containing historical experimental lowering. +All owned builds, model checks, AOT controls and preflight processes completed. +The <=10 ms gate remains failed; no twenty-batch matrix or promoted baseline. + +## One-step state settling, 2026-10-13 + +New owned candidate: event-fold-run-settle.ss computes per-transition constant +post-state facts, joins both branches and preserves the entry transition. +Ordinary-run admission now accepts a stable overwrite residual after that +entry, rejects read/write recurrences and keeps prefix/entry guards unwritten +across both entry and residual. Lowering executes entry at the first byte and +residual at the last byte; the trigger remains for the original transition. +No runtime cache, event-owner transfer, kernel/SDK change or Automation restart. + +Seven native admission cases pass, including first-byte capture, external +stable reads, recurrences and event/query barriers. Static real-IR run coverage +is 47/285 versus 33/285, not a performance measurement. Full actual Org AOT +product .real-settled-plan passes all 24 controls. Native preflight is linking +as exec 95907. Source-bound Lean/Quint/AOT collector is running as exec 12801 +in .settled-evidence-controls/.settled-evidence-real; do not mutate source +inputs or duplicate these processes. Lean settled_run_refinement explicitly +requires stable-region agreement/preservation; Quint owns peeled-entry epoch +and commit ordering. Focused Lean and Quint checks pass; neither proves speed. +The original .real-buffer baseline SHA256 remains unchanged. + +Settled v1 completed all source-bound model/actual-AOT controls through native +Ascent: Lean ff668b2f8f4379edd59aa4be8b3f2eb79f064168f38668ecc92137ab7d7d2900, +Quint f96bb76c75a2647997c0f1e08f786226b3f12fa9c7a24a7547369e3bf6734000, +Scheme e1b46397fe28968588df1644859c60f6e61152de1c50fb3d14537b7c1499699d. +Matched-performance stays missing, executionAuthorized=false. Sequential ABBA: +Frame7 46.374 ms, P95 62 us/P99 97 us, 21563.807 req/s, RSS 28819456 bytes; +Settled1 48.930 ms, P95 65 us/P99 137 us, 20437.359 req/s, RSS 28819456; +Settled2 47.077 ms, P95 65 us/P99 98 us, 21241.795 req/s, RSS 28819456; +Frame8 47.903 ms, P95 65 us/P99 119 us, 20875.519 req/s, RSS 28835840. +All complete 1000 requests/independent full tapes, five GCs, gate exit 70. +V1 is rejected, immutable product/receipts retained. V2 restores unchanged +already-admitted overwrite kernels and removes duplicate entry/residual +execution on singleton runs. V1 source-bound receipt is historical after this +revision. Do not claim a whole-engine speedup or start the three-by-twenty matrix. + +V2 completed native source-bound closure after fast-forward to PR #13 exact +head b5e1895c8bc9a79dcb3d4e845130cdc113b7d2be. The upstream delta is one +request-owned LR commit; verified no Fold compiler/tape/source-lines delta. +No unrelated dirty edits were discarded; no push or path-pin mutation occurred. +V2 bundle: Lean 94acb95e2f31b961a211ddff3ded9a10f593a5ef766a467e20e9d2daee050d27, +Quint ded044e2bfdf63df7180ae40b3836ec34199e2c548e102b31d271a5b997052cc, +Scheme 1c356314bab03672f9a558211fc2a0fd3cbbc443b12cc659af56e3c8fca43149; +review 7b8cd2d742e3e3134ecbdaceb89ad5879ea7478f3aa63cc28a53eec5a989a64e +still lacks matched-performance and executionAuthorized=false. + +V2 ABBA1: Frame9 137.279 ms (CPU64.381, max27865 us), candidate +57.128/46.096 ms (P95 79/61 us, P99 394/71 us), Frame10 46.635 ms. +No speedup inferred from the disturbed control. Repeated calmer ABBA2: +Frame11 44.537 ms, P95 60 us/P99 73 us, 22453.241 req/s, RSS28786688; +V2-3 45.936 ms, P95 62 us/P99 86 us, 21769.418 req/s, RSS28803072; +V2-4 45.341 ms, P95 62 us/P99 85 us, 22055.094 req/s, RSS28803072; +Frame12 46.412 ms, P95 63 us/P99 117 us, 21546.152 req/s, RSS28819456. +All 1000 complete independent tapes, five GCs, gate exit70. No robust physical +benefit; V2 is rejected as well. Baseline mean45.4745 ms versus45.6385 ms +candidate in this calm sequence; do not turn an individual faster lane into +an admitted improvement. Raw allocation counter deltas remain unadmitted. + +Removed only this turn's settling production path/new module, extra test cases, +Justfile projection and model changes. Original ordinary-run/test files show +no tracked diff; frame-fact/condition and prior work remain. Prototype moved +to .rejected-event-fold-run-settle.ss; .rejected-settling-source.diff includes +other task diffs versus Git head and is NOT a standalone rollback patch. +Both compiled products, generated source and negative timing receipts remain. +Fresh restored latest-head source/Ascent/AOT qualification will use +.restored-latest-controls/.restored-latest-real; older V2 packets are historical +after rollback. No full matrix, promoted baseline, kernel/SDK change or Automation. + +Restored latest-head exec11018 completed exit0. Current source-bound native +Ascent packet re-establishes Lean22b42bd804ff2ede969e742c556e907507b17c2537da011aae924e63ee080a9f, +Quint b3b72589621a30c1bbadfacd84a1419dd54cc14e39f5175d3e41b947fedfacee, +Scheme c290d4bff26ccba06fc95cc74557686cd4189077a538e76ae943a1c11b5ad175. +Review de405019967899a253a0a1616fafd6445a076f5a0666bec36711910659effbcb +retains matched-performance missing and executionAuthorized=false. Prefer fresh +.restored-latest-controls/lib for continued compilation. The candidate packets +remain historical; this is Fold qualification, not new LR/FFI/Tokio/downstream +acceptance. All task processes completed. No three-by-twenty matrix was started +because every real 1000-request preflight failed the unchanged <=10 ms gate. +Next implementation must target shared recognition across consumers/frames with +explicit action-order and ownership admission; static loop eligibility alone +did not identify the dominating dynamic costs. Do not repeat either settling +candidate or interpret host scheduling spikes as a causal optimizer benefit. + +## Cross-consumer delimiter facts, 2026-10-13 + +Exact parser PR13 head remains b5e1895c8bc9a79dcb3d4e845130cdc113b7d2be. +The first prototype propagated immutable horizontal facts across declared +helpers, but actual IR inspection found zero eligible scans in all 27 helpers. +It was removed, not reported as a runtime improvement. Its generated product +.real-cross-consumer is historical and must not qualify current source. + +The active prototype lowers dominating start-anchored delimiter/word/key scans +to lexical facts across ordered consumers. It does not hoist a conditionally +reached first observation or cross helper, loop, opaque or frame barriers. +The first consumer must have no query/error effect; state-dependent scans are +not admitted. Existing rejected unconditional content/trim sharing is not +re-enabled. No runtime lookup/cache, event list or new owner is introduced. + +Directed tests caught an incorrect traversal of the statement list: the first +statement was not rewritten and its scan remained duplicated. Fixed by mapping +expression rewriting over each statement. The obsolete generator PID8791 was +terminated, preserving unrelated Ascent builds. .real-cross-delimiter is an +incomplete historical product. Corrected .real-cross-delimiter-v2 is active +in exec session65618; do not duplicate it. Five frame and five statement tests +now pass. Lean owns observation/ordered fold equality; Quint owns same-extent +consumer lifecycle and invalidation, not throughput. Actual AOT/Ascent and +physical preflight remain required. Baseline/corpus hashes remain unchanged. + +Corrected AOT generated nine lexical delimiter facts: 3499 definitions, +7423 private call sites, 2079 syntactic tail calls, 70 event assignments. +Frame control has 3494/7443/2081/70. No dynamic cost inferred from these counts. +The new prepared executable SHA256 is +a65833e33e410f6f2ebc2022a41177241df153dea745953d67b050d0cfd2f155. +Actual 24 independent Org tape controls pass. Source-bound native Ascent join +exec92813 is active; current Lean observation is +3da42191790121abb3a98e0ec602d78d0dcbad452ecfb3b58ba1844a77e49dbf, +Quint746813527f5a2be1d8b8a9ed5528a651caa1f58722ce362e3782ef0368b31b5d. +Nine directed Fold lifecycle controls pass; bounded histories are not exhaustive. +Do not edit bound source while this collector runs. Physical ABBA must wait +for compilation to finish. No full matrix or baseline promotion occurred. + +Installed official optimize-call.ss:239-296 explicitly checks procedure effect +signatures before discarding evaluation and preserves evaluation with begin +otherwise. This prototype uses a stricter no-query/no-error/no-allocation +admission rather than treating successful model tests as a purity oracle. + +Source-bound Ascent exec92813 completed exit0. Scheme conformance observation +a2b60a430cd22f3e0bc0f7732d6e0e77c02e185e790bc72a84dadc8b1f9ab532; +review20b3dccbdad6120c1da21a9bb30725230cf34d0dd4fed277252eb3d53f1a6f08 +still lacks matched-performance, executionAuthorized=false. Directed tests, +bounded Quint histories, Lean and 24 actual AOT tapes are distinct from speed. + +All owned compilations ended before sequential ABBA preflights. Same pinned +SDK, executable controls, 1000 changing inputs, 2856770 bytes, independent +complete expected tapes and original 10 ms/100000 requests/s gate: + +| Lane | Wall ms | P95 us | P99 us | Requests/s | Peak RSS bytes | CPU ms | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Frame1 | 52.707 | 63 | 353 | 18972.812 | 28835840 | 50.031 | +| Candidate1 | 45.609 | 62 | 79 | 21925.497 | 28770304 | 45.606 | +| Candidate2 | 45.573 | 62 | 71 | 21942.817 | 28753920 | 45.548 | +| Frame2 | 46.113 | 63 | 85 | 21685.859 | 28819456 | 46.099 | +| Frame3 | 46.906 | 64 | 91 | 21319.234 | 28819456 | 46.805 | +| Candidate3 | 44.855 | 64 | 86 | 22294.059 | 28737536 | 44.810 | +| Candidate4 | 45.843 | 62 | 83 | 21813.581 | 28721152 | 45.785 | +| Frame4 | 45.827 | 62 | 81 | 21821.197 | 28819456 | 45.787 | + +All eight lanes complete full semantic comparison, have five GCs and reject +with exit70. Allocation counter delta23037048 is NOT allocated-byte evidence +across GC. Candidate RSS27.390625-27.4375 MiB includes initialization, retained +inputs/oracles and native code; not per-request memory. Request percentiles +cover compiled Fold calls; batch wall includes full tape comparison, not Rust, +Tokio, Python or public Org::parse. No claim of a stable large speedup: Frame1 +is disturbed and Frame4 nearly ties Candidate4. No consistent regression was +observed, so the source remains an unpromoted local candidate, not a new baseline. +Original .real-buffer and .real-frame-plan hashes remain unchanged. No push, +SDK/kernel change, Automation restart or three-by-twenty matrix occurred. + +Next work must address ordinary-text recognition dispatch across consumers, +not mistake nine delimiter facts for whole-engine fusion or repeat the unused +declared-helper contract. Do not rebuild current products merely to repeat +the same diagnostic. Full workload matrix remains blocked by the unchanged +preflight gate, not by missing model evidence. All owned sessions completed. + +## Whole-frame necessary-prefix rejection experiment (2026-10-13) + +Existing PR13 still points to b5e1895c8bc9a79dcb3d4e845130cdc113b7d2be +(live GitHub head check). The experiment stages a shared necessary-prefix trie +across ordered consumers and selects a negative residual program. Positive +classification retains the original program. Pure state reads before the +necessary prefix may disappear only on the negative path. Queries/errors, +helpers, loops and source-frame changes remain opaque. No request cache, +kernel change, SDK rebuild, Automation restart or dependency path pin is added. + +Changed source owners are byte dispatch, prefix-family and frame-statement +lowering. Lean negative_line_consumer has explicit necessity/effect premises; +Quint NegativeConsumer concerns current frame/owner histories only. The model +collector now binds the prefix-family implementation and byte-dispatch tests. +The directed byte-dispatch suite has two CASE-OK markers and final OK after +compiling the current prefix implementation. An earlier module load observed +the stale first-byte experiment; neither that failed check nor its historical +.real-line-negative product is a qualified current-source result. + +Fresh .line-prefix-controls build completed. Actual .real-line-prefix contains +3811 definitions, 8697 private calls, 2637 syntactic tail calls and 70 event +assignments. These are static counts, not execution counts or speed evidence. +Native links are active sessions3627 and85884; do not duplicate them. +Prepared independent corpus and all three historical baseline executable +SHA256 values rechecked unchanged. Concurrent unrelated Ascent/parser-quality +compilation was observed; no affected timing is admitted as an idle-host result. +Current candidate has not yet received source-bound full AOT closure or timing. + +The duplicate-program prefix experiment is rejected, not promoted. Actual24 +complete tapes passed, but matched elevated ABBA lanes were control45.078ms, +candidate90.471ms, candidate47.987ms, control51.529ms. Candidate peak RSS +29360128/29409280bytes exceeds control28753920/28770304bytes. Candidate P95 +122/67us, P99 818/117us, throughput11053/20839requests/s, fiveGCs. The slow +candidate's wall-minus-CPU36.447ms does not identify a cause. Earlier sandboxed +lanes1/2 cannot qualify RSS because time's sysctl was denied. All raw results +remain. Immutable rejected executable SHA256 is +c29d3d31d514096711ece4dba129242be342735ee9c54121a0b6211d8b685c44. + +Production prototype now removes that duplicate-program expansion: one lexical +prefix classification guards admitted original conditions, keeping each +transition once. It does not introduce a runtime cache. The internal immutable +predicate is admitted as total and discarded on nested source-frame entry. +Fresh .line-shared-controls compiler and directed tests completed with finalOK. +Analysis was recompiled after the internal fact's totality declaration before +actual generation. Session17453 is preparing .real-line-shared; do not duplicate. +This replacement still requires current-source AOT closure and timing. No +three-by-twenty workload matrix is authorized by the failed preflight. + +Final current-turn boundary: the single-body variant is an unpromoted source +prototype, NOT qualified AOT or performance. The independent executable link +failed with truncated generated C during overlapping links; subsequent owned +CLI and official compile-exe parallel:#f retries stopped making progress with +no compiler children. All were terminated with TERM, not left as active jobs. +.line-shared-linker.sample records select waiting; it does not establish a +kernel root cause. No SDK/kernel source was edited. Future links sharing static +outputs must be sequential. Retry logs include .line-shared-product.log, +.line-shared-preflight-serial-build.log and .line-shared-official-serial-build.log. + +Correction: the 24 GXI checks import generated Scheme source, NOT a native +compiled module. They establish source-level complete-tape agreement only. +The attempted module preflights failed at import (illegal declare in ~0.scm), +before timing, and overlapped; they provide NO performance/RSS qualification. +The temporary module driver was removed from Justfile. Do not present these +receipts as AOT acceptance or compare them to the standalone baseline. + +Current source-bound Lean identity28d41357d95bec7df065c5a2ac86152d510f6250a2a097674f054d7f88ec6645 +and Quint identity674445d5c530f4235895970f9976c98dfed5c90575075109d5b978c4ce1b0c10 +passed. Native review3e319af898280354eab2479757e42d6f1f070622d61a6e7954ae45010fd21008 +correctly lacks scheme-conformance and matched-performance, executionAuthorized=false. +No source-bound full current AOT packet, standalone single-body performance, +twenty-batch matrix, push or baseline promotion is claimed. Historical baseline +executable hashes remain unchanged. No owned benchmark/compiler sessions remain. +Next work must close the generated-product linking gate without kernel/SDK +modifications, then measure this single-body variant under matched conditions; +do not duplicate an active matrix or revive the rejected duplicate-program path. + +## Independent AOT lock recovery and horizontal anchors (2026-10-13) + +Live PR13 remains b5e1895c8bc9a79dcb3d4e845130cdc113b7d2be. Official driver.ss +compile-obj retries while its exact .o.lock exists. Process inventory and lsof +found no owner for the retained part-11 lock under .real-line-shared/lib/static. +Moved only that lock to .real-line-shared/part-11.stale-object-lock.evidence. +No SDK/kernel source changed. Sequential existing CLI links then completed. +The prior select-only sample was a lock wait, not a proved kernel deadlock. +No active unrelated compiler was terminated or baseline binary overwritten. + +Just now exposes link-org-fold-candidate for resume without regeneration. +Both link recipes call the small t/benchmarks/parser-throughput/link-locks.ss +guard: any lock rejects the build, no automatic deletion. Its native GXI +fixture checks clean admission, dirty rejection, unchanged evidence and clean +re-admission. Final OK passed; initial import-syntax failures were fixed before +qualification. qualify-fold-plan includes this regression check. + +Recovered .real-line-shared/org-fold SHA256: +c14618c3a26ccf542dffa2ba668bd95d9dc6a54de3b6b8361f47e9a95835e4dd. +Recovered prepared-tape-fold SHA256: +21f8b36cb45965c5d4a121eb1ed8e6093ce07445e5e047997fc0d78aa42f5078. +All 24 independent native Org tapes pass. Eight sequential delimiter-control +ABBA preflights have candidate43.006-45.728ms, P9560-65us, P9966-87us; +control43.494-47.975ms. Candidate RSS28803072-28917760bytes. All semantic1000 +requests pass, all have fiveGCs, all fail the unchanged10ms gate with exit70. +No large/stable speedup or promoted baseline is claimed. + +Current unpromoted prototype extends necessary prefix classification to total +line-skip-horizontal start anchors and expanded byte membership predicates. +The horizontal offset is bound once and reused in original consumer bodies; +mutable state offsets and query/error predecessors cannot admit a guard. +Nested source frames clear both prefix and horizontal facts. No runtime cache, +duplicated full/residual program, kernel edit or Automation restart is added. +Fresh .line-anchored-controls compilation and two directed byte controls pass. +Actual .real-line-anchored has3499defs/7420privatecalls/2081syntactic-tailcalls/ +70eventassignments, static observations only. Its native24complete tapes pass. +org-fold SHA256 b7424e97e412c293cf3a0f83b66f3b17825fdfc968eb53f3a05578aab5a17bcb; +prepared-tape-fold71d478a55a052d0bf04f658aa3671bcd79c788fa3c9ed7264f7b74b32ebb691b. + +Sequential ABBA versus delimiter control:47.230/44.626/45.312/46.060ms. +Sequential ABBA versus single-body predecessor:51.264/44.320/44.919/49.605ms. +Four anchored lanes P9563-64us, P9996-124us, throughput22069.209-22563.177req/s, +RSS28901376-28917760bytes, CPU44.209-45.212ms, fiveGCs. The later predecessor +controls are slower than earlier43ms lanes; no stable large benefit inferred. +Candidate memory exceeds delimiter controls. All full1000independent tapes +pass and all gate exits are70. Counterdelta23037048 acrossGC is not allocated +bytes. Request percentiles are compiledFold calls; wall includes complete tape +comparison; wholeprocessRSS includes retained inputs/oracles/nativecode. No +FFI/Tokio/Python/publicOrg::parse or SMP parallelism acceptance is implied. + +Source-bound Lean and Quint model checks passed. Current real AOT join is +running as exec15974, using .line-anchored-evidence-controls and immutable +.line-anchored-real-evidence. Do not change bound Scheme/Just/model/test inputs +or duplicate this qualification while active. There is no active benchmark +matrix. No20batchmatrix, push or baseline promotion occurred. README and the +research report record explicit units, failed gates and evidence boundaries. +Historical baseline hashes rechecked unchanged. Unsupported SMP remains open, +not passed. Continue whole-engine ordinary-loop work after this evidence gate; +the anchored frame classifier alone does not eliminate that remaining work. + +Final evidence exec15974 completed exit0 within the unchanged240s gate: +Lean cd0c4843cf7df4969a5401eef5e21f4ee1f5deca1059544e83a0d531cf4fa2a1; +Quint e2dd4fab0166eb1c9eeca6763400822a2eaef991d3d146748d9329999d46e869; +Scheme-to-actual-AOT9b9a8362dd437b28c14522da24796ea14c1d603afee00ba8a0945d71a01009ba. +Native review627eafa6dddbe7fc767fcae6571e9886d3331024b62a8a6b8c818c8236ffff04 +has realAotControlsBound=true, matched-performance missing and +executionAuthorized=false. This is conditional model plus differential +conformance, not a complete compiler proof or speed admission. All owned +builds/models/benchmarks have ended. Three historical executable hashes remain +unchanged; git diff --check passes. No push, baseline promotion, kernel/SDK +change or Automation restart. Retain the prototype unpromoted: current matched +receipts do not establish a consistent regression against its predecessors, +but also do not qualify the hard gate or eliminate ordinary-loop dispatch. diff --git a/README.org b/README.org index f23b7e05..849b3b68 100644 --- a/README.org +++ b/README.org @@ -1,247 +1,433 @@ #+title: gerbil-parser -=gerbil-parser= is a Gerbil-owned programmable syntax engine built on hygienic -macros, POO composition, parser-flow normalization, and AOT-compiled parser -artifacts. The optional Rust/Rowan product executes those same immutable -Parser IR tables without running Scheme at parse-request time. - -The language-neutral compiler and runtime live under =src/=. Their public -language-import API is the top-level =language-support.ss= facade beside -=build.ss=, so package imports do not expose the physical =src/= directory. -Its implementations remain under =src/language-support/=. Bundled and downstream language packs live under -=languages///=; grammar, parser entry, fixtures, tests, and -native corpus stay together. POO and =poo-flow= -are used only while compiling grammar object graphs; generated parser hot paths -do not perform dynamic POO dispatch. - -The repository owns the generic Rust/Rowan runtime and generated language -products under =rust/=. Downstream repositories consume these products rather -than rebuilding ParseArtifact-to-Rowan sinks. Gerbil remains the sole grammar, -normalization, conflict-resolution, and table-generation authority. The first -executable reference grammar proves both the pure Scheme runtime and the -Rust/Rowan AOT path from the same immutable Parser IR. - -* Rust/Rowan AOT product - -=src/compiler/rust-rowan.ss= lowers canonical Parser IR v1 to immutable Rust -tables. =rust/gerbil-parser-rowan= owns their generic lexer/LR executor, -lossless Rowan green-tree construction, diagnostics, and source/grammar-bound -receipt. Generated modules are ordinary Cargo source artifacts; downstream -parse requests do not initialize Gambit and do not interpret Scheme. - -The arithmetic v1 reference grammar and ISO/IEC 39075:2024 GQL are generated -products. The generic runtime closes deterministic LALR and bounded -selective-GLR execution, the closed native lexical algebra, dynamic precedence, -equivalent-completion merging, trivia-preserving byte roundtrip, and -fail-closed ambiguity rejection. The pinned 14-source OpenGQL corpus is the -first GQL admission gate. Pinned external scanners, recovery parity, -generated field accessors, and wider differential corpora remain explicit -downstream-cutover gates; directory presence alone does not claim them. - -** Rust consumer contract - -An upstream-maintained language is a generated Rust crate layered over the -small generic runtime. A Rust consumer imports its =LANGUAGE= value (or its -convenience =parse= function); it does not rebuild a lexer, precedence table, -Rowan sink, or ParseArtifact adapter, and a custom DSL does not compile the GQL -tables: - -#+begin_src rust -use gerbil_parser_rowan_gql::LANGUAGE; - -let parsed = gerbil_parser_rowan::parse(&LANGUAGE, source)?; -let root = parsed.syntax(); +A programmable syntax engine in Gerbil Scheme, with declarative language packs, +lossless parse artifacts, and Rust integration. + +Languages declare syntax and metadata; the engine owns scanning, grammar +normalization, LR construction, parsing and result publication. Hygienic macros +and POO prototypes provide constrained extension points. Gerbil remains the +compiler authority for generated Rust products. + +This is an active implementation and research project. Supported syntax and +portable products are qualified per language and target; a shared authoring +interface does not imply complete language coverage or backend parity. + +* Experimental Fold qualification (not accepted) + +The engine-owned [[file:t/benchmarks/parser-throughput/README.org][parser throughput qualification protocol]] +defines hard 1000/10000/100000-request wall gates of 0.010/0.100/1.000 seconds +and a minimum 100000 requests/s. It reports per-request P95/P99 with explicit +units, fails inadequate candidates at preflight, and requires twenty batches +per load for full acceptance. Its included generated line fixture is NOT full +Org, Rust/Tokio or Python acceptance; real adapters must retain their own +compiled source identity and semantic oracle. No achieved Org speed is claimed. + +Consumer-local selector facts now have explicit read/write support and barrier +invalidation, with separate Lean semantic lemmas and bounded Quint scope tests. +Actual AOT controls pass, but matched changing-input preflights remain below +the throughput gate. The latest supported-run experiment measures 41.504-44.692 +ms per 1000 requests (P95 60-64 us, P99 75-171 us; 22375-24094 requests/s; +process RSS 28884992-28934144 bytes), versus 42.048-44.423 ms for its control. +This is not a qualified improvement or twenty-batch acceptance. Full lane +results, artifact hashes and source-bound model receipts are retained in +[[file:FOLD-TASK-RECEIPT.md][the task receipt]]. No baseline promotion follows from model success. + +An additional scalar-expression inlining candidate passed the actual AOT and +model chain but failed matched performance: 39.846-128.127 ms per 1000 requests, +with process RSS 30883840-30916608 bytes versus 28753920-28819456 bytes for the +control. It was withdrawn; isolated faster lanes do not justify promotion. +The twenty-batch matrix remains gated on the unchanged 10 ms preflight. + +Whole-program single-use call fusion is now connected to production tape AOT, +before module partitioning. Six directed controls and 24 complete actual tapes +pass with source-bound Lean/Quint/AOT receipts. The generated product has 1300 +functions instead of 3310 and 5371 private call sites instead of 7381. These +are static counts, not proof of fewer dynamic scans or allocations. + +Six matched changing-input 1000-request preflights per product (NOT full +acceptance) have wall medians 36.666 ms candidate versus 43.726 ms control, +CPU medians 36.339 versus 43.615 ms, and process RSS medians 26550272 versus +28803072 bytes. Candidate call P95 is 51-55 us, P99 63-156 us; throughput +21368.43-27753.11 requests/s. The 46.798 ms candidate spike remains a negative +observation, not excluded; five GCs occur in every lane. The 10 ms hard gate +still fails. Baseline is unchanged; no large-load twenty-batch matrix or +public Org/FFI/Python/SMP speed admission. Full controls and negative results +are in the task receipt, with logical/model versus runtime boundaries. +Serial allocation diagnostics show identical observed bytes per no-GC request +for all three fixtures (15904/24976/31888 bytes); this subset includes observer +overhead and is not total allocation. Call fusion has not demonstrated reduced +allocation or GC pressure. + +An additional exclusive-publication candidate follows official =std/io/bio=: +seal the tape owner, clear its buffer reference, shrink the privately owned +=u8vector=, and transfer it instead of copying the complete output. Real AOT +tapes agree. Three fixture no-GC-request allocation observations decrease from +15904/24976/31888 to 14608/23456/28528 bytes/request (observation overhead is +included, GC-crossing requests excluded). Eight diagnostic lanes per product +have wall medians 35.6985 versus 37.3205 ms and CPU medians 35.6645 versus +36.279 ms; candidate P95 50-56 us, P99 60-133 us, 25957.84-28354.32 requests/s. +GC count is 4 versus 5. Four RSS-qualified lanes each have medians 26583040 +versus 26558464 bytes: slightly higher RSS, not memory reclamation evidence. +Concurrent compiler activity and retained negative lanes preclude a stable +speed admission. Every 1000-request lane still fails the 10 ms gate. + +A subsequent shared-next-byte lowering passed model/AOT and NUL/empty/end +controls, but was withdrawn: CPU median 36.5015 versus 36.2355 ms and P50 +30 versus 29 us, with slightly higher RSS. Fewer source reads alone did not +offset the added per-byte binding/guard. Independent semantic controls remain. +Stall probes captured an 8157 us call with only 319 us CPU, no GC and three +involuntary switches under compiler contention; they do not retrospectively +classify the earlier 46.798 ms batch. Neither trial promotes a baseline or +admits a large-load matrix, public Org parsing, Python, Tokio or SMP performance. + +Two whole-span recognition/counter/ordered-residual leaf implementations were +also executed and withdrawn, rather than admitted on model results. The first +four matched 1000-request lanes have CPU medians 37.066 versus 36.2945 ms; +the revised compiler-owned entry has 36.691 versus 36.001 ms, wall medians +37.041 versus 36.119 ms, and RSS medians 26648576 versus 26566656 bytes. +Revised candidate P95 is 52-56 us, P99 89-116 us, throughput +12873.66-27386.76 requests/s; its 77.678 ms slow lane is retained. Every lane +passes complete tape comparison but fails the unchanged 10 ms gate. These +are four diagnostic lanes/product, NOT the requested twenty-batch acceptance. + +The retained real-Fold count-only product now records span and consumer work: +1000 changing requests execute 45996 admitted spans over 326949 byte visits +(mean 7.108 bytes/span), plus 71640 fallback byte transitions. Span buckets +are 6335 at length1, 11332 at2-4, 25995 at5-16 and2334 over16. These are +dynamic visits, not unique source coverage or timing; diagnostic hooks are +absent from production products. Static traversal finds33 admitted loop +occurrences out of285; the inline guard cells are not loop-invariant. The next +execution-region design must account for their dependency writes, rather than +hoist stale state, cache input/results or turn each short span into an FFI call. +Lean reuses the existing counter semantics and proves length/carry encoding; +Quint separately checks borrow/commit/residual lifecycle. The retained actual +AOT conformance is source-bound; matched-performance remains explicitly open. + +The whole-engine Execution.lean model now treats ordered dependent consumers +and immutable recognition facts together; existing Quint lifecycle checks +retain source/bounds invalidation and event order. A local-zero-transition +trial passed 13 directed controls and 24 actual AOT tapes, but was withdrawn: +matched 1000-request walls were 42.116-46.836 ms (P95 63-67 us, P99 84-129 us; +21351.10-23743.95 requests/s; process RSS 28803072-28868608 bytes), versus +stable controls 43.244-43.803 ms. A 55.164 ms control spike remains recorded. +This did not establish consistent improvement or the 10 ms gate. The overall +compiler-to-model refinement and cost attribution remain open; no full matrix, +baseline promotion or downstream speed claim follows from model success. + +Dependency-selected interval execution now chooses a narrow or wide byte +classification once per ordinary interval; both use one scanner without +per-byte state guards or a maintained runtime cache. Eleven run controls and +24 real AOT tape controls pass, with Lean/Quint source binding requalified at +head 16f5399. The first 240 s qualification timeout remains recorded; a fresh +retry passed without widening the budget. Matched changing-input preflights +measure 41.011-43.931 ms per 1000 requests, Fold-call P95 58-63 us and P99 +63-109 us, throughput 22762.97-24383.70 requests/s, and whole-process RSS +28770304-28803072 bytes. Stable controls measure 42.219-42.779 ms; the first +57.987 ms control spike is retained. These observations establish neither a +stable speed improvement nor the 10 ms gate. No baseline promotion or full +twenty-batch matrix; dependent active consumer transitions remain open. + +The independent AOT link now completes after recovery of an unowned object +lock; build entrypoints reject existing locks without deleting them. The +single-body recognition prototype and its horizontal-anchor extension each +pass 24 actual Org complete-tape controls. The latter's four changing-input +1000-request preflights take 44.320-45.312 ms, with request P95 63-64 us, +P99 96-124 us, throughput 22069-22563 requests/s and whole-process peak RSS +27.5625-27.578125 MiB. The 10 ms gate still fails: no baseline promotion or +three-load twenty-batch acceptance is claimed. See the +[[file:docs/research/event-fold-recognition-architecture.org][recognition architecture and evidence boundaries]]. + +The newer dependency-derived private calling convention passes four focused +controls and the complete source-bound Lean/Quint/Scheme-to-real-AOT chain. +It removes unused helper environment arguments without request caches. One +matched 1000-request round measures 43.217-43.471 ms (P95 60-62 us, P99 95-107 us, +23004-23139 requests/s, peak RSS 28803072-28835840 bytes); a reversed round is +unstable for both control and candidate, reaching 142.624-148.276 ms for the +candidate. Neither round passes 10 ms or qualifies a speed claim. The borrowed +ordinary-run C scan was rejected and rolled back. All raw negative results are +retained in the [[file:FOLD-TASK-RECEIPT.md][task receipt]]. No baseline promotion or full matrix. + +An exact-byte reset/increment run-summary experiment also passed seven native +admission controls, 24 actual Org tapes and the complete source-bound model/AOT +chain. It reduced static inline trigger alphabets to 16/17, but strictly serial +reverse control/candidate CPU remained about 43-44 ms for 1000 requests, with no +material throughput gain and slightly higher candidate RSS (28852224-28868608 +bytes). It was withdrawn; mathematical refinement is not performance acceptance. +Live baseline probes reproduce a 111.849 ms diagnostic batch (71.842 ms CPU): +a 4.908 ms request has only 0.438 ms CPU, zero GC and two involuntary switches. +Monotonic time agrees, supporting OS descheduling for that reproduced stall; +the historical 142-148 ms batches remain unclassified. Probe overhead excludes +these results from throughput acceptance. The 10 ms gate and full-matrix rule +remain unchanged; the baseline binaries are preserved. + +Whole-residual byte-region dispatch was also tested and withdrawn. Its actual +Org controls and source-bound Lean/Quint/AOT checks passed, but the last reversed +1000-request batch regressed to 82.181 ms (P95/P99 209/603 us, 12168 requests/s, +RSS 30801920 bytes) versus 43.807 ms for the preceding control. The stable +reverse comparison improved CPU only about five percent while process RSS grew +roughly 2 MB. Lower GC counts do not establish a speedup. All eight lanes retain +complete independent tape comparisons and failed preflight; no full matrix or +promotion follows. A separate idle-state/byte-trigger admission implementation +also passes the complete source-bound model/AOT chain and 24 actual Org tapes, +but is withdrawn: eight serial matched lanes still fail 10 ms. Its candidate +range is 42.067-75.223 ms, P95/P99 reaches 157/428 us, throughput is +13294-23772 requests/s and process RSS is 28884992-28901376 bytes. Controls are +44.102-46.747 ms. These are preflight observations, not twenty-batch acceptance; +the fastest singleton is not a promoted baseline. Dependency-supported state +partitioning and executed admission coverage remain open. + +The actual generated Org Fold now has upstream =real.ss= admission and a +separate diagnostic =profile.ss=. Both compare complete independent events; +neither uses an Orgize runtime or FFI queue. The real-Fold preflight remains +failed, and the source-fact reuse experiment was reverted. See the +[[file:docs/research/event-fold-whole-path-performance.org][whole-path evidence and optimization plan]]; +diagnostic call counts are not uninstrumented performance claims. + +The guarded idle-state experiment removes irrelevant mutable-state execution +without assuming read-only configuration is zero. Two sequential changing-input +preflights measure 102.591/103.150 ms per 1000 requests (P95 135/152 us, +P99 403/392 us, 9747.444/9694.619 requests/s, 30 GCs), versus same-source control +131.982/138.163 ms and 56 GCs. No-GC request allocation counter means decrease +36-44 percent, including observer overhead; peak RSS increases slightly. +The 10 ms gate remains failed. This is not twenty-batch, FFI, Tokio, Python or +baseline acceptance. Lean transition obligations and Quint lifecycle histories +are bound to actual AOT controls by the native ASCENT evidence owner; successful +models and conformance cannot fill the missing matched-performance requirement. + +A single common-byte kernel adds necessary-byte classification without +generating a separate kernel for every byte region. Two subsequent matched +preflights measure 88.284/90.422 ms per 1000 changing requests, P95 112/113 us, +P99 312/356 us, 11327.081/11059.255 requests/s and 27 GCs; current-source control +is 122.040/120.904 ms and 56 GCs. No-GC allocation means decrease 38-52 percent +but process peak RSS increases about 1.2 MB. Real AOT complete-tape controls +pass; the 10 ms gate still fails. These opt-in experiments do not change the +accepted baseline or qualify Rust/FFI/Python/Tokio performance. + +Subsequent active-state cofactoring and loop-local guard caching both passed +24 real AOT controls but regressed matched preflight wall time. Both were +reverted; modest RSS savings do not substitute for throughput acceptance. +The existing best experimental product and production baseline remain intact. + +Direct-tape effect-only transitions remove private owner-return plumbing and +restore syntactic tail positions without adding a runtime cache. Two matched +1000-request preflights measure 72.575/73.322 ms, P95 96/96 us, P99 142/142 us, +13778.849/13638.471 requests/s, five GCs and peak RSS 29.64/29.67 MB, versus +the previous best experimental control at 96.537/90.398 ms and 27 GCs. +All 24 real AOT event-tape controls pass. This is additional preflight evidence, +not the still-failed 10 ms gate, twenty-batch acceptance or a production change. +Matched no-GC request allocation means decrease from 123/231/122 KB to +16/25/32 KB across plain-links/quote-heavy/doc inputs, including observation +overhead and excluding GC-interrupted requests. Lean, Quint and native Ascent +bind source controls to the actual AOT product; performance admission is open. + +The next experimental =tape-pruned= product removes discarded pure branches, +including ordered cursor-derived scans only inside a validated byte loop. +Errors, future queries, unknown effects and frame changes remain barriers. +Matched sequential 1000-request preflights are 69.201/71.196 ms, P95 90/96 us, +P99 144/156 us, 14450.658/14045.733 requests/s and peak RSS 29.475/29.491 MB; +the retained tail control is 74.197/73.520 ms. GC remains five per batch. +All 24 real AOT tapes and four effect/range controls pass. The <=10 ms gate +still fails: no twenty-batch acceptance or production baseline promotion. +Removing the remaining named-helper owner results or reusing token offsets +regressed wall time and was reverted despite smaller static call graphs. + +The next =tape-run= experiment coalesces overwrite-only ordinary-byte segments: +one segment scan, then the last byte's seven inline metadata overwrites. No +runtime cache; active modes, eventful/state-dependent bodies and special bytes +retain their original execution. All 24 real AOT tapes and five admission +regressions pass. Candidate 1000-request diagnostic wall is 62.059--103.421 ms, +P95 86--263 us, P99 96--575 us, 9669--16114 requests/s, peak RSS 29.426--29.458 MB, +with five GCs. Matched controls include major non-CPU pauses; one pair regresses +amid unrelated host CPU contention. These measurements do not qualify a stable +speedup, the <=10 ms gate, or the still-unrun 100000-request twenty-batch matrix. +The baseline is unchanged; the path remains an opt-in research candidate. + +The rejected dependency/effect execution-plan experiment reused official list +utilities and the compile-time list builder. Its 1000-request matched preflight +is 216.313 ms (4622.931 requests/s), versus 218.790 ms for the control; both +fail the 10 ms gate. This single batch is not an accepted speedup or a full +twenty-batch result. Its lowering integration was rolled back; conservative +analysis tests remain. Shared source classification, typed state execution +and actual event-consumer integration remain open architectural work. + +The combined frame/current-byte/dependency-ABI/event-buffer candidate was also +rejected and rolled back: 1000 real changing requests took 245.616 ms versus +219.381 ms for the matched immutable control (4071.396 versus 4558.280 +requests/s). P95/P99 were 193/6280 us versus 234/730 us; GC counts were +15 versus 10, and whole-process peak RSS was 194.859 versus 181.750 MiB. +RSS includes retained oracle/input and code; Rust capacity is outside this +Scheme-only boundary. Neither product passed the 10 ms preflight. The +canonical consumer still materialized lists after buffer production, so this +experiment did not qualify removal of consumer work. No twenty-batch or +baseline-promotion claim follows. Post-rollback source/native controls passed. + +The new direct OEV1 consumer candidate eliminates event-list materialization +and the two consumer list traversals, while prepared loop-byte regions select +only relevant ordered rules. Actual Org native wire/override controls passed +24 cases, alongside 100 canonical and 90 independent wire differential cases. +Matched single-batch changing-input results for 1000 real Org requests: + +| Fold + OEV1 path | Batch wall (ms) | Request P95/P99 (us) | Requests/s | GC | Process peak RSS (MiB) | +|-----------------+-----------------+---------------------+------------+----+------------------------| +| Canonical list + existing consumer | 198.975 | 254/304 | 5025.757 | 8 | 170.781 | +| Direct consumer + byte regions | 171.141 | 212/247 | 5843.135 | 6 | 169.875 | + +Both failed the unchanged 10 ms preflight. This is NOT a twenty-batch result, +baseline promotion, FFI/Tokio/Python acceptance or proof of memory savings. +RSS retains the input/oracle and loaded grammar; allocation totals under GC +and Rust capacity are not admitted here. Candidate code remains experimental. +Residual scan/state execution still prevents the requested throughput gate. + +The next validated-loop-byte candidate removes duplicate reads/range checks +and filters total state guards by byte before slot reads. Five freshly linked +Rust/FFI integration suites passed separately (public parse/Tokio, index, +owned projection, Org parity and host child-wait ownership). Matched sequential +single preflights, with the previous direct product as control: + +| Fold + direct OEV1 | Wall (ms) | P95/P99 (us) | Requests/s | GC | Process peak RSS (MiB) | +|-------------------+-----------+-------------+------------+----+------------------------| +| Previous direct product | 181.078 | 221/284 | 5522.482 | 7 | 177.641 | +| Validated loop byte | 142.709 | 157/192 | 7007.267 | 7 | 172.422 | + +Both fail the unchanged 10 ms gate. This 21.19 percent singleton wall reduction +is NOT twenty-batch qualification or baseline promotion. CPU was 178.319 versus +142.525 ms; equal GC counts do not explain the gain. Memory includes retained +oracle/input/code, not Rust reserved capacity. These request timings do not +include FFI, graph projection or Tokio; separate integration correctness does +not extend their boundary. Shared whole-range prescan fusion remains open. + +=parser-actor-support.ss= is the public facade for explicitly created local +Gerbil actors. Applications own worker count and lifecycle; imports start no +background service. =spawn-parser-actor= accepts an already prepared machine, +=parser-actor-parse= returns the same lossless artifact as synchronous parsing, +and =parser-actor-stop!= joins the worker. Timeout bounds reply waiting rather +than preempting an active parse. Keep the shared machine immutable after +initialization; actor scheduling does not change the LR algorithm. +* Language authoring + +A language pack exposes three Scheme interfaces: + +| File | Responsibility | +|------+----------------| +| =grammar.ss= | Declarative syntax and pinned grammar resources | +| =parser.ss= | Production POO Loader, metadata and parse entry | +| =parser-test.ss= | Development Loader, fixtures and declarative conformance tests | + +Grammar sources and corpus files remain resources beside these interfaces. +Versions, contracts and source digests belong to metadata, not versioned module +names. Scanner implementations, parser state and C/Rust bridge code belong to +the engine. + +For example, a small identifier language declares its syntax in =grammar.ss=: + +#+begin_src scheme +(import (only-in :gerbil-parser/language-support/grammar deflanguage)) +(export example-syntax) + +(deflanguage example + (syntax + (lexical + (root name) + (lex (identifier Identifier (identifier))))) + (rules (name (node Name (field value identifier))))) #+end_src -A downstream custom DSL follows the same =languages///grammar.ss= -contract. The complete pack keeps =grammar.ss=, =parser.ss=, =fixtures.ss=, -=parser-test.ss=, and its native =corpus/= together; the Rust product is a -projection of that same declared language, not a separate grammar. A direct -custom grammar imports only the narrow authoring facade: +Its =parser.ss= publishes the entry through the Loader: #+begin_src scheme -(import (only-in :gerbil-parser/rust-rowan-grammar-support deflanguage)) +(import (only-in :gerbil-parser/language-support/entry + deflanguage-parser-loader) + ./grammar) +(export example-language parse-example) + +(deflanguage-parser-loader example-language + (grammar example-syntax) + (parse parse-example) + (slots metadata: '((language . "example") + (version . "v1") + (contract . "example.syntax.v1")))) #+end_src -Repository-owned generation may import the grammar binding and the public -=:gerbil-parser/rust-rowan-support= facade, then emit an ordinary Rust module: +The Loader is a POO object. Inheritance, value overrides and extension contracts +are admitted before publishing the parse entry. Fixture values are ordinary +slots; the engine owns evaluation, catalog construction and service binding. +=deflanguage-parser-tests= constrains test declarations and checks acceptance, +lossless source, identities and structural expectations. -#+begin_src scheme -(import :gerbil-parser/rust-rowan-support - :my-dsl/languages/example/v1/grammar) +See the [[file:languages/README.org][language authoring guide]] and +[[file:docs/architecture/0018-downstream-language-products.org][downstream product guide]] for development Loaders, +POO extensions, fixtures and product generation. -(generate-language-rust-rowan-module - "src/generated/example_v1.rs" - example-v1-language-grammar) -#+end_src +* Execution and integration -The generated module is compiled into the downstream Rust crate and executes -without Gerbil, Python, a subprocess, or a runtime grammar interpreter. Gerbil -is the AOT grammar/compiler authority; Cargo consumers depend only on the Rust -engine and their generated module. - -Rust-driven authoring uses the same authority through the dedicated build-only -=gerbil_parser_rowan_compile= ABI. Build Support passes a =grammar.ss= path to -the precompiled Gerbil generator, receives the existing Rust/Rowan backend -output as owned bytes, and writes it into Cargo's =OUT_DIR=. It does not parse -Scheme or rebuild Grammar IR/LR tables in Rust, and the parse-time dependency -still contains only the Rowan engine plus generated tables. The generator is a -separate native product; the Cargo-only authoring claim remains open until that -product can be distributed and used on admitted Linux and macOS targets -without a Gerbil installation. -The same owner is exposed as the =gerbil-parser-rowan-aot= executable for -standalone bundle qualification; it initializes Gerbil's expander internally -and never invokes =gerbil=, =gxi=, or =gxc= while generating a module. -Its =defbuild-script= declaration projects a public ASP Building API package -spec into std/make, so the declared Scheme module graph and native TLS options -remain ordinary build data instead of a release script tracing and copying -Gerbil's mutable object cache. CI executes the result under =env -i= with no -Gerbil command on =PATH= and explicit expansion-metadata roots. Packaging -those immutable interfaces with the executable remains a release boundary; -runtime object-cache reconstruction is gone. - -=t/fixtures/rust-rowan-downstream= is the executable consumer contract. It has -no =gerbil.pkg= and imposes no Gerbil installation on the Rust consumer. -Upstream generation CI verifies its Scheme source pack and checks the -committed Rust module through the public AOT facade. The independent Cargo -workspace then compiles and tests that module for lossless Rowan structure, -receipts, and typed rejection using only the Rust runtime dependency. - -After building the Gerbil package, regenerate and verify the committed module: +| Product | Execution boundary | +|---------+--------------------| +| Scheme library | Prepared engine parser returns a lossless ParseArtifact | +| Independent Rust AOT | Generated immutable scanner/parser tables run in the generic Rust runtime; no Scheme VM at request time | +| C ABI and Rust FFI | A reusable Gerbil runtime and language handle execute Scheme parsing; Rust owns result buffers and borrows indexed views | -#+begin_src sh -gerbil env gxi generate-rust-rowan.ss \ - rust/gerbil-parser-rowan-arithmetic/src/generated/mod.rs -cargo fmt --all -cargo test --workspace --locked -cargo test --release -p gerbil-parser-rowan-scenarios --test performance_test -- \ - --ignored --nocapture --test-threads=1 -#+end_src +Parse artifacts preserve source text, syntax structure and spans, with source +and grammar identities. The artifact is the shared publication boundary for +traversal and downstream tooling. + +The language-independent C ABI initializes the runtime once and reuses handles +across requests. Its optional Rust Tokio integration queues requests to a +persistent Scheme owner thread. Applications own queue limits and lifecycle. +See [[file:rust/gerbil-parser-ffi/README.org][Rust FFI and Tokio ownership]] and +[[file:rust/gerbil-parser-artifact/README.org][Rust artifact admission and navigation]]. + +Independent AOT uses the same admitted Grammar/Parser IR through a separate +product gate. It does not automatically support every Scheme source parser or +POO policy. See [[file:docs/architecture/0006-rust-aot.org][Rust AOT design]] and the +[[file:t/fixtures/runtime-record-assignments/README.org][standalone record-language consumer]]. + +* Build and try -The workspace =Build Support= crate owns the ASP Rust policy and its -=tests/asp-rust-gate.rs= activation once. Product crates do not depend on Build -Support; =cargo test -p gerbil-parser-build-support --test asp-rust-gate= -activates the =Dev Gate= only in the development workspace. -The gate runs in =deny= mode, so every ASP Rust warning or error is -test-blocking. CI runs that target separately with =--nocapture= so policy -diagnostics remain visible instead of being hidden by libtest output capture. -Immutable language-crate tables are compiler output verified against Grammar IR -in native CI; policy governs their generator and handwritten runtime owners. -Normal and build dependency graphs do not contain Build Support or ASP Rust. - -* Native command - -=build.ss= is the package AOT entry. It discovers and installs the reusable -library, runtime ParseArtifact FFI, and build-only Rust/Rowan generator FFI as -separate products without forcing downstream library consumers to compile a -command they do not use. =build-gparse.ss= is the sibling first-party -=defbuild-script= entry for the optional native =gparse= command; it adds no -second command implementation or package scheduler. -=build-rust-rowan-aot.ss= independently materializes the build-time -=gerbil-parser-rowan-aot= executable. +Use the repository's Gerbil SDK environment and dependencies pinned in +=gerbil.pkg=. Scheme compilation uses the SDK's GCC configuration. #+begin_src sh +gxpkg deps --install GERBIL_BUILD_CORES=12 gerbil build GERBIL_BUILD_CORES=12 gerbil env gxi build-gparse.ss compile -GERBIL_BUILD_CORES=12 gerbil env gxi build-rust-rowan-aot.ss compile -gparse help -gparse check "1 + 2 * value" +.gerbil/bin/gparse help +.gerbil/bin/gparse check "1 + 2 * value" #+end_src -The package manager exposes its native binary directory on =PATH= for the bare -command. Before package activation, the same native artifact can be invoked -directly from that standard build directory; the repository does not own a -second shell launcher. - -=src/main.ss= owns only =:std/getopt= declarations and dispatch. -=src/cli.ss= is the thin command adapter. Grammar compilation, lexing, parsing, -ParseArtifact events, and diagnostics remain in their library owners. - -* Package layout - -The engine parser module follows the POO Flow package roles: =objects.ss= owns POO -grammar object families, =types.ss= owns schema identities, and =funcs.ss= owns -small performance-sensitive parser functions. The single public authoring -boundary is the top-level =language-support.ss= facade; its re-exported -=deflanguage-grammar= macro expands -one declaration into the grammar object, canonical Parser IR, native machine, -and immutable language descriptor. General list utilities stay at the package -utility boundary rather than becoming parser-module miscellany. - -A supported language uses one colocated package surface: - -#+begin_src text -languages/hcl/v2-24/ - grammar.ss - parser.ss - fixtures.ss - parser-test.ss - corpus/ +The command is an optional consumer of the library. It does not own a separate +parser implementation. Rust workspace checks run with: + +#+begin_src sh +cargo test --workspace --all-features --locked #+end_src -=grammar.ss= owns the versioned identity and one =deflanguage-grammar= or -=deflanguage-antlr4-grammar= declaration. =parser.ss= uses -=deflanguage-parser= with that descriptor to -generate the public parse function; it cannot repeat or replace the identity, -contract, or machine. Fixtures and tests never move into =src/= and do not -enter the =gparse= executable closure. -=defsyntax-corpus= declares all files in one expansion-time manifest. The HCL -pack vendors all 15 raw =.hcl= inputs from the v2.24.0 official specsuite. The -GQL pack pins OpenGQL grammar 1.9.0 by commit and SHA-256 digest, validates all -1,018 source rules, and AOT-materializes all 574 parser rules with 444 lexer or -fragment rules. Its 14 upstream sample programs then prove typed terminals, -structural CST kinds, case-insensitive keywords, and byte-exact roundtrip. - -Grammar-source formats are language-neutral engine adapters. ISO WG3 BNF, -ANTLR4 source admission, and fixture/corpus declaration machinery are internal -to =src/language-support/= and exported only through -=:gerbil-parser/language-support=. A language pack only -selects a pinned source and declares its versioned lowering under -=languages///=. The openCypher 2024.1 pack validates and -structures all 352 productions from the official BNF. Exact-source overlays -right-factor the result-statement path, normalize nullable prefixes and -qualified references, and remove duplicate list/map constructor parses without -mutating the pinned source. The commit-bound TCK syntax audit expands 3,897 -queries from 220 feature files: all 3,298 expected-success or runtime-error -queries are accepted with byte-exact roundtrip, while the remaining 39 parser -rejections are compile-error cases. This closes syntax acceptance for that -manifest; it does not claim execution or semantic-result conformance. - -The TLA+ =v1= pack exercises the same native generation path for module -envelopes, declarations, structured precedence expressions, conditionals, -quantifiers, collections, function constructors, theorem lines, comments, and -nested block comments. Its syntax authority is the native -=TLAPlusGrammar= in chapter 15 of /Specifying Systems/; the official SANY -=tla+.jj= source at release =v1.7.4= is pinned for executable conformance. It -does not enter the runtime or replace the Gerbil parser engine. The pack also -pins a byte-exact =HourClock.tla= source from the TLA+ Foundation Examples -repository. The pinned 99-production SANY inventory is admitted at expansion -time, while malformed recognized syntax fails closed without an unknown-token -fallback. The optional =qualification.ss= boundary composes that native -syntax admission with the official external TLC executable and returns one -typed Gerbil receipt binding source, configuration, tool and state-space -identities. TLC remains the model-checking authority and is never copied into -the parser engine; a syntax rejection stops before TLC is launched. PlusCal, -proof syntax, multiline layout, and the full official -syntax corpus remain explicit future contracts. - -* Architecture RFCs - -- =docs/architecture/0001-parser-compiler.org= fixes the pure Gerbil/AOT - authority boundary. -- =docs/architecture/0002-typed-structural-algebras.org= defines distinct - LexicalExpr, GrammarExpr, QueryPattern, and ProjectionPlan feature contracts. - The current implementation admits LexicalExpr v1 and the canonical - generic hygienic GrammarExpr machine plus fixed HCL v2.24.0 and ISO/IEC - 39075:2024 reference contracts; later features require independent executable - evidence rather than directory presence. -- =docs/architecture/0003-parse-artifact.org= establishes ParseArtifact v1 as - the sole parse publication, with lossless CST events and replay-only sinks. -- =docs/architecture/0004-versioned-language-contracts.org= defines fixed HCL - v2.24.0 and ISO/IEC 39075:2024 conformance identities plus the public - downstream DSL grammar boundary. =defsyntax-fixture= embeds versioned native - syntax files at macro expansion, while =defsyntax-corpus= owns multi-file - manifests, so downstream conformance tests and - benchmarks share one declarative source contract without runtime file I/O. -- =docs/architecture/0006-rust-rowan-aot.org= keeps =grammar.ss= authoritative - while publishing reusable Rust parser and Rowan CST products. +Native Scheme, generated products and real C/Rust hosts have separate +qualification lanes. See the [[file:docs/architecture/qualification.org][qualification guide]] for their commands and +[[file:.github/workflows/ci.yml][CI entrypoint]] for the current job graph. + +* Reference languages and implementation boundaries + +The builtin packs are Arithmetic, Bash, HCL, HL7, FHIRPath, openCypher, GQL and +TLA+. All use the three-interface authoring boundary. Their syntax coverage, +source admission and portable capabilities differ; consult the +[[file:languages/README.org][per-language status]] before selecting a product. + +Bash currently uses the shared Scheme source engine. Complete standalone Bash +Command/Source AOT remains open. Scheme-only CST policies also require a shared +representation and parity evidence before independent Rust execution. + +* Research and performance evidence + +Research informs executable engine changes and matched benchmarks. Performance +claims apply to the measured workload, runtime and control, rather than every +language or a general comparison with tree-sitter. + +- [[file:docs/research/README.org][Research map and reproduction commands]] +- [[file:docs/research/upstream-bridge-tokio-admission.org][Complete Rust request CPU evidence]]: ASCII and Unicode 64-record requests + through Tokio, FFI, Scheme scanning/LR parsing, publication and Rust admission; + includes controls, rejected experiments and the qualified comparison scope. +- [[file:docs/research/parser-engine-formal-model.org][Parser engine formal model]] and + [[file:docs/research/parser-evidence-chain.org][evidence chain]]: model-level obligations and their connection to native + observations, with explicit proof boundaries. +- [[file:docs/architecture/0001-parser-compiler.org][Compiler architecture]] and + [[file:docs/architecture/0003-parse-artifact.org][ParseArtifact design]] diff --git a/build-conformance.ss b/build-conformance.ss new file mode 100644 index 00000000..d4f92e8a --- /dev/null +++ b/build-conformance.ss @@ -0,0 +1,11 @@ +#!/usr/bin/env gxi +;;; Native preparation entry. The content admission owner performs compilation. +(displayln "CONFORMANCE-PREPARATION-START") (force-output) +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(gc-report-set! #t) +(call-with-compiled-interface-trace + (lambda () (load "scripts/preparation-cache.ss"))) +(def (main) + (prepare-conformance!) + (exit 0)) diff --git a/build-ffi-tests.ss b/build-ffi-tests.ss new file mode 100644 index 00000000..e440cd02 --- /dev/null +++ b/build-ffi-tests.ss @@ -0,0 +1,36 @@ +#!/usr/bin/env gxi +;;; Compile the C ABI consumer with the standard module build owner. +(import (only-in :std/build-script defbuild-script) + (only-in :std/source this-source-file)) +(def (compiled-spec module (objects '())) + `(gxc: ,module "-cc-options" + ,(string-append "-I" (path-expand "include" (path-directory (this-source-file))) + " -I" (path-directory (this-source-file))) + ,@(cond-expand + (darwin '("-ld-options" "-Wl,-undefined,dynamic_lookup")) + (else (if (null? objects) '() + (list "-ld-options" (string-join + (map (lambda (object) (path-expand object (gerbil-path))) objects) " "))))))) +(defbuild-script + (list + (compiled-spec "t/fixtures/ffi/abi-probe" + '("lib/gerbil-parser/src/ffi/language-abi~0.o1" + "lib/gerbil-parser/src/ffi/rust-aot-abi~0.o1")) + ;; ELF resolves each loadable module's native references independently. + ;; The generated language entry calls the runtime owner-thread guard. + (compiled-spec "t/fixtures/contextual-result-selection") + (compiled-spec "t/fixtures/lr-action-selection") + ;; The reusable library must carry the interface producer before its ABI consumers. + (compiled-spec "t/fixtures/shared-scanner/records") + (compiled-spec "t/fixtures/shared-scanner/records-engine-control") + (compiled-spec "t/fixtures/shared-scanner/records-abi" + '("lib/gerbil-parser/src/ffi/language-abi~0.o1")) + (compiled-spec "t/fixtures/shared-scanner/records-engine-abi" + '("lib/gerbil-parser/src/ffi/language-abi~0.o1")) + (compiled-spec "t/fixtures/ffi/language-probe" + '("lib/gerbil-parser/src/ffi/language-abi~0.o1" + "lib/gerbil-parser/t/fixtures/shared-scanner/records-abi~0.o1")) + (compiled-spec "t/benchmarks/source-edits/abi-call" + '("lib/gerbil-parser/src/ffi/language-abi~0.o1")) + (compiled-spec "t/benchmarks/source-edits/benchmark") + (compiled-spec "t/benchmarks/language-abi/utf8-cpu"))) diff --git a/build-language-conformance-link.ss b/build-language-conformance-link.ss new file mode 100644 index 00000000..805f3dbb --- /dev/null +++ b/build-language-conformance-link.ss @@ -0,0 +1,198 @@ +#!/usr/bin/env gxi +;;; Native object products retain Gerbil's bootstrap and Gambit's native link order. +(import (only-in :gerbil/compiler compile-exe compile-module) + (only-in :std/list/list delete-duplicates/hash) + (only-in :std/sync/wg make-wg wg-add! wg-wait!) + (only-in :std/misc/process run-process)) + +;; Report actual compiler GC work while large immutable images are lowered. +;; This changes the compiler process diagnostics, never generated program code. +(def compiler-progress-expression + "(begin (port-settings-set! (current-output-port) (list buffering: #f)) (gc-report-set! #t))") + +;; External tools start on independent pipes owned and drained by Gerbil. +;; Forward available bytes, including progress without a newline, before waiting. +(def (run-compiler command) + (run-process command stdout-redirection: #t stderr-redirection: #t + coprocess: + (lambda (process) + (close-output-port process) + (let (buffer (make-u8vector 8192)) + (let loop () + (let (count (read-subu8vector buffer 0 (u8vector-length buffer) process 1)) + (unless (zero? count) + (write-subu8vector buffer 0 count) (force-output) + (loop)))))))) + +(def (unique paths) + (delete-duplicates/hash paths from-end?: #t)) +(def (replace-extension path extension) + (string-append (path-strip-extension path) extension)) +(def (test-object-options source options) + ;; Keep the SDK's optimizer and trusted tail-host contract. Split large test + ;; modules into Gambit's per-procedure C hosts instead of one giant function; + ;; -O0 would invalidate tail-call elimination on older GCC. + (let (name (path-strip-directory source)) + (if (or (string-prefix? "gerbil-parser__t__" name) + (string-suffix? "__parser-test.scm" name)) + (string-append options " -U___SINGLE_HOST -D___MULTIPLE_HOSTS") options))) +(def (newer? source target) + (or (not (file-exists? target)) + (> (time->seconds (file-info-last-modification-time (file-info source))) + (time->seconds (file-info-last-modification-time (file-info target)))))) + +(def (run-static-jobs sources task) + ;; One standard queue keeps every worker available to the remaining closure. + (let (cores (string->number (getenv "GERBIL_BUILD_CORES" "1"))) + (unless (and (integer? cores) (exact? cores) (> cores 0)) + (error "GERBIL_BUILD_CORES must be a positive integer" cores)) + (let ((ordered (list-sort (lambda (left right) + (> (file-info-size (file-info left)) + (file-info-size (file-info right)))) sources)) + (workers (make-wg cores))) + (for-each (lambda (source) (wg-add! workers (lambda () (task source)))) ordered) + (wg-wait! workers)))) + +(def (bootstrap-identities stub) + (let (definition (call-with-input-file stub read)) + (unless (and (list? definition) (= (length definition) 3) + (eq? (car definition) 'define) + (eq? (cadr definition) 'builtin-modules) + (list? (caddr definition)) (= (length (caddr definition)) 2) + (eq? (car (caddr definition)) 'quote) + (list? (cadr (caddr definition))) + (andmap string? (cadr (caddr definition)))) + (error "unexpected pinned SDK native bootstrap" definition)) + (cadr (caddr definition)))) + +(def (link-groups gsc binary c-files) + ;; Gambit's documented incremental base chain retains SDK module order. + ;; Sixteen modules per C unit is a build partition, not a parser policy. + (let loop ((remaining c-files) (base #f) (index 0) (result [])) + (if (null? remaining) + (reverse result) + (let* ((count (min 16 (length remaining))) + (group (take remaining count)) + (rest (drop remaining count)) + (target (string-append binary "__link_" (number->string index) ".c"))) + (displayln "CONFORMANCE-LINK-GROUP " index " modules=" count) + (force-output) + (run-compiler + [gsc "-verbose" "-link" + (if base ["-l" (path-strip-extension base)] []) ... + "-o" target group ...]) + (loop rest target (+ index 1) (cons target result)))))) + +(def (main (owner "conformance")) + (let* ((library? (equal? owner "library-objects")) + (source (path-expand (if library? + "t/fixtures/ffi/library-closure.ss" + "t/conformance-main.ss"))) + (binary (path-expand (if library? "bin/gerbil-parser-library-objects" + "bin/gerbil-parser-conformance") + (getenv "GERBIL_PATH"))) + (home (getenv "GERBIL_BUILD_PREFIX" (gerbil-home))) + (library (path-expand "lib" home)) + (static (path-expand "static" library)) + (gsc (path-expand "bin/gsc" home)) + (stub (string-append binary "__exe.scm"))) + (unless (member owner '("conformance" "library-objects")) + (error "unknown native object owner" owner)) + (when library? + (compile-module source + [output-dir: (path-expand "lib" (getenv "GERBIL_PATH")) + invoke-gsc: #f optimize: #f generate-ssxi: #t static: #t keep-scm: #t verbose: #f])) + ;; Public compiler API generates the bootstrap, including admitted identities. + (compile-exe source [output-file: binary invoke-gsc: #f verbose: #f]) + (let* ((identities (bootstrap-identities stub)) + (entry (if library? "gerbil-parser/t/fixtures/ffi/library-closure" + "gerbil-parser/t/conformance-main")) + (_ (unless (member entry identities) + (error "native product entry is absent from SDK bootstrap" entry))) + (ordered + (append (filter (lambda (name) + (and (not (equal? name entry)) + (not (string-prefix? "gerbil/core" name)))) identities) + [entry])) + (roots (unique (cons (path-expand "lib" (getenv "GERBIL_PATH")) + (cons library (load-path))))) + (sources + (map (lambda (name) + (let* ((basename (string-append + (string-join (string-split name #\/) "__") ".scm")) + (path (find file-exists? + (map (lambda (root) + (path-expand basename (path-expand "static" root))) roots)))) + (or path (error "native bootstrap source is missing" name)))) ordered)) + (user-sources (filter (lambda (path) (not (string-prefix? static path))) sources))) + (when library? + (for-each (lambda (source) (displayln "NATIVE-LIBRARY-SOURCE " source)) user-sources) + (force-output)) + (let* ((includes (unique (map path-directory sources))) + (cc-options (string-append "-v -Q -ftrack-macro-expansion=0 " + (string-join + (map (lambda (path) (string-append "-I" path)) includes) " "))) + (ld-options (call-with-input-file (path-expand "libgerbil.ldd" library) read))) + (run-static-jobs user-sources + (lambda (source) + (let ((target (replace-extension source ".o")) + (c-file (replace-extension source ".c"))) + ;; One worker owns translation and its object; no global phase barrier. + (when (newer? source c-file) + (displayln "STATIC-TRANSLATE " source) (force-output) + (run-compiler [gsc "-verbose" "-c" "-o" c-file + "-e" compiler-progress-expression source]) + (displayln "STATIC-TRANSLATED " source) (force-output)) + + (let* ((options (test-object-options source cc-options)) + (stamp (and (not (equal? options cc-options)) + (string-append target ".options.sexp"))) + (identity (list gsc options))) + ;; A changed test-host policy needs explicit option admission. + ;; Production policy is unchanged: preserve its existing reuse; + ;; the preparation owner already proves source/SDK/environment. + ;; Publish an option identity only after successful compilation. + (when (or (newer? c-file target) + (and stamp + (or (not (file-exists? stamp)) + (not (equal? identity (call-with-input-file stamp read)))))) + (displayln "STATIC-OBJECT " c-file) (force-output) + (when (and stamp (file-exists? stamp)) (delete-file stamp)) + (run-compiler [gsc "-verbose" "-cc-options" options + "-obj" "-o" target c-file]) + (when stamp + (call-with-output-file stamp (lambda (port) (write identity port)))) + (displayln "STATIC-OBJECT-READY " c-file) (force-output)))))) + (for-each (lambda (source) + (unless (file-exists? (replace-extension source ".o")) + (error "native library object is missing" source))) sources) + (if library? + (begin (displayln "COMPILED-LIBRARY-OBJECTS-READY") (force-output)) + (let* ((stub-c (replace-extension stub ".c")) + (stub-object (replace-extension stub ".o")) + (objects (map (lambda (path) (replace-extension path ".o")) sources)) + (c-files (append (map (lambda (path) (replace-extension path ".c")) sources) + [stub-c]))) + (for-each (lambda (object) + (unless (file-exists? object) + (error "SDK or native object is missing" object))) objects) + (run-compiler [gsc "-verbose" "-c" "-o" stub-c stub]) + (run-compiler [gsc "-verbose" "-cc-options" cc-options + "-obj" "-o" stub-object stub-c]) + (let (links (link-groups gsc binary c-files)) + (run-static-jobs links + (lambda (link-file) + ;; Library bases omit C main; the final incremental link owns it. + (let* ((final? (equal? link-file (last links))) + (options (string-append cc-options (if final? "" " -D___LIBRARY")))) + (displayln "CONFORMANCE-LINK-OBJECT " link-file) (force-output) + (run-compiler [gsc "-verbose" "-cc-options" options "-obj" + "-o" (replace-extension link-file ".o") link-file]) + (displayln "CONFORMANCE-LINK-OBJECT-READY " link-file) (force-output)))) + ;; SDK objects are reused unchanged; only project objects are built. + (run-compiler [(getenv "GERBIL_GCC" "gcc") "-w" "-o" binary + objects ... stub-object + (map (lambda (path) (replace-extension path ".o")) links) ... + (string-append "-Wl,-rpath," library) + "-L" library "-lgambit" ld-options ...]) + (displayln "CONFORMANCE-LINKED " binary) (force-output)))))))) diff --git a/build-language-conformance.ss b/build-language-conformance.ss new file mode 100644 index 00000000..69b9459c --- /dev/null +++ b/build-language-conformance.ss @@ -0,0 +1,143 @@ +#!/usr/bin/env gxi +;;; Closed native conformance and source gxtest have distinct artifact owners. +(import (only-in :std/build-script defbuild-script) + (only-in :std/make make make-clean) + (only-in :std/source this-source-file) + (only-in :gerbil/compiler compile-module) + (only-in :gerbil/expander import-module module-context-export + module-export-name module-export-phi)) +(load "scripts/test-plan.ss") +(def conformance-modules + '("t/fixtures/fixture-release" + "t/fixtures/tla-sany-differential/worker-control" + "t/fixtures/grammar-composition-models" + "t/fixtures/language-pack-research/package-expression-runtime" + "t/fixtures/language-pack-research/composition-history" + "t/fixtures/language-pack-research/list-runtime" + "t/fixtures/language-pack-research/provenance-language" + "languages/arithmetic/parser-test" + "languages/bash/parser-test" + "languages/cypher/parser-test" + "languages/fhirpath/parser-test" + "languages/gql/parser-test" + "languages/hcl/parser-test" + "languages/hl7/parser-test" + "languages/tla-plus/parser-test" + "t/fixtures/language-diagnostics-vocabulary" + "t/fixtures/source-strategies" + "t/fixtures/bash-products" + "t/language-diagnostics-test" + "t/language-surface-test" + "t/grammar-composition-contract-test" + "t/grammar-composition-lowering-test" + "t/grammar-composition-execution-test" + "t/fixtures/language-entries" + "t/fixtures/lr1-construction" + "t/resolved-grammar-test" + "t/lr-conflict-origins-test" + "t/fixtures/layout-columns" + "t/layout-column-storage-test" + "t/language-artifact-test" + "t/benchmarks/parser-stage-cost/benchmark" + "t/fixtures/lr-action-selection" + "t/benchmarks/lr-request-stack/lookahead" + "t/lr-action-preparation-test" + "t/fixtures/ranked-publication" + "t/ranked-parser-request-test" + "t/benchmarks/ranked-literal/benchmark" + "t/benchmarks/ranked-literal/parser-cpu" + "t/benchmarks/ranked-literal/lexical-cpu" + "t/benchmarks/layout-columns/benchmark" + "t/artifact-publication-cost-test" + "t/fixtures/artifact-window" + "t/artifact-window-publication-test" + "t/language-entry-boundary-test" + "t/language-topology-test" + "t/language-loader-test" + "t/language-loader-value-test" + "t/antlr4-source-test" + "t/language-pack-metadata-test" + "t/source-strategy-test" + "t/source-grammar-test" + "t/command-profile-test" + "t/build-strategy-test" + "t/gql/benchmark-profile-test" + "t/benchmarks/versioned-languages/all-languages" + "t/benchmarks/gql/runtime/reduction-counts" + "t/benchmarks/gql/runtime/execution-counts" + "t/benchmarks/gql/runtime/matched-stages" + "t/fixtures/language-pack-research/package-expression-grammar" + "t/fixtures/language-pack-research/package-expression-parser" + "t/fixtures/language-pack-research/package-expression-parser-test" + "t/fixtures/language-pack-research/list-roles" + "t/fixtures/language-pack-research/list-origins" + "t/fixtures/language-pack-research/list-stage" + "t/fixtures/language-pack-research/list-language" + "t/fixtures/language-pack-research/list-parser" + "t/fixtures/language-pack-research/list-parser-test" + "t/fixtures/runtime-record-assignments/languages/records/grammar" + "t/fixtures/runtime-record-assignments/languages/records/fixtures" + "t/fixtures/runtime-record-assignments/languages/records/parser" + "t/fixtures/runtime-record-assignments/languages/records/parser-test" + "t/test-style-contract-test" + "t/conformance-main")) +(def (shared-test-module? name) + (or (string-suffix? "-test" name) + (member name '("t/fixtures/language-entries" "t/conformance-main" + "t/benchmarks/versioned-languages/all-languages")))) +(def (compiled-spec modules) + (map (lambda (module) + `(gxc: ,module "-cc-options" "-v -Q")) modules)) +(def (compile-static-tests!) + ;; Keep source expansion and executable publication in one compiler context. + (add-load-path! (path-directory (this-source-file))) + (for-each + (lambda (module) + (displayln "STATIC-TEST-SCHEME " module) (force-output) + (compile-module (string-append module ".ss") + [output-dir: (path-expand "lib" (getenv "GERBIL_PATH")) + ;; Stage test construction with the SDK optimizer before native + ;; lowering to reduce generated host size and compiler work. Keep + ;; every assertion and the native tail-call compilation policy. + optimize: #t generate-ssxi: #t static: #t keep-scm: #t + invoke-gsc: #f verbose: #f]) + (when (string-suffix? "-test" module) + (displayln "CONFORMANCE-EXPORT-CHECK " module) (force-output) + ;; Inspect the public expansion context without evaluating test bodies. + (let (context (import-module (string-append module ".ss") #f #f)) + (validate-test-suite-ownership! + (conformance-imported-test-names module) + (conformance-source-suite-names module) + (map module-export-name + (filter (lambda (exported) + (and (zero? (module-export-phi exported)) + (string-suffix? "-test" (symbol->string (module-export-name exported))))) + (module-context-export context))))))) + (filter shared-test-module? conformance-modules))) +(def (compile-static-conformance!) + ;; Helpers serve both the closed executable and source-mode generators. + ;; Publish complete loadable products; the link owner also consumes static SCM. + (for-each + (lambda (module) + (displayln "STATIC-HELPER-SCHEME " module) (force-output) + (compile-module (string-append module ".ss") + [output-dir: (path-expand "lib" (getenv "GERBIL_PATH")) + optimize: #t generate-ssxi: #t static: #t keep-scm: #t + invoke-gsc: #t + gsc-options: ["-verbose" "-cc-options" "-v -Q -ftrack-macro-expansion=0"] verbose: #f])) + (filter (lambda (name) (not (shared-test-module? name))) conformance-modules)) + ;; Test bodies remain source-only here; their native objects belong to the link owner. + (compile-static-tests!)) +(def (prepare-gxtest!) + ;; gxtest owns source test evaluation. The closed executable already owns + ;; native conformance bodies; production parser modules remain native. + ;; Remove only source-only test interfaces, using std/make's own output map, + ;; so gxtest imports their Scheme sources rather than an incomplete product. + (let (modules (filter shared-test-module? conformance-modules)) + (make-clean + (append (compiled-spec modules) + (map (lambda (module) `(copy: ,(string-append module ".scm"))) modules)) + libdir: (path-expand "lib" (getenv "GERBIL_PATH")) + srcdir: (path-directory (this-source-file))))) +;; Developer multicall build retains the standard native shared-module owner. +(defbuild-script (compiled-spec conformance-modules) optimize: #t) diff --git a/build-qualification.ss b/build-qualification.ss new file mode 100644 index 00000000..c15f3576 --- /dev/null +++ b/build-qualification.ss @@ -0,0 +1,14 @@ +#!/usr/bin/env gxi +;;; Formal build owner for the native POSIX qualification boundary. +(import (only-in :std/build-script defbuild-script)) +(defbuild-script + `((gxc: "t/support/qualification/process-child" + ,@(cond-expand + (darwin '("-ld-options" "-Wl,-undefined,dynamic_lookup")) + (else '()))) + (gxc: "t/support/qualification/ownership") + (gxc: "t/support/qualification/process") + (gxc: "t/support/qualification/products") + (gxc: "t/support/qualification/environment") + (gxc: "t/support/qualification/performance") + (gxc: "t/support/qualification/plans"))) diff --git a/build-rust-ffi-tests.ss b/build-rust-ffi-tests.ss new file mode 100644 index 00000000..cfbe8752 --- /dev/null +++ b/build-rust-ffi-tests.ss @@ -0,0 +1,18 @@ +#!/usr/bin/env gxi +;;; Link the real Rust static consumer through Gerbil's normal build owner. +(import (only-in :std/build-script defbuild-script) + (only-in :std/source this-source-file)) +(def archive (getenv "GERBIL_PARSER_RUST_NATIVE_ARCHIVE" #f)) +(unless archive (error "GERBIL_PARSER_RUST_NATIVE_ARCHIVE must name the Cargo staticlib")) +(defbuild-script + `((gxc: "t/fixtures/ffi/rust-language-probe" + "-ld-options" ,(string-append archive + (cond-expand + (darwin " -Wl,-undefined,dynamic_lookup") + (else (string-append " -ldl -lpthread -lm " + (path-expand "lib/gerbil-parser/src/ffi/language-abi~0.o1" (gerbil-path)) " " + (path-expand "lib/gerbil-parser/t/fixtures/shared-scanner/records-abi~0.o1" (gerbil-path)))))))) + ;; Keep this disposable transport entry out of whole-library optimizer analysis. + optimize: #f + force: #t + verbose: 9) diff --git a/build-rust-rowan-aot.ss b/build-rust-rowan-aot.ss deleted file mode 100644 index 18d06439..00000000 --- a/build-rust-rowan-aot.ss +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- -;;; Optional build-time Rowan generator; the runtime package stays independent. - -(import (only-in :std/build-script defbuild-script) - (only-in :gerbil-parser/src/build-support/rust-rowan-aot - rust-rowan-aot-build-spec)) - -(defbuild-script (rust-rowan-aot-build-spec)) diff --git a/build-rust-runtime-aot.ss b/build-rust-runtime-aot.ss new file mode 100644 index 00000000..817d192b --- /dev/null +++ b/build-rust-runtime-aot.ss @@ -0,0 +1,9 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Optional build-time Rust generator; the runtime package stays independent. + +(import (only-in :std/build-script defbuild-script) + (only-in :gerbil-parser/src/build-support/rust-runtime-aot + rust-runtime-aot-build-spec)) + +(defbuild-script (rust-runtime-aot-build-spec)) diff --git a/build-shared-library.ss b/build-shared-library.ss new file mode 100644 index 00000000..aa9d09a9 --- /dev/null +++ b/build-shared-library.ss @@ -0,0 +1,38 @@ +#!/usr/bin/env gxi +;;; SDK compiler owns static closure and job execution; no generated C rewriting. +(import (only-in :gerbil/compiler compile-module compile-exe execute-pending-compile-jobs!) + (only-in :std/os/fcntl __fcntl1 __fcntl2 F_GETFL F_SETFL O_NONBLOCK) + (only-in :std/os/error do-syscall)) +;;; Compiler children inherit these descriptors; use the SDK's native OS API. +(def (blocking-standard-streams!) + (for-each + (lambda (fd) + (let (flags (__fcntl1 fd F_GETFL)) + (unless (fxnegative? flags) + (do-syscall (__fcntl2 fd F_SETFL (fxand flags (fxnot O_NONBLOCK))))))) + '(0 1 2)) + 0) +(def (standard-streams-blocking?) + (andmap + (lambda (fd) + (let (flags (__fcntl1 fd F_GETFL)) + (and (not (fxnegative? flags)) (fxzero? (fxand flags O_NONBLOCK))))) + '(1 2))) +(def (main source output cc-options ld-options) + (displayln "COMPILER-STREAMS before-blocking=" (standard-streams-blocking?)) + (unless (zero? (blocking-standard-streams!)) (error "cannot establish blocking compiler streams")) + (displayln "COMPILER-STREAMS after-blocking=" (standard-streams-blocking?)) + (displayln "SHARED-LIBRARY-MODULE-COMPILE") (force-output) + (def output-directory (path-expand "lib" (path-directory source))) + (compile-module source + [output-dir: output-directory invoke-gsc: #f optimize: #f generate-ssxi: #f parallel: #t verbose: #t + gsc-options: ["-cc-options" cc-options]]) + (execute-pending-compile-jobs!) + (displayln "SHARED-LIBRARY-CLOSURE-LINK") (force-output) + (compile-exe source + [output-dir: output-directory invoke-gsc: #t output-file: output parallel: #t verbose: #t + gsc-options: ["-cc-options" cc-options "-ld-options" ld-options]]) + (execute-pending-compile-jobs!) + (unless (file-exists? output) (error "native bundle was not published" output)) + (displayln "SHARED-LIBRARY-COMPILED") (force-output) + (exit 0)) diff --git a/build-support/gerbil-parser-build-support/Cargo.toml b/build-support/gerbil-parser-build-support/Cargo.toml index d744a840..6945c1f7 100644 --- a/build-support/gerbil-parser-build-support/Cargo.toml +++ b/build-support/gerbil-parser-build-support/Cargo.toml @@ -8,11 +8,10 @@ license.workspace = true repository.workspace = true publish = false -[dependencies] +[dev-dependencies] asp-rust.workspace = true asp-rust-build-support.workspace = true - -[dev-dependencies] +gerbil-parser-runtime.workspace = true gerbil-scheme-rust-ir.workspace = true [lints] diff --git a/build-support/gerbil-parser-build-support/src/lib.rs b/build-support/gerbil-parser-build-support/src/lib.rs index 87d1e111..74d60795 100644 --- a/build-support/gerbil-parser-build-support/src/lib.rs +++ b/build-support/gerbil-parser-build-support/src/lib.rs @@ -1,21 +1,11 @@ -//! Shared build and development support for the `gerbil-parser` workspace. +//! Shared dependency-free test macros for the `gerbil-parser` workspace. #![forbid(unsafe_code)] mod policy; -mod rowan_scenarios; +mod runtime_scenarios; +mod test_assertions; -pub use asp_rust::{AspRustConfig, AspRustWorkspacePolicy, default_asp_rust_config}; -pub use policy::workspace_policy; -pub use rowan_scenarios::{ - ROWAN_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID, ROWAN_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID, - rowan_engine_scenario_package, +pub use runtime_scenarios::{ + RUNTIME_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID, + RUNTIME_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID, }; - -pub use asp_rust_build_support::{ - AspRustScenario, AspRustScenarioMeasurement, AspRustScenarioObservation, - AspRustScenarioPackage, asp_rust_scenario, asp_rust_scenario_package, - measure_asp_rust_scenario, render_asp_rust_scenario_benchmark_toml, -}; - -#[doc(hidden)] -pub use asp_rust; diff --git a/build-support/gerbil-parser-build-support/src/policy.rs b/build-support/gerbil-parser-build-support/src/policy.rs index 50c9b537..22fa1c0f 100644 --- a/build-support/gerbil-parser-build-support/src/policy.rs +++ b/build-support/gerbil-parser-build-support/src/policy.rs @@ -1,28 +1,16 @@ -//! ASP Rust policy owned by workspace Build Support. - -use crate::{AspRustWorkspacePolicy, default_asp_rust_config}; - -/// Return the single ASP Rust policy owned by this workspace. -#[must_use] -pub fn workspace_policy() -> AspRustWorkspacePolicy { - let mut config = default_asp_rust_config(); - // Generated language tables are immutable compiler output verified against - // Grammar IR in native CI. Policy governs their generator and the runtime - // owners instead of treating hundreds of thousands of table rows as - // handwritten module responsibilities. - config.ignored_dir_names.insert("generated".to_owned()); - AspRustWorkspacePolicy::new("gerbil-parser", config) -} +//! Expand the ASP Rust policy in a test target, never in the support library. /// Mount the workspace ASP Rust policy as a Cargo-test-only Dev Gate. -/// -/// Warnings and errors are test-blocking; ASP Rust remains a dev dependency. #[macro_export] macro_rules! asp_workspace_policy_gate { () => { - $crate::asp_rust::asp_rust_workspace_dev_gate!( - mode = deny, - policy = $crate::workspace_policy() - ); + fn workspace_policy() -> ::asp_rust::AspRustWorkspacePolicy { + let mut config = ::asp_rust::default_asp_rust_config(); + // Generated tables are checked against Grammar IR in native CI. + config.ignored_dir_names.insert("generated".to_owned()); + ::asp_rust::AspRustWorkspacePolicy::new("gerbil-parser", config) + } + + ::asp_rust::asp_rust_workspace_dev_gate!(mode = deny, policy = workspace_policy()); }; } diff --git a/build-support/gerbil-parser-build-support/src/rowan_scenarios.rs b/build-support/gerbil-parser-build-support/src/rowan_scenarios.rs deleted file mode 100644 index 0c21907d..00000000 --- a/build-support/gerbil-parser-build-support/src/rowan_scenarios.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Rust/Rowan parser-engine performance red-zone Scenarios. - -use crate::{AspRustScenarioPackage, asp_rust_scenario, asp_rust_scenario_package}; - -/// Stable identity for the deterministic engine hot-path Scenario. -pub const ROWAN_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID: &str = - "rowan-engine-deterministic-hot-path-v1"; - -/// Stable identity for Scheme-AOT structural event emission. -pub const ROWAN_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID: &str = - "rowan-engine-event-tree-hot-path-v1"; - -/// Package the generated-table Rust/Rowan latency red zone as ASP Rust Scenarios. -#[must_use] -pub fn rowan_engine_scenario_package() -> AspRustScenarioPackage { - asp_rust_scenario_package! { - package: "gerbil-parser-rowan-scenarios", - scenarios: [ - asp_rust_scenario! { - name: ROWAN_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID, - package: "gerbil-parser-rowan-scenarios", - description: "The generic engine keeps generated-table lexing, LR execution, and lossless Rowan CST construction inside the hot-path budget", - fixture_root: "rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1", - tags: ["performance", "red-zone", "rowan", "engine", "aot", "deterministic-lr"], - commands: [ - { - label: "focused", - argv: ["cargo", "test", "--release", "-p", "gerbil-parser-rowan-scenarios", "--test", "performance_test", "rowan_parse::rowan_engine_deterministic_hot_path_v1", "--", "--ignored", "--exact", "--nocapture"] - } - ], - benchmark: { - harness: "libtest", - test: "rowan_parse::rowan_engine_deterministic_hot_path_v1", - snapshot: "rowan_engine_deterministic_hot_path_v1", - target_total: "5ms", - max_total: "10ms", - regression_budget: "2ms", - memory_budget_bytes: 16_777_216, - target_rationale: "Generated-table deterministic execution is the steady-state floor shared by every downstream DSL grammar product.", - warmup_iterations: 5, - measure_iterations: 31, - metrics: [ - { name: "parse_count", unit: "count", kind: Exact, target: 256 }, - { name: "rowan_node_count", unit: "count", kind: Minimum, target: 1 }, - { name: "provider_process_count", unit: "count", kind: Exact, target: 0 } - ] - } - }, - asp_rust_scenario! { - name: ROWAN_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID, - package: "gerbil-parser-rowan-scenarios", - description: "The generic Rowan event sink builds deeply nested lossless CSTs without language-specific parser policy", - fixture_root: "rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1", - tags: ["performance", "red-zone", "rowan", "engine", "aot", "event-tree"], - commands: [ - { - label: "focused", - argv: ["cargo", "test", "--release", "-p", "gerbil-parser-rowan-scenarios", "--test", "performance_test", "rowan_parse::rowan_engine_event_tree_hot_path_v1", "--", "--ignored", "--exact", "--nocapture"] - } - ], - benchmark: { - harness: "libtest", - test: "rowan_parse::rowan_engine_event_tree_hot_path_v1", - snapshot: "rowan_engine_event_tree_hot_path_v1", - target_total: "5ms", - max_total: "10ms", - regression_budget: "2ms", - memory_budget_bytes: 16_777_216, - target_rationale: "Structural event emission must stay bounded before downstream Org parser logic is admitted.", - warmup_iterations: 5, - measure_iterations: 31, - metrics: [ - { name: "parse_count", unit: "count", kind: Exact, target: 128 }, - { name: "rowan_node_count", unit: "count", kind: Minimum, target: 16512 }, - { name: "provider_process_count", unit: "count", kind: Exact, target: 0 } - ] - } - } - ] - } -} diff --git a/build-support/gerbil-parser-build-support/src/runtime_scenarios.rs b/build-support/gerbil-parser-build-support/src/runtime_scenarios.rs new file mode 100644 index 00000000..c7ff0d09 --- /dev/null +++ b/build-support/gerbil-parser-build-support/src/runtime_scenarios.rs @@ -0,0 +1,81 @@ +//! Rust parser-engine performance red-zone Scenarios. + +/// Stable identity for the deterministic engine hot-path Scenario. +pub const RUNTIME_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID: &str = + "runtime-engine-deterministic-hot-path-v1"; + +/// Stable identity for Scheme-AOT structural event emission. +pub const RUNTIME_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID: &str = + "runtime-engine-event-tree-hot-path-v1"; + +/// Expand the generated-table Rust red zone in a dev-only test target. +#[macro_export] +macro_rules! runtime_engine_scenario_package { + () => { + ::asp_rust_build_support::asp_rust_scenario_package! { + package: "gerbil-parser-build-support", + scenarios: [ + ::asp_rust_build_support::asp_rust_scenario! { + name: $crate::RUNTIME_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID, + package: "gerbil-parser-build-support", + description: "The generic engine keeps generated-table lexing, LR execution, and lossless indexed CST construction inside the hot-path budget", + fixture_root: "build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path", + tags: ["performance", "red-zone", "runtime", "engine", "aot", "deterministic-lr"], + commands: [ + { + label: "focused", + argv: ["cargo", "test", "--release", "-p", "gerbil-parser-build-support", "--test", "performance_test", "runtime_parse::runtime_engine_deterministic_hot_path", "--", "--ignored", "--exact", "--nocapture"] + } + ], + benchmark: { + harness: "libtest", + test: "runtime_parse::runtime_engine_deterministic_hot_path", + snapshot: "runtime_engine_deterministic_hot_path", + target_total: "5ms", + max_total: "10ms", + regression_budget: "2ms", + memory_budget_bytes: 16_777_216, + target_rationale: "Generated-table deterministic execution is the steady-state floor shared by every downstream DSL grammar product.", + warmup_iterations: 5, + measure_iterations: 31, + metrics: [ + { name: "parse_count", unit: "count", kind: Exact, target: 256 }, + { name: "runtime_node_count", unit: "count", kind: Minimum, target: 1 }, + { name: "provider_process_count", unit: "count", kind: Exact, target: 0 } + ] + } + }, + ::asp_rust_build_support::asp_rust_scenario! { + name: $crate::RUNTIME_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID, + package: "gerbil-parser-build-support", + description: "The generic syntax event sink builds deeply nested lossless CSTs without language-specific parser policy", + fixture_root: "build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path", + tags: ["performance", "red-zone", "runtime", "engine", "aot", "event-tree"], + commands: [ + { + label: "focused", + argv: ["cargo", "test", "--release", "-p", "gerbil-parser-build-support", "--test", "performance_test", "runtime_parse::runtime_engine_event_tree_hot_path", "--", "--ignored", "--exact", "--nocapture"] + } + ], + benchmark: { + harness: "libtest", + test: "runtime_parse::runtime_engine_event_tree_hot_path", + snapshot: "runtime_engine_event_tree_hot_path", + target_total: "5ms", + max_total: "10ms", + regression_budget: "2ms", + memory_budget_bytes: 16_777_216, + target_rationale: "Structural event emission must stay bounded before downstream Org parser logic is admitted.", + warmup_iterations: 5, + measure_iterations: 31, + metrics: [ + { name: "parse_count", unit: "count", kind: Exact, target: 128 }, + { name: "runtime_node_count", unit: "count", kind: Minimum, target: 16512 }, + { name: "provider_process_count", unit: "count", kind: Exact, target: 0 } + ] + } + } + ] + } + }; +} diff --git a/build-support/gerbil-parser-build-support/src/test_assertions.rs b/build-support/gerbil-parser-build-support/src/test_assertions.rs new file mode 100644 index 00000000..1f2c5632 --- /dev/null +++ b/build-support/gerbil-parser-build-support/src/test_assertions.rs @@ -0,0 +1,9 @@ +//! Shared assertions that require no ASP Rust dependency in the caller. + +/// Check the lossless-source invariant for any generated AOT language. +#[macro_export] +macro_rules! assert_syntax_lossless { + ($parsed:expr, $source:expr) => { + assert_eq!($parsed.syntax().to_string(), $source) + }; +} diff --git a/build-support/gerbil-parser-build-support/tests/asp-rust-gate.rs b/build-support/gerbil-parser-build-support/tests/asp-rust-gate.rs index 9c94cf46..84c086f4 100644 --- a/build-support/gerbil-parser-build-support/tests/asp-rust-gate.rs +++ b/build-support/gerbil-parser-build-support/tests/asp-rust-gate.rs @@ -1,3 +1,3 @@ -//! Workspace-owned ASP Rust `DevGate`. +//! Cargo-test-only ASP Rust workspace policy gate. gerbil_parser_build_support::asp_workspace_policy_gate!(); diff --git a/rust/gerbil-parser-rowan-arithmetic/src/generated/mod.rs b/build-support/gerbil-parser-build-support/tests/fixtures/generated/arithmetic.rs similarity index 99% rename from rust/gerbil-parser-rowan-arithmetic/src/generated/mod.rs rename to build-support/gerbil-parser-build-support/tests/fixtures/generated/arithmetic.rs index 9576d11f..4a416a90 100644 --- a/rust/gerbil-parser-rowan-arithmetic/src/generated/mod.rs +++ b/build-support/gerbil-parser-build-support/tests/fixtures/generated/arithmetic.rs @@ -1,7 +1,7 @@ -// @generated by gerbil-parser/src/compiler/rust-rowan.ss +// @generated by gerbil-parser/src/compiler/rust-runtime.ss // Source authority: canonical Grammar IR v1; do not edit. -use gerbil_parser_rowan::{ +use gerbil_parser_runtime::{ ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, }; diff --git a/rust/gerbil-parser-rowan-scenarios/tests/integration/arithmetic_v1.rs b/build-support/gerbil-parser-build-support/tests/integration/arithmetic.rs similarity index 63% rename from rust/gerbil-parser-rowan-scenarios/tests/integration/arithmetic_v1.rs rename to build-support/gerbil-parser-build-support/tests/integration/arithmetic.rs index cdf53d05..8ea67d1c 100644 --- a/rust/gerbil-parser-rowan-scenarios/tests/integration/arithmetic_v1.rs +++ b/build-support/gerbil-parser-build-support/tests/integration/arithmetic.rs @@ -1,10 +1,13 @@ -use gerbil_parser_rowan_arithmetic::LANGUAGE; +#[path = "../fixtures/generated/arithmetic.rs"] +mod generated; + +use generated::LANGUAGE; #[test] -fn generated_arithmetic_parser_builds_a_lossless_rowan_tree() { +fn generated_arithmetic_parser_builds_a_lossless_runtime_tree() { let source = " alpha + 2 * (beta - -3) \n"; - let parsed = gerbil_parser_rowan::parse(&LANGUAGE, source).expect("accepted arithmetic"); - assert_eq!(parsed.syntax().to_string(), source); + let parsed = gerbil_parser_runtime::parse(&LANGUAGE, source).expect("accepted arithmetic"); + gerbil_parser_build_support::assert_syntax_lossless!(parsed, source); assert_eq!(parsed.receipt().language, "arithmetic"); assert_eq!(parsed.receipt().version, "v1"); assert_eq!(parsed.receipt().contract, "arithmetic-expression.v1"); @@ -14,7 +17,7 @@ fn generated_arithmetic_parser_builds_a_lossless_rowan_tree() { #[test] fn generated_arithmetic_parser_preserves_precedence_shape() { - let parsed = gerbil_parser_rowan::parse(&LANGUAGE, "1 + 2 * 3").expect("accepted arithmetic"); + let parsed = gerbil_parser_runtime::parse(&LANGUAGE, "1 + 2 * 3").expect("accepted arithmetic"); let root = parsed.syntax(); let names = root .descendants() @@ -35,11 +38,12 @@ fn generated_arithmetic_parser_preserves_precedence_shape() { #[test] fn generated_arithmetic_parser_fails_closed() { - let error = gerbil_parser_rowan::parse(&LANGUAGE, "1 + * 2").expect_err("rejected arithmetic"); + let error = + gerbil_parser_runtime::parse(&LANGUAGE, "1 + * 2").expect_err("rejected arithmetic"); assert_eq!(error.diagnostic.reason_kind, "parse-rejected"); assert_eq!(error.receipt.grammar_digest, LANGUAGE.grammar_digest); - let joined = gerbil_parser_rowan::parse(&LANGUAGE, "123abc") + let joined = gerbil_parser_runtime::parse(&LANGUAGE, "123abc") .expect_err("number and identifier cannot be fused by the Rust lexer"); assert_eq!(joined.diagnostic.reason_kind, "parse-rejected"); } @@ -47,6 +51,6 @@ fn generated_arithmetic_parser_fails_closed() { #[test] fn generated_arithmetic_identifier_matches_the_declared_scanner() { let source = "λ-value_2 + ٣"; - let parsed = gerbil_parser_rowan::parse(&LANGUAGE, source).expect("Unicode arithmetic"); - assert_eq!(parsed.syntax().to_string(), source); + let parsed = gerbil_parser_runtime::parse(&LANGUAGE, source).expect("Unicode arithmetic"); + gerbil_parser_build_support::assert_syntax_lossless!(parsed, source); } diff --git a/build-support/gerbil-parser-build-support/tests/integration/function_ir.rs b/build-support/gerbil-parser-build-support/tests/integration/function_ir.rs index adad47cf..c9b4d195 100644 --- a/build-support/gerbil-parser-build-support/tests/integration/function_ir.rs +++ b/build-support/gerbil-parser-build-support/tests/integration/function_ir.rs @@ -10,7 +10,7 @@ fn scheme_ir_compiles_to_owned_rest_function() { } #[test] -fn scheme_event_fold_compiles_to_stateful_rowan_events() { +fn scheme_event_fold_compiles_to_stateful_runtime_events() { let ir = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/../../t/fixtures/event_fold.ir.json" @@ -23,7 +23,7 @@ fn scheme_event_fold_compiles_to_stateful_rowan_events() { let digest = &rust[digest_start..digest_start + 71]; let generated = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/../../rust/gerbil-parser-rowan/tests/unit/event_fold_generated.rs" + "/../../rust/gerbil-parser-runtime/tests/unit/event_fold_generated.rs" )); assert!(generated.contains(digest), "generated Rust is stale"); } @@ -40,7 +40,7 @@ fn scheme_outline_fold_compiles_one_marker_scan_per_line() { let digest = &rust[digest_start..digest_start + 71]; let generated = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/../../rust/gerbil-parser-rowan/tests/unit/outline_fold_generated.rs" + "/../../rust/gerbil-parser-runtime/tests/unit/outline_fold_generated.rs" )); assert!( generated.contains(digest), diff --git a/build-support/gerbil-parser-build-support/tests/integration_test.rs b/build-support/gerbil-parser-build-support/tests/integration_test.rs index 4a9a7605..5b550fb9 100644 --- a/build-support/gerbil-parser-build-support/tests/integration_test.rs +++ b/build-support/gerbil-parser-build-support/tests/integration_test.rs @@ -2,3 +2,6 @@ #[path = "integration/function_ir.rs"] mod function_ir; + +#[path = "integration/arithmetic.rs"] +mod arithmetic; diff --git a/rust/gerbil-parser-rowan-scenarios/tests/performance/rowan_parse.rs b/build-support/gerbil-parser-build-support/tests/performance/runtime_parse.rs similarity index 65% rename from rust/gerbil-parser-rowan-scenarios/tests/performance/rowan_parse.rs rename to build-support/gerbil-parser-build-support/tests/performance/runtime_parse.rs index 1b2570b1..4e35c2c7 100644 --- a/rust/gerbil-parser-rowan-scenarios/tests/performance/rowan_parse.rs +++ b/build-support/gerbil-parser-build-support/tests/performance/runtime_parse.rs @@ -1,12 +1,16 @@ use std::hint::black_box; use std::time::{Duration, Instant}; +use asp_rust_build_support::{ + AspRustScenario, AspRustScenarioMeasurement, AspRustScenarioObservation, + measure_asp_rust_scenario, render_asp_rust_scenario_benchmark_toml, +}; use gerbil_parser_build_support::{ - AspRustScenario, AspRustScenarioObservation, ROWAN_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID, - ROWAN_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID, measure_asp_rust_scenario, - render_asp_rust_scenario_benchmark_toml, rowan_engine_scenario_package, + RUNTIME_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID, + RUNTIME_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID, runtime_engine_scenario_package, }; -use gerbil_parser_rowan_arithmetic as arithmetic_v1; +#[path = "../fixtures/generated/arithmetic.rs"] +mod arithmetic_v1; const ARITHMETIC_PARSE_COUNT: usize = 256; const EVENT_PARSE_COUNT: usize = 128; @@ -14,35 +18,35 @@ const EVENT_DEPTH: usize = 128; #[test] #[ignore = "runner-sensitive ASP Rust performance Scenario; run focused in release mode"] -fn rowan_engine_deterministic_hot_path_v1() { - let scenario = scenario(ROWAN_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID); +fn runtime_engine_deterministic_hot_path() { + let scenario = scenario(RUNTIME_ENGINE_DETERMINISTIC_HOT_PATH_SCENARIO_ID); let source = " alpha + 2 * (beta - -3) + gamma * (17 - delta) \n"; let measurement = measure_asp_rust_scenario(&scenario, || { let parse_started_at = Instant::now(); let mut node_count = 0_u64; for _ in 0..ARITHMETIC_PARSE_COUNT { - let parsed = gerbil_parser_rowan::parse(&arithmetic_v1::LANGUAGE, black_box(source)) + let parsed = gerbil_parser_runtime::parse(&arithmetic_v1::LANGUAGE, black_box(source)) .expect("generated arithmetic engine must parse"); node_count += parsed.syntax().descendants().count() as u64; black_box(parsed.syntax().text_range()); } AspRustScenarioObservation::default() - .with_timing("generated_tables_to_rowan", parse_started_at.elapsed()) + .with_timing("generated_tables_to_runtime", parse_started_at.elapsed()) .with_metric("parse_count", ARITHMETIC_PARSE_COUNT as u64) - .with_metric("rowan_node_count", node_count) + .with_metric("runtime_node_count", node_count) .with_metric("provider_process_count", 0) }) - .expect("measure deterministic Rust/Rowan engine Scenario"); + .expect("measure deterministic Rust engine Scenario"); assert_red_zone(&scenario, &measurement, Duration::from_millis(10)); } #[test] #[ignore = "runner-sensitive ASP Rust performance Scenario; run focused in release mode"] -fn rowan_engine_event_tree_hot_path_v1() { - use gerbil_parser_rowan::TreeEvent; +fn runtime_engine_event_tree_hot_path() { + use gerbil_parser_runtime::TreeEvent; - let scenario = scenario(ROWAN_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID); + let scenario = scenario(RUNTIME_ENGINE_EVENT_TREE_HOT_PATH_SCENARIO_ID); let source = "x".repeat(EVENT_DEPTH); let mut events = Vec::with_capacity(2 * EVENT_DEPTH + EVENT_DEPTH + 2); events.push(TreeEvent::StartNode(arithmetic_v1::LANGUAGE.root_kind)); @@ -56,51 +60,46 @@ fn rowan_engine_event_tree_hot_path_v1() { } events.extend(std::iter::repeat_n(TreeEvent::FinishNode, EVENT_DEPTH + 1)); let sample = - gerbil_parser_rowan::build_rowan_events(&arithmetic_v1::LANGUAGE, &source, &events) + gerbil_parser_runtime::build_syntax_events(&arithmetic_v1::LANGUAGE, &source, &events) .expect("sample event stream must build"); - assert_eq!( - gerbil_parser_rowan::SyntaxNode::new_root(sample) - .descendants() - .count(), - EVENT_DEPTH + 1 - ); + assert_eq!(sample.root().descendants().count(), EVENT_DEPTH + 1); let measurement = measure_asp_rust_scenario(&scenario, || { let started_at = Instant::now(); for _ in 0..EVENT_PARSE_COUNT { - let green = gerbil_parser_rowan::build_rowan_events( + let index = gerbil_parser_runtime::build_syntax_events( &arithmetic_v1::LANGUAGE, black_box(&source), black_box(&events), ) .expect("generated structural events must form one lossless tree"); - black_box(green); + black_box(index); } AspRustScenarioObservation::default() - .with_timing("scheme_aot_events_to_rowan", started_at.elapsed()) + .with_timing("scheme_aot_events_to_runtime", started_at.elapsed()) .with_metric("parse_count", EVENT_PARSE_COUNT as u64) .with_metric( - "rowan_node_count", + "runtime_node_count", (EVENT_PARSE_COUNT * (EVENT_DEPTH + 1)) as u64, ) .with_metric("provider_process_count", 0) }) - .expect("measure structural Rust/Rowan engine Scenario"); + .expect("measure structural Rust engine Scenario"); assert_red_zone(&scenario, &measurement, Duration::from_millis(10)); } fn scenario(name: &str) -> AspRustScenario { - rowan_engine_scenario_package() + runtime_engine_scenario_package!() .scenarios .into_iter() .find(|scenario| scenario.name == name) - .unwrap_or_else(|| panic!("missing Rowan engine Scenario {name}")) + .unwrap_or_else(|| panic!("missing AOT engine Scenario {name}")) } fn assert_red_zone( scenario: &AspRustScenario, - measurement: &gerbil_parser_build_support::AspRustScenarioMeasurement, + measurement: &AspRustScenarioMeasurement, max_total: Duration, ) { let receipt = render_asp_rust_scenario_benchmark_toml(scenario, measurement) diff --git a/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/expected/README.org b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/expected/README.org new file mode 100644 index 00000000..0757a718 --- /dev/null +++ b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/expected/README.org @@ -0,0 +1,3 @@ +#+title: expected + +Every generated-table engine execution must produce a complete lossless indexed CST without external processes. diff --git a/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/inputs/README.org b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/inputs/README.org new file mode 100644 index 00000000..e029993d --- /dev/null +++ b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/inputs/README.org @@ -0,0 +1,3 @@ +#+title: inputs + +The engine workload is the fixed arithmetic source executed through the AOT-generated Rust tables and generic AOT engine. diff --git a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/scenario.toml b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/scenario.toml similarity index 54% rename from rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/scenario.toml rename to build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/scenario.toml index 385df150..a05f3d92 100644 --- a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/scenario.toml +++ b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_deterministic_hot_path/scenario.toml @@ -1,8 +1,8 @@ -id = "rowan-engine-deterministic-hot-path-v1" -title = "Generated deterministic parser-engine execution stays inside the Rowan latency red zone" -policy_ids = ["RUST-AGENT-ROWAN-ENGINE-PERFORMANCE-001"] -agent_goal = "Preserve bounded generated-table lexing, LR execution, and lossless Rowan CST construction latency." -reference_repositories = ["rust-analyzer/rowan", "tree-sitter/tree-sitter"] +id = "runtime-engine-deterministic-hot-path-v1" +title = "Generated deterministic parser-engine execution stays inside the AOT latency red zone" +policy_ids = ["RUST-AGENT-RUNTIME-ENGINE-PERFORMANCE-001"] +agent_goal = "Preserve bounded generated-table lexing, LR execution, and lossless indexed CST construction latency." +reference_repositories = ["rust-analyzer/runtime", "tree-sitter/tree-sitter"] reference_patterns = [ "Engine benchmarks measure complete lossless CST construction", "Warm samples retain percentile evidence and exact work counts", diff --git a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/expected/README.md b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/expected/README.org similarity index 55% rename from rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/expected/README.md rename to build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/expected/README.org index dd5f6923..2115df14 100644 --- a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/expected/README.md +++ b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/expected/README.org @@ -1,5 +1,5 @@ -# Expected +#+title: Expected -Every sample builds 128 lossless Rowan trees with at least 16,512 nodes, no +Every sample builds 128 lossless indexed syntaxs with at least 16,512 nodes, no provider process, and a 10 ms p95 ceiling. The scenario measures the generic engine sink, not Org-specific recognition. diff --git a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/inputs/README.md b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/inputs/README.org similarity index 93% rename from rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/inputs/README.md rename to build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/inputs/README.org index ac95d699..bdf5c17a 100644 --- a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/inputs/README.md +++ b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/inputs/README.org @@ -1,4 +1,4 @@ -# Input +#+title: Input The focused benchmark constructs 128 nested nodes over 128 source bytes and repeats the generated event stream 128 times per sample. The event sequence is diff --git a/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/scenario.toml b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/scenario.toml new file mode 100644 index 00000000..4127f041 --- /dev/null +++ b/build-support/gerbil-parser-build-support/tests/performance/scenarios/runtime_engine_event_tree_hot_path/scenario.toml @@ -0,0 +1,11 @@ +id = "runtime-engine-event-tree-hot-path-v1" +title = "Scheme-AOT structural events stay inside the AOT latency red zone" +policy_ids = ["RUST-AGENT-RUNTIME-ENGINE-PERFORMANCE-001"] +agent_goal = "Preserve bounded lossless indexed syntax construction from deeply nested generated parser events." +reference_repositories = ["rust-analyzer/runtime", "tree-sitter/tree-sitter"] +reference_patterns = [ + "Structural parser decisions remain downstream-owned", + "The generic sink validates exact source coverage without copying source text", +] +inputs = "inputs" +expected = "expected" diff --git a/build-support/gerbil-parser-build-support/tests/performance_test.rs b/build-support/gerbil-parser-build-support/tests/performance_test.rs new file mode 100644 index 00000000..e28f1b91 --- /dev/null +++ b/build-support/gerbil-parser-build-support/tests/performance_test.rs @@ -0,0 +1,2 @@ +#[path = "performance/runtime_parse.rs"] +mod runtime_parse; diff --git a/build-test-driver.ss b/build-test-driver.ss new file mode 100644 index 00000000..9aeef9ea --- /dev/null +++ b/build-test-driver.ss @@ -0,0 +1,8 @@ +#!/usr/bin/env gxi +;;; Compile the test runtime C exit binding. +(import (only-in :std/build-script defbuild-script)) +(defbuild-script + `((gxc: "t/fixtures/tla-sany-differential/exit-child-process" + ,@(cond-expand + (darwin '("-ld-options" "-Wl,-undefined,dynamic_lookup")) + (else '()))))) diff --git a/build.ss b/build.ss index eca48764..b6f0fcc1 100755 --- a/build.ss +++ b/build.ss @@ -2,7 +2,8 @@ ;;; -*- Gerbil -*- ;;; Thin package-build entrypoint; the PackageSpec owns project topology. -(import (only-in :std/build-script defbuild-script) +(import (only-in :std/source this-source-file) + (only-in :std/build-script defbuild-script) (only-in :asp-gerbil-scheme/building-api default-exclude-dirs asp-gerbil-scheme-library-package-prototype @@ -11,43 +12,65 @@ ;; Reference corpora are not Gerbil package sources. clan's native defaults ;; already exclude t/, .git/, and .gerbil/. (def gerbil-parser-exclude-dirs - (cons ".data" (cons "target" default-exclude-dirs))) + (cons "tools" (cons "benchmarks" (cons ".data" (cons "target" default-exclude-dirs))))) ;; These source files belong to executable and test owners, not the production ;; library catalog. Keep the boundary declarative so PackageSpec still performs ;; the single source discovery pass. (def gerbil-parser-exclude-modules - '("build-gparse.ss" - "build-rust-rowan-aot.ss" - "generate-rust-rowan.ss" + '("build-qualification.ss" "qualify.ss" "build-conformance.ss" + "test-processes.ss" + "scripts/tests/process-qualification-test.ss" + "scripts/tests/history-commit-test.ss" + "scripts/generate-source-parser.ss" + "scripts/generate-reductions.ss" + "scripts/preparation-cache.ss" + "scripts/tests/preparation-test.ss" + "scripts/tests/compiler-process-test.ss" + "scripts/test-plan.ss" + "scripts/tests/test-plan-test.ss" + "scripts/tests/conformance-runner-test.ss" + "build-test-driver.ss" + "build-language-conformance.ss" + "build-language-conformance-link.ss" + "build-ffi-tests.ss" + "build-rust-ffi-tests.ss" + "build-shared-library.ss" + "build-gparse.ss" + "build-rust-runtime-aot.ss" + "generate-rust-runtime.ss" "src/main.ss" "src/cli.ss" - "src/ffi/parse-artifact-v1-native.ss" - "src/ffi/rust-rowan-aot-v1-native.ss" - "src/ffi/rust-rowan-aot-main.ss" - "languages/arithmetic/v1/parser-test.ss" - "languages/cypher/opencypher-2024-1/parser-test.ss" - "languages/gql/iso-39075-2024/parser-test.ss" - "languages/gql/iso-39075-2024/query-syntax-test.ss" - "languages/hcl/v2-24/parser-test.ss" - "languages/fhirpath/v2.0.0/parser-test.ss" - "languages/hl7/v2-2.5.1/parser-test.ss" - "languages/tla-plus/v1/parser-test.ss")) + "src/ffi/language-abi.ss" + "src/ffi/language-abi-context.ss" + "src/ffi/rust-aot-abi.ss" + "src/ffi/rust-runtime-aot-main.ss" + "languages/bash/parser-test.ss" + "languages/arithmetic/parser-test.ss" + "languages/cypher/parser-test.ss" + "languages/gql/parser-test.ss" + "languages/hcl/parser-test.ss" + "languages/fhirpath/parser-test.ss" + "languages/hl7/parser-test.ss" + "languages/tla-plus/parser-test.ss")) ;; Gambit compiles loadable modules as Mach-O bundles on Darwin. The ;; Homebrew GCC toolchain needs the standard unresolved-symbol policy for an ;; FFI bundle; the final AOT consumer resolves these symbols when it links the ;; native runtime. -(def gerbil-parser-native-ffi-specs - (cond-expand - (darwin - '((gxc: "src/ffi/parse-artifact-v1-native" - "-ld-options" "-Wl,-undefined,dynamic_lookup") - (gxc: "src/ffi/rust-rowan-aot-v1-native" - "-ld-options" "-Wl,-undefined,dynamic_lookup"))) - (else - '((gxc: "src/ffi/parse-artifact-v1-native") - (gxc: "src/ffi/rust-rowan-aot-v1-native"))))) +(def gerbil-parser-ffi-specs + (let ((include-option + (string-append "-I" (path-expand "include" + (path-directory (this-source-file))))) + (link-options + (cond-expand + (darwin '("-ld-options" "-Wl,-undefined,dynamic_lookup")) + (else '())))) + (map (lambda (module) + `(gxc: ,module "-cc-options" ,include-option ,@link-options)) + '("src/ffi/language-abi" + "src/ffi/language-abi-context" + "src/ffi/rust-aot-abi")))) ;; PackageSpec remains here because the Build API derives project ownership ;; from this declaration's source location. Its default native projection owns @@ -60,7 +83,7 @@ (spec gerbil-parser-build-spec) (exclude-dirs gerbil-parser-exclude-dirs) (exclude-modules gerbil-parser-exclude-modules) - (extra-spec gerbil-parser-native-ffi-specs)) + (extra-spec gerbil-parser-ffi-specs)) ;; Keep the standard multicall entrypoint at top level. std/build-script owns ;; spec/compile/clean and delegates the projection to one std/make scheduler. diff --git a/docs/architecture/0001-parser-compiler.org b/docs/architecture/0001-parser-compiler.org index 3e07703a..e7a378c6 100644 --- a/docs/architecture/0001-parser-compiler.org +++ b/docs/architecture/0001-parser-compiler.org @@ -59,10 +59,10 @@ $ =build.ss= discovers Gerbil modules and owns AOT staging. A language's =grammar.ss= contains one declaration that generates grammar, parser IR, machine, and descriptor bindings. Its =parser.ss= contains only the public -=deflanguage-parser= entry. Tests live under =t/= or beside a versioned +=deflanguage-parser-loader= entry. Tests live under =t/= or beside a versioned language pack and benchmarks use the ASP Gerbil Scheme benchmark API. -The repository owns no Rust, Rowan, language-specific AST, semantic-analysis, +The repository owns no language-specific AST, semantic-analysis, or backend authority. Downstream systems consume ParseArtifact v1. =gparse= is a thin command adapter over the same public Scheme parser API; it never owns grammar or parser implementation. diff --git a/docs/architecture/0002-typed-structural-algebras.org b/docs/architecture/0002-typed-structural-algebras.org index 9ffb9075..a15af501 100644 --- a/docs/architecture/0002-typed-structural-algebras.org +++ b/docs/architecture/0002-typed-structural-algebras.org @@ -48,7 +48,7 @@ flowchart TD F --> O[Generated parser machine] O --> T[ParseArtifact v1] T --> E[CST event projection] - E --> R[Rowan or Gerbil or custom sink] + E --> R[Borrowed Rust view or Gerbil or custom sink] T --> Q Q --> P #+end_src @@ -94,6 +94,9 @@ The canonical constructors are: (field name expression) (alias NodeKind expression) (prec left 10 expression) +(layout-start "/\\") +(layout-next "/\\") +(layout-end ")" "]") #+end_src Constructors lower immediately to validated canonical data. =repeat= and @@ -103,6 +106,22 @@ SyntaxKind categorized as =token=. =extras= is the sole trivia authority; terminal rows do not duplicate that policy. Duplicate role rows with different values are rejected before parser generation. +Layout markers retain a branch-local source-column reference at the inclusive +last character of the marker, matching JavaCCs end column. An empty +=layout-end= declaration closes at the enclosing column boundary. A language +may declare non-empty literal spellings in =layout-end= to close at explicit +delimiters even when they occur to the right of the marker. This is a +zero-width reduction: the enclosing production still consumes the delimiter, +and an aligned continuation marker always remains in its current list. +The existing grammar macro projects the validated closing spellings into the +ordinary LR reduction data; the runtime does not infer a language's delimiters. + +Layout-enabled tables additionally retain a precedence-suppressed reduction +behind a =layout-guard= shift. The shift keeps its ordinary precedence while +its source column is admissible; an outdent selects the retained reduction. +Ineligible layout-start/next shifts are filtered without losing reductions +needed to close nested constructs. Non-layout tables have no such guard. + =prec= is a canonical GrammarExpr v1 constructor admitted by the sole LALR(1) compiler. Static =left=, =right=, and =none= declarations resolve only the conflicting shift and reduction owned by that production. Equal-rank @@ -174,7 +193,7 @@ contracts. * ParseArtifact and CST events The parser owns a backend-neutral immutable parse artifact so structural truth -does not depend on Rowan. RFC 0003 owns the admitted event algebra, identity, +is defined by Scheme events. RFC 0003 owns the admitted event algebra, identity, failure boundary, validator, and replay projection. Only implemented variants belong to v1: @@ -210,7 +229,7 @@ matcher. - POO dispatch in generated parser or matcher hot paths. - Pratt parsing as a second grammar authority. - Query predicates that execute arbitrary effects during matching. -- Rowan ownership inside =gerbil-parser=. +- A second structural authority in downstream syntax views. * Executable feature contracts diff --git a/docs/architecture/0003-parse-artifact.org b/docs/architecture/0003-parse-artifact.org index e4620ef9..20a4e377 100644 --- a/docs/architecture/0003-parse-artifact.org +++ b/docs/architecture/0003-parse-artifact.org @@ -13,7 +13,7 @@ delivery schedule. stream owns source coverage, structural nesting, field scopes, and stable event identities. The generated parser may create private recognition values while a request is active, but those values are never published. Gerbil CST values and -downstream Rowan or custom trees are replay projections, not parser authority. +borrowed Rust views and custom trees consume those events. #+begin_src mermaid flowchart LR @@ -132,7 +132,7 @@ authority. - Hashing the complete Parser IR on every parse request. - Revalidating a trusted builder result with a second full hot-path replay. - Adding dormant recovery or incremental event variants. -- POO dispatch, Rowan ownership, or language-specific behavior in artifact code. +- POO dispatch or language-specific recognition in artifact code. * Executable admission @@ -145,7 +145,7 @@ Gerbil CST projection as one assertion. A failure reports the child path and the first mismatched property. #+begin_src scheme -(check-parser-ast (parse-arithmetic-v1 "1") "1" +(check-parser-ast (parse-arithmetic "1") "1" (node SourceFile 0 1 (field expression 0 1 (node NumberExpression 0 1 diff --git a/docs/architecture/0004-versioned-language-contracts.org b/docs/architecture/0004-versioned-language-contracts.org index ff88d623..b2d4931e 100644 --- a/docs/architecture/0004-versioned-language-contracts.org +++ b/docs/architecture/0004-versioned-language-contracts.org @@ -24,7 +24,7 @@ flowchart LR CA --> C C --> N[Canonical Grammar IR v1] C --> M[Hygienic native machine] - M --> E[deflanguage-parser] + M --> E[deflanguage-parser-loader] S[Source bytes] --> M M --> A[ParseArtifact v1] A --> R[Replay-only CST sinks] @@ -83,10 +83,21 @@ bind consumers to the internal =src/= layout. A supported language owns one directory at =languages///=. Its =grammar.ss=, =parser.ss=, =fixtures.ss=, =parser-test.ss=, and =corpus/= are colocated. The parser module -declares its public boundary with =deflanguage-parser= and consumes the sole +declares its public boundary with =deflanguage-parser-loader= and consumes the sole descriptor emitted by =grammar.ss=; it cannot introduce a second language identity, contract, machine, or language switch. +A source language with context-sensitive lexical obligations can declare a +=source-language= descriptor in =grammar.ss= instead of manufacturing an LR +grammar for a different syntax. Its declaration binds the language, version, +contract, scanner, and parser once. The same =deflanguage-parser-loader= public entry +accepts =(source descriptor)=; the engine checks that every returned +=ParseArtifact= is valid and carries the declared grammar and source digests. +The parser can retain immutable scanner checkpoints with byte offsets and +language context, including deferred tokens such as Bash here-documents. A +scanner-backed language has no implicit LR fallback or checkpoint-reuse claim; +it owns the source grammar in its versioned directory. + Native examples are real source files, not Scheme string fragments. A downstream test declares =defsyntax-fixture= with an immutable logical fixture identity, language version, contract identity, expected terminal state, root, @@ -132,31 +143,39 @@ The TLA+ contract takes the native =TLAPlusGrammar= from chapter 15 of /Specifying Systems/ as its language specification. The official SANY =tla+.jj= source is pinned at release =v1.7.4=, commit =5a47802b5c391f59ecdd44117981f4ff8c0656ba=, and Git blob -=bf9e7acb5337f4b6c2a4d6a973a1a65c95e72f56= for executable conformance. SANY +=bf9e7acb5337f4b6c2a4d6a973a1a65c95e72f56= for executable conformance; the +checked-in source digest is +=sha256:15edd079cf16cf91556ba66b496c9ffc16f26ab30856753ed58e60b0d54f2d07=. SANY is not a runtime dependency or a second parser engine: native TLA+ productions are lowered into the same Grammar IR and compiled by gerbil-parser. The TLA+ Foundation Examples corpus is pinned at commit -=ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10=. The pack version is =v1= and its -admitted identity is =tla-plus.native-core.v1=. The executable slice covers +=ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10=. Grammar metadata gives the +core slice version =v1= and contract =tla-plus.native-core.v1=. It covers module envelopes, EXTENDS, constant and variable declarations, single-line operator definitions, theorem declarations, precedence-bearing expressions, IF/THEN/ELSE, CHOOSE, bounded and unbounded quantification, tuples, sets, function constructors, strings, line comments, and nested block comments. It includes the byte-exact official =HourClock.tla= source plus focused structural and negative contracts. The 99-production SANY inventory is digest-bound and -materialized during expansion. The contract does not yet claim PlusCal, TLAPS -proof syntax, indentation-sensitive or multiline definitions, or the complete -official SANY syntax corpus. +materialized during expansion. The separate layout grammar has metadata +version =v2= and contract =tla-plus.native-layout.v2=. It admits aligned +conjunction and disjunction lists, nested column grouping, and the checked +POO Flow temporal-model corpus. Both grammars live in the unversioned +=languages/tla-plus/= namespace; their identities come from grammar metadata. +Neither contract claims PlusCal, TLAPS proof syntax, or the complete official +SANY syntax corpus. Model qualification is a language-pack boundary, not a parser-engine feature. -=languages/tla-plus/v1/qualification.ss= exposes one -=qualify-tla-plus-model= entry that first requires the native v1 ParseArtifact +=languages/tla-plus/parser.ss= exposes a layout entry +=qualify-tla-plus-model= and an explicit core entry +=qualify-tla-plus-core-model=. Each first requires its ParseArtifact to be valid, accepted and byte-exact, then invokes the official external TLC provider over that exact source and configuration. Its typed receipt binds both digests, the resolved tool identity, TLC version, exit status and complete state-graph summary. Missing summary fields, remaining states, nonzero exit, or syntax rejection fail closed. The language pack does not embed SANY or TLC, -and syntax rejection prevents the provider process from starting. +and syntax rejection prevents the provider process from starting. The receipt +syntax contract is read from the selected grammar metadata. Version identities are never inferred from a current branch or latest release. Changing an upstream edition, tag, commit, corrigendum, or admitted production @@ -196,11 +215,28 @@ ParseArtifact event stream is published. * Downstream author contract -A downstream DSL author normally declares one =deflanguage-grammar= form. An +A downstream DSL author normally declares one concise =deflanguage= form. An advanced author may instead compose POO grammar roles through the compiler-only API, but must call =compile-grammar= to cross into canonical Grammar IR before -=compile-parser=. POO modules are absent from the generated native runtime -closure. Every terminal has exactly one lexical rule, every reference +=compile-parser=. POO assembles and validates the loader when the language +module loads; the bound recognition method performs no POO lookup or admission. +Standalone generated Rust AOT products do not include the Scheme loader. + +Loader declarations normalize typed =grammar= / =source= input and POO prototype +extensions to one admitted form. Engine identity and parse slots remain sealed; +extension contracts check the effective object once when the module loads. + +A generated descriptor may bind an explicit Scheme parser policy with +=language-grammar-with-parser-policy=. The engine scopes its branch budget and +assembles lossless rejection artifacts from a language-owned successful-CST +checker. The public entry exports that configured descriptor for native product +admission. Scheme loader and contextual/native C ABI entries consume the same +policy, including the byte publication path. Its identity is separate from the +syntax digest. Standalone Rust descriptor generation rejects the unlowered +callback policy before emission; raw recognition IR is a narrower product. +See [[file:../../languages/README.org][the language extension guide]] for the authoring forms and product boundaries. + +Every terminal has exactly one lexical rule, every reference resolves, repetitions are non-nullable, the flow is connected, and the entrypoint yields exactly one unfielded root. The author does not modify =runtime/parser.ss=, =compiler/lr.ss=, or =compiler/machine.ss=. @@ -208,14 +244,18 @@ entrypoint yields exactly one unfielded root. The author does not modify The language's =parser.ss= then declares exactly one public entry: #+begin_src scheme -(deflanguage-parser hcl-v2-24-language - (grammar hcl-v2-24-language-grammar) - (parse parse-hcl-v2-24)) +(deflanguage-parser-loader hcl-language + (grammar hcl-language-grammar) + (parse parse-hcl)) #+end_src -The entry metadata does not own or copy the machine. Identity and machine are -read from the immutable descriptor generated by =grammar.ss=, and the public -parse function is bound to that sole machine. +The entry is a POO =LanguageLoader.= object admitted by +=LanguageLoaderContract=. Identity and machine are read from the descriptor +emitted by =grammar.ss=. The macro seals engine identity and dispatch slots, +checks optional extension contracts after prototype inheritance, and captures +the descriptor in the parse method once. Parsing does not repeat slot lookup, +POO admission, compilation or runtime initialization. The public authoring and +extension examples are in =languages/README.org=. The author's source-level scenarios use =defsyntax-fixture=, or =defsyntax-corpus= when a fixed upstream example set contains multiple files. diff --git a/docs/architecture/0005-v1-sidecar-evolution.org b/docs/architecture/0005-sidecar-evolution.org similarity index 88% rename from docs/architecture/0005-v1-sidecar-evolution.org rename to docs/architecture/0005-sidecar-evolution.org index 95da4a95..1e81aa3e 100644 --- a/docs/architecture/0005-v1-sidecar-evolution.org +++ b/docs/architecture/0005-sidecar-evolution.org @@ -139,3 +139,20 @@ Gerbil benchmark contract independently enforces elapsed-time and RSS policy. These receipts qualify development and compiler-side POO construction. They do not enter the generated parser machine, do not instrument the runtime hot path, and do not claim that a source-only observation executed runtime behavior. + +* Grammar boolean admission + +The exact builtin GrammarRole and Grammar descriptors specialize successful +=.element?= and =.validate= calls with pure slot predicates and the same native +prototype and composition obligations. This avoids constructing a classification +and validation object for every scalar responsibility just to return a boolean. +The descriptors are privately captured at owner initialization. Derived contracts +continue through Core's complete admission, including custom classifiers, +responsibilities and obligations. Explicit =.admit= always returns the complete +Core evidence. Rejected validation uses Core's typed error and diagnostics. + +Declaration lists are checked on every invocation; no mutable candidate is +cached as admitted. =t/grammar-contract-admission-test.ss= compares membership +with complete evidence for every responsibility, missing ancestry, descriptor +refinements, mutated lists and malformed composition. The existing 50-pair, +20-sample ASP construction gate retains its 100 ms ceiling and 50 ms target. diff --git a/docs/architecture/0006-rust-aot.org b/docs/architecture/0006-rust-aot.org new file mode 100644 index 00000000..cb1cb986 --- /dev/null +++ b/docs/architecture/0006-rust-aot.org @@ -0,0 +1,188 @@ +#+title: RFC 0006 — Scheme artifacts and independent Rust AOT +#+startup: overview + +* Authority and public interfaces + +Language authors define syntax in =grammar.ss=, parser identity and production +entry points in =parser.ss=, and corpus/test services in =parser-test.ss=. +Macro expansion and POO composition feed admitted canonical products. The engine +owns validation, compilation, publication and execution; language-specific +recognition data remain in the language package or explicit common adapter. + +The primary Rust path is: + +#+begin_example +Rust caller -> generic C ABI -> Scheme parser engine -> canonical GPA1 result + <- C-owned result bytes + borrowed Rust node/field/token access +#+end_example + +The independent AOT path is: + +#+begin_example +Scheme descriptor -> admitted compiler/build product -> generated Rust tables + -> generic Rust executor -> validated source-range index +#+end_example + +These products have distinct dependencies and receipts. Native access never +imports generated LR tables, invokes the independent executor, or constructs +its owned syntax index. There is no conversion facade between the products. + +* Ownership and dependency direction + +| Package | Responsibility | Runtime dependency | +|---------+----------------+--------------------| +| gerbil-parser-artifact | Inert descriptor reader, GPA1 admission, borrowed navigation; separate validated flat index for AOT | SHA-256 and descriptor data representation | +| gerbil-parser-ffi | Generic FFI, handle/result lifetimes, owner-thread sessions, optional standalone VM ownership | Artifact package | +| gerbil-parser-runtime | Independent generated-table/event execution, diagnostics and source indexes | Artifact package and SHA-256 | +| gerbil-parser-runtime-external | Optional external event producer and provider receipt | Generic runtime | +| gerbil-parser-build-support | Source policy, integration tests and benchmark scenario data | Development dependencies only | +| Language crates/fixtures | Generated language data and independent semantic expectations | Selected public backend | + +Development scenarios and arithmetic integration tests live in the existing +=gerbil-parser-build-support= package. A separate Cargo package with a comment-only +library is unnecessary: the test runner, policy and fixture data have one owner. +The arithmetic reference tables are a generated test fixture in that same package; +their integration and performance tests do not require a separate language crate. + +Public APIs retain current names only. Cargo manifests, independent fixture +lockfiles, Scheme generators, C headers, CI commands and documentation must +use the same names. No package redirect or compatibility implementation exists. + +* Descriptor and admission contract + +=NativeLanguage::catalog()= reads the immutable descriptor from its exact C ABI +handle once. The existing inert Scheme datum representation is retained; the +reader does not evaluate expressions. Duplicate keys, invalid escapes, trailing +forms, excessive depth and malformed catalog rows are rejected. + +The descriptor publishes =language=, =grammarDigest=, =rootKind=, =syntaxKinds=, +=terminals= and =fields=. Source providers publish their declared root and result +catalog; LR descriptors publish the root of their canonical IR. A contextual +handle publishes its contextual product digest. Native consumers obtain these +values from the handle, rather than supplying a separate generated specification. + +GPA1 admission checks magic/version/status, exact payload length, product and +source digests, valid categories and symbol indexes, sequential node/token ids, +reserved positions, balanced node/field scopes, UTF-8 boundaries and complete +ordered token coverage. Fields must belong to their nearest declaring node kind. +Canonical and contextual product identities cannot be substituted. + +Rejection can contain an engine-owned opaque =unknown= token for unscanned +source. Native publication includes this terminal when the language does not +already declare it, in both the descriptor and the encoder index. Existing +terminal indexes and grammar recognition remain unchanged. Source-engine roots +are admitted against their result catalog and derived from the compiled command +projection. Aliased LR entry rules determine their root kind independently of +catalog ordering, and the compiler requires that kind to be a declared node. + +* Borrowed AST and token contract + +=NativeLanguage::parse -> NativeParsed::view -> root/children/fields/tokens= +uses the original source and C result buffers. A flat scope/parent index supports +navigation; it carries no duplicated token text. Fields remain associations, +including repeated scopes and zero-width node values. An absent association is +not fabricated as an empty field. + +=NativeToken::class()= preserves the scanner terminal class; =kind()= returns +its declared syntax kind, or the class when no syntax mapping exists. =id()= +is the scanner ordinal, independent of node ids. =tokens()= preserves trivia +and source order for accepted and rejected inputs. Rejected syntax has no accepted +root and no invented node parent. It is successful transport with an explicit +syntax-rejection status, not a transport failure or recovery result. + +The C/Scheme boundary copies input and allocates the output payload. Rust views +borrow those buffers and the original Rust source; this is not a claim of zero +allocation across the ABI. The caller releases views before releasing the result. + +Strict input admission uses the pinned upstream Scheme caller-buffer UTF-8 encoder. +Its C scratch allocation is released on success and exceptions; it is distinct +from the source and result buffers. See [[file:../research/upstream-ffi-utf8-admission.org][the complete request CPU qualification]]. + +* Runtime and lifetime contract + +An application with an initialized VM uses =NativeSession::attach=. A standalone +Cargo application enables =standalone= and uses =NativeRuntime::start=. C hosts +use =runtime.h= and the generated factory. Initialization, handles and parsing +remain on the initializing OS thread. A second VM or VM restart in one process +is unsupported. + +Rust borrows prevent dropping the language or closing its runtime while results +or views survive. Native results release their C allocation once; language +handles unregister once. Standalone shutdown performs normal Gambit cleanup. +Independent C resources must be released before shutdown; a live-resource +rejection retains the runtime. Libraries remain loaded until all owners end. + +* Independent AOT and publication + +=rust-runtime-support= exposes ordinary and contextual language generators, +shared scanner generation and the bounded Command/Source generator. Descriptor +products delegate to =language-rust-runtime-module-source= for identity, portable +policy admission and execution emission. A Scheme-only CST policy is rejected +before AOT product publication; native parsing executes it in Scheme. + +The build ABI lives in =src/ffi/rust-runtime-aot.ss= and +=include/gerbil-parser/rust-runtime-aot.h=. The standalone +=gerbil-parser-runtime-aot= command takes a downstream grammar module path; +=GERBIL_PARSER_RUNTIME_AOT_LIB= identifies its installed metadata bundle. +Generated Rust source imports =gerbil_parser_runtime=. The normal Gerbil build +owner controls compilation and dependency closure. + +The independent executor validates event ancestry, source ranges, complete +coverage and subtree intervals before exposing =SyntaxTree=. Iterative traversal +and flat debug output avoid recursive tree walks. Recognition, external provider +identity and language policy remain separate obligations; an index or table hash +alone does not establish semantic conformance. + +* Qualification + +| Gate | Local evidence | +|------+--------------------------------------------------| +| Runtime | 130 core tests, including canonical/contextual native token controls | +| Artifact admission | Three inert-reader tests and twenty controls from all ten production entries | +| Language interfaces | 28 bounded DSL/conformance child processes, including the independent records consumer | +| Native transport | Full FFI owner suite, final four-module suite and explicit OK | +| Host ownership | Fresh C, Rust-owned VM and borrowed-session processes, 100 calls, two handle releases, 113 result releases and normal cleanup | +| AOT consumers | Locked records fixture (two tests) and ASCII fixture (one test) | +| Quality/publication | Strict workspace Clippy, source-policy gate and replacement crate package verification | + +The 113 result releases include two lazy catalogs, one explicit descriptor and +110 parse results. Compile-fail doctests retain thread and lifetime restrictions. +All twelve workspace test executables passed individually under the existing +ten-second completion limit. These are local receipts; CI must qualify each +pushed commit separately. + +Earlier failures remain bounded evidence: an aggregate ten-second Cargo run did +not finish; mixed temporary optimizer interfaces and native object versions +caused a factory crash. Consistent specialized producer/consumer builds and +removal of older owned temporary object versions resolved the crash. The loader +selects the highest contiguous object version. No timeout was widened. + +* Remaining architecture obligations + +Language-wide ABI conformance now generates accepted/rejected controls from all +ten production entries in the eight language packages. The generic artifact crate +admits those bytes without importing language tables. The controls are binary test\ndata, not generated Rust parser source. CI regenerates and compares +the controls through the native conformance executable. Backend capability admission must remain explicit; supporting native +source parsing does not qualify every source strategy for independent AOT. + +Aliased entry roots derive from their entry rules rather than syntax catalog +order. Both parser and declaration generator contracts are version 3 so changed +compiler semantics cannot reuse older cache receipts. Source providers must +publish a root in their admitted result catalog. Opaque rejection tokens are +engine-owned publication vocabulary; language grammars need no fallback rule. + +A unified Contextual Parser IR, selector specificity/overlap witnesses, macro +lineage, general recovery and incremental subtree certificates remain separate +open contracts. Benchmark execution, ABI transport, digest admission, navigation +index construction and field traversal separately. Representation changes alone +establish no end-to-end latency or throughput improvement. + +* Build-host module snapshots + +File-based generation inputs are admitted by complete content identity. +Unchanged source reuses the official Gerbil module context; changed source +triggers official reload before descriptor selection. Library inputs retain +SDK snapshot semantics and accept an explicit =reload?: #t= request through +=language-module-descriptor= or =rust-runtime-source=. Imported helper changes +require refreshing those dependencies or the build host; root reload is not +recursive invalidation. See [[file:../research/module-loading-reuse.org][module loading controls and CPU evidence]]. diff --git a/docs/architecture/0006-rust-rowan-aot.org b/docs/architecture/0006-rust-rowan-aot.org deleted file mode 100644 index ff5dbefb..00000000 --- a/docs/architecture/0006-rust-rowan-aot.org +++ /dev/null @@ -1,196 +0,0 @@ -#+title: RFC 0006 — Gerbil-owned Rust/Rowan AOT Product -#+startup: overview - -* Status - -The generic deterministic LR and bounded selective-GLR engine, arithmetic -reference product, and an independent downstream custom-language fixture are -executable. GQL and Cypher remain Scheme-owned language descriptions consumed -through the generic engine boundary. - -The lexical algebra also admits =(line)= as a generic, allocation-free -whole-line primitive. Scheme and Rust both consume LF, CRLF, bare CR, or a -final unterminated line losslessly. A bounded POO =LineStructure= declaration -now projects heading, block, list, table, and text-line rules to an immutable Rust -table. Each block declares either opaque contents or recursively parsed -elements. The generic engine executes section nesting, block recovery, and -nested element/opaque block contents over that -table; all markers and syntax-kind identities remain language-owned. This is -not yet a complete contextual Parser IR or an Org parser cutover. - -* Decision - -=grammar.ss= remains the single language-authoring authority. Its hygienic -declaration expands into canonical Grammar IR v1 and Parser IR v1 exactly as it -does for the Scheme runtime. A compiler-owned Rust backend serializes the -already resolved syntax-kind catalog, lexical rules, productions, LR actions, -gotos, field actions, aliases, identity, and grammar digest into an immutable -Rust module. Rust never resolves grammar conflicts or reparses a grammar -declaration. - -The generic =gerbil-parser-rowan= crate executes those static tables and -publishes a lossless Rowan green tree plus a receipt binding language, version, -contract, grammar digest, and source digest. The parser does not publish a -second vertical Rust package for GQL: consumers load the Scheme-owned language -description through this generic boundary. A parse request runs no fallback -Rust grammar, JSON grammar DSL, or S-expression interpreter. - -For grammars whose structure cannot be reduced to a finite flat terminal -stream, the same crate also exposes a generic structural-event Rowan sink. -An AOT-generated downstream parser may emit node-start, source-backed token, -and node-finish events. The sink validates generated kind categories, balanced -nesting, and exact ordered UTF-8 coverage before returning one green tree. It -contains no language-specific recognition or recovery policy. The event sink -constructs nodes bottom-up without Rowan's node-cache hash recursion, so deep -but valid hierarchies do not incur quadratic hashing in the generic layer. - -The sink is not itself an AOT compiler for procedural =parser.ss= code. A -downstream script that prints a handwritten Rust scanner or event algorithm is -not an implementation of the single-authority decision above, even when it -embeds a Scheme-source digest and passes differential fixtures. The remaining -compiler boundary is a fuller contextual Parser IR derived from Scheme-owned -language declarations. The current =LineStructure= table validates kind -references against the canonical grammar digest and covers headings, tables, -marker-led nested lists, keyed lines, paragraphs, inline links, opaque blocks, -and recursively parsed block contents. The keyed-line rule binds a bounded key -set or prefixed dynamic key, case/indent policy, heading adjacency, and -single/repeated values to the parser digest. The list rule binds marker sets, -ordered-item admission, -visual tab width, and typed bullet/trivia kinds to the parser digest; checkbox, -counter, description-tag, and full continuation policy are not admitted yet. -Other line/byte recognition, lexical state -transitions, structural actions, and typed recovery still need explicit -admission. The Rust runtime executes the admitted table and hands source-backed -events to the generic Rowan sink. - -The table-backed =parse_structural_lines= route is an interim compatibility -slice, not the target parser engine for context-sensitive languages. Its Rust -implementation currently chooses the order of block, list, heading, keyed-line, -table, and paragraph decisions. Consequently, a downstream =parser.ss= can -configure those recognizers but cannot define or test their control flow as a -Scheme algorithm. Adding more Rust recognizers would deepen the wrong ownership -boundary. The public =:gerbil-parser/rust-rowan-event-support= facade now admits -one first, statically checked Scheme event-fold subset: boolean state, line-prefix -conditions, node/token events, branching and final transitions. Its declaration -runs in Scheme tests and lowers to a versioned, closed IR; =gerbil-scheme-rust-ir= -constructs Rust syntax with =quote=/=syn= rather than source templates. The -generated Rust function builds a lossless Rowan tree in a UTF-8, CRLF and -multiline-state fixture. This proves the compiler path, not a full contextual -parser: dynamic offsets, nested state, arbitrary Org recognition and recovery -still require admitted IR operations and downstream parity. The full compiler -gate requires the same Scheme functions to match their Rust event streams on -UTF-8, malformed input, recursion, and recovery fixtures. Only event validation, -tree construction, and typed failure transport remain handwritten in the -generic Rust/Rowan runtime. A digest over a static rule table, by itself, does -not satisfy this gate. -The AOT test contract checks the typed Rust syntax tree through a parser-specific -macro; generated source text is not used as an assertion language. -Org-like languages may use a scan phase internally, but there is no separately -authored =generate-scanner= program, second Rust grammar, or downstream event -algorithm. Arbitrary Scheme procedures are not silently translated to Rust; -unsupported procedural behavior fails the AOT contract until it is expressed -in the admitted Parser IR. A downstream Cargo consumer never invokes Scheme. - -Cargo-only custom generation crosses a narrow native ABI: build support passes -the downstream =grammar.ss= path to =gerbil_parser_rowan_compile=. The -precompiled Scheme product loads that module through Gerbil's own expander, -requires exactly one exported language descriptor, and returns the existing -=compiler/rust-rowan.ss= output as owned bytes. Rust writes those bytes to -=OUT_DIR=; it never reparses Scheme, implements Grammar IR, or constructs LR -tables. The normal parse-time dependency remains =gerbil-parser-rowan= and -does not link the generator. - -The native generator product must be distributed for every admitted target. -Its =defbuild-script= owner projects a public ASP Building API package spec -into std/make. The runtime modules, executable, and native TLS options remain -declarative build data, and release support does not reconstruct that graph -from mutable cache objects. -Until a clean Cargo consumer can use that product on Linux and macOS without a -Gerbil installation, the native ABI is an executable boundary but the -Cargo-only authoring claim remains open. - -The dedicated =gerbil-parser-rowan-aot= executable qualifies the same owner as -a standalone product. It calls =gerbil-load-expander!= before loading the -downstream module and can replace the runtime load path with an explicit -metadata bundle through =GERBIL_PARSER_ROWAN_AOT_LIB=. Its public authoring -surface is the minimal =:gerbil-parser/rust-rowan-grammar-support= facade, so a -direct custom grammar does not pull the source-import and fixture facades into -its build product. CI consumes the closed executable under =env -i= with no -Gerbil command on =PATH= and explicit expansion-metadata roots. A source module -still requires =.ssi= interfaces; packaging those immutable interfaces is a -release concern, not a reason to reconstruct the runtime object graph. - -#+begin_src text -grammar.ss - -> precompiled Gerbil expander FFI - -> Grammar IR v1 - -> Parser IR v1 and grammar digest - -> compiler/rust-rowan.ss - -> generated immutable Rust tables - -> gerbil-parser-rowan runtime - -> lossless Rowan CST plus receipt - -> downstream semantic lowering -#+end_src - -Downstream projects may own CST-to-domain-IR lowering, but they do not own -another GQL parser package, syntax-kind catalog, lexer, precedence map, -recovery table, or parser runtime. - -The downstream qualification fixture owns the complete versioned language-pack -shape: =grammar.ss=, =parser.ss=, =fixtures.ss=, =parser-test.ss=, native -=corpus/=, and a committed generated Rust module. It deliberately has no -=gerbil.pkg= and requires no Gerbil installation in its Rust consumer path. -Upstream generation invokes only the public Gerbil facade; the resulting Rust -module imports =gerbil_parser_rowan= as a normal dependency and compiles outside -the root Cargo workspace. This is the acceptance boundary for custom DSL -authors; an in-workspace generated module alone is not sufficient evidence. - -* Stable v1 boundary - -This work adds a backend for the existing v1 declarations. It does not add a -second grammar schema or a v2 compatibility route. Generated numeric Rowan -kinds follow Grammar IR declaration order and remain valid only with the exact -grammar digest carried by the product. Catalog overflow, unknown kinds, -malformed tables, invalid UTF-8 boundaries, uncovered source, and missing LR -actions fail closed. - -Rowan is a lossless syntax-tree representation, not the grammar authority and -not a parser generator. The Rust runtime therefore consumes only tables whose -conflicts and precedence were already resolved by Gerbil's compiler. - -* First vertical slice - -Arithmetic v1 is the executable reference grammar. The generic runtime admits -the complete native LexicalExpr v1 algebra: whitespace, newline, decimal and -profiled numbers, identifiers, quoted strings, heredocs, line and block -comments, ordered choice, literals, precedence projection, and fallback. It -executes deterministic LALR actions, applies generated field and alias -reductions, preserves trivia, constructs one balanced root, and verifies that -the Rowan tree reproduces the exact source. Selective-GLR forks enumerate all -admitted actions within a bounded speculative-depth budget. Complete branches -are ranked by dynamic precedence and maximal consumption; equivalent CSTs -merge, while distinct equal-score CSTs fail with a typed ambiguity diagnostic. -The parse publishes a request-local =SelectiveGlrReceipt= for those decisions. - -The ISO/IEC 39075:2024 GQL Scheme language pack must losslessly accept the -pinned OpenGQL 1.9.0 corpus through the generic consumer boundary. That corpus -exercises real fork exploration and equivalent-completion merging; it is an -admission floor, not yet the complete downstream cutover proof. - -The initial slice deliberately rejects unsupported generated features instead -of silently interpreting them differently. Before publishing GQL or Cypher, -the Rust backend must add and qualify: - -- pinned external scanners; -- case-insensitive keyword behavior over the complete corpora; -- recovery behavior and diagnostic parity; -- generated typed node and field accessors; -- exact Scheme/Rust acceptance, rejection, CST-shape, roundtrip, and receipt - parity for the pinned language corpora. - -* Downstream cutover gate - -=gql-rust= may remove its native Gerbil runtime and local Rowan sink only after -the generated GQL product passes the full pinned corpus and malformed-input -differential gates. Its private CST-to-=MetaQueryIR= lowering remains a -downstream semantic owner and is not moved into =gerbil-parser=. diff --git a/docs/architecture/0007-contextual-parser-ir.org b/docs/architecture/0007-contextual-parser-ir.org new file mode 100644 index 00000000..c699ebab --- /dev/null +++ b/docs/architecture/0007-contextual-parser-ir.org @@ -0,0 +1,1297 @@ +#+title: RFC 0007 — Contextual Parser IR and Compiler Dispatch +#+startup: overview + +* Status + +Proposed. This RFC specifies the missing compiler boundary for languages whose +tokens and productions depend on context. It does not admit the experimental +Bash source parser as a public language product. Acceptance requires the +executable gates below; a Scheme-only callback parser does not satisfy them. + +* Problem and authority + +Grammar IR v1 owns terminal identities, productions, precedence and lexical +rules. Parser IR v1 currently compiles a fixed lexical rule set and LR machine. +The bounded =LineStructure= product handles some contextual structure, but its +runtime chooses a fixed recognizer order. Bash exposes the gap: quoting changes +word scanning, reserved words depend on command position, and a here-document +delimiter commits a later line-scanning obligation. Encoding those facts in a +handwritten =parser.ss= creates a second grammar authority and makes each new +language pay for another parser. + +The language declaration remains the sole syntax authority. POO grammar roles +compose declarations before canonicalization. The compiler validates and +resolves them into immutable Contextual Parser IR; Scheme and Rust backends +consume that same IR. Runtime POO objects, MOP lookups, arbitrary callbacks, +grammar normalization, and conflict resolution are excluded from parsing. + +* Independent axes + +The compiler treats these as distinct typed axes, rather than adding one +language-specific branch per combination: + +| Axis | Declares | Example Bash values | Runtime representation | +|------+----------+---------------------+------------------------| +| Lexical context | Which tokens can be scanned | command, word, arithmetic, here-body | finite mode id | +| Syntactic position | Which meaning a token can have | command-start, assignment-prefix, argument, redirect-target | parser state and expected terminals | +| Delimiter obligation | A future source boundary | quote, substitution, here-document marker | bounded typed stack or queue | +| Reduction action | Node, field, and context transition | redirection queues here-body | compiled action id | +| Recovery site | Error boundary and synchronization | unterminated quote, missing =fi= | compiled policy id | + +POO receiver dispatch is appropriate for one extensible declaration family. +Compiler-side multiple dispatch is appropriate when two axes grow +independently: for example, lexical context × token form, or syntactic position +× token role. A method contributes a *declaration*, not a parsing procedure. +Role ancestry must use one validated C4 order. The existing grammar-role +normalizer's recursive parent walk does not by itself establish this guarantee +for a diamond, so contextual role migration includes a distinct linearization +check. Method applicability is the product of the three explicit axes; a +method is more specific only when every selector is at least as specific and +one is strictly more specific. Methods are resolved during compilation, and +the winning resolution and its origins are recorded. If two applicable methods +have incomparable specificity and disagree, compilation fails with both +owners; C4 or declaration order never silently chooses the result. An +equal-result tie retains both origins in the receipt. Methods may not mutate +another role's declaration. The resulting table contains only closed opcodes +and scalar or bounded data. + +* Canonical Contextual Parser IR + +The new schema is a versioned extension of Parser IR, not a replacement grammar +format. Each row has an identity and a source role. Canonical ordering is by +validated role linearization, then local declaration order; numeric ids are +assigned only after validation. The grammar identity hashes the normalized +Grammar IR, contextual rows, scanner implementation versions, and dispatch +resolution receipt. Generated Scheme and Rust products embed that digest. + +#+begin_src scheme +(contextual-parser-ir + (schema "gerbil-parser.contextual-parser-ir.v1") + (grammar-digest SHA256) + (modes (command ...) (word ...) (here-body ...)) + (state (mode-id obligation-stack pending-here-queue)) + (scan-rules (rule-id mode-id terminal-id match-op priority transition-id) ...) + (reductions (rule-id node-kind field-actions context-action) ...) + (dispatch (parser-state mode-id expected-terminal-set rule-id) ...) + (recoveries (site-id policy-id diagnostic-id) ...)) +#+end_src + +A scan rule has an explicit match opcode from a closed, typed scanner algebra: +literal, class run, bounded delimiter, balanced delimiter, and line-until-marker. +Its inputs and output are byte offsets, finite mode, and bounded serializable +obligations. No rule receives a Scheme procedure or a pointer into a mutable +parser object. Maximal munch, declared lexical precedence, and stable rule id +resolve ordinary matches. Remaining overlaps are compile-time errors. A parser +state's expected terminals may narrow the admitted rules, but cannot silently +change a token's bytes. The trace records that contextual choice. + +A reduction action is a closed sequence of node/field events and state +operations (push/pop delimiter, enqueue/dequeue here marker, change mode). +Actions are validated for kind references, stack effect, byte coverage and +boundedness. No branch publishes events until its parse branch commits. + +* Checkpoints and recovery + +A checkpoint contains source byte offset, LR stack or branch set, lexical mode, +delimiter stack, pending here queue, event cursor, and the exact grammar digest. +The stack and queue are bounded by the source length plus an explicit resource +budget; exceeding the budget is a typed rejection, never silent truncation. +All fields have a canonical serialization; no closures or process-local object +identity may appear. Reuse requires both digest equality and an equivalent +state at the edit boundary. A source edit that can affect a pending delimiter +or here marker invalidates scanning through the matched body boundary. + +Recovery is declared per site and compiled to typed operations. Missing token, +unterminated delimiter, and unexpected token are separate diagnostics with +source spans. A failed selective branch rolls back scanner state and events; +the accepted branch alone publishes a ParseArtifact. On rejection, the source +remains lossless and the diagnostic identifies the first certified failure. + +* Authoring and compilation path + +#+begin_src text +grammar.ss: GrammarRole + contextual roles + -> POO/MOP composition and multi-axis method resolution + -> canonical Grammar IR + contextual rows + resolution receipt + -> validated Contextual Parser IR and grammar digest + -> generated Scheme machine / generated Rust tables + -> one generic runtime executor and ParseArtifact +#+end_src + +The compiler first rejects unknown kinds, nonserializable state, unbounded +transitions, conflicting methods, ambiguous scan rules, and actions with an +invalid stack effect. It then resolves dispatch and compiles tables. Both +backends receive the same numeric ids and state transitions; neither backend +derives grammar decisions from source declarations. Generated source is an +artifact of the compiler, never an author-maintained parser. + +The existing =LineStructure= recognizers should become declarations and rules +in this IR. Its current order is preserved as explicit precedence during +migration, then verified against the old product before the compatibility +route is removed. The existing event-fold subset supplies a model for +statically checked Scheme-to-Rust action lowering; it is not itself a second +syntax grammar. + +* Qualification gates + +1. Two language packs use the same compiler path for one contextual mechanism + each: Bash quoted/parameter words and deferred here bodies; HCL quoted + templates or heredocs. Neither runtime contains language-name branches. +2. The same declarations produce Scheme and Rust event streams that agree on + acceptance, rejection, token byte spans, node/field structure, diagnostics, + source roundtrip, and grammar digest over UTF-8, CRLF, nested delimiters, + incomplete input, and recovery fixtures. +3. A test creates two competing, incomparable methods and proves compilation + rejects them with both origins. Another changes a scanner version and proves + the product digest and checkpoint admission change. +4. A clean downstream build consumes the generated Rust product without + Gerbil at parse time. The =parser.ss= of each language pack contains only its + public =deflanguage-parser-loader= entry. + +Until these gates pass, the Bash prototype is an experimental behavioral +oracle. Passing its Scheme tests alone does not qualify a versioned language +product or a reusable engine. + +* Current executable slice + +=src/compiler/contextual-dispatch.ss= admits POO contextual roles and resolves +finite lexical mode × syntactic position × token form methods before runtime. +It rejects incomparable methods with different results, records all origins in +a same-result tie, and hashes the closed resolution table. The focused suite +exercises Bash and HCL *declaration examples* through the same compiler. + +=src/compiler/contextual-scanner-ir.ss= validates closed scan matchers and +state actions; =src/runtime/contextual-scanner.ss= executes the same IR for +both examples. The scanner digest binds the semantic opcode contract and the +runtime rejects a different contract before scanning. Its immutable state owns source byte offsets, lexical mode, +pending and active delimiters, and a canonical checkpoint whose scanner and +source digests are checked on restoration. =src/runtime/scan.ss= owns generic +balanced word scanning. Language packs may declare delimiters and actions but +must not ship a scanner procedure. Bash nested words, multiple deferred here +bodies and HCL template/raw delimiter fixtures pass through this one runtime. + +The internal =parse-source/contextual= path drives the same scanner from an LR +machine. A focused HCL fixture reaches a real ParseArtifact through an existing +ParserMachine; the scanner IR is bound to its grammar digest, and the compiler +maps each LR state to a declared syntactic position using +required and forbidden =(token name)=, =(literal text)=, and =(eof)= +lookaheads, with specificity and ambiguity checks. Runtime preparation checks +state uniqueness in one pass and installs a vector for state-position lookup. The LR +executor exposes the state to this path without changing the existing lexer +callback. Scanner compilation rejects duplicate exact match candidates and a +dispatch table whose digest no longer matches its contents. + +The prepared scanner indexes rules by mode and token results by the closed +mode/position/form key. Equal-ranked matches must agree on both token form and +state action. The source-only scale comparison over 512 cells and 255 tokens +preserved tokens and canonical checkpoints; five alternating pairs reduced +median process CPU from 16.402 ms to 2.271 ms. This is a scanner execution +receipt, not native compilation or a versioned language qualification. + +=just benchmark-contextual-scanner= reproduces the execution workload with +independent CPU and wall samples. =just test-performance-before-compile= runs +the eight existing performance contracts from current source through a private +load-path overlay. Each suite has a 120-second deadline, streamed progress, +complete logs, and required case/module/harness completion markers. A failed +or timed-out suite fails the preflight; the command does not start GXC. + +These remain compiler fixtures, not migrated language packs. The declarations +are currently constructed in tests; the compiler derives the LR state-position +table from those declarations. Expansion-time +binding to =deflanguage-grammar=, contextual GLR branch state, incremental +checkpoints, generated Scheme/Rust machines, and the parity gates above remain +unimplemented. The internal LR path rejects a selective-GLR fork instead of +publishing a parser result with an incorrect scanner state. + +* Source performance preflight before the lexical/reduction optimization, 2026-10-02 + +These are individual current-source runs with unchanged admission budgets, +not a successful invocation of all eight gates and not native qualification. +The source overlay deliberately excludes this checkout's compiled parser +objects. A deadline includes source loading and the test harness; p95 below +measures the contract's action after loading and warmup. + +| Contract | Observed p95 | Original ceiling | Result | +|----------+--------------+------------------+--------| +| LALR fixed point | 22.623 ms | 500 ms | pass | +| Incremental suffix | 105.379 ms | 250 ms | pass | +| Recovery frontier | 78.331 ms | 250 ms | pass | +| Selective GLR | 381.145 ms | 250 ms | fail | +| Parser hot path | no complete receipt | 900 ms | 120 s deadline | +| ISO GQL runtime | 1027.924 ms | 150 ms | fail | +| openCypher 100 clauses | 404.126 ms | 250 ms | fail | +| Versioned language corpus | 235.443 ms | 500 ms | pass | + +The new scanner's independent scale check passes byte coverage, token counts, +and source roundtrip over 512 dispatch cells and 255 tokens. Five execution +samples report process CPU of 1.816–2.094 ms. This check does not replace any +existing parser contract. No GXC step is admitted by these partial results. + +The host was concurrently busy (observed load averages 72.69, 77.94, 90.01). +The wall-time failures remain failures; source execution overhead and host +scheduling must be isolated before assigning an algorithmic cause. CPU +diagnostics are evidence for that investigation, not alternative admission. +An independent three-sample GQL diagnostic (100 accepted queries per sample) +reports process CPU of 296.880, 281.457, and 287.631 ms, with wall times of +752.520, 536.935, and 592.869 ms. CPU alone exceeds the 150 ms ceiling in this +source execution path, so host scheduling does not fully explain that failure. + +* Matched optimization receipt, 2026-10-02 + +The generated lexical first-character filter now admits numeric profiles by +their declared prefix starts, ASCII digits and optional leading period; +quoted rules by their declared delimiter starts; and heredocs by =<=. It +still tests every admitted candidate, preserving maximal munch and lexical +precedence. Unicode continues through the complete candidate set. A generated +witness agrees with the unfiltered scanner at every offset over ordinary and +custom radix prefixes, escaped/doubled quotes, incomplete strings, Unicode, +and here bodies. + +A same-process comparison constructs the baseline lexer from immutable +=82cebf6= source with the exact current GQL Parser IR. All five alternating +pairs preserve complete token streams and improve CPU time; median CPU for +100 global lexical scans falls from 44.820 ms to 21.997 ms. This is global +lexing, not a claim that complete directed GQL parsing became twice as fast. + +The LR runtime handles zero/one-operand stack pops directly and retains its +prepared width and lexical-mode vectors for the execution loop. Five +alternating prepared-GQL comparisons preserve complete recognition values; +four improve CPU time, with mean CPU 130.279 ms to 118.475 ms and median +134.556 ms to 124.214 ms per 100 parses. Full selective-GLR receipts match the +baseline for equivalent branches, dynamic precedence, fragment interning and +unresolved ambiguity. The functional source run passes 33 cases across parser +runtime, lexical admission and contextual parsing/scanning. + +Lookahead-fold caching, an alternative general reduction loop, and semantic +stack reuse were removed after their matched CPU results were inconsistent. +The existing admission ceilings remain unchanged. =just benchmark-gql= +reports independently checked prepared and full-source CPU/wall samples; +it is a diagnostic, not an admission gate. The preflight records a source +digest and rejects a run if its input files change while the suites execute. + +* Complete source preflight after optimization, 2026-10-02 + +All eight suites ran sequentially with unchanged admission ceilings and a +120-second deadline per suite. Their source digest remained +=sha256:cdfbcaa670726b8c67516cea098f7a1be72f8e4c26bb27fc2bc8ef94d2fbc3d3=. +The preflight returned failure; the earlier individual passing wall results +do not qualify this later snapshot. + +| Contract | Observed p95 | Ceiling | Result | +|----------+--------------+---------+--------| +| LALR fixed point | 19.044 ms | 500 ms | pass | +| Incremental suffix | 342.594 ms | 250 ms | fail | +| Recovery frontier | 463.587 ms | 250 ms | fail | +| Selective GLR | 764.256 ms | 250 ms | fail | +| Parser hot path | 3434.241 ms | 900 ms | fail | +| ISO GQL runtime | no complete receipt | 150 ms | 120 s deadline | +| openCypher 100 clauses | no complete receipt | 250 ms | 120 s deadline | +| Versioned language corpus | 1083.055 ms | 500 ms | fail | + +Host load averages +were observed at 150.27/139.36/122.95 and 134.82/133.28/125.22 during this run. +Scheduling is a material confounder, but it does not explain away the failed +admission or the baseline source GQL CPU result above. + +The repository's existing CI executes =gerbil build= before its native +Gerbil test suite (=.github/workflows/ci.yml=, lines 150 and 177–179). Running +the same numerical ceilings through a source overlay therefore adds an +interpreted execution gate beyond that workflow. The user subsequently approved +source equivalence, complexity, and matched CPU checks as the pre-compilation +gate, followed by bounded GXC and the original native wall-time ceilings. No +numerical ceiling has been changed. + +* Native post-build qualification, 2026-10-02 + +This section records the pre-rebase source snapshot. Its native artifacts and +performance receipts do not qualify the later PR #13 revision described below. + +The parser cache was rebuilt in an isolated output directory, without +reusing the checkout's earlier parser binaries. Existing external dependency +artifacts were linked into this cache. Module resolution was checked through +both =gxi= and =gerbil env gxi=; the scanner, LR runtime, and machine resolved +to this cache's =.ssi= files. + +Standard =std/make= imports its target catalog before beginning compilation. +Cold GQL/openCypher expansion in this phase triggered the initial idle +deadlines. The core was therefore bootstrapped in dependency order, postponing +the language-bound FFI module. The complete original 122-target package spec +was then built with the standard optimized native settings. A temporary copy +of the standard scheduler added dependency begin/end logging; a subprocess +observer reported actual generated-file changes. The core completed with +=NATIVE-CORE-BUILD-OK= in 572.78 seconds. After a 30-second idle interruption +in the C queue, incomplete module interfaces were invalidated and the full +spec resumed with a 600-second total deadline and a 60-second idle deadline. +It completed with =NATIVE-BUILD-OK= and exit 0 in 404.39 seconds. These are local +library-build receipts, not optional-command or remote-CI qualification. + +All eight original native performance contracts then ran against source digest +=sha256:cdfbcaa670726b8c67516cea098f7a1be72f8e4c26bb27fc2bc8ef94d2fbc3d3=. + +| Contract | Native p95 | Ceiling | Result | +|----------+------------+---------+--------| +| LALR fixed point | 7.033 ms | 500 ms | pass | +| Incremental suffix | 6.390 ms | 250 ms | pass | +| Recovery frontier | 40.407 ms | 250 ms | pass | +| Selective GLR | 12.543 ms | 250 ms | pass | +| Parser hot path | 206.422 ms | 900 ms | pass | +| ISO GQL runtime | 277.461 ms | 150 ms | fail | +| openCypher 100 clauses | 50.288 ms | 250 ms | pass | +| Versioned language corpus | 16.761 ms | 500 ms | pass | + +The GQL minimum, +median, and maximum were 10.985, 40.399, and 424.968 ms. The complete native +qualification failed; seven passing contracts do not close the eighth. + +The same 33 focused functional cases passed natively, with five =MODULE-OK= +markers, =HARNESS-OK=, and final =OK= in 35.17 seconds. + +Native stage diagnostics retained complete artifact equality. For 100 full +parses, one sample measured 13.395 ms CPU versus 355.403 ms wall; another was +14.639 ms CPU versus 345.905 ms wall. Artifact publication also measured +2.539 ms CPU versus 94.178 ms wall. This observes substantial elapsed time +outside measured process CPU; it does not establish the cause or waive the +150 ms admission ceiling. Prepared stages are still not a decomposition of +the streaming parse. + +A further experiment bound the lexer once per parse rather than obtaining it +through a wrapper for every token. Six complete artifacts agreed; 33 source +functional cases passed. Five alternating source pairs improved mean CPU +227.735 to 203.460 ms and median 231.170 to 204.253 ms, with four wins. +However, five native pairs of 1,000 parses regressed mean CPU 203.395 to +209.229 ms and won only twice. The experiment was discarded, the exact source +snapshot was restored, its digest rechecked, and the parser was recompiled +with standard =gxc -O= settings. No lexer-binding change remains. + +After this exact restoration, a focused rerun of the unchanged GQL contract +again failed: p95 304.289 ms against 150 ms, minimum 11.041 ms, median +38.300 ms, maximum 401.057 ms. The callback's parse-success checks completed; +the receipt assertion failed at =t/gql-runtime-benchmark-test.ss:40=. This +rerun did not publish =CASE-OK= or final =OK=, and cannot qualify the native +runtime. The remaining +gate is GQL wall-time admission; its cause is not fully attributed and no +threshold or host scheduling policy was changed. + +* Rebase onto PR #13, 2026-10-02 + +The contextual branch was rebased from common ancestor =9ceca64= onto the +verified PR head =85f62fea55b2868eb3c0adff29ce08220bdf9b8f=. The pre-rebase +implementation is preserved at =4dbab6b2c291b862511af4356771a351112baff0= +by =backup/contextual-checkpoint-pr13-20261002=. The earlier design commit +was replayed unchanged. The only textual conflict was the compiler's lexical +first-character rules: the merged rules retain upstream =character-run= +admission and this branch's numeric, quoted-string, and heredoc admission. +Upstream layout-aware LR execution and parser routing remain present. Export +formatting and a redundant LR documentation example were condensed to keep +the runtime at 1,000 lines without changing execution. + +The final current-source digest is +=sha256:98bc033e5d2d843044f4ad4203452808782203c1e2c497608e74dae18e65f83c=. +An explicit source overlay preceded compiled parser modules. Nine suites +executed 75 cases, including contextual dispatch/scanning/parsing, Unicode +lossless failures, upstream character runs and layout boundaries, LR/GLR +artifact behavior, recovery, and incremental fresh equivalence. All published +=CASE-OK=, =MODULE-OK=, =HARNESS-OK=, and final =OK=; the bounded runner exited +zero in 23.86 seconds. The grammar-composition module +was also loaded in an earlier run, but its plural suite export prevented case +execution; it is excluded from the 75-case receipt. + +The remote PR head was rechecked after replay and remained =85f62fe=. +This is local rebase and source functional evidence. Native compilation and +the eight performance gates have not been rerun for this source revision; +the earlier native GQL failure is historical, and its current status remains +unqualified. No remote branch was pushed. + +* Latest PR revision and native qualification, 2026-10-03 + +PR #13's history was subsequently rewritten to +=7c1533356585154c8b5d340f64ffefb289d18907=. Only this branch's two commits +were replayed onto that revision. The overlap retained upstream's configurable +=current-lr-branch-budget= and the SANY corpus command. The prior local state +remains reachable through =backup/contextual-before-pr13-refresh-20261003=. +The remote revision was rechecked after validation and remained unchanged. + +The standard =gxi build.ss compile= library projection completed all 128 +targets in 101.65 seconds, using four build workers and the existing 1 GiB +heap fence. The independent cache at =.data/native-pr13-7c15333= initially +contained no parser binaries; only external dependency artifacts were linked. +The parser, LR runtime, and machine resolve to this cache's =.ssi= files. +This receipt does not cover optional commands or remote CI. + +Current-source functional validation executed 75 cases in nine suites, with +real LR/admission traces and a five-second output watchdog. The same 75 cases +then passed against the newly compiled native modules in 25.03 seconds. Both +runs published all case/module/harness markers and final =OK=, with exit zero. + +The final qualification snapshot is +=sha256:1dcd1645eede849f2d973bc1a0c8bfaefed8011d54ea17f78435c857cb2db5ba=. +All eight original native wall-time contracts passed without changed inputs, +sample counts, or numerical ceilings: + +| Contract | p95 ms | Existing ceiling ms | Result | +|----------+--------+---------------------+--------| +| LALR fixed point | 3.635 | 500 | PASS | +| Incremental suffix | 0.769 | 250 | PASS | +| Recovery frontier | 0.891 | 250 | PASS | +| Selective GLR | 5.317 | 250 | PASS | +| Arithmetic hot path | 16.305 | 900 | PASS | +| ISO GQL | 33.006 | 150 | PASS | +| openCypher 100 clauses | 8.466 | 250 | PASS | +| Versioned language batch | 6.155 | 500 | PASS | + +Cold admission is logged through the upstream compiled-module preloader. +Early attempts failed the silence watchdog before test execution; a later +attempt also retained a process after final =OK=. The final runner preloads +the relevant production and benchmark dependencies and explicitly exits after +the harness returns. These failed attempts are retained, not counted as +qualification. The openCypher test now reports each completed 100-clause +scenario under =GERBIL_PARSER_LR_TRACE=, matching the upstream GLR progress +pattern. Its logging cost remains inside the timed callback. Each final +contract had a 120-second total deadline, five-second output deadline, final +=OK=, and exit zero. + +The complete build, source/native functional logs, per-contract logs, runner +scripts, and machine-readable receipt are retained under +=.data/qualification-pr13-7c15333/=. The final performance logs are in +=qualified/=; =receipt.json= binds the source digest and timings. Earlier +native failures above remain historical. Host load was about 27 during this +run versus over 130 in the earlier run; these receipts establish current local +admission, not a matched causal speedup from the rebase. Declaration-macro +binding and contextual scanner state across GLR branches remain outside this +implemented vertical slice. No remote branch was pushed. + +* Deferred FIFO complexity and matched optimization, 2026-10-03 + +A fresh fetch/rebase confirmed that PR #13 still points to =7c15333=; the +optimization baseline is the local qualified commit =718ffc5=. The existing +scanner appended each new delimiter to its immutable pending list, copying +k(k-1)/2 existing cells over k consecutive declarations. This is quadratic +queue-maintenance work even though token scanning itself can remain linear. + +The candidate uses a private two-list FIFO: an ordered front and reversed rear. +Enqueue allocates one rear cell; dequeue reverses a rear only when the front +is empty. Queue maintenance is O(k) along one enqueue/drain execution path. +Individual reversals and canonical serialization remain O(k). Independent +futures can repeat a reversal, so this does not claim an amortized bound across +arbitrary persistent branching. The distinction follows Okasaki's discussion +of strict queues and multiple futures in /Purely Functional Data Structures/ +(1996), https://www.cs.cmu.edu/~rwh/students/okasaki.pdf, sections 3.1–3.2. +This is not a lazy banker queue or a worst-case real-time queue. + +The seven scanner-state fields, opcode contract, canonical schema, and ordered +pending-delimiter receipts are preserved. Restoring a checkpoint builds an +ordered front; subsequent enqueues can populate its rear. A new semantic test +compares complete token/checkpoint suffix traces from original, restored, and +independently replayed checkpoints, including mixed front/rear states. Source +validation passed all 23 focused scanner/parser cases before native compilation. + +The added =t/benchmarks/contextual-scanner/deferred-scale.ss= extends the +existing scanner benchmark family with full deferred batches, Unicode payloads, +CRLF, token order, UTF-8 coverage, and final pending/active checks. Preparation +and IR compilation are untimed. The exact old runtime was saved and imported +under a separate namespace; both native modules used =gxc -O=. Paired scans +alternate execution order, discard three warmups, and compare complete public +token traces and canonical final checkpoints. Each cell below summarizes five +paired samples; small native cases batch multiple scans to reduce timing noise. +Timing includes final checkpoint serialization and source digest computation. + +| Delimiters | Native scans/batch | Before CPU ms/scan | After CPU ms/scan | Median change | Wins | +|------------+--------------------+--------------------+-------------------+---------------+------| +| 1 | 2048 | 0.005601 | 0.005331 | -4.83% | 3/5 | +| 8 | 256 | 0.035730 | 0.026258 | -26.51% | 3/5 | +| 32 | 64 | 0.107906 | 0.113578 | +5.26% | 2/5 | +| 128 | 16 | 0.425438 | 0.411938 | -3.17% | 4/5 | +| 512 | 4 | 2.169000 | 1.987750 | -8.36% | 3/5 | +| 2048 | 1 | 13.057000 | 7.865000 | -39.76% | 5/5 | +| 4096 | 1 | 125.764000 | 18.692000 | -85.14% | 5/5 | + +Source execution at 2048 delimiters improved median CPU 169.514 to 143.288 ms +with five wins; at 32 and 128 it regressed 2.54% and 5.09%, while 512 was +approximately unchanged. The optimization is retained for removing quadratic +copying and reproducible large-batch gains. These results do not establish a +universal small-input speedup or improved ordinary GQL throughput. No peak +memory improvement is claimed from the structural allocation argument alone. + +The retained snapshot is +=sha256:b01800c0dca6af19f423931df9444408a4231a8977c509a5a0207ddc799651ac=. +The standard 128-target native dependency build passed in 13.55 seconds after +the focused native compilation. All 76 native functional cases passed in nine +suites, with exit zero, final =OK=, and the existing heap/silence fences. +The original performance ceilings and workloads were unchanged: + +| Contract | p95 ms | Existing ceiling ms | Result | +|----------+--------+---------------------+--------| +| LALR fixed point | 2.279 | 500 | PASS | +| Incremental suffix | 0.689 | 250 | PASS | +| Recovery frontier | 0.798 | 250 | PASS | +| Selective GLR | 3.398 | 250 | PASS | +| Arithmetic hot path | 18.904 | 900 | PASS | +| ISO GQL | 16.725 | 150 | PASS | +| openCypher 100 clauses | 8.001 | 250 | PASS | +| Versioned language batch | 9.309 | 500 | PASS | + +The first GLR run published a passing numerical receipt and final =OK= but +timed out during process termination. It was excluded from qualification; +only this remaining gate was repeated. The focused rerun completed with exit +zero under the same five-second output fence. Other gates were not repeated. +Raw evidence, baseline source, source/native paired runners, both GLR attempts, +=matched-receipt.json=, and =qualification-receipt.json= are preserved under +=.data/deferred-queue-experiment/=. The scale entrypoint is +=just benchmark-contextual-deferred 4096 5=. + +These are current local implementation and admission receipts. Remote CI, +declaration-macro binding, and contextual GLR scanner-state admission remain +separate gates. No remote branch was pushed. + +* 2026-10-03: bounded contextual literal catalog tries + +The exact PR #13 remote revision at this session's start was +=7c1533356585154c8b5d340f64ffefb289d18907=. The local experiment starts at +=81ae6ebea466af2557f6b4ca6f36e772839dcb46=, which already contains that +revision. No additional rebase was required at that boundary. + +A position-admission hash index was tested first against the existing 1/4/16/64 +axis scanner scenarios, comparing full token and final checkpoint traces. +ASCII source CPU medians changed by +0.44%, -0.31%, +0.20%, and +0.21%; Unicode +changes were -3.18%, +0.19%, -2.04%, and -0.80%. This did not establish a +consistent material gain. The runtime change was discarded before GXC; the +negative undeclared-position/EOF admission test remains. + +The next candidate reuses =make-literal-end-scanner= from =src/runtime/scan.ss=, +already used by the ordinary scanner compiler. A linear catalog scan tests all +literal prefixes at each position; the trie follows the matching character +prefix and retains its longest accepting node. For one catalog this changes +recognition from work proportional to all candidate prefix comparisons to +work proportional to the source prefix traversed, assuming ordinary hash +lookup cost. Trie preparation requires work and space proportional to catalog +characters. This does not reduce the number of contextual rules considered. + +The first unconditional candidate exposed native small-catalog and one-word +preparation regressions. The retained variant constructs a trie only when the +catalog has at least 128 literals and the input has at least 64 characters. +These are conservative workload guards, not a proven optimal crossover or a +universal throughput guarantee. Equal catalogs share one trie within each +prepared scanner; equal-key hashing/comparison and IR/digest admission still +contribute to preparation cost. There is no global cross-request cache. + +External IR remains raw literal data with the same schema and opcode contract. +The prepared matcher is engine-owned; input IR explicitly rejects its private +=literal-trie= tag, including a forged but self-consistent digest. Tests exercise +longest complete Unicode prefixes, an incomplete longer alternative, +UTF-8 offsets, cross-mode catalog reuse, and the existing no-match failure. +Rule precedence and conflicting-action ambiguity remain in the existing +selection path. Checkpoint layout and canonical data are unchanged. + +=t/benchmarks/contextual-scanner/matched-scale.ss= now exports the existing +IR fixture and accepts an optional catalog count. Its default scenario is +unchanged. =just benchmark-contextual-scanner 16 5 128 512= reproduces the +catalog scanning workload with token-count/kind/lexeme and byte coverage +checks. Its timing excludes compilation and scanner preparation. The separate +paired experiment also measures preparation plus scanning, using the same IR +and source, exact baseline runtime under another namespace, =gxc -O=, three +warmups, five alternating pairs, and complete token/checkpoint equality. + +Raw experiments, discarded candidates, final native pairs, short-input probes, +source admission failures/reruns, and native qualification logs are retained in +=.data/literal-trie-experiment/=; the position probe is in +=.data/position-index-experiment/=. The first expanded semantic test incorrectly +expected an unknown fallback where this scanner's established contract raises +=no match=; it was fixed. One source harness startup exceeded the five-second +silence fence and was excluded, then the remaining scanner gate completed. +The source guarded timing run overlapped a short semantic rerun near its end; +those figures are not used as the final performance claim. Native pairs and +qualification run sequentially. + +Final native measurements below use 128 words (positive literal matches, +identifier fallback, and Unicode words), 20 scans per sample, and five pairs. +CPU medians are per scan; equality covers kind/lexeme/start/end and canonical +final checkpoint, including UTF-8 coverage. + +| Axes | Literals | Includes preparation | Before CPU ms | After CPU ms | Change | Wins | +|------+----------+----------------------+---------------+--------------+--------+------| +| 1 | 128 | no | 0.648300 | 0.276750 | -57.31% | 5/5 | +| 1 | 128 | yes | 0.696750 | 0.343150 | -50.75% | 5/5 | +| 1 | 512 | no | 1.919300 | 0.282900 | -85.26% | 5/5 | +| 1 | 512 | yes | 2.492350 | 0.687150 | -72.43% | 5/5 | +| 16 | 128 | no | 0.872750 | 0.442750 | -49.27% | 5/5 | +| 16 | 128 | yes | 3.638500 | 3.220700 | -11.48% | 3/5 | +| 16 | 512 | no | 2.109300 | 0.473050 | -77.57% | 5/5 | +| 16 | 512 | yes | 6.982950 | 3.658400 | -47.61% | 3/5 | + +Prepared scanning improves 49.27% to 85.26% in all four retained large-catalog +cells, with five wins each. Single-axis preparation plus scanning also has five +wins. The 16-axis preparation cells have only three wins, and earlier probes +included a +23.2% regression at 512 literals; their final medians do not +establish a stable end-to-end preparation gain. IR/cell size, repeated admission, +and allocation/GC are material remaining costs. There is no whole-language or +public contextual-parser throughput claim. + +Short-input probes use one word and 100 scans per sample. The source-length +guard keeps them on the linear path; measured small deltas remain, so no +universal no-regression claim is made. Small catalogs remain linear as well. +The threshold of 128 uses the observed large-catalog native gain boundary; +64-character source admission is a conservative preparation guard rather than +a measured optimal length crossover. Peak memory was not measured. + +The final qualified source digest is =sha256:97e5a427498c6fb583f8e4eddfddf145f3757138ad3de8dc0dacf29c1718e894=. The standard 128-target +native library build and all 79 native functional cases passed with exit zero. +All eight original native wall contracts passed at unchanged ceilings: + +| Contract | p95 ms | Ceiling ms | Result | +|----------+--------+------------+--------| +| lalr | 3.894 | 500 | PASS | +| incremental | 0.694 | 250 | PASS | +| recovery | 1.325 | 250 | PASS | +| glr | 3.390 | 250 | PASS | +| hot-path | 13.842 | 900 | PASS | +| gql | 19.362 | 150 | PASS | +| opencypher | 11.077 | 250 | PASS | +| languages | 8.195 | 500 | PASS | + +=matched-receipt.json= and =qualification-receipt.json= bind these local +results. Public declaration-macro binding, contextual GLR scanner-state +admission, optional CLI builds, and remote CI remain separate. The remote PR +head was checked again and is still =7c15333=, already an ancestor of this +branch. No remote push was performed. + +* 2026-10-03: scanner preparation study and bounded dispatch factoring + +This experiment starts at =038aec59f15dafda647ed39c10957e7a164f5e64= and uses +the existing =matched-scale.ss= fixture with 1/16/64 mode and position axes, +0/128 literals, ASCII identifiers and Unicode fallback words. The exact runtime +is retained separately for source/native pairs, with complete public token +traces and canonical final checkpoint equality. Compilation is untimed. + +Source preparation was first split into digest admission, rule preparation, +and cell indexing. At 64 axes with 128 literals, the three post-warmup median +CPU times per preparation were 22.815, 0.543, and 7.172 ms respectively. +The digest remains the largest component; no validation or digest check was +removed, memoized against mutable caller data, or treated as trusted. + +Three representation candidates were considered: + +- Direct UTF-8 byte serialization removes the intermediate canonical string. + The default byte port changed Unicode digests and failed equivalence before + any production change. Explicit UTF-8/LF settings restored equality, as + documented by the Gambit byte-port/settings interface: + https://gambitscheme.org/4.9.8/manual/#U8vector_002dports . Native digest-only + medians at 64 axes improved for the zero-literal cell but the Unicode catalog + had only two of five wins. This candidate was rejected; the original + canonical string and SHA-256 path remain. +- A full Cartesian vector avoided composite keys, but constructing ordinal + maps and performing three lookups per cell increased source costs. The + zero-literal 64-axis preparation-plus-scan median regressed 37.70%, with zero + wins. This candidate was rejected before its native compilation. No dense + product allocation or new IR axis catalog was retained. +- Factoring dispatch by mode and position leaves a short form association row. + This removes the three-pair composite key allocation at token dispatch and + reduces prepared cell key storage. It uses two symbolic hash lookups plus a + bounded form row lookup. The prepared index is private and read-only after + construction; scanner/state field counts, canonical checkpoints, IR schemas and opcode + contracts retain their existing representation and meaning. + +The retained eligibility guard requires multiple mode/position rows, symbolic +keys, and at most eight distinct forms. A single-row native probe regressed +7.95% even after batching 100 scans per sample, so that shape retains the +original hash path. Wide form catalogs and structural keys also retain it. +The row search is bounded by eight; sparse rows do not allocate a Cartesian +product. Duplicate detection checks presence independently of the result, +so a negative cell cannot hide a duplicate. New tests exercise negative and +missing cells, duplicate negative cells, sparse dispatch, single-row, wide +catalog and structural-key fallback while preserving admitted token results. + +The preliminary grouped source probe was mixed: the 16-axis zero/128-literal preparation-plus- +scan cells improved median 14.65%/8.24%, with 3/5 and 5/5 wins. The 64-axis +zero-literal cell regressed 29.93%, and the 128-literal cell regressed 3.44%. +These observations precede the single-row and primitive type guards and do +not establish final source acceleration. +Native execution is measured separately. There is no peak-memory claim. + +The existing public scanner benchmark now accepts =scan= (default) or +=prepare-scan= as its fifth argument. The latter times fresh scanner admission +and index construction with each complete scan; IR compilation remains outside +measurement. For example: + +#+begin_src sh +just benchmark-contextual-scanner 16 5 128 128 prepare-scan +#+end_src + +The default token workload and original wall ceilings are unchanged. All +experiments and exact baseline source live under +=.data/scanner-prepare-experiment/=. One native startup exceeded the five-second +silence fence; explicit native module loading supplied genuine progress for +the rerun. An initial grouped grid later timed out in its last large cell; +that run is excluded from final qualification. Each completed timed half now +reports genuine scan count and CPU progress after its timer stops. The final +native grid uses three warmups, five alternating pairs and five scans per +sample. A combined source harness startup also timed out; its remaining +scanner gate passed a focused rerun under the same fence. + +The final implementation tests =table?= directly at dispatch. The prior generic +record predicate retained a roughly 9% single-row scan penalty despite choosing +the original index, so this extra cost was removed. The final matrix below +includes this change. CPU medians are per scan; preparation rows include fresh +IR digest admission and scanner indexes on every scan. + +| Axes | Literals | Includes preparation | Before CPU ms | After CPU ms | Change | Wins | +|------+----------+----------------------+---------------+--------------+--------+------| +| 1 | 0 | no | 0.229200 | 0.230600 | +0.61% | 1/5 | +| 1 | 0 | yes | 0.244600 | 0.251000 | +2.62% | 1/5 | +| 1 | 128 | no | 0.268200 | 0.264200 | -1.49% | 3/5 | +| 1 | 128 | yes | 0.410800 | 0.370000 | -9.93% | 3/5 | +| 16 | 0 | no | 0.368000 | 0.213000 | -42.12% | 5/5 | +| 16 | 0 | yes | 2.553000 | 2.007000 | -21.39% | 4/5 | +| 16 | 128 | no | 0.417200 | 0.259400 | -37.82% | 5/5 | +| 16 | 128 | yes | 3.128800 | 3.119000 | -0.31% | 3/5 | +| 64 | 0 | no | 0.262200 | 0.228000 | -13.04% | 5/5 | +| 64 | 0 | yes | 27.548200 | 25.553400 | -7.24% | 3/5 | +| 64 | 128 | no | 0.299000 | 0.277000 | -7.36% | 3/5 | +| 64 | 128 | yes | 26.547200 | 22.180800 | -16.45% | 5/5 | + +The 16-axis prepared-scan gains are 37.82%/42.12%, with five wins each. At 64 +axes with 128 literals, preparation plus scanning improves 16.45%, also with +five wins. Other preparation cells have only three or four wins; the 16-axis +128-literal cell is effectively unchanged. Single-row cells still have small +positive/negative changes (including a +2.62% preparation median), so no +universal latency improvement is established. Source interpreter regressions +and the untouched digest bottleneck remain explicit limitations. This is a +bounded contextual scanner result, not ordinary-language parser throughput. + +The first native functional harness completed all 80 cases and printed =OK= +but timed out during exit, and is excluded. The final native functional rerun +and its process exit are qualified separately. Final source scanner validation +used explicit standard-library admission logs to keep cold imports within the +same five-second progress policy; parser modules remained on the source overlay. + +Final source execution was measured again after both guards. Its results are +mixed: at 16 axes, preparation-plus-scan medians improve 8.82%/20.73% for +zero/128 literals, but only three pairs win. At 64 axes those medians regress +4.29%/15.30%, with two/zero wins. Prepared source scanning at 16 axes regresses +17.51%/8.27%; the 64-axis 128-literal cell regresses 18.53%. These source results +are not replaced by the stronger native receipts. The optimization is retained +for the measured native multi-row workloads; interpreter acceleration and +universal fallback no-regression remain unestablished. + +Final local qualification binds =sha256:d2f7a27a1e1ce98cf02f3e5742bd6f1efe30c639cb09b9e0168cf515a7835497=. The standard 128-target native +library build and all 80 native functional cases completed with exit zero. +All eight unchanged wall contracts passed: + +| Contract | p95 ms | Existing ceiling ms | Result | +|----------+--------+---------------------+--------| +| lalr | 3.435 | 500 | PASS | +| incremental | 1.549 | 250 | PASS | +| recovery | 1.321 | 250 | PASS | +| glr | 3.276 | 250 | PASS | +| hot-path | 14.112 | 900 | PASS | +| gql | 20.078 | 150 | PASS | +| opencypher | 7.607 | 250 | PASS | +| languages | 8.294 | 500 | PASS | + +The language batch twice printed passing receipts and =OK= but timed out in +its trailing expression/exit phase. Both attempts are excluded and preserved. +A diagnostic rerun put harness execution, recording its actual return status, +and normal =##exit= into one pre-expanded expression. It reported +=HARNESS-RETURN status=0= and completed with process exit zero under the +unchanged five-second silence policy. This isolates successful harness return +and exit; it does not prove an operating-system cleanup root cause. The seven +already completed wall gates were not repeated for this remaining gate. + +=matched-receipt.json= and =qualification-receipt.json= record the final local +boundaries. PR #13 was rechecked and remains open at =7c15333=, already included +in this branch. Remote CI, optional CLI qualification, public declaration-macro +binding and contextual GLR scanner-state admission remain separate. No remote +push was performed. + +* 2026-10-03: explicit reusable contextual scanner plans + +The remaining preparation profile motivated separating validated static scanner +work from per-input state. The ownership pattern is comparable to the fixed +compiled pattern and separate match data described by the [[https://www.pcre.org/current/doc/html/pcre2api.html][PCRE2 API]]. This +transfers a staging/ownership idea; it supplies no scanner algorithm or Gerbil +concurrency proof. + +=prepare-contextual-scanner-plan= takes an explicit owned snapshot of compiled +scanner IR and validates its schema, opcode contract and digest. It copies +pairs and strings, preserves DAG sharing and symbol identities, and rejects +cycles and values outside closed scalar/list data. Rule and dispatch indexes +are built once. Catalogs of at least 128 literals retain both the linear short +input variant and the trie variant selected at 64 source characters. No +process-global identity cache is introduced. + +#+begin_src scheme +(def plan (prepare-contextual-scanner-plan compiled-scanner-ir)) +(def first (prepare-contextual-scanner plan first-source)) +(def second (prepare-contextual-scanner plan second-source)) +#+end_src + +Each binding creates a new scanner and initial state; delimiter queues remain +input-local. The digest string is copied per binding because canonical +checkpoint receipts expose it. Subsequent mutation of the original IR or a +bound scanner's checkpoint digest cannot change the plan or other bindings. +Caller changes to the original IR require a new plan to take effect. Callers +must avoid modifying IR concurrently during plan creation. Source strings +retain the existing ownership contract and must stay stable while scanned. +The raw IR entrypoint continues digest admission on every call. + +The plan is opaque through the exported API; constructors, predicates and +accessors are private. It is not a serialized receipt. Scanner IR, opcode and +checkpoint schemas, state field count and scan semantics remain unchanged. +This is an opt-in scanner interface: =parse-source/contextual= still accepts +its existing product and prepares raw scanner IR. Full parser product plans, +public declaration-macro binding and contextual GLR state admission are not +implemented by this change. + +** Source gate before native compilation + +The two scanner suites passed with normal exit under a 120-second total and +five-second silence fence. Three additional cases cover mutable input and +checkpoint isolation, rejection of cycles/callbacks, and independent deferred +queues. An exact =568c317= runtime copy was then measured against the candidate +in the existing =fixture-ir= scale family: axes 1/16/64, catalogs 0/128 and +request batches 1/4/16. Sources alternate a one-word short input and a 128-word +input containing Unicode. One-request batches contain only the short input. +IR compilation is outside the timer. Baseline batches prepare raw IR for each +input. Candidate batches include snapshotting, validation and index creation +once, then fresh binding and scanning for each input. Both timers include +complete token trace and canonical final checkpoint construction. + +Each cell has three warmups and five alternating paired measurements. Full +kind/lexeme/start/end traces and final checkpoints match for every request; +concatenated lexemes and UTF-8 byte coverage are checked outside the timer. +The source runner completed normally in 30.48 seconds. Source CPU median +changes, including upfront plan construction, were: + +| Axes | Literals | Requests | Change | Wins | +|------+----------+----------+--------+------| +| 1 | 0 | 1 | +24.53% | 1/5 | +| 1 | 0 | 4 | -2.18% | 4/5 | +| 1 | 0 | 16 | -21.84% | 5/5 | +| 1 | 128 | 1 | +186.43% | 0/5 | +| 1 | 128 | 4 | -7.52% | 5/5 | +| 1 | 128 | 16 | -10.94% | 5/5 | +| 16 | 0 | 1 | +113.68% | 0/5 | +| 16 | 0 | 4 | -23.20% | 5/5 | +| 16 | 0 | 16 | -60.76% | 5/5 | +| 16 | 128 | 1 | +167.29% | 0/5 | +| 16 | 128 | 4 | -17.42% | 5/5 | +| 16 | 128 | 16 | -53.31% | 5/5 | +| 64 | 0 | 1 | +195.15% | 0/5 | +| 64 | 0 | 4 | -27.12% | 5/5 | +| 64 | 0 | 16 | -76.28% | 5/5 | +| 64 | 128 | 1 | +182.53% | 0/5 | +| 64 | 128 | 4 | -27.61% | 5/5 | +| 64 | 128 | 16 | -78.25% | 5/5 | + +Single-use snapshot costs regress. The source reuse gate passed for the 16/64 +axis cells at four and sixteen requests; the one-axis zero-catalog four-request +cell is small and has only four wins. No universal source acceleration claim +is made. + +** Matched native evidence + +The same pair was compiled with =gxc -O= into the dedicated native cache. +The native runner completed normally in 38.31 seconds. CPU medians are per +whole request batch, including plan creation, rather than per token or scan: + +| Axes | Literals | Requests | Before CPU ms | Plan CPU ms | Change | Wins | +|------+----------+----------+---------------+-------------+--------+------| +| 1 | 0 | 1 | 0.070 | 0.087 | +24.29% | 0/5 | +| 1 | 0 | 4 | 0.489 | 0.410 | -16.16% | 5/5 | +| 1 | 0 | 16 | 1.694 | 1.371 | -19.07% | 5/5 | +| 1 | 128 | 1 | 0.113 | 0.197 | +74.34% | 0/5 | +| 1 | 128 | 4 | 0.768 | 0.613 | -20.18% | 5/5 | +| 1 | 128 | 16 | 2.839 | 1.831 | -35.51% | 5/5 | +| 16 | 0 | 1 | 5.861 | 4.209 | -28.19% | 3/5 | +| 16 | 0 | 4 | 28.281 | 4.515 | -84.04% | 5/5 | +| 16 | 0 | 16 | 59.327 | 5.378 | -90.93% | 5/5 | +| 16 | 128 | 1 | 6.403 | 5.947 | -7.12% | 3/5 | +| 16 | 128 | 4 | 28.580 | 6.458 | -77.40% | 5/5 | +| 16 | 128 | 16 | 169.058 | 9.118 | -94.61% | 5/5 | +| 64 | 0 | 1 | 25.598 | 48.214 | +88.35% | 0/5 | +| 64 | 0 | 4 | 119.396 | 42.030 | -64.80% | 5/5 | +| 64 | 0 | 16 | 720.086 | 55.090 | -92.35% | 5/5 | +| 64 | 128 | 1 | 25.680 | 43.655 | +70.00% | 0/5 | +| 64 | 128 | 4 | 118.600 | 41.089 | -65.35% | 5/5 | +| 64 | 128 | 16 | 519.275 | 39.717 | -92.35% | 5/5 | + +Four- and sixteen-request batches win all five pairs in every native cell. +The 16/64-axis four-request improvements range from 64.80% to 84.04%; their +sixteen-request improvements range from 90.93% to 94.61%. One-request 16-axis +cells have only three wins and do not establish single-use benefit. One-axis +and 64-axis single requests regress. Use a plan when IR is reused across +inputs; do not switch one-shot preparation to this API based on these results. +Host scheduling and allocation/GC variation remain visible in raw samples. +No memory-footprint or concurrent-execution bound is claimed. + +The existing =matched-scale.ss= / =benchmark-contextual-scanner= entrypoint adds +=plan-batch= and a request count. =prepare-scan= with the same count provides +a current-runtime raw-preparation control. Plan creation is included in each +sample; IR compilation stays outside. For example: + +#+begin_src sh +just benchmark-contextual-scanner 16 5 128 128 prepare-scan 4 +just benchmark-contextual-scanner 16 5 128 128 plan-batch 4 +#+end_src + +The default =scan= phase and one-request default remain. A source smoke run of +the new batch phase passed. Frozen baseline, paired drivers, raw logs and +JSON medians are in local =.data/scanner-plan-experiment/=. These exploratory +receipts are not a replacement for the unchanged eight wall gates. + +** Qualification boundary + +The standard native build projected 128 targets and completed with exit zero. +Nine focused native modules passed 83 cases with =MODULE-OK=, =HARNESS-OK=, +=HARNESS-RETURN status=0= and normal process exit. All eight original wall +ceilings passed at source digest =sha256:58089d751ad7a37916e2a94114b591fd3408a0e1022611e7b5add6c648862b2f=: + +| Gate | p95 ms | Ceiling ms | Result | +|------+--------+------------+--------| +| lalr | 3.725 | 500 | PASS | +| incremental | 1.785 | 250 | PASS | +| recovery | 0.736 | 250 | PASS | +| glr | 2.989 | 250 | PASS | +| hot-path | 14.870 | 900 | PASS | +| gql | 32.541 | 150 | PASS | +| opencypher | 10.136 | 250 | PASS | +| languages | 6.247 | 500 | PASS | + +The first language gate attempt printed passing receipts and harness return +but triggered the unchanged five-second silence fence during process completion. +Its runner exit 124 is excluded and its log is retained as +=native-gates/languages-idle-timeout.log=. Only this remaining gate was rerun; +the rerun completed normally with process and runner exit zero. No operating +system cleanup cause is claimed. + +The latest PR #13 head was read again as =7c15333= and verified to be an ancestor +of this local branch. No new rebase was necessary at that remote head. Native +qualification and local commits do not establish remote CI or deployed behavior; +no push was performed. Optional CLI qualification remains separate. + +The existing benchmark entrypoint also passed native smoke checks for +=prepare-scan= and =plan-batch= at four requests, and default =scan= at one +request, with normal process and runner exit zero. + +* 2026-10-03: prepared contextual parser requests + +The scanner plan is now usable through a complete contextual parser request. +=prepare-contextual-parser machine product= validates the product's schema, +base grammar, machine Parser IR digest, product digest, LR position table and +scanner admission once. It binds the generated machine, an owned scanner plan, +a private position vector and a copied product digest. Position IDs must be +symbols, matching the compiler's existing declaration contract; even a +self-consistent forged string-position product is rejected. Copying the table +spine alone would leave mutable position values shared. + +#+begin_src scheme +(def plan (prepare-contextual-parser machine compiled-product)) +(def first (parse-source/contextual/prepared plan first-source)) +(def second (parse-source/contextual/prepared plan second-source)) +#+end_src + +Every request creates fresh scanner state, token accumulation and LR checkpoint. +Raw and prepared entrypoints share the same deterministic drive and artifact +construction. =parse-source/contextual= still admits raw product data on every +call. The prepared entrypoint accepts only an opaque plan and a source, so it +cannot be used with another machine argument. It copies the published product +digest per request: mutating an artifact's =grammarDigest= cannot poison future +plan requests. Scanner plans already isolate their checkpoint digest strings. + +Generated =ParserMachine= AOT IR and runtime must remain immutable after module +initialization, as declared by =src/compiler/machine.ss=. This plan references +the machine; it does not snapshot its executable runtime or protect against +violating that machine contract. Prepare after initialization and driver +installation. Caller-owned product data may change after preparation without +changing the plan. Avoid concurrent mutation during preparation. Input source +strings retain their existing stability requirement while being parsed. +Changes to the product require a new plan to take effect. + +This extends the internal contextual deterministic LR slice. Public contextual +declaration-macro binding, contextual GLR branch state and optional CLI +qualification remain separate. Existing generated language parsers do not +switch to this API automatically; the HCL grammar supplies the bounded +integration fixture, rather than a full language scanner implementation. + +** Full artifact source and native matched gates + +The HCL product fixture was moved unchanged into +=t/benchmarks/contextual-scanner/parser-fixture.ss= and is shared by integration +tests and =matched-parser.ss=. This remains in the existing contextual scanner +benchmark family. Each batch cycles through an accepted six-byte assignment, +32 Unicode assignments, an incomplete assignment and a rejected input with an +unscanned Unicode suffix. A one-request batch covers only the short accepted +assignment; four-request batches cover all shapes, and sixteen repeat them. +IR and product compilation are outside the timer. Raw batches admit and +prepare on every request; prepared batches include plan creation once followed +by request binding, scanning, LR drive and full ParseArtifact construction. + +The frozen baseline is the exact parser runtime at =5548a0e=. Both sides use the +same machine, scanner runtime and artifact implementation. Three warmups and +five alternating pairs are run per cell. Full ParseArtifacts match field for +field for every request, including events, diagnostics, identities and byte +coverage. Artifact validity and lossless roundtrip are checked outside timing. +Final source and native matched runners completed normally in 9.70 and 12.01 +seconds, under the unchanged 120-second total and five-second silence fences. +Native libraries were compiled with =gxc -O= in the dedicated cache. + +Final source CPU medians per whole batch, including plan creation: + +| Requests | Raw CPU ms | Prepared CPU ms | Change | Wins | +|----------+------------+-----------------+--------+------| +| 1 | 3.941 | 4.940 | +25.35% | 2/5 | +| 4 | 33.370 | 10.369 | -68.93% | 5/5 | +| 16 | 103.928 | 27.009 | -74.01% | 5/5 | + +Final native CPU medians per whole batch: + +| Requests | Raw CPU ms | Prepared CPU ms | Change | Wins | +|----------+------------+-----------------+--------+------| +| 1 | 4.013 | 4.509 | +12.36% | 0/5 | +| 4 | 21.467 | 6.651 | -69.02% | 5/5 | +| 16 | 78.253 | 10.826 | -86.17% | 5/5 | + +Four- and sixteen-request native batches reduce CPU by 69.02% and 86.17%, with +all five pairs winning. The source cells reduce CPU by 68.93% and 74.01%, also +with five wins each. Single-use plans regress on both final runs. The earlier +native probe before the symbolic position guard showed larger reuse medians +and a three-win single-use cell; those figures are not the final-code evidence. +Allocation/GC and host variation are visible in raw samples. No single-use, +all-language throughput, memory-footprint or concurrent-execution improvement +is claimed. + +The durable current-runtime command compares raw and prepared full requests: + +#+begin_src sh +just benchmark-contextual-parser 16 5 +#+end_src + +It reports both CPU and wall times and rejects any full-artifact mismatch. +Frozen source, paired drivers, raw logs and JSON medians are retained locally +under =.data/contextual-parser-plan-experiment/=. These measurements complement +the existing wall-time gates; they do not replace their workloads or ceilings. + +** Final qualification and exit boundary + +The standard native build projected 128 targets and completed with normal +exit. The same nine focused functional modules were ultimately qualified +individually: 86 unique cases, one =MODULE-OK= per module, =HARNESS-OK=, +=HARNESS-RETURN status=0=, and normal process and runner exit zero. Batch and +individual attempts that printed passing checks but exceeded five seconds of +silence during completion are excluded and retained in their console logs; +the original nine-module raw log is =native-functional-idle-timeout.log=. +Functional receipts bind each qualified module to its successful console/log. +No assertion failure is being counted as a successful exit. + +A single real =##gc= after harness return was added to diagnostic wrappers, +with =FINAL-GC-OK= emitted only after collection completes. This runs outside +benchmark measurement intervals and remains under the same silence fence. +It did not eliminate every completion timeout. The GLR, hot-path and GQL wall +attempts still exceeded the fence after passing receipts and final collection; +their =*-idle-timeout.log= files are excluded. Only these remaining gates were +rerun: the harness status was checked, then the interpreter completed through +its ordinary CLI return rather than an explicit =##exit=. Each rerun invoked +the harness exactly once and completed normally. This establishes an exit +boundary, not an operating-system cleanup or garbage-collector root cause. +No timeout was enlarged and no heartbeat was counted as child progress. + +All original eight wall workloads and ceilings passed at final source digest +=sha256:4133021870234cea00e7862e0bca5f17f940355c48446332ca5b517e117c158a=: + +| Gate | p95 ms | Ceiling ms | Result | +|------+--------+------------+--------| +| lalr | 3.800 | 500 | PASS | +| incremental | 1.761 | 250 | PASS | +| recovery | 2.102 | 250 | PASS | +| glr | 4.298 | 250 | PASS | +| hot-path | 16.203 | 900 | PASS | +| gql | 22.596 | 150 | PASS | +| opencypher | 14.427 | 250 | PASS | +| languages | 8.246 | 500 | PASS | + +The documented =just benchmark-contextual-parser 4 2= command also completed +normally. Its first cold launch timed out before a receipt; the recipe now +uses genuine standard-library preload events from the existing preload helper. +A subsequent direct-script fixture import failed because it lacked the module +context needed for a relative import; the fixture is now imported by its full +package identity. Both failed command consoles are retained and excluded. +The verified command uses the final source and checks full artifact equality. + +=qualification-receipt.json= records the qualified gates, functional receipts, +and excluded attempts. PR #13 remains open at =7c15333=, already included in +this branch. Local qualification does not establish remote CI; no push was +performed. + +* 2026-10-03: prepared request phase study and two rejected scanner candidates + +This study starts from the prepared contextual parser at =4dadaf7=. The HCL +fixture already installs its generated =direct-step= during grammar-module +initialization; adding another action-dispatch layer is not the next +experiment. An isolated copy of =src/runtime/parser.ss= records CPU around +=lr-checkpoint-drive/contextual= and successful artifact construction. An +independent scan-only loop uses the same compiled scanner product and constant +=line= position, which is the fixture's position for every LR state. +Production instrumentation and runtime code are unchanged by this profile. + +The existing =parser-fixture.ss= family supplies accepted ASCII and Unicode +assignments at 1/32/256 lines. Plans and fixture compilation are outside +measurement. Each sample performs eight prepared requests, with three warmups +and five measured samples per cell. Artifacts satisfy validity and lossless +roundtrip; independent scans cover exactly six tokens per line. The source +profile completed normally in 30.73 seconds and the native profile in 12.20 +seconds under 120-second total and five-second silence fences. Only profiling +copies were compiled with =gxc -O=; neither proposed production candidate below +reached native compilation. + +CPU medians per eight-request sample: + +| Runtime | Lines | Text | Total ms | Drive ms | Artifact ms | Scan-only ms | +|---------+-------+------+----------+----------+-------------+--------------| +| source | 1 | ASCII | 2.284 | 1.850 | 0.318 | 0.503 | +| source | 1 | Unicode | 1.955 | 1.591 | 0.281 | 0.499 | +| source | 32 | ASCII | 53.828 | 49.491 | 4.102 | 16.501 | +| source | 32 | Unicode | 57.444 | 51.496 | 5.450 | 17.303 | +| source | 256 | ASCII | 615.094 | 578.936 | 38.013 | 190.241 | +| source | 256 | Unicode | 540.467 | 507.871 | 32.489 | 154.691 | +| native | 1 | ASCII | 0.204 | 0.144 | 0.041 | 0.052 | +| native | 1 | Unicode | 0.196 | 0.137 | 0.040 | 0.050 | +| native | 32 | ASCII | 3.202 | 2.848 | 0.311 | 0.992 | +| native | 32 | Unicode | 2.818 | 2.500 | 0.286 | 0.966 | +| native | 256 | ASCII | 23.000 | 20.560 | 2.180 | 8.249 | +| native | 256 | Unicode | 26.441 | 23.853 | 2.247 | 7.978 | + +Drive includes contextual scanning and reduction. Scan-only is a separate run, +not an instrumented component that can be subtracted to prove exact LR cost. +Each column's median is computed independently, so medians need not add up. +The profile adds timer and counter work; short-input percentages are especially +sensitive to that overhead. Artifact timing excludes argument evaluation such +as reversing the token list. Source behavior and native behavior remain +separate evidence. + +At 256 lines, native drive accounts for approximately 89%/90% of total for +ASCII/Unicode; artifact construction is approximately 9.5%/8.5%. Independent +scan-only medians are approximately 36%/30% of total. Thus substantial work +remains inside LR drive beyond scanning. This is a bounded hotspot hypothesis, +not an allocation count, concurrency proof or all-language attribution. + +** Rejected candidate 1: scalar match selection + +=best-match= currently constructs =scan-match= records while folding over rules. +The prototype removed that private record, retained selected rule/end in loop +variables and hoisted source/start reads. It preserved longest extent, rank +priority, equal form/action tie handling and ambiguity rejection. The three +existing scanner/parser/deferred suites passed in source with normal exit. +Full prepared request pairs also matched every ParseArtifact, including +rejected inputs and Unicode suffix coverage. + +** Rejected candidate 2: ASCII opcode byte width + +=horizontal-whitespace+= accepts only space/tab; =newline= and =newline-one= +accept CR/LF. Their consumed character width is therefore their UTF-8 byte +width. The prototype used this closed-opcode fact to avoid allocating a UTF-8 +vector solely for token length, and retained the existing encoder path for +other matchers. It did not cache literal widths from mutable caller IR. +Full prepared request pairs again matched every artifact. This establishes +conformance only for the measured shapes, not every opcode or invalid input. + +** Source admission gate + +Both candidates used the exact =4dadaf7= parser and scanner copies as baseline. +Both sides prepared their plans once before timing: these are steady prepared +request batches, not the prior raw-versus-plan amortization comparison. +The existing four-shape fixture cycles short accepted input, 32 Unicode +assignments, incomplete input and an unscanned Unicode suffix. Request counts +4/16/64 cover all shapes. Three warmups and five alternating pairs are used; +complete ParseArtifact equality, validity and roundtrip are checked outside +measurement. The scalar run completed normally in 10.75 seconds; the byte +width run in 12.90 seconds. + +| Candidate | Requests | Before CPU ms | Candidate CPU ms | Change | Wins | +|-----------+----------+---------------+------------------+--------+------| +| scalar-selection | 4 | 6.183 | 6.031 | -2.46% | 3/5 | +| scalar-selection | 16 | 23.023 | 23.670 | +2.81% | 1/5 | +| scalar-selection | 64 | 95.989 | 94.950 | -1.08% | 4/5 | +| ascii-byte-width | 4 | 6.139 | 9.194 | +49.76% | 1/5 | +| ascii-byte-width | 16 | 24.912 | 30.314 | +21.68% | 2/5 | +| ascii-byte-width | 64 | 105.924 | 104.270 | -1.56% | 3/5 | + +Neither candidate has stable whole-request gains. Scalar selection regresses +at sixteen requests and wins only three/four pairs in the other cells. ASCII +byte width regresses at four/sixteen requests and has only three wins at +sixty-four. Both were withdrawn before GXC. No native speedup or reduced total +allocation is claimed; eliminating an explicit object or encoding call does +not itself establish whole-parser improvement. + +The prototypes, paired console logs and JSON receipts are retained locally in +=.data/prepared-parser-hot-path/=. =scalar-rejected.ss= and =byte-rejected.ss= +record the attempted code. The final production scanner is byte-for-byte equal +to the committed baseline. Only this research record is retained in the repo. +Existing production build/functional/wall qualification belongs to =4dadaf7=; +it was not rerun or represented as a new gate for this documentation change. + +** Next falsifiable experiment: reduction and recognition materialization + +The next probe should count the actual semantic operations before changing +representations: + +1. Count generated =direct-step= production frequencies and classify pass, + field and alias reductions, including reduction widths. +2. Count =recognition-children-field= singleton versus fragment-wrapping paths, + and =recognition-sequence->list= calls that actually flatten a rope. The + existing rope already gives constant-time concatenation; introducing a + second concatenation representation is not justified by this profile. +3. Select a measured hot class, then test fusion of adjacent field/alias work + only where the Parser IR proves the shape. Nested field scopes, empty + reductions, byte boundaries and immutable shared children must be preserved. + The public field setter means mutating or reusing children cannot be assumed + safe merely because the final tree looks similar. +4. Compare complete prepared artifacts against the indexed/generated baseline + on short, long, Unicode and rejected shapes. Source CPU admission comes + before native compilation; native paired evidence and the original wall + ceilings remain required for retaining a runtime candidate. Operation counts + alone cannot qualify a change. + +The authoritative HCL reduction emitter is +=src/compiler/fused-reduction.ss=, exposed through the generic +=scripts/generate-reductions.ss= producer. The generated header identifies the +engine POO strategy. CI passes the HCL grammar module identifier and checks +=src/compiler/hcl-reductions.ss=. Reduction experiments +must change this emitter and regenerate the product, preserving the Parser IR +and exact artifact boundary. The earlier directory-scoped producer search was +incomplete. These are proposed experiments, not implemented reduction capabilities. + +At the profiling checkpoint, PR #13 remained open at =7c15333=, already +included in this branch; that checkpoint did not include a remote push. + +* Latest revision and comprehensive native test admission + +On 2026-10-03 the accumulated branch was pushed to PR #13 at =07ae542=. +Fetching and rebasing against both that PR head and =origin/main= at +=c6ecb40= reported that the branch was already up to date. The exact-head +Rust CI passed, while both native jobs hit the five-second silence +watchdog in =t/benchmark-test.ss=. + +The test batch now reports only completed 1,000-parse batches. Its output cost +remains inside the measured thunk; the 900ms ceiling, twenty samples and +five-second silence fence are unchanged. A local completed batch run admitted +p95 =191.662ms=. This is a gate receipt, not a matched optimization comparison. + +The full native entry now preloads the imports declared by each test rather +than unconditionally admitting the TLA+ candidate. It validates completed +=CASE-OK= output so a module running zero cases cannot pass. Sixteen existing +suites exported only plural names (or no suite); gxtest discovers only exported +names ending in =-test=. Each now has one discoverable suite binding. Newly +admitted ANTLR cases passed six cases; the build-product contract passed two +after replacing its obsolete POO Flow dependency assertion with the current +complete immutable dependency identities from =gerbil.pkg=. + +A disposable native test child publishes the harness return status, completes +its final GC, flushes both output streams, then exits with that exact status. +The child boundary uses C11 =_Exit=; native library teardown is outside this +harness receipt. A separate status probe completed with process exit =42=, +confirming that a nonzero result is retained. Five-second silence enforcement +continues throughout imports and test execution. This does not qualify native +library unload behavior or resource-finalizer integration. + +The inventory contains 59 native test modules. Full-suite admission and the +new GitHub revision's CI must be recorded separately; the focused receipts +above do not claim completion of that inventory. diff --git a/docs/architecture/0007-scheme-runtime-concurrency.org b/docs/architecture/0007-scheme-runtime-concurrency.org new file mode 100644 index 00000000..0fbe3168 --- /dev/null +++ b/docs/architecture/0007-scheme-runtime-concurrency.org @@ -0,0 +1,51 @@ +#+title: RFC 0007 — Scheme parser request concurrency +#+startup: overview + +* Decision + +A parse of one source is a synchronous call over one immutable compiled +ParserMachine. The scanner, LR/GLR executor, layout frames, diagnostics, and +ParseArtifact stay in that call. Gerbil actors or threads may schedule +independent parse requests around this boundary; they do not become grammar +operators, LR actions, or a second parsing algorithm. The public parse API +remains a function from a parser and source to a ParseArtifact. + +The host may use =:std/actor= for a service with a bounded request queue, +reply routing, deadlines, and supervision. A library caller may instead use +=spawn= and =thread-join!= directly. Neither policy is imposed by the parser +package. Capacity and queue limits belong to the host because an actor per +source without admission could multiply heap use during GLR parses. Throughput +and CPU parallelism must be measured on the selected Gambit build and workload; +the existence of actor threads alone is not a multicore performance receipt. + +* State and failure boundary + +- A request owns its source bytes, token stream, diagnostics, observability, + GLR configuration memo, and layout-column vector. Layout frames are copied + into each GLR branch. =current-layout-columns= and =current-layout-frames= + are dynamically scoped for the request; no request writes a shared parser + descriptor while parsing. +- An actor message identifies the requested grammar artifact and source and + returns one success or failure ParseArtifact. An actor must not expose its + mutable work buffers through a reply. A caller can discard a late reply + after its deadline; hard interruption of an active parser call requires a + separately validated cancellation boundary. +- Incremental checkpoints must carry all context needed to resume a request. + Layout parsing currently reparses the full source on edits and leaves + recovery disabled because its layout-frame state is not serialized into + checkpoints. An actor must not make that incomplete checkpoint portable by + passing it between requests. +- Native FFI is a separate ownership and thread-affinity contract. Before + concurrent FFI calls are admitted, generated and native artifacts need an + explicit Send/Sync or single-owner rule, lifetime rule for source buffers, + and differential parse receipts. The Scheme actor decision grants no + implicit permission to share native handles across threads. + +* Acceptance + +The TLA+ layout test runs nine simultaneous requests against one compiled +parser, using three sources with different marker kinds and nesting. It checks +that each result equals its sequential ParseArtifact and remains valid. This +is a reentrancy check, not a throughput claim. A production actor adapter +would additionally require bounded admission, deadline and shutdown tests, +error delivery, and workload measurements before becoming a public API. diff --git a/docs/architecture/0008-extensible-language-packs.org b/docs/architecture/0008-extensible-language-packs.org new file mode 100644 index 00000000..bb038fcf --- /dev/null +++ b/docs/architecture/0008-extensible-language-packs.org @@ -0,0 +1,669 @@ +#+title: RFC 0008 — Extensible language packs and selective Rust builds +#+startup: overview + +* Decision + +The parser core has no built-in language registry, list of supported language +names, or per-language dispatch branch. A language pack owns its syntax +authority, metadata identity, corpus, expected failures, and any semantic +lowering. A native pack compiles language-neutral lexical and Grammar IR into +Parser IR and an immutable parser artifact. An external pack may keep its own +parser and publish source-backed lossless syntax events through the source-index sink. +The two authorities have distinct receipts. For a native pack, version and +contract live in configured parser metadata; neither a namespace nor a Rust module path +is the version authority. + +The Rust runtime crate accepts an explicit =LanguageSpec= and source bytes on +the independent AOT path. Its event sink also accepts a minimal kind catalog and +source-backed events without requiring LR tables. It depends on the artifact index library and +generic runtime support only. Generated language tables, external adapters, +and optional semantic adapters live in separate crates. A downstream author +can create a native language crate from the public Scheme grammar facade or +adapt a maintained external parser. Adding either never requires an edit to +the engine or an upstream enum. The existing independent records fixture +exercises the native boundary outside the root Cargo workspace. + +The opt-in =gerbil-parser-runtime-external= crate owns the second receipt type. +=ExternalLanguageSpec= supplies only provider identity and an =EventCatalog=; +=parse_external_events= checks the generic event sink and returns +=ExternalParseReceipt= with =authority = external-parser=. The adapter does +not require =LanguageSpec=, LR tables, or Scheme tooling at parse time. Its +runtime check establishes that the declared digest is well formed; a build +gate must separately prove that it matches the actual external provider +artifact. + +* Selective distribution + +A convenience language-support crate may re-export first-party language crates +behind additive Cargo features. Its default feature set is empty; each +language feature enables one optional language dependency and gates its public +re-export. A downstream consumer can instead depend directly on one generated +language crate, which is the smallest supported path. The generic runtime +does not acquire these language features or dependencies. + +Cargo unifies features enabled for the same package by all dependency paths. +Therefore minimal-build evidence must inspect the resolved dependency graph, +not just one =default-features = false= declaration. A support crate's =all= +feature may be a convenience for tools, but it is never a prerequisite for +downstream custom languages. Features select *which artifacts compile*; they +do not change a grammar's identity, parsing semantics, or receipt schema. + +* Public authoring and runtime contract + +1. =grammar.ss= uses the public declarative facade and publishes one explicit + grammar identity. Expansion emits canonical Grammar IR and Parser IR or a + typed rejection. The author does not implement another scanner, LR table, + conflict resolver, or hand-written Rust parser behind the facade. +2. The build product generates immutable Rust tables and their grammar digest. + The generator is a build or release dependency, never a parse-time + dependency. A committed generated artifact is permitted when it is checked + against the Scheme owner and source digest. +3. =parse(&LANGUAGE, source)= returns a lossless CST and a receipt tied to the + selected artifact. The language crate may expose a semantic adapter, but + generic runtime output remains available without that adapter. +4. An external parser adapter emits one balanced source-exhaustive event tree + whose token ranges are ordered UTF-8 boundaries and whose node/token kinds + belong to its declared catalog. Its receipt binds the external provider, + provider version and artifact digest, syntax contract, and source digest. + It does not fill the native =grammar_digest= or =parser_digest= fields with + an unrelated provider identity, and it cannot claim Scheme Parser IR parity + without a separate differential gate. +5. A service chooses a descriptor explicitly. A registry for discovery or + user configuration belongs in the host and is populated from linked + language crates. It cannot force every installed language into a minimal + executable. +6. The Scheme actor/request layer may schedule independent calls. Native FFI + handles need a separately verified ownership, lifetime, and thread-safety + contract before they are shared across such calls. + +* Declarative source scanner profiles + +=language-source-support= exports =ScannerProfile.=, =ScannerProfileContract= +and =defscanner-profile=. A profile is a POO object with ordinary =modes=, +=initial-mode=, =tokens= and =rules= values. A rule is +=(name mode form matcher rank action)=. The compiler validates effective +inherited values, resolves token dispatch and emits closed contextual Scanner IR. +Callbacks and private prepared-runtime objects are not profile data. + +#+begin_src scheme +(defscanner-profile (letters :: self ScannerProfile.) + (modes main) (initial-mode main) (tokens letter) + (rules (letter main letter (literal "α") 0 keep))) + +;; Inherit modes, token roles and initial state; replace the rule value. +(defscanner-profile (digits :: self letters) + (rules (digit main letter (literal "1") 0 keep))) +#+end_src + +Ordinary POO value overrides and extensions pass the same contract when bound. +There is no public thunk slot or language-owned scanning loop. The macro handles +region-plan extraction internally with =(region-word plan initial-stop?)=. +The compiler handles serialization, worker ownership and backend emission. + +The production Bash grammar declares command/marker/body modes, quote removal, +FIFO markers, tab stripping, comments and LF boundaries. =ShellSourceStrategy.= +requires its =scanner= profile in addition to its =regions= value. Both enter +source identity. The engine prepares mode/dispatch/region indexes once per +bound strategy; input workers hold source-local immutable checkpoints. The +SourceScanner adapter forwards existing tokens and byte ranges without another +UTF-8 encoding pass or duplicate token construction. + +| Closed operation | Meaning | +|---+---| +| =(unless-prefix blocked exceptions child)= | Evaluate one admitted child unless a blocked prefix matches; an explicit exception admits it. Nested guards reject. This prevents region errors inside comments and shields process substitutions from operator prefixes. | +| =(line-prefix prefix separator)= | Match prefix-led content up to one declared separator, excluding it. Prefix must not contain that separator. | +| =(marker-line-at separator)= / =(body-line-at separator)= | Match active obligation lines with an explicit separator; existing CRLF-normalizing line operations retain their own meaning. | +| =(expect-marker-in strip-tabs? mode)= | Enter a declared delimiter mode; reject a second outstanding expectation. | +| =(enqueue-marker-in policy mode)= | Require an expectation, decode its marker with a closed policy, queue it and enter a declared base mode. | +| =(activate-next mode)= / =(finish-marker base body)= | Activate/drain the immutable FIFO and select declared modes. | + +Longest match precedes rank; equal-width conflicts use rank and unresolved +ambiguity rejects. Mode names, dispatch coverage, matcher progress and action +compatibility are checked before execution. Scanner/loader schemas are unchanged. +The Scheme generator emits the actual Bash scanner profile plus token/span and +rejection controls; Rust consumes this data without Scheme at scan time. Native +CI regenerates and compares the artifact. + +This closes Bash scanner state execution. Structured word and command recognition +still use engine procedures. A scanner artifact alone is not a complete Bash Rust +parser or a source-descriptor C ABI product; those require recognition/result +lowering and native catalog admission. These remain S3 in the convergence plan. + +** Source-backed recognition publication + +The engine's =recognition.ss= prepares an owned source span from one scanner +token. It validates the declared byte length and Unicode scalars once. ASCII +uses origin plus character offset without allocating an index; other sources +share one character-to-byte index across every recursive result. Node and token +publication use checked constant-time endpoint lookup. Returned source views +are copies, so callers cannot invalidate admitted offsets. + +=region-scanner.ss= prepares a source-local boundary context only when matching +is requested. Closing an outer region records every nested quote/pair endpoint. +Later decomposition reuses those endpoints. Quote and pair identities have +separate indexes even when their openers share an offset. Source text and +indexes belong to the request, while the admitted declaration plan is reusable. +This is deliberately demand-driven: literal quote characters in a here-document +must not be interpreted before an expansion requests a declared region. + +Bash word, assignment and expanding here-document results use these shared +primitives. Their recursive token publication shares the existing persistent +recognition sequence and materializes one ordered list at the public result +boundary. It does not repeatedly encode source prefixes, rescan nested regions +or copy nested token suffixes. Ordinary words do not allocate region contexts. +These mechanisms are engine capabilities; language authors retain the same +three interfaces and do not supply endpoint or token-list workers. + +Word/command recognition is still procedural. Result construction and its +derived catalog now use the shared POO operations described below; source +descriptors also use the generic native C ABI. Standalone Rust recognition +requires a separate producer and consumer qualification. + +* Acceptance gates + +- Build the generic Rust runtime with no first-party language crate linked. +- Build and test exactly one first-party language selection and the independent + record assignments fixture without a Gerbil installation at parse time. +- Build one independent external adapter against the minimal event catalog. + Reject wrong kind categories, unbalanced events, overlapping or missing + source bytes, invalid UTF-8 boundaries, and malformed provider identity. + Bind and verify the provider artifact digest at build time. Verify + that its receipt names the external authority and never impersonates native + Grammar IR. +- Build two selected languages together and verify that identities, kind + catalogs, and receipts do not cross; inspect the resolved Cargo feature and + dependency graph for each build. +- Regenerate a custom language from its public Scheme declaration and reject + stale generated tables, wrong grammar digests, invalid kind references, + malformed source, and unsupported Parser IR constructs. +- Compare source bytes and structural CST projections with the Scheme parser + on the admitted native corpus before claiming AOT parity. An external + provider has its own independent conformance corpus. Measure binary size + and cold build cost for minimal and selected-language builds separately. + +The language feature packaging is owned by the Rust build support. This RFC +fixes the common interface it must preserve; it does not add language cases to +the generic engine. + +* Current evidence + +The generic runtime depends on the artifact index library and SHA-256 hashing. The standalone records fixture resolves only the generic +Rust runtime crate as its direct runtime dependency and passes its two Cargo tests. +The external adapter's two focused tests cover source-exact output, external +receipt identity, invalid UTF-8 token range, and malformed provider digest. +The event sink separately tests root, kind, and source-coverage rejection. +An independent ASCII digit parser fixture builds outside the root workspace, +derives its provider digest from its parser table in =build.rs=, consumes that +same table when recognizing a digit run, and passes its lossless-tree/receipt +test. Production providers must apply the same artifact-binding gate to +their actual parser artifact. The optional language-support feature crate, +two-language selection gate, and production-provider conformance remain +acceptance work. + +** Source result construction through POO slots + +The three pack interfaces remain =grammar.ss=, =parser.ss= and +=parser-test.ss=. Source packs declare their result operations in the grammar: + +#+begin_src scheme +(defresult-profile (pack-results :: self ResultProfile.) + (scanner pack-scanner) + (nodes (File item) (Item text)) + (tokens item-text unparsed-source)) +#+end_src + +=nodes= declares each node constructor and its legal child field names. +=tokens= declares generated terminal kinds. =scanner= admits the existing +=ScannerProfile.= and derives its terminals; the author does not repeat that +list. These are ordinary POO value slots. A declaration can inherit a profile, +replace =nodes=, =tokens=, =scanner= or =projections=, or assign an extended list through +=.o=. There are no user thunks and no callback result builders in this facade. +The same =ResultProfileContract= checks inherited and replaced values. + +The bound engine owns immutable copies and indexed operations. Its node builder +rejects undeclared kinds/fields, invalid child values and children outside the +parent byte span. Its token builder rejects undeclared terminals and invalid +byte bounds. Published catalog copies cannot mutate the execution plan. +=source-language-result-catalog= derives syntax kinds, fields and terminals +from those operations; the native encoder has no Bash-specific symbol table. +The operations participate in source identity, without changing the source or +Loader schema versions. This is a construction contract, not recognition rule +order, field multiplicity or a parser algorithm declaration. + +=ShellSourceStrategy.= requires a =results= slot; the common shell compiler +produces it from Bash's =deflanguage= syntax/rules declaration. +=LineSourceStrategy.= derives its default result profile from =root-kind= and +=token-kind= and permits a POO override of =results=. Both use the same node +and terminal admission path. =generate-language-pack= accepts the +source descriptor export through the existing generic handle registration; +no builtin language dispatcher or handwritten C transport is needed. Source +parsing currently executes the Scheme engine behind the C ABI. Standalone +Rust Word recognition is qualified below; standalone Command/Source parsing +remains an open producer obligation. + +** Internal source compilation products + +The profile constructors below describe internal source compiler and executor +contracts. They are not language author declarations and are no longer exported +by =language-source-support=. Authors use =deflanguage= syntax/rules through the +common library; see [[file:../research/scheme-bash-source-frontend.org][the complete Bash frontend migration]]. + +** Executable part recognition slots + +=PartProfile.= owns =contexts=, =rules= and =literal= value slots. +=defpart-profile= creates the same ordinary POO object as the other profiles; +a child replaces =rules= or =literal=, and =.o= can extend inherited rule data. +No procedure, matcher callback or user thunk is admitted. + +#+begin_src scheme +(defpart-profile (template-parts :: self PartProfile.) + (contexts word HereDocument) + (bindings (identifier (.o (:: self BindingProfile.)))) + (rules ((word HereDocument) (prefix "@" next) + (name SimpleParameter identifier))) + (literal LiteralPart)) +#+end_src + +Each row names its applicable contexts, a prefix and a following-character +condition (=any=, =next= or a string of allowed characters), and an operation. +Rows retain declaration order within a context; this is explicit recognition +precedence, including intentional shorter-prefix overrides. Operations are +=quote=, =quoted-body=, =pair=, =parameter=, =name= and =escape=. Quote/pair +operations declare opening/closing widths; =name= refers to a named +=BindingProfile.= value. Its =name= slot uses the existing closed TextProfile +DSL, including numeric runs, character classes, bounded sequences and guarded +branches. =parameter= uses the required =parameter= binding. The engine owns +the constrained parameter/result projection. + +Binding checks required contexts, operation fields and terminals, quote +contexts and delimiters, and pair prefixes/closing widths against the admitted +ResultProfile and RegionPlan. It owns declaration copies and compiles ordered +first-character buckets per context, with direct indexes for ASCII and a +separate index for other characters. The same matcher drives both literal +boundaries and part selection. A name prefix followed by an invalid name still +ends the preceding literal; the following literal retains that prefix. This +preserves lossless grouping for inputs such as =a$%b$x=. + +=ShellSourceStrategy.= now requires =parts= alongside =regions=, =scanner= and +=results=. The common shell compiler derives Bash's part profile and executes the +compiled plan through the shared Word, assignment and here-document paths. +The rule recipe participates in Source identity; schema versions do not +change. The independent template configuration above executes the same part +operations in tests. Parameter/assignment name, prefix, operator and subscript +recognition now comes from binding values. Command recognition and complete +standalone Rust Source recognition remain open obligations. + +** Binding values and scoped embedded regions + +=BindingProfile.= has four value slots: =name= (nonnullable TextProfile IR), +=prefixes=, =operators= and =subscript=. =defbinding-profile= constrains the +corresponding declarations. POO inheritance can override any value; no +procedure or thunk is accepted. =PartProfile.bindings= associates names with +these objects. A =name= operation chooses its declared binding. The required +=parameter= binding supplies parameter prefixes/operators and optional array +index delimiters; the optional =assignment= binding supplies identifier and +operator recognition at command positions. Assignment bindings require +operators and exclude prefixes/subscripts, matching their execution role. + +#+begin_src scheme +(defbinding-profile (assignment-binding :: self BindingProfile.) + (name (seq (run (union (alphabetic) (characters "_")) 1 1) + (run (union (alphabetic) (numeric) (characters "_")) 0 #f))) + (prefixes) (operators "=" ":=") (subscript #f)) +#+end_src + +The engine compiles names with the existing TextProfile implementation and +prefix/operator sets with the existing literal trie. Both now accept an +explicit local end bound. The trie retains the longest accepted prefix inside +that bound, even when a longer candidate exists beyond it. A parameter prefix +requires a following body character. Every binding recipe is owned and enters +Source identity with the other admitted operations. + +Array indexes use an engine-derived region context. Added index delimiters +apply in index frames; ordinary substitution frames retain their declared +nesting policy. Thus =$(echo [)= inside an index does not create an unmatched +array frame. The ordinary command scanner is unchanged. Closed pair captures +are indexed by the actual declared opener row and source offset, avoiding +collisions between overlapping prefixes; repeated nested captures use their +stored endpoint. Quote and pair caches remain independent. + +=PreparedTextProfile= exposes the existing Rust matcher as a reusable bounded +recognizer. Preparation admits a nonnullable static profile once. Matching +returns an absolute byte end or =None= for no match/invalid UTF-8 boundaries; +it borrows a bounded source slice and allocates no intermediate string. The +Scheme producer lowers the production Bash simple/parameter/assignment name +recipes through the same TextProfile emitter and generates their boundary +controls. CI regenerates this artifact and the Rust executor checks every +control. This qualifies binding-name AOT; complete Word/Command Source AOT, +Command execution remains an open obligation; Part/Binding composition is qualified below. +Result projection and scoped-region execution are qualified below. + +** CommandProfile roles and projections + +=defcommand-profile= constructs an inherited =CommandProfile.= POO value. +The =roles= slot declares every closed command role exactly once: trivia, +separator, redirect, here-redirect, strip-redirect, reserved, +conditional-operator, pipeline, and-or and case-end. Each role contains token +clauses =(terminal literal ...)=; a clause without literals accepts that +terminal kind. Role matchers bind once when the engine prepares the source +strategy; execution uses owned token-kind/literal indexes. + +The =texts= slot declares =(descriptor TextProfile)= using the existing +nonnullable TextProfile DSL. Full-string admission keeps =123x= and +={name}tail= from becoming file descriptors. There is no user procedure slot. +The =projections= slot maps closed engine node roles to declared ResultProfile +kinds. The base value derives identity projections for all command shapes; +ordinary grammar packs omit this clause. POO inheritance can replace the +value with a complete mapping, and loader admission checks required fields +before installing the parser. Unknown roles, duplicates, callback values, +undeclared scanner terminals and missing projection fields reject. + +A shell strategy now requires =commands:= together with =scanner:=, =parts:=, +=regions:= and =results:=. No fallback builds a former command profile. +The owned command recipe participates in Source identity under the existing +Source v1 schema; loader and C ABI versions stay unchanged. Effective inherited +slot values are compiled internally and require no thunks from the author. + +Command result construction reuses engine recognition sequences: compound +children append in constant time and materialize once per published node. +Deferred here-document marker receipts use a two-stack FIFO, preserving lexical +order and independent quote policies without copying the existing marker list. +This layer declares token roles and node projections. Control-flow grammar +execution remains in the engine; complete standalone Rust Command execution still requires lowering those +grammar operations to shared IR. Word execution is qualified below. + +** Closed control programs + +=CommandProfile.= adds a =forms= value slot. =defcommand-profile= now requires +=(forms ...)= after =texts=, before the optional =projections= clause. There +is no legacy macro form and no callback slot. Effective inherited forms are +copied, admitted and included in the Source recipe using the existing schema. + +Each form is =(id selector priority node-role operation ...)=. Literal +=word=/=operator= selectors compile to token-kind/literal indexes; bounded +=lookahead= and =adjacent= selectors handle implicit function headers and +balanced starts. Higher priorities decide literal versus lookahead selection. +Duplicate literals/lookaheads reject. =manual= entries provide the constrained +array and pipeline-header hooks. =array-tail= declares its opener in the +manual selector; =pipeline-prefix= may contain zero or more guarded consuming +operations. Its FIRST set skips the interpreter on ordinary commands. + +| Operation | Engine meaning | +|---+---| +| =(take field trigger)= / =(raw field)= | Consume a matching/raw source token | +| =(word field)= / =(command field)= | Invoke the admitted Word/Command capability | +| =(list field terminator allow-empty?)= | Parse an ordered compound list with typed termination | +| =(call field form-id)= | Invoke a nonnullable declared form | +| =(node field role operation ...)= | Build a nested admitted node | +| =(as role)= | Select a compatible projection for this branch | +| =(optional trigger ...)= / =(branch trigger yes no)= | Guard or select a program body | +| =(many trigger ...)= / =(until trigger ...)= | Run a consuming repeated body | +| =(choose (trigger ...) ...)= | Select an ordered declared alternative | +| =(balance open close depth open-field close-field ...)= | Match balanced nested tokens with a consuming body | + +Triggers are closed token predicates, role references, ordered =or= values, +and lookahead/adjacency values bounded to 32 tokens. All fields resolve against engine +node shapes before execution. Unknown opcodes, fields, form references, +cyclic direct calls, zero-consumption repetitions and invalid projections +reject during loader admission. Call-graph DFS memoizes visiting/done states, +so shared callee bodies are visited once rather than expanding every path. +Runtime repetition additionally checks cursor +progress, and missing terminators reject at EOF. + +If/elif/else, while/until, for/select including arithmetic headers, case +clauses, explicit/implicit functions, conditionals, arithmetic commands, +groups, array tails and pipeline prefixes all use this program executor. +Their dedicated parser functions are removed. POO overrides can define an +independent Unicode loop syntax without editing the executor. A declarative +refactor gives failures new explanatory text; diagnostic codes, reason kinds, +source identity, rejected status and source retention stay qualified. +This is real Scheme execution of the closed IR. Full Rust control interpretation remains separate work; Word execution is +qualified below; Word result-projection IR executes in +the Scheme engine as described below. + +** Shared result-projection IR + +=ResultProfile.= has a =projections= POO value slot. A program declares its +identity, result kind, ordered typed captures and closed construction operations: + +#+begin_src scheme +(defresult-profile (template-results :: self ResultProfile.) + (projections + (atom Atom ((text span)) (token content glyph text required)) + (word Word ((parts parts)) (many fragment parts)))) +#+end_src + +Projection-only profiles derive node kinds, fields and token kinds. Existing +explicit =nodes= constrain legal fields; =tokens= supplies additional terminals, +and =scanner= derives scanner terminals. Different programs can construct the +same derived kind; its fields are the ordered union of their declarations. +Authors do not repeat a projection's catalog. Unknown operations, duplicate +programs or captures, invalid cardinalities and captures used with the wrong +type reject before execution. Each capture is consumed exactly once in order. +=token= consumes a character-span capture, =one= consumes a nested node and its +persistent token sequence, and =many= consumes an ordered list of parts with +its sequence. =token= and =one= declare =required= or =optional=; =many= admits +an empty part list. There are no author procedures or thunks. + +The engine resolves fields, terminals and capture indexes once at binding. +Single required-token and single part-list programs specialize internally; +other programs use the same closed executor. These are engine-generated +execution paths for admitted IR, not compatibility modes. Runtime capture arity, +character bounds, UTF-8 byte offsets and child containment remain checked. +Token sequences share immutable branches and materialize at public publication; +a nested projection never copies an expanding token suffix. + +Word, here-document line, assignment, literal/name/escape, quoted part, +substitution, parameter and array-subscript construction all execute this IR. +The word recognizer supplies typed spans and nested captures; it no longer owns +result-field names or terminal-kind names. =admit-part-plan= checks capture types and required/optional cardinalities +against the projection signatures required by its operations. An independent =@name= +language replaces Word, body, literal and identifier result kinds, fields and +terminals through the same macro, without modifying the engine. + +Projection recipes and returned copies are owned independently and participate +in Source identity. Source v1, Loader v2 and C ABI v2 remain unchanged. This +closes Scheme Word result projection; full standalone Rust Source control execution still needs a shared interpreter +and qualification. Shared Word recognition is qualified below. + +** Engine-owned fixture metadata + +The Loader's ordinary =fixtures= slot remains the authoring value. Its derived +=fixture-catalog= slot is sealed at the public macro leaf, including against a +prototype override. =LanguageLoaderContract= now admits the cached catalog and +its descriptor/origin ownership. Foreign identities, duplicate IDs, bad source +digests, unsupported expectations and non-fixture values reject during loading. +The engine neither runs a parser nor invokes a fixture callback at admission. + +The bound snapshot owns copied strings/lists, status partitions, counts and an +ID index. =language-loader-fixtures= selects all/accepted/rejected and publishes +copies; =language-loader-fixture-count= reads a stored count; +=language-loader-fixture= returns an indexed copy or =#f=. Queries do not filter +or rehash the full corpus. POO replacement/extension recomputes the effective +catalog, while repeated access to one object uses POO's cached slot. The named +fixture test service parses the owned snapshot and retains complete roundtrip, +expected status, root and required-node obligations. Test strategy selection and +fixture-catalog assertions use this same shared capability. + +HCL, openCypher and TLA+ remove every hand-written fixture partition and their +five exported group aliases. Their declarations remain corpus data in +=grammar.ss= and one =fixtures:= value in the Loader. All affected test and +benchmark consumers migrate directly; no compatibility names remain. Loader v2, +Source v1, native metadata v1 and C ABI v2 do not change. + + +** Shared Rust result projection execution + +=result-profile-rust-source= lowers any admitted POO =ResultProfile.= or bound +result plan through the existing Rust compiler. It emits an immutable +=ResultProfileSpec=: category-separated kinds, declared node fields, typed +span/node/parts captures and ordered token/one/many instructions. Kind indexes +are bounded before emission. Language directories gain no extra interface or +Rust implementation. Result field and terminal renaming remains declaration data. + +=PreparedResultProfile= admits the entire table once, including entries not +selected by a particular parse. =bind= resolves a projection during source +engine preparation; its =PreparedResultProjection= executes fixed instructions +without another identifier lookup. The bound handle owns its static table +identity and outlives the preparer, so a Source engine needs no self-referential +borrow. Captures move into private =ProjectedNode= +values. Child composition checks the exact result catalog, source borrow and +parent byte range. Span captures require UTF-8 boundaries, correct arity and +required/optional cardinality. Node and token spellings inhabit separate +categories, as in Scheme. + +Projected children retain their declared fields. Nested captures own their +subtrees without cloning accumulated token/event suffixes. An iterative walk +materializes events once for the existing source-index sink, which separately verifies +ordered, exhaustive token coverage. Projection construction does not decide +syntax recognition or recovery. + +The production Bash result profile and thirteen actual Scheme Word/assignment/ +heredoc captures form a reproducible cross-backend witness. Rust independently +executes the same construction instructions; tests compare every event, field +order and final source text. This closes portable result construction. Shared +Rust Part/Binding recognition is qualified below. Command program interpretation, +Source admission and end-to-end standalone parser receipts remain subsequent engine obligations. +No Loader, Source, native metadata or C ABI schema changes are introduced. + + +** Shared Rust region recognition + +The Scheme =region-plan-rust-source= producer emits an immutable =RegionSpec= +from the admitted region declaration. Scanner RegionWord and downstream +quote/pair decomposition share one explicit-stack execution algorithm and one +admission predicate; languages supply no Rust recognizer or callback. +=PreparedRegionPlan= admits static delimiters and pair scopes once, then borrows +independent full sources through =source=. Every offset is an absolute UTF-8 +byte offset. Escape advancement follows Unicode scalar boundaries, and word +stops use the pinned Scheme whitespace class. + +=PreparedRegionSource= lazily memoizes every nested quote/pair boundary closed +by a requested match. Pair keys include the opener prefix and source offset; +quote keys are separate. Scope restrictions apply to nested pairs, quote scopes +remain declared on the quote, and longest-prefix selection resolves overlaps. +Source preparation leaves unrequested quotes inert. Caches live with the source +and are never shared across sources. Scanner-only matching disables cache writes +because it publishes a word boundary without decomposing the interior. + +Scheme controls cover the production Bash declaration and a distinct Unicode +configuration with restricted nested scopes. Rust recognizes the actual strings +and compares exclusive boundaries and rejection outcomes. This qualifies region +recognition. Word/assignment/parameter composition is qualified below; Command +program execution remains the subsequent engine boundary. All loader/Source/C ABI +schema versions remain unchanged. + + +** Shared Word and Binding composition in Rust + +=part-profile-rust-source= admits the inherited POO PartProfile against the +ResultProfile and region declaration, then lowers binding TextProfiles, +prefix/operator vocabulary, scoped subscript regions and ordered recognition +rules to =PartProfileSpec=. The language still declares grammar values; no Rust +recognizer, callbacks or extra language interfaces are required. + +=PreparedPartProfile= admits all static tables once. It binds result signatures, +name recognizers and quote contexts, then prepares per-context ASCII first- +character dispatch with a Unicode map. Declaration order determines matching, +including shorter-prefix overrides. It does not choose the longest part rule. +Binding prefix/operator selection retains the declared longest literal match, +and a prefix requires a following body character. Nullable names, duplicate +rules, missing contexts, wrong projection signatures and conflicting derived +subscript scopes reject before source execution. + +=word=, =assignment= and =here_content= borrow a full immutable source and an +absolute UTF-8 capture range. Assignment returns no node for ordinary words. +All nested captures borrow that same source/catalog owner. Explicit recognition +and construction tasks compose parts, quotes, parameters, indexes, opaque +substitutions and assignment values without recursive Rust parser calls or +copies of accumulated token lists. Optional captures remain absent instead of +becoming empty required tokens. ResultProfile instructions move the completed +subtrees into the declared fields; ordered TreeEvents are published once. +Region matches that cross the requested capture boundary reject. + +The Scheme generator executes the production Bash profile and an inherited +POO profile with Unicode parameter prefixes, operators, index delimiters, +quote/escape openers and a shorter-prefix rule override. Rust independently +recognizes 110 bounded sources with nonzero byte origins. Complete events, +preorder fields, rejection/no-assignment outcomes and lossless source text are +compared. A 512-level parameter control uses the same explicit task stack. +This qualifies shared Word/Binding execution. Command interpretation and full +Source ParseArtifact admission/parity remain open, and no end-to-end parser +speed improvement is inferred from these controls. No schema version changes. + + +** Ordered catalog construction for Command integration + +=PreparedResultProfile.bind_node= binds a declared node and its allowed fields +once. =PreparedResultNode.build= moves ordered =ResultChildCapture= values into +that node. Unlike fixed projection instructions, this constructor preserves +interleaved and repeated fields in their supplied lexical order, matching the +Scheme =result-plan-node= contract. It validates UTF-8 parent/child ranges, +declared token categories and exact source/catalog ownership for nested nodes. +The shared result catalog remains the authority; no language-specific Rust node +builder or separate constructor schema is introduced. + +Scheme-executed catalog controls cover interleaved SimpleCommand children, +repeated Pipeline commands, Unicode fields and an empty root. The Rust tests +compare complete events and lossless source-index output. A separate integration +recognizes Words in Rust and moves them into Command and Source nodes through +one shared catalog. These controls qualify construction and composition only; +they do not establish Command syntax recognition or full Source AOT. The next +executor must preserve these ordered captures while interpreting command +control programs, then qualify complete ParseArtifact parity. + + +** Shared Command control and Source recognition in Rust + +=CommandProfile.= lowers through the engine-owned producer to closed trigger, +form and instruction tables. =PreparedCommandProgram= binds ordered result +constructors, descriptor text and literal/fallback selectors once. Its explicit +frame/task stack interprets =as/raw/take/word/command/list/call/node= and +=optional/many/until/branch/choose/balance=. Lookahead skips declared trivia; +adjacent selectors also require contiguous byte spans. Literal entries win +priority ties; fallback selectors retain Scheme's reverse declaration ordering. +Roles and all Source projection fields are admitted before execution. Duplicate +selectors, call cycles, foreign fields and nullable loop/call bodies reject. +Consumption widths follow a topological form order: a shared call DAG is not +recursively expanded during admission. A 32-level, two-edge-per-level DAG +control exercises this boundary. + +An engine-owned cursor consumes scanner tokens, recognizes Word/Binding values, +constructs command/list/pipeline/and-or and redirection results, and associates +heredoc bodies with their markers in FIFO order. Quoted marker bodies retain raw +tokens; unquoted bodies use the shared PartProfile. =CommandParse.events= inserts +scanner trivia at the same recognition boundaries as Scheme, validates source +order and publishes complete syntax events. Languages supply data; the internal +cursor/VM bridge is private and requires no downstream callbacks. + +The public =rust-runtime-support= facade exports +=generate-command-source-rust-module=. Its input is one admitted POO +=ShellSourceStrategy.= value, including inherited slots; its output is one +immutable =CommandSourceSpec= combining scanner, regions, parts, commands and +results. The Rust host prepares the product once and reuses it: + +#+begin_src scheme +(import :gerbil-parser/rust-runtime-support) +(generate-command-source-rust-module my-source-strategy "src/generated.rs") +#+end_src + +#+begin_src rust +let engine = PreparedCommandSource::new(&generated::SOURCE)?; +let parsed = engine.parse(source)?; +let events = parsed.events()?; +#+end_src + +The actual Scheme Source engine generates 54 accepted/rejected controls. +Production controls cover all 14 Bash forms and nested constructs; a second POO +value overrides Unicode command keywords, pipeline membership, root projection +and the nullable prefix declaration. Rust compares complete events, preorder +fields and marker/body links, then verifies lossless source text. Negative gates +cover malformed tapes, source/catalog ownership, selectors and cyclic/nullable +programs. Regeneration is byte-exact after rustfmt under the existing 90/5 fence. +These controls qualify standalone Source recognition and publication, including +its syntax outcomes; they do not establish full versioned ParseArtifact/hash/ +diagnostic parity or switch the public C ABI from the Scheme Source engine. +The cursor currently rejects command/list nesting above 256; replacing that +remaining Source recursion with an explicit task stack is a separate execution +obligation. Region and Word stacks retain their existing deeper-nesting controls. +No schema bump or end-to-end speed improvement is claimed. diff --git a/docs/architecture/0009-tla-sany-candidate.org b/docs/architecture/0009-tla-sany-candidate.org new file mode 100644 index 00000000..0d347d80 --- /dev/null +++ b/docs/architecture/0009-tla-sany-candidate.org @@ -0,0 +1,112 @@ +#+title: RFC 0009 — Independent TLA+ SANY syntax candidate +#+startup: overview + +* Contract boundary + +The published =tla-plus.native-layout.v2= grammar remains the syntax +contract used by POO Flow. Its namespace is unversioned and its identity +lives in =languages/tla-plus/grammar.ss=. Further SANY syntax work +uses =languages/tla-plus/grammar.ss= with a separate +=p4-draft= metadata version and =tla-plus.native-sany-candidate.p4= +contract. The candidate has its own parser entry and test suite. Neither +entry silently changes the other grammar's identity or accepted syntax. + +The candidate inherits the generic source-column junction semantics. It +adds variable-width module and separator borders, function-set expressions, +operator constants, =EXCEPT= updates, record-field selection, instance +qualification, infix operator definitions, symbolic qualified names, and +several arithmetic and relation operators. Record-set types with =:= +fields remain distinct from record values with =|->= fields. Temporal +leads-to is recognized as one =~>= infix operator. These are grammar-owned +constructs. The language-neutral engine only gained =character-run=, a +bounded lexical primitive for repeated source characters. + +* Evidence and limits + +The independent candidate test checks structural nodes and source-exact +round trips for each added family. The pinned =tlaplus/Examples= corpus at +commit =ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10= is a regression +source, with per-file accepted, rejected, validity, and round-trip outcomes. +The candidate now accepts all 424 files at that immutable pin, with zero +rejections, invalid artifacts, or source round-trip failures. The final +195 rejections were handled together, retaining all 229 previously accepted +files. The published layout entry and the candidate each retain all 28 +POO models. These counts describe this exact corpus and parser contract. + +The final slice covers nested and concatenated modules, preserved document +text, digit-prefixed identifiers, higher-order formals, symbolic operator +formals and definitions, LAMBDA, labels, subexpression selectors, LET +recursion and instantiation, proof statements, ASSUME/PROVE, and original +proof-step lexemes. A pure validator checks proof levels, QED completion, +which steps admit subproofs, and label parameter syntax. Proof levels remain +explicit in the CST rather than being erased as trivia. Embedded PlusCal +remains preserved comment text; this receipt does not execute or translate it. + +Three engine fixes account for the nested-list failures: the source reference +uses the markers inclusive final-character column; reductions under an +ineligible layout marker remain available when ordinary actions are absent; +and layout tables retain precedence-suppressed reductions behind a guarded +shift for outdent. Ordinary non-layout tables keep their precedence behavior. +The candidate uses a request-local speculative-depth budget of 4096, while the +runtime default remains 256. The 258793-byte TCP proof exercises the larger +budget; actual branch progress is emitted under =GERBIL_PARSER_LR_TRACE=. + +The repository-owned Scheme =corpus.ss= verifies the checkout pin, clean +tracked state, exact 424-file count, accepted syntax, artifact validity, and +original bytes. The Scheme =qualify.ss run= kills a check after five seconds +without child output and bounds the batch to ten minutes. The Scheme +=preload.ss= reports actual dependency admission; +artifact tracing reports decode, digest verification, and schema admission. +Child stdin is closed so completed noninteractive gxi scripts cannot wait +in the REPL. The native suite runs each test module in its own process, +retaining the 1 GB heap fence and existing performance gates without retaining +every module's expansion context in one heap. Both the corpus +and structural differential are required Linux CI gates. Import/dependency +and semantic checks remain separate oracle outcomes. + +A Scheme-owned structural differential runs the official v1.7.4 SANY +XML exporter against the candidate on temporal and arithmetic modules. It +verifies the pinned =tla2tools.jar= SHA-256 and compares seventy-two normalized +expression trees across =~>=, =/\\=, =\\/=, ==>=, =+=, =-=, =*=, =/==, and ==<=. +This check exposed a real =A + B - C= grouping difference; the candidate now +follows SANY's precedence for that expression. Six action-subscript trees +cover square and angle actions, tuples, a single name, a parenthesized tuple, +and the =[]= / =<>= prefixes. +The lexer follows SANY's =]_= / =>>_= tokens. Further cases cover local +function definitions, repeated LET definitions, weak/strong fairness, +shared-domain and tuple bindings, function/set constructors, and bounded, +unbounded, and temporal quantifiers. The comparison retains every bound +name, its domain, and the tuple marker, rather than comparing bodies alone. +Further operator cases cover sequence aliases, function merge/pair grouping, +and both explicit groupings of division and exponentiation. +Ten negative differentials require both parsers to reject =[A]x=, a +bounded temporal quantifier, ungrouped =/=, =\\div=, =^=, and =:>= chains, +an incorrectly indented junction body, missing QED, inconsistent proof levels, +and a forbidden subproof of HAVE; +a SANY semantic/import error does not satisfy +that negative gate. Two controls verify that missing imports and undeclared +identifiers fail the oracle without being counted as syntax rejection. +Tuple-binding support exposed a set-filter/set-map ambiguity in =Digraph.tla=. +The candidate now follows SANY's +=BraceCases/matchFcnConst= selection for that overlap, with ambiguous and +explicitly parenthesized expression regressions. +Aligned junctions exposed a different boundary error: both bounded quantifiers +parsed successfully, but a same-line =]_= to the right of the marker could not +close the list. The candidate now declares =)=, =]=, =}=, =>>=, =]_=, and +=>>_= as explicit layout closing boundaries. Eight structural cases compare +the resulting conjunction/disjunction lists, including nested lists and +function/set/tuple/action closers. The indentation negative verifies that a +closing boundary does not admit an incorrectly indented body token. +Seven more trees retain lists that end before a comma, colon, function arrow, +constructor arrow, or =THEN= / =ELSE= / =IN= keyword. Those spellings are +also declared by the candidate grammar as explicit boundaries. +Five CASE trees additionally check greedy final-arm arithmetic, function +application, record selection, explicit grouping, and multiple ordinary arms. +This remains a bounded structural gate. The 424-file syntax receipt does not +prove semantic equivalence or complete structural equivalence of every corpus +expression, proof hierarchy, operator precedence interval, or PlusCal program. + +The candidate is not a replacement for the layout grammar in POO Flow. +Language-pack authors select a descriptor explicitly and retain the grammar +metadata identity in syntax receipts. A future promotion requires a +separate contract review and a corpus with exact structural expectations. diff --git a/docs/architecture/0010-persistent-recognition-snapshots.org b/docs/architecture/0010-persistent-recognition-snapshots.org new file mode 100644 index 00000000..db2ab47e --- /dev/null +++ b/docs/architecture/0010-persistent-recognition-snapshots.org @@ -0,0 +1,600 @@ +#+title: RFC 0010 — Persistent recognition snapshots and explicit publication +#+startup: overview + +* Status and decision + +Research and implementation decision, 2026-10-03. The connected deterministic +capture/relative-position/certified-nonterminal-transfer/publication path is now +implemented in explicit capture sessions. Balanced repetition indexes, local +snapshot queries and equivalent generated streaming capture remain unfinished. +Measured runtime baseline: =21510a44cadb0d6dcdc3c2c4f973bcf9c372bc43=. +Topology benchmark coverage and its receipts are retained. The implementation +receipt below distinguishes captured structure from skipped recognition. +It does not replace the ParseArtifact v1 authority or introduce a public tree API. + +The structural work follows one dependency chain: capture reusable grammar +structure, retain it with relative coordinates, certify reuse during recognition, +and materialize the existing canonical artifact at an explicit boundary. +Changing action dispatch alone does not address the measured suffix replay. + +The completed grammar-construction work stays in place: grammar-wide prediction, +shared LR(0) prediction templates, kernel-first identity, and sparse LALR graphs. +Earlier matched complete TLA construction measured 46.5365 ms versus 15.229 ms; +that receipt is in the linked performance framework. It is construction evidence, +not a claim about parse throughput or incremental updates. + +* Cost model and authority + +Let B be source bytes, T significant tokens, E published events, K affected input, +and H affected grammar ancestry. Separate these costs: + +| Operation | Existing costs and ownership | +| Grammar generation | LR state construction, closure/template work, lookahead propagation; compiler/lr-automaton.ss | +| Fresh parse | Lexing, deterministic recognition or bounded ambiguity search, event construction, publication | +| Session edit | Source patch, token convergence, checkpoint selection, suffix recognition, event relocation, publication, receipt | +| Complete publication | Flat events, absolute offsets and token identities, whole-source SHA256, diagnostics | + +=src/runtime/artifact.ss= constructs a canonical alist with grammarDigest, +sourceDigest, sourceByteLength, status, events and diagnostics. Its sourceDigest +is SHA256 of the complete source bytes; events are an actual flat list. +Consequently full publication remains Omega(B + E), even if recognition becomes +local. =make-certified-window-artifact= still rebuilds the flat event result. +The incremental receipt's complete =reusedTokenIds= list can itself be linear. +Source patching currently also traverses UTF8 text. These are distinct costs, +not one LR-loop problem. + +RFC 0003 makes ParseArtifact the sole published authority. RFC 0005 governs +sidecar evolution. A private snapshot may accelerate production of that artifact; +it cannot silently become a different published authority. Merkle source identity +must not be substituted for the existing SHA256 identity. Lazy event objects must +not be returned in the v1 flat events field. + +Existing private recognition sequences already use O(1) concatenation ropes. +Adding a rope around semantic concatenation alone would repeat existing work. +What is missing is reusable grammar reduction structure and its admissibility +proof before semantic actions erase grammar nonterminals through pass/alias/inline. + +* Executable evidence + +| Family / size / insertion position | Fresh CPU median ms | Session CPU median ms | Remaining significant tokens | +| Arithmetic / 3200 / first | 6.805 | 10.442 | 6401 | +| Arithmetic / 3200 / middle | 8.579 | 6.743 | 3201 | +| Arithmetic / 3200 / last | 7.045 | 3.343 | 33 | +| Independent HCL attributes / 800 / first | 0.671 | 8.266 | 3204 | +| Independent HCL attributes / 800 / middle | 0.674 | 4.851 | 1636 | +| Independent HCL attributes / 800 / last | 0.689 | 1.751 | 36 | + +HCL first insertion relexes only 12 bytes and converges on 4800 suffix tokens, +but the recognition reuse count is zero. Token convergence is insufficient for +recognition convergence. Counters report the receipt's remaining-token measure; +they are not independently instrumented transition or reduction counts. + +HCL fresh parsing and captured sessions have different executor paths: generated +language parsing versus generic LR capture. This is an end-to-end API comparison, +not a matched inner-loop comparison. The architecture must cover both owners; +otherwise sessions can retain an executor penalty after subtree reuse is added. + +Arithmetic has a separate semantic constraint: inserting the first operand into +a left-associated expression changes its ancestor chain. Rebalancing operator +semantics is inadmissible. HCL sibling declarations isolate repeated independent +structure; they do not prove that the current grammar already admits balanced +reductions. A repeat transformation must preserve its published output exactly. + +Existing same-width generic-token edits, 400 to 3200 arithmetic terms, increase +cached best wall time from about 0.252 to 1.954 ms despite the event-window fast +path. This supports the publication concern; it is not CPU median evidence. +Prepared-token stage timings are a different executor path and cannot be summed +or used as percentages of the direct streaming fresh-parser total. + +** Incremental LR and balanced sequences + +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Wagner and Graham, Efficient and Flexible Incremental Parsing]] provides the +primary incremental-LR model. Its local-update complexity depends on maintained +structural information and balancing suitable sequences. Context-dependent +productions can invalidate locality. Transfer the reusable grammar fragments and +context checks; do not transfer a logarithmic bound to arbitrary CFGs, arithmetic +ancestry, or whole-artifact publication. + +** Maintained implementations + +[[https://raw.githubusercontent.com/tree-sitter/tree-sitter/master/lib/src/parser.c][Tree-sitter parser implementation]] validates reuse against parser/lexical +state, changed ranges, fragile or erroneous nodes and external scanner context. +Its useful seam here is reuse admission before ordinary recognition work. +Matching byte positions alone cannot certify it. Its public tree contract differs +from the existing flat artifact contract. + +[[https://raw.githubusercontent.com/lezer-parser/lr/main/src/parse.ts][Lezer LR implementation]] tests goto availability for a cached node and strict +context compatibility. Transfer the nonterminal transition check plus context +binding. A matching exported AST kind alone is insufficient: semantic actions +can erase the grammar LHS. Neither implementation is a drop-in executor for the +current Scheme-owned language machines. + +** Generation, ambiguity and recovery + +[[https://arxiv.org/abs/2209.08383][langcc: A Next-Generation Compiler Compiler]] is relevant to generated parser +control flow and grammar transformation. Evaluate complete language behavior and +build cost before transferring its generation strategy. It does not resolve the +flat-publication bound or supply current session reuse certificates. + +The current bounded ambiguity executor is not a graph-structured-stack/shared +packed-forest implementation. A future GSS/SPPF change needs its own equivalence +experiment: all viable alternatives, scoring, deterministic selection, budget +exhaustion and diagnostics. Reusing a selected AST alone can hide alternatives. +That work is outside the first deterministic snapshot admission. + +[[https://soft-dev.org/pubs/html/diekmann_tratt__dont_panic/][CPCT+ recovery study]] addresses repair quality. Synthetic token values and +semantic actions require an explicit recovery contract before adoption. Recovery +quality, search completeness and valid-input throughput are different metrics. +Recovered or ambiguous regions initially veto snapshot reuse. + +** Private source and recognition snapshot + +Keep a session-owned immutable source snapshot and a grammar-annotated forest. +Record production/LHS, subtree extent, executor binding, boundary context and +recognition evidence before executing erasing semantic projections. Represent +positions relative to persistent source spans. Unchanged suffix nodes then need +no per-node absolute-offset rewrite during private updates. + +Persistent internal identities are not canonical exported token or node IDs. +Export derives the existing IDs and absolute coordinates in canonical order. +Store the internal event projection in an indexed persistent sequence so local +queries and updates can avoid flattening. Retain grammar annotations separately +from public semantic node kinds. + +Balance only compiler-certified, context-independent repetition structure. Keep +left/right operator associativity and ordered events unchanged. Materialization +must reconstruct the original grammar/semantic fold; a balanced physical repeat +index is not permission to reassociate semantic expressions. + +** Reuse certificate + +Admit a cached fragment only when all applicable checks hold: + +1. Exact grammar and executor/control-table binding; grammarDigest alone does + not identify an executor representation or checkpoint schema. +2. Unchanged covered source and an exact relocation mapping; UTF8 byte boundaries + remain valid and zero-length fragments cannot stall traversal. +3. The current entry control admits the fragment's actual grammar nonterminal; + its exit transition and relevant lookahead agree. +4. Lexical mode, external scanner state where present, and layout/frame context + agree at boundaries. Edits inside comments/strings invalidate relevant context. +5. No ambiguity, recovery, error fragility or incomplete recognition proof is + hidden by reuse. Initially restrict to certified deterministic regions. +6. Projected canonical events and diagnostics retain their lossless ordering, + extents, identity rules and grammar-specific semantics. + +A mismatch falls back to ordinary recognition. Evidence must make the reason +visible in internal counters rather than reporting token reuse as tree reuse. + +** Publication and consumption + +Continue returning ParseArtifact v1 from current entrypoints. Explicit full export +traverses the private snapshot, derives canonical events and hashes the full +source. It remains linear and must be timed independently. + +A future local-query or snapshot-update public API could expose a lower-cost +consumption boundary, but it requires RFC 0003/0005 authority design first. +Until that exists, report only private recognition gains plus actual complete +API latency. Do not advertise sublinear end-to-end edits from internal counters. + +* Migration order and falsifiable gates + +These steps are dependent parts of one architecture, not independent tuning tasks. + +| Step | Owner / concrete change | Required gate before the next step | +| 1. Preserve structure | Compiler production metadata; generic LR reduction capture; generated parser equivalent metadata | Snapshot-to-v1 projection equals fresh complete artifacts, including aliases/pass/inline, nullable productions and ordered trivia | +| 2. Persistent representation | Session source spans, relative event/grammar positions, certified repeat indexing | Sequential edits and inverse edits preserve every event, byte offset, token ID, digest and diagnostic; bounded retained history | +| 3. Certified recognition reuse | LR entry/goto/lookahead checks and lexical/layout boundary evidence | HCL first/middle/last insertion/deletion shows real skipped recognition fragments, with fresh equality and explicit rejection tests | +| 4. Executor coherence | Generated-language and generic LR capture share snapshot invariants | Same workload reports executor identity, local update CPU and complete export CPU separately; no hidden session capture slowdown | +| 5. Broaden semantics | Additional languages, layout, recovery/ambiguity only with explicit proofs | Existing native conformance and owned language corpus gates; selected-tree equality alone cannot admit GLR reuse | + +At each step, run the existing relevant test modules and scenario infrastructure. +When runtime or compiler owners change, requalify the complete native suite and +applicable owned language corpus. The 53-module prior baseline qualification is +historical evidence, not validation of this proposed migration. + +Mandatory edit cases: repeated sequential edits, inverse edits, UTF8 boundaries, +length-changing and zero-width edits, newline/layout changes, lexical mode shifts, +comment/string boundary crossings, malformed input, nullable reductions and +same-kind nodes with incompatible grammar context. Include an ambiguity veto and +fallback equivalence before any selective reuse crosses ambiguity boundaries. + +Measure K and affected ancestry, actual reused fragments, shifts/reductions, +source bytes copied, events materialized, digest bytes, allocations, GC time and +retained snapshot memory. Report initial construction, local update/query and +full publication separately. Accept locality only on certified grammar families; +a flat full-export slope is expected, not a reason to change digest semantics. + +Stop or redesign if complete output changes, memory grows with unbounded session +history, certificates cannot distinguish erased nonterminals, or private gains +are hidden by executor/materialization overhead. A public snapshot API without +an explicit authority decision is not an acceptable shortcut. + +* Reproduction and evidence boundary + +#+begin_src sh +just benchmark-incremental-topology +just benchmark-incremental-siblings +# Boundary size and original mode compatibility: +gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss topology 2 +gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss topology-hcl 2 +gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss 400 +#+end_src + +The recorded topology sweep covers 24 arithmetic and 30 HCL cases; every case +passed complete-artifact and inverse-edit equality. Timings are five-sample +single-machine observations, not universal speedup claims or remote CI evidence. +The source metadata and corpus/construction receipts remain in +[[file:../research/scheme-parser-performance-framework.org][the existing Scheme parser performance framework]]. + +* Implementation receipt: deterministic grammar capture + +The generic LR capture part of the first dependency gate is executable in Scheme. +Equivalent capture inside generated direct-source/drive executors remains pending: + +- =src/runtime/lr-parser.ss= maintains an optional grammar stack parallel to LR + states and semantic values. Reductions retain production ID, exact prepared + runtime and actual generated/indexed reduction executor identity, entry/goto + state, child grammar fragments/token leaves, significant + token extent/count, lookahead kind/lexeme, cached semantic value and the exact + default reduction offset. It captures grammar information erased by semantic + projection; semantic AST kinds are not substituted for grammar nonterminals. +- Prefix snapshots persist the grammar stack alongside semantic values. Their + immutable list tails and retained child nodes share unchanged prefix structure. + They do not retain a historical complete source token list. +- =lr-recognition-project= evaluates the captured productions in iterative + postorder, without reading cached semantic values. It handles deeply left + recursive grammar structure without recursive Scheme descent. +- =make-incremental-session machine source #t= enables capture. The option persists + across resumed, fallback and event-window edits. Default sessions keep capture + disabled while generated-executor coherence remains unqualified; ordinary reductions avoid + fragment-construction calls and allocate no grammar nodes. +- =incremental-session-project-artifact= publishes the independently projected + root through the existing artifact owner using the session's exact token + objects. Token object identity is part of artifact admission; retokenizing the + source for this check would violate that pre-existing contract. +- GLR/fork and recovery continuations discard deterministic grammar evidence; + layout sessions do not produce this root. Event-only edit shortcuts clear the + stale root; a later captured LR resume rebuilds it from a valid saved prefix. + +The internal metadata and conformance projection live in runtime modules; they +are not a new public tree authority or a change to ParseArtifact v1 fields. +Positions in this first slice remain absolute. No suffix fragments are skipped +by this implementation, and no incremental speedup is claimed. + +=t/recognition-snapshot-test.ss= also covers nullable input, nested aliases/fields, +UTF8 and trivia, physically shared prefix pieces, successive edits, event-fast +path invalidation followed by recapture, deep left recursion, GLR and layout veto, +invalid-input veto and default capture-disabled behavior. + +#+begin_src sh +just benchmark-incremental-capture +just benchmark-incremental-siblings-capture +GAMBOPT=max-heap=1G,debug=q gerbil test -v 3 t/recognition-snapshot-test.ss +#+end_src + +Next gate: retain relocatable positions and canonical token/event projection +without copying every unchanged grammar node, then admit fragment reuse using +this captured runtime/production/control information. Flat publication remains +a separate linear cost throughout that migration. + +** Historical capture-only qualification + +Capture-only source built successfully with =GERBIL_BUILD_CORES=12 gerbil build=. +All 54 native test modules passed in nine bounded batches, each with exact +=MODULE-OK= counts, =HARNESS-OK= and final =OK=, without error markers. +This includes the new nine-case snapshot suite. The original 400-term benchmark +entry also completed its unchanged fresh-equivalence checks. + + + +This is local bounded native qualification. No remote CI run or push is implied. + +* Implementation receipt: connected relative-position and reuse path + +This implementation extends the capture-only slice rather than establishing +another published representation: + +1. Grammar fragments and semantic sequences accept immutable position views. + Repeated moves collapse the outer view. A zero-net displacement still records + source rebinding: returning to an old offset does not restore old token identity. +2. The source owner catalogs unedited prefix/suffix fragments with a precise + relocation map, source-token boundaries and captured lexical-mode evidence. + LR entry state indexes each position's candidates; incompatible ancestor + states are not linearly retried. Traversal prunes complete subtrees ending + before the resume point and subtrees with fewer than two significant tokens. +3. Before shifting a candidate's first token, deterministic pending reductions + are normalized. Admission checks the exact runtime, installed reduction + executor, entry state, actual grammar LHS goto/exit state, first token/mode, + following significant token kind/spelling and exact reduction lookahead offset. + A positive-width certified yield transfers one nonterminal; token convergence + alone never increments the recognition-reuse counters. +4. Transfer shares cached grammar/semantic subtrees through position views, + advances conceptual consumed-token counters and retains an immutable resumable + checkpoint at the transferred boundary. It skips internal LR recognition. +5. The existing artifact owner walks views, validates translated leaf spelling, + kind and byte extent against the current canonical token stream, and emits + those current token instances. IDs, flat events and full-source SHA256 keep + their existing contract. Independent grammar-action replay rebinds the same + current token stream and must publish the identical complete artifact. + +Admission is deliberately narrower than successful deterministic input: the +whole LR table must contain no fork, no dynamic/layout execution, and a lowered +concat recurrence for repeated structure. Short remaining source-token tails +(up to twice the existing checkpoint spacing) use ordinary replay; building a +catalog there costs more than the bounded replay. Arithmetic operator ancestry +stays on the existing path. These are performance-family admission rules, not a +claim that repetition proves arbitrary context independence. Every admitted +individual nonterminal still requires the complete certificate above. + +Catalog construction still takes O(T + visited grammar nodes), and the current +left-recursive ancestry can be linear. Each independent sibling transfer also +has constant-boundary probes and a checkpoint. No O(K log N) local-update claim +is established. Position views retain relevant immutable subtrees, not historical +session/source objects; retained heap size over a long edit stream remains a +separate measurement gate. + +** Rejected attempt and resulting algorithm change + +The first connected prototype skipped approximately 75 percent of significant +HCL sibling tokens but rebuilt/scanned all candidates without entry-state indexing. +At 800 declarations, first insertion was approximately 10.75 to 9.99 ms, middle +insertion 5.61 to 8.29 ms, and last insertion 1.47 to 6.31 ms. This was rejected as +a performance result. These diagnostic prototype observations do not qualify the +final implementation. The structural response was state-indexed candidate lookup, +spatial/size pruning and explicit short-tail/nonrepeat replay admission. + +** Reproduction + +The benchmark control changes only the transfer admission parameter. Capture, +executor, initial session, edit topology, heap and complete publication are the same. +Five CPU samples per case, explicit GC before each timed call, session creation +outside the timer; receipt construction remains inside. Preflight checks compare +complete fresh artifacts, independent forward projection, inverse artifacts and +independent inverse projection. Repeat the HCL 400/800 runs in reverse order to +check whether the observed direction survives ordering. + +#+begin_src sh +gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss no-reuse topology-hcl-capture 2 100 200 400 800 +gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss topology-hcl-capture 2 100 200 400 800 +gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss no-reuse topology-capture 2 400 800 1600 3200 +gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss topology-capture 2 400 800 1600 3200 +#+end_src + +** Remaining structural work, in dependency order + +- Make the reuse index persistent and suitable repeat structure balanced, with + a certified grammar fold. Current spatial pruning still traverses affected + left-recursive ancestry; it does not establish logarithmic update cost. +- Qualify equivalent capture in generated streaming executors. The HCL fresh + direct parser and generic captured session remain different execution paths; + their timing difference cannot be credited to the transfer optimization. +- Separate an explicitly admitted local snapshot/query interface from complete + v1 publication. Full event serialization, source patching, token projection, + digest and receipt lists remain linear. A faster private recognizer cannot + remove those costs without a separately decided contract. +- Expand to other deterministic repeat grammars only with owned corpus gates; + layout, external scanner state, dynamic precedence, ambiguity and recovery + require additional proofs and stay excluded from this admission. + +** Final matched measurements and native qualification + +For HCL 800, ranges below contain the two five-sample CPU medians, in milliseconds. +The control and enabled runs use the same capture executor and full API boundary: + +| Edit | Transfer disabled ms | Transfer enabled ms | Remaining significant tokens, disabled -> enabled | +| First insertion | 11.685–12.855 | 7.456–7.515 | 3204 -> 804 | +| First deletion | 9.745–9.840 | 7.270–7.288 | 3196 -> 799 | +| Middle insertion | 5.654–5.708 | 4.642–4.735 | 1636 -> 412 | +| Middle deletion | 5.579–5.627 | 4.528–4.605 | 1628 -> 407 | +| Last insertion | 1.465–1.488 | 1.462–1.469 | 36 -> 36 | +| Last deletion | 1.408–1.428 | 1.442–1.463 | 28 -> 28 | + +First insertion transfers 800 fragments / 2400 significant tokens; middle +insertion transfers 408 / 1224. The direction repeats in reversed execution order; +these are bounded single-machine observations. Control raw samples show substantial +spread (for first insertion, approximately 9.68 to 20.56 ms in the first pass); +enabled samples also vary. Report the recorded paired medians and repeated +direction, rather than a universal fixed percentage gain. Last edits admit no transfers: +small timing differences are not a recognition speedup. Arithmetic admits no +transfers and retains the exact remaining-token counts of the control. + +Certificate probes are separately counted: first HCL insertion probes 4800 bytes. +The existing relexedByteCount field does not include speculative certificate +probes. It must not be read alone as total scanner work. At this historical source head, boundary probing and ordinary scanning could +read a boundary token twice. The implementation receipt below completes the +bounded, context-bound probe handoff and measures it separately. + +HCL direct fresh parsing remains approximately 0.6 ms for 800 declarations, much +faster than the captured session path. The table proves the transfer improvement +within that session path; it does not establish parity with the separate direct +executor or qualify capture as the default API mode. Complete v1 publication +still materializes approximately 27,000 events and hashes the complete source. + +No push, merge or exact-head remote CI result is implied by this local receipt. + +** Rebase and primary-source bindings + +The working branch was fetched and rebased onto PR #13's head +=7c1533356585154c8b5d340f64ffefb289d18907=. Git reported that it was already up to +date; the rebase did not rewrite the existing implementation commits. This is +remote revision evidence, not remote CI qualification or push authorization. + +Primary implementation sources inspected for this continuation are immutable: + +- [[https://github.com/tree-sitter/tree-sitter/blob/20cf25c10f13ed9b8c499dd966b4e7a57bedbf81/lib/src/parser.c#L703][Tree-sitter token cache]], revision =20cf25c10f13ed9b8c499dd966b4e7a57bedbf81=. + Its cached token is tied to byte position, external-scanner state and a current + parse-state admissibility check. This supplies a context-checking pattern; + its external-scanner contract is not implemented by the current closed scanner. +- [[https://github.com/lezer-parser/lr/blob/ed59b8b9c0c26164d6483f4c881a8c200184894e/src/parse.ts#L105][Lezer TokenCache]], revision =ed59b8b9c0c26164d6483f4c881a8c200184894e=. + Token position, tokenizer mask, context and contextual-tokenizer behavior + participate in admission. Numeric mode IDs alone are not a transferable + context proof. The current runtime's immutable, interned mode objects provide + a narrower exact-identity key. +- [[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Wagner and Graham, sections 7 and 7.1]] requires associative sequence + representation and appropriate context independence for locality. An earlier + context marker can change an entire sequence's grammatical interpretation. + Generic grammar subtree rebalancing does not establish that guarantee. + +** Implemented source-probe handoff + +=src/runtime/probe.ss= holds one successful token result per edit request, bound +to the exact machine/source, character position, byte position and interned +lexical-mode object. =source-probe-scan= clears the previous slot before calling +the source scanner and records only a successful return. Exceptions still +propagate to the existing speculative veto handler; canonical scanning and parse +failure retain diagnostic authority. A source token returned by an unsuccessful +fragment certificate can still be consumed only when its exact lexical context +matches; fragment validity and token-scan validity are different proofs. + +=source-probe-take!= returns the exact scanned token object to ordinary source +consumption. The slot is cleared when consumed or when the cursor passes it; +intervening trivia can precede a future-position slot. It does not retain a +source-size cache or any cross-session history. This closes the boundary between +certification and canonical token consumption while leaving the nonterminal +entry/goto/lookahead certificate intact. It does not cache errors or admit +external scanner, layout, dynamic precedence or ambiguous execution. + +The default parameter is enabled only inside the already-admitted experimental +capture/fragment path. =current-lr-probe-reuse-enabled?= provides the same-runtime +control. Normal capture-disabled sessions allocate no probe cache. + +The receipt adds =fragmentProbeReuseTokenCount= and =fragmentProbeReuseByteCount=. +Probe/control rejection evidence is retained even when no fragment is accepted; +fresh fallback clears transfer/probe receipts. The historical =relexedByteCount= +field remains the token-convergence measure, not the complete scanner workload. +On successful benchmark cases, separately report the scanner token-yield extent +sum =relexedByteCount + fragmentCertificateProbeByteCount - fragmentProbeReuseByteCount=. +This is token extent accounting, not an instruction/character-probe count for +all internal scanner alternatives, and it does not include failed exploratory +reads discarded by fresh fallback. + +** Executable paper boundary and next structural gate + +=t/recognition-snapshot-test.ss= now compiles an =A c+ | B d+= grammar where both +=c= and =d= consume =X X= and export the same =Item= kind. The runtime contains +admissible repeat structure. Changing the distant marker to =B= nevertheless +rejects the old =c= fragment via its actual LR entry/goto context; ordinary parsing +continues successfully. Public node-kind equality cannot substitute for this +proof. The existing full-artifact/projector tests also compare cache enabled and +disabled with the same transfer counts and remaining significant-token count. + +The next balanced-repeat implementation needs an explicit grammar fold contract: + +| Required owner / artifact | Concrete gate | +| Grammar lowering / Parser IR | Preserve the explicit repeat origin, item operand actions and concat fold; distinguish associative list construction from operator alias reductions | +| Repeat representation | Group immutable items in a balanced physical index while preserving left-to-right field/trivia order and the original semantic fold | +| LR transfer | Certify the repeat segment's actual entry/exit control and prefix-semantic transformation; an old left-recursive production cannot be relabeled as a binary-group production | +| Independent publication | Replay the original grammar fold to the identical canonical flat artifact, IDs, offsets and SHA256 | +| Context / scale | Reject the A/B counterexample; compare sequential first/middle/last edits, construction cost, index work, retained memory and complete export separately | + +Balancing an auxiliary index can improve lookup yet leave linear grammar ancestry +reconstruction. Changing the actual production to a binary repetition can create +ambiguity and different LR tables. Neither is a proof of local recognition under +the existing deterministic certificate. A compressed, certified repeat fold must +be designed and qualified before claiming logarithmic updates. Operator ancestry +and the complete flat publication bound remain outside that locality claim. + +* Monotone grammar cursor: eliminate per-edit whole-forest enumeration + +The catalog transfer implementation above rebuilt candidate start/end tables and +significant-token prefix counts by enumerating the captured grammar forest on +every edit. The continuation replaces that catalog with a request-owned, +preorder cursor, following the admission/traversal separation in +[[https://github.com/tree-sitter/tree-sitter/blob/20cf25c10f13ed9b8c499dd966b4e7a57bedbf81/lib/src/parser.c][Tree-sitter's pinned reuse-node implementation]]. The existing LR certificate +remains the admission authority; a source interval alone is insufficient. + +The cursor retains a frontier of grammar nodes with cumulative relative offsets. +It skips complete already-passed subtrees, descends through edit-overlapping or +rejected candidates, retains future candidates until scanning reaches them, and +pops an accepted subtree without enumerating its interior. Old source-token +lookup advances monotonically; a rejected ancestor leaves the lookup index +available for a smaller candidate at the same start. Deletion-interior positions +map conservatively to the edit boundary, including whole-source deletion. + +=fragmentCursorVisitCount= counts inspected frontier frames, including position +views and repeated future peeks. It is not a unique-node count. The topology +benchmark records the original grammar-fragment count outside the timed region +and adds nested HCL sibling blocks to distinguish skipping block interiors from +merely transferring flat declaration leaves. Tests cover nested edits, bulk +removal/restoration, empty-source transitions and the context-dependent repeat +counterexample from Wagner and Graham. + +This removes whole-forest candidate enumeration, not all linear work. Token/mode +list-to-vector conversion remains O(T); accepted segments still bind real source +tokens; left-recursive affected ancestry can remain linear; the frontier uses +O(H) space. Flat event publication and whole-source hashing retain their previous +linear lower bound. A balanced repeat semantic fold remains a separate, explicitly +unimplemented step under the sequence-semantics contract above. + +** Qualified native cursor experiment + +Each flat/nested matched case uses 20 fresh and 20 cached samples, followed by +another complete matched pass. Initial session capture, forest counting, +explicit GC and completion printing are outside the timer. Complete artifact +publication and edit receipts remain inside. Every row checks fresh artifact +equality, independent forward projection, inverse edit equality and independent +inverse projection. Five-sample scale/guard rows cover flat HCL 2/100/200/400/800 +and arithmetic 2/400/800/1600/3200. All eleven batches ran serially with a 1G heap, +closed stdin, five-second output inactivity limit and 180-second batch limit. + +Representative complete-edit CPU medians (milliseconds): + +| Workload and edit | Catalog pass 1 | Cursor pass 1 | Catalog pass 2 | Cursor pass 2 | +|---------------------------------+----------------+---------------+----------------+---------------| +| HCL 800, first insertion | 7.809 | 5.543 | 7.498 | 5.048 | +| HCL 800, middle insertion | 5.206 | 3.302 | 4.843 | 3.240 | +| HCL 16 nested blocks, first insert | 9.043 | 3.825 | 9.065 | 3.692 | +| HCL 16 nested blocks, first delete | 7.678 | 2.408 | 7.594 | 2.394 | + +For the 400/800 flat cases, first/middle edits improve by 27.5–45.2% across the +two passes; short-tail cases do not establish a stable algorithm gain. In the +16-block nested case, first insertion improves by 57.7/59.3%, and first deletion +by 68.6/68.5%. Its initial forest has 28,354 grammar fragments; the cursor +inspects 567 frames for first insertion and 76 for first deletion. These are +frame observations rather than unique-node counts. Matched controls retain +identical reused/remaining significant-token counts: the gain comes from +candidate discovery cost, not a broader admission rule. + +The scanner-probe off/on control has mixed CPU differences and does not establish +an independent throughput gain. Its bounded token handoff and context checks +are qualified capabilities; no separate CPU improvement is claimed. Arithmetic +still uses the existing guarded path and does not acquire repeat transfers. + +Reproduce from a checkout whose runtime, language and benchmark sources match +=7cb1194=; the receipt commit supplies the runner and fixture: + +** Remaining structural work after the cursor + +1. Replace per-edit token/mode vector construction with persistent ordered source + storage. Measure setup allocation and lookup work separately from LR replay. +2. Introduce balanced repeat storage only with the sequence-origin and semantic + fold contract above. Preserve the context-dependent repeat veto and original + reduction semantics; tree shape alone cannot prove admissibility. +3. Separate persistent recognition output from flat artifact materialization. + The current complete-publication API and SHA256 identity still impose linear + work even when recognition transfer is local. + +The cursor removes one full-forest preparation pass. It does not close these +three gaps or establish logarithmic whole-edit complexity. + +** Follow-up: persistent source provenance experiment + +[[file:0011-persistent-source-provenance.org][RFC 0011]] connects measured token/mode storage to the existing grammar cursor +and explicit publication boundary. Its integrated immutable chunk storage passes +semantic checks, but 112 same-head native rows reject default admission for +complete-artifact CPU time. The source index is explicitly experimental and +defaults off; the existing vector/certified recognition path remains the measured +fast path. Mode churn on unchanged text and flat token binding are now separate, +falsifiable compiler/runtime research seams. Neither balanced repeat folds nor +local publication has been claimed complete. + + +* Scheme/ASP verification owner + +Execute correctness assertions through the shared native Scheme harness. +Performance admission uses =t/core-observability-test.ss=, +=t/gql-runtime-benchmark-test.ss= and =t/incremental-suffix-benchmark-test.ss=, +with their existing ASP =benchmark.ss= contracts. The repository's =Justfile= +selects diagnostic microbenchmarks when a specific algorithm phase needs study. +Workloads, samples and thresholds remain owned by those Scheme sources. diff --git a/docs/architecture/0011-persistent-source-provenance.org b/docs/architecture/0011-persistent-source-provenance.org new file mode 100644 index 00000000..34b124f3 --- /dev/null +++ b/docs/architecture/0011-persistent-source-provenance.org @@ -0,0 +1,222 @@ +#+title: RFC 0011 — Persistent source provenance through certified recognition +#+startup: overview + +* Decision and scope + +This extends RFC 0010's existing deterministic capture pipeline. The connected +path is source token/mode storage -> recognition certificates -> original +production replay -> canonical ParseArtifact publication. This implementation +changes storage and provenance while preserving grammar productions. + +The implementation uses an eager, chunked AVL rope. It remains explicitly experimental: matched whole-edit measurements reject +default admission for the current complete-artifact API. Normal sessions +still pass capture? =#f= and allocate neither this index nor a grammar forest. +The parameter =current-lr-source-index-enabled?= independently controls captured +sessions. The vector control uses the same compiled scanner, LR runtime, +grammar cursor, certificates and artifact publisher. + +* Algorithm families and actual transfer limits + +| Primary source | Transfer to this code | Limit and falsifiable gate | +| [[https://www.cs.tufts.edu/comp/150FP/archive/hans-boehm/ropes.pdf][Boehm, Atkinson and Plass, Ropes (1995)]] | Ordered immutable leaves, cached length measures, shared slices and copied boundary paths; source-index.ss | A balanced AVL join is used, rather than the paper's deferred balancing policy. Verify leaf identity, tree measures, UTF8 positions and complete artifacts after sequential edits. | +| [[https://www.cs.tufts.edu/comp/150FP/archive/ralf-hinze/finger-trees.pdf][Hinze and Paterson, Finger Trees (2006)]] | Additive token/byte/character measures separate physical sequence shape from semantic fold; source-index.ss and funcs.ss | Their persistent amortized bounds rely on laziness. This eager implementation cannot inherit those bounds or rebalance arbitrary semantic reductions. Test the existing context-dependent repeat veto and preserve projector equality. | +| [[https://rtheunissen.github.io/bst/docs/references/2012_bagwell_rompf.pdf][Bagwell and Rompf, RRB Trees (2012 preprint)]] and [[https://github.com/TiarkRompf/rrbtrees][author implementation]] | Chunk locality and a focused traversal cursor avoid repeated root lookups; measured native cursor access | This is a binary AVL rope, not an RRB implementation. Wide-node copying, prefix-size search and Gerbil's object costs require their own experiment. No borrowed constant-factor or radix bound is claimed. | +| [[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Wagner and Graham, incremental parsing]] | Retain recognizable subtrees and verify context at reuse boundaries; reuse.ss and lr-parser.ss | Source identity does not prove a valid LR transfer. Runtime, stack/state, original production semantics, lexical mode and lookahead admission remain mandatory. | + +The papers suggest distinct seams, not interchangeable implementations. Source +sequence balancing is lawful because token order and additive measures are +preserved. Rebalancing a recognition repeat changes a production fold unless +its sequence origin and original reduction order are retained. Arithmetic's +nested reductions therefore remain unchanged. A balanced token tree does not +close the repeat semantic contract in RFC 0010. + +* Storage and provenance contract + +=source-index.ss= stores paired token/mode vectors in leaves of at most 64 +slots. Internal nodes cache token count, byte span, character span and height. +A relative position view wraps shared storage; nested adjacent translations +collapse and zero translation returns the underlying piece. This uses UTF8 byte +positions for edits and character counts for scanner movement separately. + +Leaves and published nodes never mutate during parser operation. Fresh builder +buffers and request cursors mutate only before publication. This follows the +existing token-vector ownership convention; it is not defensive deep immutability +against callers mutating private token objects. + +The parser's private source-stream observer records fresh scanner tokens, +successful nonterminal transfers and ordinary lexical convergence. Accepted old +source ranges coalesce before tree slicing. Rejected candidate cursors are +copies and cannot consume the owner. Prefix storage is sliced from the previous +index. The next index is assembled from provenance while parsing; there is no +post-parse whole-token traversal to reconstruct it. + +Certificate scanning can hand a boundary token directly to the source driver. +The observer reconciles that token with the old source only outside the changed +byte interval, after exact kind, lexeme and shifted byte-span checks. This is +storage provenance for an already scanned token, not recognition admission. +Its actual captured mode must be retained even when the old token's mode differs. +Sorted sparse mode patches copy one bounded mode vector per touched leaf while +retaining token backing vectors. Adjacent source spans still coalesce. + +Certified event-window edits splice this index too, even when their grammar +forest is dropped. Resumed deterministic parses can capture a new forest later. +Failure, layout parsing and GLR without a complete deterministic source origin +do not publish a partial index. Source-index modes are checked against the +actual captured stream. Independently fresh parsing verifies token contents and +coordinates; historical/normalized modes need not equal those of a fresh path. + +* Why the first representation failed + +Sharing grammar fragments alone left scanned boundary newlines classified as +fresh input. On 400 sibling declarations, this made 401 small fresh token chunks +for a first insertion; 800 declarations made 801. Each boundary split the old +origin, causing tree joins and bounded leaf work repeatedly. + +Matching mode IDs as a condition for token sharing did not remove that pattern: +the scanner boundary path can record a different mode for the identical token. +The correction separates token provenance from mode updates, without dropping +the mode check during future lexical reuse. The source-index tests include an +actual mode 0 override across the 63/64 leaf boundary and prove the old version +is unchanged and token backing vectors are physically shared. + +A second locality issue was a root-based byte seek at every nearby declaration +boundary. The cursor now advances within its current chunk, using the tree +search only when the target lies beyond that chunk. Both experiments are +reported as intermediate failures, not qualified speedups. + +* Cost model and evidence boundary + +Let N be source tokens, C reachable chunks, F fresh tokens, R disjoint reused +origin ranges, P changed mode positions, and L touched mode chunks. + +- Indexed count and height use cached measures; byte seek outside the current + chunk descends a balanced tree and scans at most one bounded leaf. +- Slice and join copy boundary paths and bounded leaf descriptors. An unchanged + accepted source interval retains token backing storage. +- The builder assembles coalesced ranges rather than flattening all N old tokens + into two vectors for each edit. Fresh token chunks contain only F new entries. +- Sparse mode relabeling traverses affected paths and copies at most 64 slots + per touched mode chunk. P or L can still scale with the unchanged suffix. +- Certificates and flat publication still enumerate accepted tokens. Relative + token materialization allocates absolute tokens for published artifacts. + Counters describe builder storage operations, not total heap allocation. +- Source patching, checkpoint prefix lists, absolute event publication, complete + reused-ID receipts and whole-source SHA256 remain linear. Whole-edit time is + not logarithmic. +- Sliced backing vectors retain up to a bounded leaf's unused slots. Repeated + edits can fragment leaves; tree balance alone does not prove a live-heap bound + independent of edit history. No heap/resident-memory win is claimed. + +* Validation and matched experiment + +The source-index unit suite checks empty input, UTF8 translation, cursor trials, +sliced storage identity, coalesced sources, sparse modes and 180 deterministic +splices against an independently rebuilt flat reference. The integrated snapshot +suite checks real sequential edits, complete fresh artifacts, independent +production projection, captured mode equality, retained old versions, event +windows and the same-head vector control. + +The existing =t/benchmarks/incremental-session/benchmark.ss= now supports explicit +=source-index= and =no-source-index= switches. Topology workloads retain first, +middle and last insert/delete scenarios. History workloads apply six edits to +successive sessions; validation and index audits run outside measured calls. +Each timed call publishes complete artifacts. The history timing is the complete +six-edit chain, not a per-edit distribution. + +** Fixed-head results and admission + +Runtime/harness measurement head: =99dee96f43587bdb72125a982c2983c69e756fce=. +PR #13 was fetched/rebased again at +=7c1533356585154c8b5d340f64ffefb289d18907=; Git reported the branch up to date. +Both benchmark controls run native runtime modules from that same HEAD. + +All 16 jobs completed with exit zero, no watchdog failure and exactly 112 rows. +Each row has 20 CPU samples. Full artifact equality, inverse edits and source +mode audits pass. The two variants agree on recognition fragment reuse, skipped +significant tokens, resumed shifts, remaining tokens, grammar cursor visits and +probe reuse. The changed storage does not buy extra LR work skipping. + +Selected complete-edit CPU medians in milliseconds; history rows are SIX edits: + +| Workload | vector pass 1 | index pass 1 | vector reversed pass | index reversed pass | +| HCL 400 first insertion | 2.5550 | 3.6420 | 2.5290 | 3.6720 | +| HCL 800 first insertion | 6.0515 | 7.1870 | 4.8285 | 7.3190 | +| Nested 16 first deletion | 2.4375 | 3.7565 | 2.3830 | 3.7970 | +| HCL 800 six-edit history | 17.9635 | 26.2425 | 18.5220 | 25.5110 | +| Nested 16 six-edit history | 17.3645 | 22.9615 | 17.3235 | 22.8865 | + +The flat workload's 24 paired medians all regress. All eight history comparisons +regress. Three of the 24 nested comparisons have a lower index median; the best +first-pass ratio, 0.824 on the last deletion at 16 blocks, becomes 0.996 on the +reversed pass. That does not support a stable useful gain. + +The storage result remains real and narrower: the 400 first insertion creates +one fresh token chunk, shares all 2400 old source tokens and copies 38 bounded +mode chunks. The 800 case shares 4800 old tokens and copies 75 mode chunks. +Mode churn spans the unchanged suffix, so removing full token vectors does not +remove all suffix-sized metadata work. These counters are not heap measurements. + +Admission commit =8f003df06917853e2b31a5f0a3474fadd0eedcab= sets the source-index +parameter to =#f= by default. Explicit =source-index= experiments still exercise +the fully integrated path. Existing certified grammar cursor/probe reuse stays +available on the faster vector source path. No source-index CPU speedup is claimed. + +Receipts: + +The data pin measures the candidate before default admission was rejected; the +later switch changes the default, and benchmark variants explicitly override it. +The reproduction entrypoint records the calling checkout's clean runtime HEAD. + +** Implications for the next algorithm change + +The experiment falsifies the assumption that persistent token storage alone +makes this complete-artifact edit path faster. It does not isolate CPU time among +cursor access, mode patches, token relocation and publication. Their separate +phase timings must be measured before claiming a dominant cause. + +There are two concrete compiler/runtime seams to study next: + +1. Mode provenance: normalized grammar transfers can change boundary modes on + unchanged text. Exact numeric mode certificates are conservative and simple, + but source locality is not mode-update locality. Factoring lexical behavior + into compiler-proved equivalence classes could avoid some churn. Any such + quotient must account for literal priority, longest-match behavior, trivia, + accepted terminals and scanner closure; equal token contents alone are not + a universal mode certificate. Opaque scanners and contextual repeat vetoes + remain counterexamples until proved otherwise. +2. Recognition/publication: the current injector builds flat token segments and + binds absolute token instances even for reused grammar interiors. A future + local-recognition operation must retain certified token ranges and replay the + original semantic fold only at the required boundary. Generated scanners also + consume flat source strings today. Source-rope scanning, fragment range + binding and an explicit materialization operation therefore form one path; + hiding a flattening pass in any of them would invalidate local-update claims. + +The current API still publishes the complete authoritative ParseArtifact, with +whole-source SHA256. A private retained snapshot or local query is not a second +canonical artifact. Changing this boundary must preserve exact published events, +current token identities, original production folding and invalidation behavior. + +* Following the full optimization path + +1. Admit persistent source storage only at an explicit measured boundary. Treat + source/token sharing, mode copying and complete edit CPU as separate facts. +2. Retain sequence origins in grammar repetition and replay the original fold; + test the context-dependent veto before balancing recognition ancestry. +3. Expose local persistent recognition queries separately from complete flat + publication. Keep ParseArtifact v1 digest, token identity and event order as + authority, and measure the remaining whole-source work explicitly. + +This sequence keeps source storage, LR context, semantic folding and publication +connected. It cannot be completed by renaming the flat artifact or merely adding +a tree beside a later full rebuild. + + +* Scheme/ASP verification owner + +Execute correctness assertions through the shared native Scheme harness. +Performance admission uses =t/core-observability-test.ss=, +=t/gql-runtime-benchmark-test.ss= and =t/incremental-suffix-benchmark-test.ss=, +with their existing ASP =benchmark.ss= contracts. The repository's =Justfile= +selects diagnostic microbenchmarks when a specific algorithm phase needs study. +Workloads, samples and thresholds remain owned by those Scheme sources. diff --git a/docs/architecture/0012-lexical-context-quotients.org b/docs/architecture/0012-lexical-context-quotients.org new file mode 100644 index 00000000..e98fa5ca --- /dev/null +++ b/docs/architecture/0012-lexical-context-quotients.org @@ -0,0 +1,278 @@ +#+title: RFC 0012 — Compiled lexical context quotients at incremental boundaries +#+startup: overview + +* Connected pipeline and proof boundary + +RFCs 0010–0011 connect source provenance, certified recognition, original +production replay and canonical ParseArtifact publication. This experiment +addresses the lexical boundary in that same pipeline. It does not add another +parse authority. Its implementation is in compiler/machine.ss; ordinary +suffix convergence and certificate probing consume it in incremental.ss and +reuse.ss. The request-owned probe slot keeps the actual requested LR mode. + +An LR mode names a set of admitted declared lexical rules. Numeric mode identity +is sufficient for reuse but can reject equivalent scanner behavior. Declared +rule ordinals retain token identity, expression, precedence and declaration +order. Longest consumption wins; equal consumption uses precedence, then order. +Within one prepared machine, equal complete admission vectors therefore permit +sharing one prepared scanner closure. Arithmetic has eight modes and three +such plans; HCL has 33 modes and ten plans. + +This proves lexical behavior, not equivalence of LR stacks or original production +semantics. Fragment admission still checks the prepared runtime, state/stack, +original production transfer, lookahead and its existing lexical context. The +single request-owned probe slot retains its exact mode-object guard, including +its cross-runtime veto. External scanners that may consult changed source +prefixes still exclude ordinary suffix convergence. These guards are required +at their respective interfaces. + +* Primary algorithms, transfer and falsifiable limits + +| Primary source | Transfer at an existing interface | Limit and gate | +| [[https://github.com/tree-sitter/tree-sitter/blob/20cf25c10f13ed9b8c499dd966b4e7a57bedbf81/crates/generate/src/build_tables/build_lex_table.rs][Tree-sitter build_lex_table, pinned source]] | Associate parse states with lexical token sets, build shared lexical states and minimize afterward. Here: exact declared-rule admission vectors share preparation. | Tree-sitter also merges nonidentical sets using token-conflict, prefix and coincident-token checks. This implementation does not adopt that union or claim its complete DFA minimization. Compare every mode's complete token and next offset with separate preparation. | +| [[https://re2c.org/manual/manual_c.html][re2c official C manual]] | Maximal munch and declaration-order ties constrain which competitors a lexical quotient must preserve. Here: conservative first-transition candidates and guaranteed-acceptance dominance. | Our declared lexical precedence adds another tie condition. Retain rule identity and competing longer matches, and reject same-spelling different-token counterexamples. Unknown expressions remain possible candidates. | +| [[https://github.com/python/cpython/blob/v3.14.0/Objects/listsort.txt][CPython listsort.txt, v3.14.0]] | Exponential bracketing from a prior position followed by binary search in the bracket; the separate speculative source finger in reuse.ss | This is ordered point lookup, not a sorting implementation or its adaptive merging policy. Backward queries require a full lower bound. Comparison/call costs can outweigh the bound, so complete-edit timing remains the admission gate. | + +The Tree-sitter receipt pins revision +=20cf25c10f13ed9b8c499dd966b4e7a57bedbf81=, Git blob +=ef4d162c1178c04fda8b283a82b2f131fd3fe7a8= and source SHA256 +=e91a1ad27c7d30bd0c34ceb6df559d0af531ede2f34ea67a7809a93e12a2da1c=. +The exact source groups token sets before creating lexical states and then +minimizes transitions. No performance number from that project is transferred. + +** Conservative first-transition projection + +This projection still missed the common all-rule fallback. When the directed +mode has no matching candidates, generated-lexer scans with all rules to produce +the offending token or failure frontier. One mode can admit a newline directly, +while another obtains the same newline through that fallback. Candidate-mask +identity alone is too conservative for this actual executor. + +** Effective scanner projection with fallback dominance + +The final certificate normalizes the executor's effective candidates: + +1. Project admitted rules through P(c). If the result is empty, no directed + rule can match, so use the machine's common all-rule projection instead. +2. Separately derive G(c), a lower bound on guaranteed nonempty acceptance. + Closed run scanners and matching one-character literals can guarantee a + match. Incomplete strings/comments, numeric prefixes and external expressions + cannot supply this guarantee. P and G have opposite proof directions. +3. A pure fallback consumes exactly one character. Remove it only when an + admitted guaranteed nonfallback rule consumes at least one character and + wins a length-one tie by higher precedence or earlier declaration order. + Keep every other competitor, including possible longer matches. +4. Intern the normalized vectors. Equal canonical identities prove equal + scanner outputs and next character positions at this boundary. Any failed + directed scan that remains possible reaches the same all-rule scanner. + +First-character classes include both P and G, so characters with different +acceptance guarantees cannot accidentally share a normalization class. The +best guaranteed rule is computed in one traversal; dominance is not a quadratic +pairwise rule search. For M LR modes, C unique complete masks, D ASCII classes +and R rules, certificate preparation costs O(MR + 128R + CDR); metadata stores +shared per-class vectors for C masks and references them from M modes. Scanner +preparation itself occurs C times when sharing is enabled, instead of M times. +All certificates belong to one machine. No cross-machine numeric IDs are compared. + +Ordinary old-token reuse remains outside the changed byte interval and at an +exact shifted byte boundary. The old lexeme supplies c from an unchanged suffix. +Successful indexed reuse records the actual new LR mode when it differs; the +certificate does not claim historical mode IDs are identical. Fresh parsing, +full artifact equality and the independent original projector verify results. + +* Carrying the proof through speculative probes + +Ordinary source convergence alone did not reduce measured scanner work, even +when all HCL modes were proved equivalent at newline. The grammar reuser had +already scanned boundaries while certifying fragments, before ordinary source +convergence could supply a token. The same lexical proof must therefore be +consumed at that earlier source interface. + +A closed-source owner now admits old-source tokens to those speculative probes. +The byte must be after the new changed interval, map to an old token start after +the old changed interval, and have a compatible lexical certificate. Prefix +reuse is excluded: changing later characters can extend a longest match that +formerly ended before the edit. Source bytes after the suffix boundary are +unchanged; external prefix-dependent scanners remain excluded. + +=fragmentCertificateProbeByteCount= counts successful fresh scanner probe bytes. +=certifiedLexicalProbeTokenCount= counts old-source-certified probe requests, +including speculative requests; it is not a count of distinct published tokens. +The existing relex receipt describes source-stream coverage and is not replaced +with a fabricated zero when speculative scanning is avoided. The shared policy +parameter independently enables both consumers for matched controls. + +* Native qualification and matched measurements + +An attempted scanner-row extension failed because generated-lexer's ranked +normalization copied only the original seven fields from bare-rules. The added +metadata was dropped before certificate construction. The initial stale-cache +hypothesis was incorrect; a separately compiled nine-element vector also +preserved all nine elements. The final design places pure-fallback flags and +guaranteed-acceptance predicates in a separate AOT certificate column aligned +by declared rule ordinal, preserving the scanner's existing internal layout. +There is no legacy-row adapter. + +The complete matched run contains 120 admitted rows at runtime head +=27ec5c694336be773da581f2cc2c7ae0f28c2b96=. Twenty jobs exited normally with +unchanged five-second/180-second gates. Two output-complete jobs were killed +by the inactivity watchdog during native shutdown; each was retained as a failed +attempt and rerun once in a fresh process at identical pins. Their rows are +excluded. An earlier sampled shutdown showed =___cleanup_dyn_module= through +dyld =dlclose= waiting for debugger-unload notification; this explains the +observed failure class, but does not turn a failed job into a passing receipt. + +| Complete-request scenario | Strict median ms, repetitions 1 / 2 | Proof reuse median ms, repetitions 1 / 2 | Fresh certificate probe bytes, strict → reuse | +| Flat 800, first insert | 6.8470 / 6.7035 | 5.1835 / 5.0775 | 4800 → 0 | +| Flat 400, first insert | 2.7235 / 2.9690 | 2.5355 / 2.5035 | 2400 → 0 | +| Nested 16, first insert | 3.7845 / 4.1965 | 4.5395 / 3.8005 | 96 → 0 | +| Flat 800, six edits | 23.6595 / 23.6465 | 23.0965 / 23.2135 | 14406 → 0 | +| Nested 16, six edits | 21.7860 / 21.9555 | 20.4760 / 22.3175 | 300 → 6 | + +The 800-line flat first-insert gain repeats at approximately 24.3%. It does not +establish a universal gain: nested first insertion changes from approximately +19.9% regression to 9.4% improvement, and nested 16 history changes from 6.0% +improvement to 1.6% regression. Across 56 paired request medians, changes range +from 31.8% improvement to 19.9% regression; the median paired ratio improves +approximately 2.4%. This descriptive summary is not a statistical confidence +interval. The source-stream relex coverage is still 812 bytes for the flat +800 first insertion; zero denotes eliminated *fresh certificate probes*, not +zero source-stream coverage or zero parser work. + +Fresh stateless ParseArtifact medians in the two enabled 800-line first-insert +rows are approximately 0.6150 / 0.6155 ms, while the incremental requests retain +reusable session state as well as the complete artifact. No speedup over those +fresh parses is claimed. The lexer proof does not close the complete incremental +publication cost. + +Admission commit =384255e= leaves runtime lexical-plan reuse disabled by default. +Explicit =plan-reuse= controls still exercise both proof consumers; equal LR +mode reuse and prior fragment admission remain available. Scanner sharing stays +enabled. ASCII certificate tables are now machine-owned promises: full-plan +identity does not force them, and the first ASCII query requiring stronger +equivalence prepares and caches the table. Normal parsing and disabled reuse +therefore avoid the unused experimental table's construction cost. The +all-mode-pair tests force the lazy table and verify its actual outputs. + +The final preparation gate has 16 rows at +=7fb9cba974b3ab5d855c436e89713949c1065277=, with twenty samples, reversed +control orders and normal exits for every job: + +| Factory scope | Family and mode plans | Separate median ms, repetitions 1 / 2 | Shared median ms, repetitions 1 / 2 | +| Scanner only, proof promise unforced | Arithmetic 8 → 3 | 0.0870 / 0.0870 | 0.0680 / 0.0680 | +| Scanner only, proof promise unforced | HCL 33 → 10 | 0.4435 / 0.4420 | 0.2035 / 0.2030 | +| Scanner plus first proof-table force | Arithmetic 8 → 3 | 0.1595 / 0.1550 | 0.1355 / 0.1320 | +| Scanner plus first proof-table force | HCL 33 → 10 | 0.6415 / 0.6320 | 0.3975 / 0.3905 | + +Default scanner construction improves approximately 21.8% for arithmetic and +54.1% for HCL against separate preparation on the same head. These are factory +measurements, not whole-parser startup or complete-edit improvements. Forcing a +fresh table still costs work; the stronger proof is computed once per owning +machine when first requested. + +The historical receipt loaded 57 native test modules and passed 315 selected +test cases in 19 normally exiting batches without watchdog events. A subsequent +suite-selection audit (RFC 0013) found that only 34 modules actually executed +cases: 22 exported names ended in =-tests= rather than gxtest's =-test= suffix, +and one language-pack wrapper registered no cases. Consequently this receipt +is not full-suite qualification. Its log hashes and the 315 actual passed +cases remain valid; the original sealed receipt is preserved. The five lexical-plan cases cover complete scan +outputs and offsets across modes, fallback precedence/order counterexamples, +Unicode shifted boundaries in vector/indexed owners, six-edit artifacts and the +independent original projector. Existing cross-runtime probe guards also pass. + +The edit data pin precedes admission and lazy preparation. Its controls already +explicitly override the runtime policy; the original warmed proof lookup and +source-owner algorithm remain unchanged. Final native tests validate the lazy +construction and disabled default, and the separate final factory experiment +measures the changed preparation boundary. No fresh full-edit CPU number is +attributed to a different source head. + +Clean qualification first explicitly compiles =t/fixtures/lr1-construction.ss= +and =t/line-structure-assertions.ss=. +The production PackageSpec excludes =t/=, so a clean library rebuild alone +cannot satisfy the reference/AOT tests' imported support modules. The failed +pre-repair missing-module errors and exit 42 remain environment failures, not +semantic passes. The complete import audit identifies these two bare local +module imports; source-relative quoted scenario imports are loaded by gxtest. + +On Darwin, qualification additionally reports actual dyld API operations through +process cleanup. The Homebrew system-shell launcher drops =DYLD_*= diagnostics; +the recipe preserves its literal =GERBIL_HOME=, =GERBIL_PREFIX= and Gambit path +mappings and invokes that same native executable directly. =DYLD_PRINT_APIS=, +documented by the installed =dyld(1)= manual, emits actual =dlopen=, =dlsym= and +=dlclose= operations. The previously twice-failed fourth batch then exited 0 +with a continuous real unload trace and the unchanged watchdog. There is no +timer heartbeat, fast-exit replacement or relaxed inactivity limit. CPU +measurements use their original executor configuration, separately from this +native qualification progress setting. +Preparation rows measure the generated lexer factory, including its certificates; +they exclude LR table preparation, grammar expansion and whole-parser startup. +The 120-row run prepared certificates eagerly. Final preparation controls +separately measure scanner-only factories and factories whose returned proof +promise is forced; deferring a computation is not eliminating its cost when +requested. +Edit rows measure complete ParseArtifact requests, including canonical events +and source digest. Preparation gains cannot be reported as whole-edit gains. + +Run from the corresponding receipt's recorded source head. Reproduction uses the existing flat/nested topology and six-edit history +scenarios, twenty CPU samples, both control orders and two repetitions. Controls +explicitly select =plan-reuse= or =no-plan-reuse= on the same compiled head. +Preparation controls select =no-plan-sharing= independently. Raw logs are sealed +by digest, process exit, row count, watchdog outcome and exact source/harness +pins. Only normally exited jobs are admitted. =--resume= validates exact source, +sample count, harness, fixtures, arguments and retained logs; it permits one +fresh retry per output-complete final watchdog failure, preserves that failed +log and its hash, and rejects incomplete-result or semantic failures. The fixtures report actual nested native-module loads as well as import/eval +operations. No timer heartbeat substitutes for actual child progress; five seconds of silence or 180 seconds per batch fails. The final native gate also reports real unload operations on Darwin. + +* Remaining structural limit and next experiment + +The source/provenance index in RFC 0011 stays experimental. Its balanced storage +and chunk sharing did not produce a matched whole-edit gain. Lexical equivalence +cannot repair that representation cost, and scanner equivalence cannot authorize +a different semantic reduction order. + +The current complete-result boundary still copies source, publishes a flat event +sequence with absolute byte positions, and computes the standard whole-source +SHA256 in artifact.ss and incremental receipts. A changed prefix changes SHA256's +subsequent chaining state; a cached suffix hash cannot preserve this contract. +An insertion before a flat event suffix changes all its absolute positions. +Returning that entire materialized suffix retains a cost proportional to its +output size. Recognition-local bounds must therefore be measured separately +from complete publication, without substituting a Merkle digest or dropping +canonical events. + +The next structural experiment belongs at original production replay and +publication: retain immutable recognition children and original fold order, +represent translated event windows internally, and materialize canonical rows +only at the existing publication boundary. Measure accepted local recognition, +original fold, event materialization and digest separately on the established +first/middle/last and sequential-history cases. Admit a new public local-result +interface only with an explicit contract and a full-artifact compatibility path. +This RFC implements lexical certificates; it does not claim that local-result +API or sublinear complete publication already exists. + +** Concrete next interfaces and falsification gates + +| Interface | Current retained authority | Next measured change and rejection condition | +| =lr-parser.ss: reduce-value= and =funcs.ss: recognition-sequence->list= | Original production order and semantic child sequence | Retain translated immutable event windows through original reductions. Reject any context-dependent repeat reassociation, altered field order or mismatch with the independent original projector. | +| =artifact.ss: recognition-events= and =make-success-parse-artifact/raw-event-tape= | Current source-token identity, trivia order, canonical IDs and absolute UTF8 offsets | Materialize translated windows once at publication. Require exact full event rows after first/middle/last insertion/deletion and inverse edits; distinguish event allocation cost from recognition cost. | +| =artifact.ss: artifact= and =incremental.ss: directed-edit-fields= | Standard source SHA256 and existing receipt schema | Measure UTF8 conversion and digest separately. Reuse only an owner-proven immutable source digest; caller-supplied mutable source/artifact validation cannot be skipped on an assumed identity. Reject any changed digest contract. | + +The independent =lr-recognition-project= is a validation oracle here; benchmark +semantic preflights run it outside the warmed timed edit samples. Its cost must +not be attributed to the timed request without an actual phase measurement. +The listed seams identify experiments, not a measured CPU profile or completed +local-result implementation. + + +* Scheme/ASP verification owner + +Execute correctness assertions through the shared native Scheme harness. +Performance admission uses =t/core-observability-test.ss=, +=t/gql-runtime-benchmark-test.ss= and =t/incremental-suffix-benchmark-test.ss=, +with their existing ASP =benchmark.ss= contracts. The repository's =Justfile= +selects diagnostic microbenchmarks when a specific algorithm phase needs study. +Workloads, samples and thresholds remain owned by those Scheme sources. diff --git a/docs/architecture/0013-semantic-sequence-publication.org b/docs/architecture/0013-semantic-sequence-publication.org new file mode 100644 index 00000000..f96d9c31 --- /dev/null +++ b/docs/architecture/0013-semantic-sequence-publication.org @@ -0,0 +1,207 @@ +#+title: RFC 0013 — Ordered semantic sequence publication +#+startup: overview + +* Algorithm and actual seam + +RFCs 0010–0012 reduce scanner and grammar replay work. The complete artifact +still requires every canonical event, current token binding, trivia and digest. +This experiment targets the remaining intermediate semantic children lists. + +[[https://homepages.inf.ed.ac.uk/wadler/topics/deforestation.html][Wadler, Deforestation, TCS 73 (1990), 231–248]] supplies the idea of +eliminating intermediate trees between a producer and its consumer. Its +syntactic transformation applies to a restricted function form. This runtime +implements an explicit producer/consumer fusion at a known seam; it does not +implement that general compiler transformation or inherit its performance. + +The semantic producer is =reduce-value= / =apply-operand-action= in +=src/runtime/lr-parser.ss=. The consumer is =recognition-events= in +=src/runtime/artifact.ss=. Previously each field/alias action flattened its +ordered recognition sequence before storing node/fragment children. Publication +then traversed that intermediate list. Generated HCL/arithmetic operand actions +also use the same =recognition-sequence-for-action= boundary; root acceptance +still explicitly materializes and validates the singleton root. + +The candidate retains the ordered sequence in multi-child fields and aliases. +Publication consumes leaves directly with an explicit traversal stack and a +cumulative position delta. It allocates neither an intermediate children list +nor relocated child/value wrappers for those leaves. Single-child fields still +materialize the singleton because nested field wrapping has distinct semantics. + +* Nonempty concatenation invariant and cost model + +=recognition-sequence-append= removes empty operands. Relocating an empty +sequence returns the empty sequence. Consequently a branch contains at least +two semantic children, and field cardinality is determined by shape alone: +empty list, singleton list, other list or branch. No count field is necessary. + +Both controls now retain the original two-field concatenation nodes. There is +no fusion switch read, cached summary update or extra summary allocation on +concatenation. Field/alias bounds follow only the relevant left/right edge with +a tail-recursive walker and compose position deltas. The bound cost is O(edge +depth), and an arbitrary proper-list leaf still requires its last element for +an end bound. Publication consumes the ordered leaves with an explicit stack. The walker +descends left in local variables and saves only the deferred right side, using +one pending vector/cons per branch instead of two. List materialization also +uses this walker; that common improvement is not isolated by the fusion switch. + +An explicit traversal stack avoids Scheme recursion on a left-associated +sequence. Position views compose translation without copying a subtree. +Publication remains O(events + tokens); source hashing and canonical output +remain linear. This experiment cannot remove that output lower bound. + +* Semantic contract and independent oracle + +- Original production and operand action order is unchanged. +- Empty fields, already-named singleton field wrapping, alias bounds and + existing nested fields retain their behavior. +- The event publisher accumulates sequence and value translations, validates + moved leaves against current tokens, and retains strict identity for unmoved + tokens. A view remains a moved view even when cumulative delta cancels to zero. +- Trivia interleaving, root clamps, event IDs, digest and artifact schema remain + the existing publisher's responsibility. +- =lr-recognition-project= explicitly disables fusion while independently + replaying original productions and rebinding current tokens. This preserves + a materialized semantic oracle for the candidate. +- Selective GLR explicitly disables fusion while constructing alternatives: + candidate tree equality must compare canonical children lists. A pre-fork + deterministic prefix is shared by all alternatives. The new scenario test + requests fusion and checks the established fork completion evidence. +- Recognition child fields have a public mutator. No object-identity event tape + cache or assumption of deep immutability is introduced. + +The experimental switch is =current-recognition-sequence-fusion-enabled?=, +initially false. Semantic children exposed by private runtime accessors may be +ordered sequences under this switch; callers needing a list use the existing +=recognition-sequence->list= boundary. + +* Reproduction and evidence + +Use the existing native incremental session matrix, with both variant orders +and two repetitions. Each measurement uses 20 CPU samples. The 112 rows cover +flat HCL 400/800, nested HCL 8/16 (80 declarations per group), first/middle/last +insert/delete, and six-edit histories. Fresh full artifacts, inverse edits and +independent production replay are checked outside timed samples. + +The existing five-second real child-output inactivity and 180-second batch +limits are unchanged. Only normal-exit completed rows qualify. Retained failed +attempts are not admitted as timing or test passes. + +Focused native qualification checks a 10,000-child left-associated sequence, +nested fields, positive/negative views, Unicode source edits, canonical LR +references and selective GLR evidence. Full qualification additionally runs +all repository native test modules. Suite selection and remaining qualification +gates are recorded below; a loaded module is not a passed suite. + +* Completed fusion matrix and admission + +Fusion is not admitted as the production default. Flat and nested results do +not show a stable complete-request benefit. In particular, the flat 800-unit +six-edit history is 19.1–33.0% slower, and the nested 16-unit history is +20.4–20.9% slower. The switch remains false. The common single-continuation +walker reduces pending frame creation by construction; the fusion matrix does +not isolate its production throughput effect. + +A separate comparison therefore builds the preceding production head +=44aab77710ce79fd3952588803650732360211b3= in an isolated archive/output +namespace and compares it with the current default, using the exact same +preceding-head benchmark source in both processes. Runtime/language/fixture +sources are checked against the archived Git tree. This comparison includes +all current default-path overhead; it does not attribute a combined result +to the walker alone. + +* Next structural seam + +The repository already provides +=make-success-parse-artifact/raw-event-tape= in =src/runtime/artifact.ss= and +uses it in =src/compiler/hcl-source.ss=. This publishes a flat +operation/name/offset tape, resolves trivia and assigns canonical IDs once. +The next falsifiable algorithm experiment is to lower original LR operand +semantics to that existing event-program seam and retain reusable relative +program fragments. Its acceptance conditions must include named singleton +field nesting, multi-child wrapping, source-token identity after relocation, +root clamps, exact canonical IDs, trivia and digest. Original production replay +remains the independent artifact oracle. Dynamic/GLR alternatives retain their +canonical semantic merge path until equivalence is separately proved. + +No direct LR event lowering or reusable event program is claimed implemented +by this RFC. A blind event cache indexed by mutable semantic child identity +would not establish the required contract. Full canonical output and source +digests still impose a linear publication cost. + +* Native suite-selection audit and remaining gates + +The native harness selects exported names ending in =-test=. Twenty-two old +suites exported only =-tests=, and the language-pack wrapper constructed an +empty suite instead of registering its six imported language suites. The new +sequence suite initially had the same naming error. These are now corrected; +qualification separately counts each module's started and passed cases and +rejects empty modules. The four new sequence cases actually execute, including +the 10,000-child case. RFC 0012's historical 57-module claim is corrected; +its 315 passed selected cases and original sealed logs are preserved. + +Performance thresholds remain 100ms p95 for 50 checked GrammarRole/Grammar +pairs and 150ms p95 for 100 complete GQL parses, with the original 20/40 +samples. The complete intermediate audit observes approximately 251ms and +321ms respectively. Independent processes do not close these gates: object +construction triggers the five-second real-output watchdog, while GQL exits +normally with a failed timing receipt. Several unrelated long-running host +processes were simultaneously consuming CPU. This observation does not prove +that contention explains every timing failure. No thresholds, sample counts, +GC policies or inactivity limits are relaxed, and no production speedup or +fully green native gate is claimed. The next required performance receipt is +an unchanged-gate run in an adequately available execution environment. + +* Default-path regression and correction + +The experiment had routed already materialized ordinary children through a +general sequence visitor, adding a callback and zero-delta composition for +every child even with fusion disabled. =recognition-events= now retains the +preceding direct list consumer whenever children are a proper list, and uses +the deferred-sequence visitor only for branches/views. Token proofs, event +ordering, nested fields, canonical IDs, trivia and digest are unchanged. +A fresh full default comparison and native audit qualify this correction. + +* Completed default comparison and overall decision + +| Six-edit history | Preceding CPU ms | Candidate CPU ms | Paired changes | +|------------------+------------------+------------------+----------------| +| Flat 400 | 10.153 | 10.489 | +5.2%, +1.0% | +| Flat 800 | 19.964 | 19.762 | -1.9%, -0.1% | +| Nested 8 | 11.508 | 11.587 | -0.7%, +2.2% | +| Nested 16 | 19.638 | 19.220 | -7.4%, +4.0% | + +Times in the table are medians of the two row medians; paired changes retain +each run order rather than using a ratio of pooled values. Across all 28 +workloads, 16 slow in both orders, one improves in both and 11 are mixed. +Several flat workloads still slow by approximately 2–6% in both orders after +the direct list consumer is restored. These small effects are not a sufficient +basis for claiming a throughput benefit. The single-continuation walker still +halves pending frame creation at each branch by construction, but that source +fact does not justify admitting the combined runtime change. + +Accordingly neither fusion nor the combined default-path candidate is +performance-admitted. Fusion remains disabled; the implementation and receipts +are retained in this local research branch, and no remote production update is +made. In particular, “fusion disabled” does not imply that the combined default +has no overhead. The remaining representation/action boundaries need further +work before publication. The raw LR event-program seam described above is the +next structural hypothesis, with the existing canonical publisher and original +production replay oracle as its required contract. No claim of implemented +LR event lowering, fully green native timing gates or remote CI is made. + +* Subsequent event-program implementation + +RFC [[file:0014-persistent-lr-event-programs.org][0014]] records the subsequent persistent LR event program implementation, +its runtime ownership correction and its measured evidence. The negative +sequence-fusion evidence above remains a historical receipt for its pinned +source. The new implementation has a separate performance admission boundary. + + +* Scheme/ASP verification owner + +Execute correctness assertions through the shared native Scheme harness. +Performance admission uses =t/core-observability-test.ss=, +=t/gql-runtime-benchmark-test.ss= and =t/incremental-suffix-benchmark-test.ss=, +with their existing ASP =benchmark.ss= contracts. The repository's =Justfile= +selects diagnostic microbenchmarks when a specific algorithm phase needs study. +Workloads, samples and thresholds remain owned by those Scheme sources. diff --git a/docs/architecture/0014-persistent-lr-event-programs.org b/docs/architecture/0014-persistent-lr-event-programs.org new file mode 100644 index 00000000..fe91092d --- /dev/null +++ b/docs/architecture/0014-persistent-lr-event-programs.org @@ -0,0 +1,167 @@ +#+title: RFC 0014 — Persistent LR event programs +#+startup: overview + +* Structural target and transfer from research + +The rejected sequence fusion experiment in RFC 0013 retained recognition +nodes and changed their children container. It did not remove the intermediate +semantic representation between LR reductions and canonical publication. +This experiment replaces that representation at the deterministic LR action +boundary: field and alias actions produce persistent event instructions. + +[[https://homepages.inf.ed.ac.uk/wadler/topics/deforestation.html][Wadler's deforestation]] motivates eliminating an intermediate tree between +producer and consumer. This implementation supplies a dedicated semantic +backend; it does not implement the paper's general compiler transformation. +[[https://lezer.codemirror.net/docs/ref/][Lezer's tree representation]] motivates compact instruction storage and +retaining reusable pieces by identity. Lezer TreeBuffer uses 16-bit quadruples; +this backend uses private Scheme records and full UTF-8 byte positions. No +16-bit packing or performance result from Lezer is transferred. The existing +LR runtime, entry/exit states, lookahead and lexical mode proofs remain the +reuse authority. An event program alone is not a reuse certificate. + +* Producer, storage, consumer + +=src/runtime/event-reduce.ss= implements the original pass/concat operand +actions using field frames, node instructions, concatenation branches and +position views. Tokens themselves are token instructions. A node value is +also its instruction, eliminating an extra opcode packet. Frame bodies are +shared; concatenation does not copy either operand. Fragment instructions +preserve a trivia boundary before an enclosing field closes. + +The backend freezes the field label when lowering an action. Recognition +children expose a field setter, so no identity cache indexed by those mutable +children is introduced. Legacy recognition values can be converted, but the +native event path directly produces programs. Legacy conversion is recursive +and is not claimed to be stack safe for arbitrary external trees. + +=event-program-walk= uses an explicit continuation stack. Its traversal is +O(instructions), preserves source order, and composes position views without +allocating translated token objects. Cancellation to zero retains moved +provenance. Program bodies are private; public constructors and visitors do +not expose mutable instruction storage. + +=event-program-events= in the existing artifact owner consumes the program +directly into the final canonical event list. No flat intermediate event tape +is constructed. IDs are allocated at publication, root bounds are clamped, +and trivia is interleaved at the same semantic boundaries. Existing artifact +schema, hashing and source ownership apply unchanged. Unmoved token operands +must match the current token by identity; moved operands must match kind, +lexeme and translated byte bounds. The final publication remains linear in +output events and tokens. It cannot remove the complete-artifact lower bound. + +* Runtime authority and fallback + +=current-lr-event-program-enabled?= defaults to false. The prepared-runtime +factory selects a separate runtime identity, sharing original LR tables, +action indexes and lexical modes. Checkpoints and fragments retain that +identity. Captured sessions bind the complete ParserMachine to that selected +runtime, including the token parser closure. This keeps the machine, +checkpoint and recognition fragment on the same exact runtime identity; +fragment admission retains its existing eq? ownership checks. Semantic +reductions do not read the switch per action. + +Only deterministic pass/concat runtimes are admitted. Layout, dynamic +production semantics and selective GLR forks retain their existing semantic +backend. A generic event reducer supports eligible runtimes. HCL additionally +installs a digest-checked generated =direct-event-step=, generated from the +same original step definition with field/alias actions lowered to programs. + +The original production projector calls the original materialized reducer. +It independently replays grammar reductions and current token binding; it +does not project the retained event program. Complete fresh artifacts and +inverse edits supply additional independent conformance checks. + +* Falsifiable experiment and qualification + +The existing full request benchmark covers flat HCL 400/800 declarations, +nested HCL 8/16 groups, first/middle/last insertion and deletion, and edit +histories. Both variant orders use 20 CPU samples per row. Physical assertions +require both an event root and the generated HCL event executor. Every edit +compares the complete artifact against fresh parsing and original-production +projection outside timed samples. Retained program identity is separately +checked by the native event-program suite. + +Measurements require all 112 normal-exit rows and preserved log hashes. +Five seconds without real child output and 180 seconds per test/benchmark +batch remain failure conditions. A partial build or loaded module is not a +semantic test receipt. Implementation alone does not qualify performance or +justify enabling the backend by default. + +* Current evidence boundary + +At source =e1cb12d=, focused native qualification selected three modules and +47 cases, all with normal exits and explicit completion. Native generated +section completion separately recorded 276 generated source sections and +manifest SHA256 =861003d6a37ebbc535a08363dadd7ead22f7c65f97f20e761c2fddd02275c471=. +The first focused run failed while loading a stale/missing native section; +that failed receipt remains preserved and is not counted as a pass. + +The subsequent correction binds the selected machine at captured-session +creation and adds a positive transfer/count equality test plus backend +persistence after changing the dynamic switch. ASP Scheme is upgraded from +=v0.1.2.1= to =v0.1.2.2=, whose annotated tag resolves to +=f5b7c009d2cab61144a8af18008a0bfba3056584=. +Updated native qualification and matched performance must be recorded below +before drawing a performance conclusion from the corrected implementation. + +* Next structural experiment if the corrected comparison remains negative + +The current lowering still produces recognition-child sequences, then +=children-code= visits those sequences to create event branches and field +frames. This removes late recognition-node replay, but keeps two sequence +representations and an extra producer traversal. No benefit follows merely +from calling the resulting representation an event program. + +A subsequent producer/consumer fusion experiment should share the semantic +sequence and event instruction DAG: token/program leaves, frozen field +labels, append branches with cached semantic bounds, and composed position +views. Alias reduction can retain the existing sequence as its body; field +reduction must preserve empty/singleton/named-singleton trivia boundaries. +The original-production projector remains independent, and existing +unrelated-runtime, lexical boundary, GLR and layout rejection proofs remain +unchanged. This is a proposed implementation seam, not an implemented +capability or measured speedup. + +That experiment must compare the same complete requests with equal transfer, +scanner and certificate counters. Its first gate is positive physical sharing +and complete artifact equality across Unicode, inverse and sequential edits; +its performance gate is the entire 112-row, two-order matrix. Microbenchmarks +of append or visitor dispatch alone do not establish request improvement. + +* Corrected complete-request comparison + +All 28 workloads were slower in both orders. Geometric mean request cost +increases were 24.3% / 23.2% for flat topology edits, 21.8% / 27.0% for nested +topology edits, 19.0% / 17.8% for flat histories and 21.0% / 21.9% for nested +histories. These are same-head experimental-backend comparisons, not +before/after production-default performance claims. The event backend remains +disabled by default; performance admission is rejected. + +The ownership correction restores certified transfers but does not make the +current event representation cheaper. The producer fusion experiment above +is the next concrete structural seam; it remains proposed until implemented +and measured. Generated HCL source consistency passed at this same source. + +* Whole native audit on the upgraded dependency + +The corrected package contract was separately rerun on its new commit: both +cases passed with normal exit and explicit MODULE/HARNESS/final completion. +Runtime, language and dependency source is unchanged since the whole audit. +The whole audit and this focused correction are distinct receipts; the +remaining observed gate failure is grammar-object construction time. + +* Subsequent producer/storage/consumer experiment + +RFC [[file:0015-unified-lr-semantic-event-sequences.org][0015]] records the shared LR/event sequence, bounded blocks, specialized +consumer and persistent AVL joins. Its independent 112-row result is mixed, +so it does not supersede the canonical default admission decision above. + + +* Scheme/ASP verification owner + +Execute correctness assertions through the shared native Scheme harness. +Performance admission uses =t/core-observability-test.ss=, +=t/gql-runtime-benchmark-test.ss= and =t/incremental-suffix-benchmark-test.ss=, +with their existing ASP =benchmark.ss= contracts. The repository's =Justfile= +selects diagnostic microbenchmarks when a specific algorithm phase needs study. +Workloads, samples and thresholds remain owned by those Scheme sources. diff --git a/docs/architecture/0015-unified-lr-semantic-event-sequences.org b/docs/architecture/0015-unified-lr-semantic-event-sequences.org new file mode 100644 index 00000000..a68ee5b1 --- /dev/null +++ b/docs/architecture/0015-unified-lr-semantic-event-sequences.org @@ -0,0 +1,259 @@ +#+title: RFC 0015 — One persistent sequence for LR semantics and events +#+startup: overview + +* Question and fixed comparison + +RFC [[file:0014-persistent-lr-event-programs.org][0014]] established that the first event backend added request cost even +when certified transfer and scanner counters matched. All 28 matched requests +were slower in both orders. This experiment targets the duplicated semantic +sequence construction, rather than changing dispatch syntax or relaxing proofs. +The production default remains canonical while qualification is pending. + +* Research transfer and its limits + +[[https://homepages.inf.ed.ac.uk/wadler/topics/deforestation.html][Wadler's deforestation]] identifies intermediate producer/consumer trees as +a transformation target, under restrictions on the input functions. We do not +implement that compiler transformation or inherit a speed claim. We specialize +the known field/alias LR algebra so the semantic sequence already is the event +sequence. This is an implementation inference from that structural principle. + +[[https://github.com/lezer-parser/common/blob/main/src/tree.ts][Lezer's tree implementation]] makes storage, traversal and reusable fragments +explicit. Its TreeBuffer stores 16-bit type/start/end/end-index quads. Our +immutable Scheme records retain full UTF-8 byte offsets, field labels, original +token references and position views; there is no port of the packing scheme. +The existing LR and lexical certificates still admit every reused fragment. + +* Representation and operation contracts + +| Semantic value | Shared instruction | LR operation | +|----------------+--------------------+--------------| +| Empty sequence | false | append identity | +| Unnamed token | original token | shift retains the token | +| Node/fragment | program value | opaque semantic singleton | +| Named child | immutable field frame | label frozen at construction | +| Multiple children | balanced append branch with cached start/end/height | both operands retained | +| Moved sequence | composed position view | body retained; zero-sum moves remain moved | + +Branch bounds are calculated once from operand bounds. Field construction +uses capped arity to distinguish empty, unnamed singleton, named singleton +and multiple children. Named singleton/multiple fields retain a fragment +boundary so the final trivia flush agrees with canonical publication. Alias +construction retains the sequence directly as its body. Neither operation +visits all children on the normal event path. + +A private semantic visitor is the explicit inspection/materialization boundary. +It treats nodes and fragments as leaves, preserves field names and moved +provenance, and constructs conventional recognition children only on demand. +The event publisher walks the same DAG directly and still allocates the full +canonical event list. Complete-output construction remains linear; this +experiment cannot remove the output-size lower bound. + +* Runtime, generated code and proof ownership + +The selected ParserMachine owns the selected LR runtime, lexical plans and +rebound token parser closure. The LR loop binds its semantic mode once; +event shifts retain raw tokens, and the original branch retains its ordinary +children. Generic pass/concat and generated HCL event reductions both use +=event-program-append=. The HCL generator derives this from the sole Parser IR. +GLR, dynamic semantics and layout remain on their existing admitted backend. + +The strict prepared-runtime and executor identities, entry/exit LR state, +lookahead, lexical modes and token-binding checks are unchanged. The original +production projector still uses the original reducer; it cannot validate the +new representation by replaying that same representation. Legacy child +conversion snapshots mutable field labels and is limited to explicit legacy +inputs, not used for normal event reductions. + +* Physical and semantic qualification + +The native suite checks alias body identity on a 10,000-leaf sequence, +cached bounds after composed/cancelled moves, and the representation of +actual captured HCL fragment values. Existing checks cover named fields, +mutable field snapshots, moved token proofs, UTF-8, inverse/history edits, +positive transfer count equality, cross-runtime rejection, canonical LR, +GLR/layout boundaries and full original-production projection. + +At implementation source =2d6011fac87303744b79229a237fadb515efa753=, +five native modules completed 60 cases with normal exits, explicit module, +harness and final completion. All generated native sections were complete. +The first development build failed on a bracket error and remains preserved; +it is not a qualification receipt. + +* Measurement and admission + +Use the entire existing 112-row matrix, 20 CPU samples, both variant orders: +flat 400/800 declarations, nested 8/16 groups, first/middle/last insert/delete +and six-edit histories. Require physical event-root/executor assertions, +complete fresh artifacts, inverse edits, independent original-production +projection and equality of every scalar transfer/scanner counter. + +Five seconds without real test/benchmark output and 180 seconds per batch +remain failures. Native compilation has its separately bounded silent C +phases. Local conformance, same-head backend cost, production-default impact +and remote CI remain separate qualifications. No default admission or speedup +is claimed before the results below are complete. + +* Unchunked result and the identified consumer cost + +The unchunked =2d6011f= experiment completed all 112 rows. All semantic +and scalar transfer/scanner counters matched, but all 28 workloads were +slower in both orders. Family geometric mean increases were 20.6% / 17.0% +for flat topology edits, 21.2% / 24.4% for nested topology edits, 10.3% / +10.0% for flat histories and 20.6% / 21.7% for nested histories. + +A separate same-token publication experiment reconstructs the canonical +root through the original production projector outside timing. Both roots +must reproduce the entire captured artifact. It measures only +=make-success-parse-artifact=, with 20 CPU samples and both orders on flat +400/800 and nested 8/16 sources. All 16 rows passed equality, but the event +consumer was 79.1% to 86.4% slower. These are publication-only measurements, +not a profile claiming that all request cost belongs to publication. They +establish that producer fusion alone leaves a substantially costlier consumer. + +* Bounded blocks and persistent traversal + +The next implementation retains the shared semantic/instruction algebra, +but packs up to 16 semantic leaves into a private vector block. Append +copies a bounded leaf block and the boundary path of an AVL join, sharing +unchanged subtrees. Join cost is logarithmic in sequence size, with at most +16 leaf references copied per block. Cached branch height excludes opaque +node/field bodies, which own their own sequence balance. An existing block +is never rewritten. Moving a multi-leaf +block retains a position view, rather than eagerly translating its leaves. +Node, field and fragment boundaries remain explicit instructions. + +A traversal creates one mutable, private cursor per block and advances it +through the immutable leaves. It no longer allocates a continuation vector +for every binary concatenation inside that block. The cursor never escapes +or mutates the program. The materialization visitor follows the same block +contract; output order, capped arity and cached bounds agree across both +consumers. + +The first block regression run revealed that a deferred right-branch frame +can point to a block without being a block cursor. That caused an out-of-range +index and failed native cases; this failed receipt is retained. Cursor and +deferred-frame forms are explicitly distinguished in the correction. Updated +semantic, request and publication results must all qualify this correction. + +* Specializing the consumer and controlling traversal depth + +The block correction passed five native modules and 61 cases, but isolated +publication still cost 80.5% to 87.5% more than canonical publication. Blocks +alone did not solve the consumer cost. The next experiment expanded a single +traversal template into the publisher, with literal operation consumers. The +general inspection walker uses the same template. Generated native Scheme +contained no call to the generic event walker. This passed 61 cases, but +publication still cost 61.5% to 73.5% more. Neither intermediate candidate +was given an additional full request matrix after this phase gate failed. + +The template now traverses ordinary code directly up to depth 64 and falls +back to explicit continuations beyond that bound. Opening a node returns +its publication ID; the corresponding traversal activation retains that ID +until closing. This removes the duplicate node-ID stack. The fallback stores +that same context in its node footer, including when nested inside blocks +and moved views. Root clamps, token proofs and complete-output checks remain +in the consumer. + +A 10,000-node artifact must validate with all 20,001 events, matching root +opening/closing ID zero. Together with the prior cases this passed 62 native +cases. Publication at 400 declarations was mixed (-4.0% / +1.45%); nested +8/16 groups improved 5.1% to 7.1%, while 800 declarations still cost 31.2% +to 33.4% more. The linear sequence height was a plausible cause of early +fallback, rather than an observed profile of fallback calls. + +The final candidate uses persistent AVL joins so broad sequences have +logarithmic height. Native cases additionally require a height at most 12 +for 10,000 leaves and asymmetric joins of 1/5,000, 257/4,096 and their +reverse orders. They require complete order and unchanged old operands. +The implementation now qualifies the entire producer, storage and consumer +pipeline together; its final receipts are recorded below. + +* Completed balanced candidate and admission decision + +At source =7570df3df745605a520f59e51717e4bee6d986eb=, all native sections were +complete and five focused modules passed 63 cases. The complete publication +experiment produced 16 equal-artifact rows. Flat 800 publication changed ++0.83% / -1.87%, eliminating the large observed 31%–33% regression of the +unbalanced predecessor. Flat 400 changed +0.58% / -4.02%; nested 8 changed +-3.58% / -5.07%; nested 16 changed +2.04% / -4.90%. These mixed orders do +not establish a universal publication speedup. + +The complete same-head request matrix produced 112 rows and 28 paired +workloads. Fresh output, inverse output, independent original-production +projection and every scalar transfer/scanner counter matched. Fourteen +workloads improved in both orders, ten regressed in both, and four were +mixed. Family geometric means by execution order were: + +| Family | Order 1 | Order 2 | +|--------+---------+---------| +| Flat topology | +12.99% | +12.86% | +| Nested topology | -3.88% | -2.21% | +| Flat six-edit histories | +19.22% | +16.92% | +| Nested six-edit histories | +0.42% | -1.02% | + +This qualifies the shared representation and removes the earlier broad +consumer regression, but rejects a general performance/default-admission +claim. The canonical production default remains selected. These are local +same-head comparisons, not a production-default before/after experiment +or remote CI qualification. Runtime certificates and existing performance +thresholds remain unchanged. + +* Next structural hypothesis and falsifiable experiment + +AVL joining makes the traversal height logarithmic but copies the boundary +path for repeated singleton append. A flat reduction stream can therefore +pay logarithmic path allocation at each append. The request regression and +near-canonical publication make producer/storage cost a concrete candidate; +this is a hypothesis, not an allocation profile establishing sole causation. + +[[https://www.staff.city.ac.uk/~ross/papers/FingerTree.html][Hinze and Paterson's 2–3 finger trees]] provide persistent sequence end access +in amortized constant time and concatenation/splitting logarithmic in the +smaller side. That is a relevant alternative for the prefix-plus-singleton +LR reduction pattern. The Scheme implementation must establish its own +strict evaluation/scheduling and allocation behavior; the Haskell design's +complexity claims cannot be inherited by a direct eager transcription. +The authors' published correction explicitly warns that a smart measured +constructor can force a recursive call and lose laziness crucial to the +analysis; a Scheme adaptation needs a scheduling proof or a more limited +complexity claim. + +The next experiment should replace only the shared sequence join policy, +retaining frozen field/node/token/view leaves and the original independent +projector. It should measure constructor allocations and producer-only cost +for 400/800/1,600 sequential appends and uneven concatenations, retaining all +prior roots. Admission still requires the entire 112-row request matrix, +all counter invariants, deep fallback and zero-sum movement checks, and +separate production-default impact. A lower isolated publication median is +insufficient. No finger-tree implementation or speed claim is made here. + +* Full native and generated-source boundary + +The same implementation head completed the full native audit of 59 modules: +492 of 493 started cases passed. The only failure was the existing +=GERBIL-PARSER-POO-GRAMMAR-OBJECTS= performance gate: p95 106.286 ms against +100 ms. The build/dependency version contract passed. GQL's unchanged +150 ms gate passed at p95 53.815 ms. The audit propagated the failure with +nonzero exit; harness output alone is not presented as success. There is +no threshold relaxation or claim of complete local/remote qualification. + +The HCL generated-source check completed with normal exit zero and +=GENERATED-HCL-STEP-OK=. Focused 63-case, full-audit, generated-source and +native-section manifests are separately sealed. No PR push or merge was +performed. The ASP dependency remains pinned to formal release =v0.1.2.2=. + +* Subsequent end-append experiment + +RFC [[file:0016-persistent-lr-append-buffers.org][0016]] implements the subsequent immutable end buffer and binary block +carry experiment. It records constructor allocation evidence and its separate +112-row request comparison; it preserves this RFC's historical results and +keeps production-default admission separate. + + +* Scheme/ASP verification owner + +Execute correctness assertions through the shared native Scheme harness. +Performance admission uses =t/core-observability-test.ss=, +=t/gql-runtime-benchmark-test.ss= and =t/incremental-suffix-benchmark-test.ss=, +with their existing ASP =benchmark.ss= contracts. The repository's =Justfile= +selects diagnostic microbenchmarks when a specific algorithm phase needs study. +Workloads, samples and thresholds remain owned by those Scheme sources. diff --git a/docs/architecture/0016-persistent-lr-append-buffers.org b/docs/architecture/0016-persistent-lr-append-buffers.org new file mode 100644 index 00000000..02f18f7a --- /dev/null +++ b/docs/architecture/0016-persistent-lr-append-buffers.org @@ -0,0 +1,189 @@ +#+title: RFC 0016 — Persistent end append for the shared LR/event sequence +#+startup: overview + +* Evidence and target + +RFC [[file:0015-unified-lr-semantic-event-sequences.org][0015]] completed a 112-row comparison. The shared DAG and balanced +publisher retained all semantics and counters, but flat topology requests +regressed about 13% and flat histories about 17%–19%. Publication itself +was near canonical, with mixed order results. This motivates a producer +experiment, rather than claiming that a publication improvement fixes all +request cost. The canonical production default remains selected. + +* Algorithm transfer + +[[https://aleksandar-prokopec.com/publications/conc-trees/][Prokopec and Odersky's Conc-Trees]] distinguish balanced concatenation from +an append organization with amortized constant end operations. This Scheme +experiment uses an immutable tail block and equal-rank binary carries; it +is not a complete Conc-Tree implementation and inherits no performance +ratio or arbitrary persistent branching guarantee. + +Small action sequences remain one block until they spill. Every full block +has 16 semantic leaves. Complete blocks are stored newest +first with strictly increasing binary ranks. Appending one semantic leaf +copies at most a 16-reference tail block and one envelope. A full tail is +carried through equal-rank blocks; each merge retains both old operands. +If B full blocks are carried along one linear history, the merge count is +B - popcount(B). Thus carry work is linear in B across that history. No +previously published tail, digit list, or branch is mutated. Repeatedly +forking the same carry root may repeat logarithmic carry work. This bound +is explicitly narrower than arbitrary persistent amortization. + +For multi-sequence append, the end buffer is normalized from newest/smallest +blocks toward older/larger blocks before AVL concatenation. This prevents +alternating singleton/multiple append from nesting unbounded envelopes. +A moved sequence remains an opaque position view with zero-sum movement +provenance; complete token proofs remain in the publisher. Program nodes +and fields remain opaque semantic leaves with their existing bodies. + +* Producer, storage and consumer ownership + +Only the private shared sequence join policy changes. LR reductions and +HCL generated reductions still call the single =event-program-append=. +Field/alias algebra, exact prepared-runtime and executor identity, lexical +certificate, token binding, source ownership, and original-production +projection are unchanged. Traversal visits prefix, chronological completed +blocks, then tail, retaining the existing depth-64 iterative fallback. +The materialization visitor and specialized publisher use the same order. + +The internal =event-program-append/avl= is an experiment control for the +previous join policy on the same leaves; it is not a second parser or +semantic reducer. The production event path uses the end buffer. GLR and +layout keep their existing backend. The event backend itself stays opt-in. + +* Falsifiable qualification + +The focused suite retains carry-boundary roots at 16/17, 32/33, 255/256/257 +and 1,024 leaves, forks old roots, compares complete semantic leaves with +the AVL control and exercises append blocks within a 100-level fallback. +It also alternates singleton and multi-sequence joins for 3,000 tokens, +requiring bounded rope height, original token order and immutable operands. +All previous UTF-8, fields, cancelled moves, 10,000-node publication, +HCL executor ownership, history/inverse and original-projector checks remain. + +The construction harness prebuilds tokens and retains every intermediate +root for 400/800/1,600 append chains. It separately forks the same carry +root 128 times at 256/512/1,024 leaves. Both policy orders use 20 samples. +CPU timing includes construction and root retention, excluding collection, +semantic checking, token construction and source publication. Post-timing +collection settles Gambit allocation and retained-live counters; those byte +receipts include the observation/collection overhead and are not pure +constructor instruction counts. The same overhead applies to both policies. + +Five-second inactivity and 180-second batch gates remain. Native compilation +has its separate bounded C phase. Constructor improvement alone is not an +admission result: the full 112 rows must preserve all scalar counters and +complete independently projected artifacts, and publication must still +qualify. Same-head backend comparison, production-default impact and remote +CI remain distinct. Results and remaining gates are recorded below. + +* Completed constructor experiment + +At source =fc330f3= (runtime unchanged from =51d4787=), the focused native +suite passed all 65 cases and all native sections were complete. A preceding +fixture compilation hit the old 30-second limit before executing tests; its +failed receipt is preserved. The corrected fixture driver independently +bounds native compilation at 180 seconds, kills only that build process +group and saves metadata even on failure. Test/benchmark inactivity and +batch limits remain 5 and 180 seconds. + +All 24 constructor rows completed with original token identities and retained +root checks. Linear retained-root construction improved in both orders: + +| Leaves per chain | CPU change order 1 / 2 | Allocated bytes change | +|------------------+------------------------+------------------------| +| 400 | -57.81% / -59.86% | -32.08% | +| 800 | -66.40% / -66.36% | -39.48% | +| 1,600 | -69.40% / -68.27% | -45.47% | + +The eight-chain candidate medians grew approximately 1.1 / 2.2 / 4.4 ms, +while AVL grew approximately 2.6 / 6.5 / 14 ms. The allocation observations +and scaling support the targeted linear-append hypothesis. These are +construction-only results, not parser/request speedups. + +Repeated carry-root forks improved CPU about 24%–28%, but allocated bytes +changed +6.45% / +3.67% / 0% at 256/512/1,024 leaves. This is why neither +arbitrary persistent constant-time appends nor universal allocation reduction +is claimed. Publication and complete request qualification follow below. + +* Completed publication phase + +The final candidate completed all 16 same-token publication rows with full +artifact equality and 20 samples per row. Relative to independently projected +canonical roots, flat 400 changed +1.07% / +5.23%, flat 800 -0.95% / +3.50%, +nested 8 +0.87% / -8.03%, and nested 16 +0.47% / +0.65%. There is no broad +publication speed claim. These observations are separate from the producer +experiment; all request and admission results are recorded next. + +* Completed request matrix and admission + +The complete 112-row matrix at =fc330f3ef67334d6dcfcb2055b13018ab93646e4= +passed fresh artifact equality, inverse edit equality, independent +original-production projection and every scalar scanner/transfer counter. +Of 28 paired workloads, 17 improved in both orders, seven regressed in both +and four were mixed. Family geometric means relative to same-head canonical: + +| Family | Order 1 | Order 2 | +|--------+---------+---------| +| Flat topology | +1.96% | +3.06% | +| Nested topology | -5.05% | -8.47% | +| Flat six-edit histories | -6.62% | -14.93% | +| Nested six-edit histories | -6.81% | -5.00% | + +This demonstrates that the end-append constructor improvement transfers to +history requests and some topology requests. It does not establish universal +request improvement. The flat topology regressions and seven consistently +slower workloads prevent broad performance/default admission. The production +default remains canonical, with all original ownership and reuse checks. +No production-default before/after or remote CI qualification is claimed. + +* Next evidence boundary + +The current publication phase uses fresh roots, whereas topology edits can +publish moved reused bodies. It cannot establish publication cost for that +changed distribution. The next structural investigation should isolate +publication of captured first/middle/last edited roots with current tokens +and the original projector, then observe singleton versus multi-sequence +normalization work during actual LR reductions. This separates the remaining +moved-publication and normalization hypotheses before another representation +change. Neither is presented as the sole measured cause of the residual +flat topology cost. + +* Full native boundary + +The full native audit completed 59 modules and 495 started cases; 494 passed. +The sole failure was the existing =GERBIL-PARSER-POO-GRAMMAR-OBJECTS= gate, +p95 146.752 ms against the unchanged 100 ms ceiling. This observation does +not establish a causal regression from the append policy; the scenario +constructs grammar objects and was already failing in the prior experiment. +The GQL runtime gate passed at p95 131.728 ms against 150 ms. Package/build +contracts, all newly added semantic cases and every remaining module passed. +The audit retained its nonzero failure status; it is not a full green receipt. + +No threshold changes, push or merge occurred. PR #13 was read-only verified +at remote head =7c1533356585154c8b5d340f64ffefb289d18907=, already contained +by the local branch. The source, constructor samples, complete request +samples, native failure and remaining qualification boundaries are preserved +for review in this single RFC and its receipt family. + +The HCL generated-source check completed with normal exit zero and +=GENERATED-HCL-STEP-OK= at the same source head. Its executor command and +log hash are sealed separately. The production default remains canonical. + +* Follow-up: checkpoint history integrity + +[[file:0017-checkpoint-history-cursor-integrity.org][RFC 0017]] measures actual moved edited roots and native LR construction +operations. It identifies and repairs a resumed-checkpoint token cursor defect +that caused tail inverse edits to discard successful incremental work and +reparse in full. Its operation counts and publication-only measurements have +separate evidence boundaries from this RFC's constructor and request matrix. + + +* Scheme/ASP verification owner + +Execute correctness assertions through the shared native Scheme harness. +Performance admission uses =t/core-observability-test.ss=, +=t/gql-runtime-benchmark-test.ss= and =t/incremental-suffix-benchmark-test.ss=, +with their existing ASP =benchmark.ss= contracts. The repository's =Justfile= +selects diagnostic microbenchmarks when a specific algorithm phase needs study. +Workloads, samples and thresholds remain owned by those Scheme sources. diff --git a/docs/architecture/0017-checkpoint-history-cursor-integrity.org b/docs/architecture/0017-checkpoint-history-cursor-integrity.org new file mode 100644 index 00000000..90aa3ac0 --- /dev/null +++ b/docs/architecture/0017-checkpoint-history-cursor-integrity.org @@ -0,0 +1,72 @@ +#+title: RFC 0017 — Checkpoint history cursor integrity +#+date: 2026-10-04 +#+status: Implemented; verification owned by Scheme tests and ASP benchmarks + +* Structural invariant + +A resumed LR checkpoint owns a source byte cursor, the number of already +consumed significant tokens, and a prepared-runtime identity. These values +must describe the same parser prefix. Treating a resumed prefix as zero tokens +produces an inconsistent initial capture even when its byte cursor is late in +the source. Subsequent inverse edits can then lose a valid earlier checkpoint +and fall back to fresh parsing. + +=src/runtime/parser.ss= captures the initial significant-token position as the +length of the resumed prefix. =src/runtime/incremental.ss= retains earlier +prefix checkpoints when the newly captured checkpoint collection is empty. +The correction preserves checkpoint history across tail insertion/deletion +and their inverses. It applies to canonical and event execution. + +Checkpoint reuse still requires the selected prepared runtime, source/token +binding and fresh-equivalent complete artifacts. A label, digest alone or +matching token text is insufficient to admit reuse. Failed parsing must clear +reusable recognition state. The event backend remains opt-in. + +* Semantic verification owner + +=t/recognition-snapshot-test.ss= owns the short-tail forward/inverse regression +for both backends, retained prefix sharing, independent grammar projection, +Unicode shifts and rejection of foreign runtime identities. +=t/recovery-incremental-test.ss= owns sequential edit history, checkpoint byte +cursors, token insertion, lexical boundary changes, rejection and repair. +=t/event-program-test.ss= owns immutable sequence joins, moved roots and +complete publication equivalence. + +These assertions execute against ordinary compiled modules. Verification does +not rewrite implementation source, replace runtime bindings or decode generated +reports. Benchmark data is produced by the existing Scheme/ASP test owner at +execution time and is not checked into documentation as result snapshots. + +* Performance verification owner + +=t/core-observability-test.ss= uses the ASP benchmark contract in +=t/benchmarks/poo-grammar-objects/benchmark.ss=. +=t/gql-runtime-benchmark-test.ss= uses +=t/benchmarks/gql-runtime/benchmark.ss=. +=t/incremental-suffix-benchmark-test.ss= uses +=t/scenarios/performance/incremental-suffix/benchmark.ss=. + +Keep workload, sample count and admission thresholds in those Scheme contracts. +Do not duplicate them in a document-side runner, results schema or analysis +pipeline. Functional checks may run concurrently; timing gates run separately. +An unsuccessful harness/child process is a failed run even if some cases finish. + +* Existing execution path + +#+begin_src sh +gerbil env gxi build-test-driver.ss compile +GERBIL_PATH="$PWD/.gerbil" GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" \ + GAMBOPT=max-heap=1G,debug=q \ + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log .gerbil/parser-checkpoint-history.log \ + --require 'TEST-SUITE-OK modules=3' --require '^OK$' \ + -- gxi t/fixtures/tla-sany-differential/test-suite.ss \ + t/recognition-snapshot-test.ss t/recovery-incremental-test.ss t/event-program-test.ss + +just test-local gql +#+end_src + +The first command selects functional owners using the shared native harness. +The second selects the existing ASP GQL gate. The native package and declared +dependencies must be built first. Compiler work has its own budget; test +execution retains the five-second real-output watchdog. Neither local command +substitutes for Linux/macOS integration CI. diff --git a/docs/architecture/0018-downstream-language-products.org b/docs/architecture/0018-downstream-language-products.org new file mode 100644 index 00000000..a8b25ff0 --- /dev/null +++ b/docs/architecture/0018-downstream-language-products.org @@ -0,0 +1,880 @@ +#+title: RFC 0018 — Downstream language products and the Bash admission boundary +#+date: 2026-10-04 +#+startup: overview + +* Status and scope + +Product contract and implementation track. The original proposal was grounded in PR 13 head +=675755009e46824327a1c05d5c787087c1297af4=. The interfaces and commands described +as targets below remain targets unless listed in the implementation section. This proposal extends RFCs +0004, 0006, 0007 and 0008; it does not introduce another grammar authority. +The original audit table below retains the names and limitations of that head. +The implemented Rust/native access contract is now maintained in +[[file:0006-rust-aot.org][RFC 0006]]; later executable examples use current =rust-runtime= names. +Bash 5.3 source syntax is the first demanding consumer. Shell execution, +expansion evaluation and alias execution are outside its parser contract. + +* Original audited seams and gaps + +| Seam | Implemented at the audited head | Missing product behavior | +|------+---------------------------------+--------------------------| +| Authoring | Public =rust-runtime-grammar-support.ss= exports =deflanguage= | One downstream repository initializer/product declaration | +| Generation | =rust-runtime-source= takes a grammar module path with exactly one exported LanguageGrammar | Capability preflight and complete generated package products | +| Rust runtime | =parse(&LANGUAGE, source)=, generated LR tables, operand fields, lossless Rust syntax events | Contextual/deferred scanner IR parity and generated typed node access | +| Scanner | Scheme contextual dispatch, balanced-word and deferred FIFO scanner tests | A Bash grammar with structured word interiors and matching Rust execution | +| Distribution | Independent records Cargo fixture, no Gerbil at parse time | Release/package automation, minimal C product and Rust FFI consumer | +| Runtime C ABI | Owned result allocation/release, UTF-8 error payloads, real C callback probe | Language-independent handles, explicit byte lengths and runtime-owner facade | + +The parser FFI uses explicit language handles and the language-independent +=src/ffi/language-artifact-codec.ss=. The former builtin name-dispatch facade +and its C entry are removed; production FFI imports no GQL/openCypher package. +The build-only =src/ffi/rust-runtime-aot.ss= accepts an external grammar path +and generates immutable Rust source. These two services have distinct duties. +=src/ffi/schema.ss= centrally declares their ABI/schema identities; file paths, +module namespaces and C result names do not encode versions. +The current Rust lexical expression enum includes a heredoc expression, but +that does not establish Bash queued marker, quote removal or =<<-= semantics. + +* One declaration, multiple release products + +A language author maintains =grammar.ss=, one package product manifest, and a +corpus. The grammar alone owns language identity, syntax version, node/field +catalogs, lexical behavior and conflict/recovery policy. The manifest selects +output targets, package names, toolchain pins and corpus paths; it cannot +silently redefine the grammar's identity or semantics. + +Conceptual author-owned repository (names are proposed): + +#+begin_example +grammar.ss # public declarative grammar facade +language-package.toml # output selection and pinned generator/runtime +corpus/ # accepted, rejected, incomplete, edit histories +#+end_example + +The release tool generates =Cargo.toml=, the Rust module/re-export, optional +typed node wrappers, C header, native build declaration and product receipt. +Authors never copy the upstream source tree, maintain generated LR tables, +write an FFI dispatcher case, or duplicate a parser in Rust. Generated artifacts +may be committed for release and verified by regeneration in language CI. +A downstream Cargo build then uses the generated language crate and generic +runtime only. Gerbil is a producer/release tool, not a required consumer tool. + +| Product | Consumer dependency | Runtime authority | Admission | +|---------+---------------------+-------------------+-----------| +| Pure Rust AOT crate | Generic Rust runtime + selected generated language | Generated Grammar/Parser/Scanner IR | All required operations admitted by Rust backend | +| Native C library + Rust safe facade | Selected Gerbil AOT closure + native runtime dependencies | Same compiled language IR executed in Gerbil | C lifecycle, result ownership and parity gates | + +Both products use the same language artifact identity. Packaging may produce +separate backend digests. A missing Rust opcode is a build rejection naming +the source rule, normalized operation and supported target; it cannot silently +turn the native product into an AOT-parity claim. Native C packaging is not +assumed to be standalone merely because Gerbil can export C functions. + +The generator must finish validation before publishing a product directory. +Outputs are staged and atomically published only after all selected targets +succeed; failed generation cannot leave half-new tables beside an old header. +The receipt binds transitive grammar inputs, toolchain/runtime pins, artifact +schema versions, required capabilities and each output digest. Changing a +shared scanner role is therefore a detectable language artifact change. + +* Bash syntax granularity + +More nodes is not itself a better syntax contract. Expose stable named nodes +and fields for tooling decisions, preserve punctuation/trivia as source-backed +leaves, and hide LR helper productions. Do not impose evaluated shell values +on the CST. Generated typed accessors remain optional; generic tree/event APIs +must retain every source byte and diagnostic. + +| Syntax | Named nodes / useful fields | Required distinction | +|--------+-----------------------------+----------------------| +| Lists | Script, CommandList, AndOrList, Pipeline; commands/operators | =&&=, =||=, =|&= and asynchronous separators | +| Commands | SimpleCommand; assignments, command, arguments, redirects | Reserved word vs ordinary argument; =a=b= position | +| Words | Word, WordPart, QuotedPart, Escape | Adjacent parts form one word; quotes retain exact spans | +| Expansion | ParameterExpansion, CommandSubstitution, ProcessSubstitution, ArithmeticExpansion | Nested source trees, parameter operator and subscript fields | +| Compounds | If/Case/For/Select/While/Until, Function, Subshell, BraceGroup | Bodies/conditions/branches with stable field identities | +| Redirection | Redirection, HereDocument; fd, operator, target/body/delimiter | FIFO marker binding, quoted delimiter, tab stripping policy | +| Incomplete input | Missing/Error nodes plus diagnostics | Editor recovery must preserve bytes and admit no false success | + +A single opaque balanced-word token passes lexical recognition tests but does +not supply a refactoring-quality tree for nested substitutions. Bash admission +needs normalized word/interpolation modes and grammar entrypoints that expose +interior tokens and fields. Subparse boundaries must share UTF-8 byte ownership +with their parent; emitting overlapping parent/child token events is rejected. + +The old source parser is a corpus/reference candidate, not the implementation +authority. Port its cases into the declarative language contract and independently +check canonical artifacts. Bash syntax checking and Tree-sitter projections can +provide separate references; disagreements must be classified rather than +resolved automatically by copying either tree. + +* Contextual scanner portability + +Scheme owns the normalized scanner operations. Rust implements their specified +execution, not Bash-specific callbacks. Compile grammar roles into immutable +mode/dispatch/transition tables; source state is per parse/session. + +Bash requires command-position keyword/assignment admission; quote and nested +substitution stacks; here-document marker queues; delimiter quote-removal and +=<<-= tab policy. Checkpoint identity includes scanner artifact identity, +source binding, byte cursor, mode/stack and pending marker state. Restoring a +checkpoint against a different source/artifact fails closed. Incremental reuse +requires matching scanner state, not merely equal token text or length. + +The generic IR extension must carry these states across both backends before +Bash pure Rust AOT is advertised. A bounded state limit produces a typed limit +diagnostic; it must never truncate state or silently change recognition. +The capability inventory and golden transition traces are shared conformance +data. Existing Scheme balanced-word/deferred tests are starting seams, not +proof that the Rust lexer or complete Bash grammar implements them. + +* C ABI product boundary (language handles) + +Preserve published v1 result initialization/replacement/release behavior. +A new length-bearing API uses =const uint8_t*= plus =size_t= for source bytes; +NUL-containing and invalid UTF-8 inputs must be handled explicitly without +string truncation. ABI version, header struct sizes and capability negotiation +are independent from syntax and artifact versions. + +A language handle is admitted from a selected compiled language product. +Parsing accepts that handle rather than a global language-name registry. +The handle's descriptor binds grammar/parser/scanner digests, node and field +catalogs, recovery policy and supported entrypoints. Admission validates the +artifact before any parse; arbitrary grammar modules are build-time trusted +code and are not loaded as data in a parse request. + +Result buffers remain C-owned, with explicit byte lengths and idempotent +release. No GC-managed pointer escapes to Rust. A Rust guard owns the result +and copies/borrows within its lifetime according to the declared API. Language +handles cannot outlive their runtime/library owner. Library unload, runtime +shutdown, cancellation, panic/exception transport and repeated release need +separate tests; the current disposable test-child exit is not teardown evidence. + +The first native facade serializes calls through one initialized Gerbil runtime +owner thread. It is not Send/Sync by assertion. Worker isolation and parallel +runtime use require dedicated qualification; callbacks may not enter Gerbil +from arbitrary Rust threads. Transport success is distinct from syntax +acceptance: a rejected source can still produce a valid lossless artifact and +structured diagnostics. Byte limits, state limits and malformed artifacts +remain distinct transport/admission failures. + +* Measurable comparison with Tree-sitter + +Tree-sitter already generates C and Rust scaffolding with =tree-sitter init=; +matching that workflow is the minimum authoring gate. Its external scanner +contract has create/destroy/serialize/deserialize/scan callbacks, and Bash +heredocs are explicitly an external-scanner use case. Those are actual +comparison points, not claims that Tree-sitter lacks language packaging. + +The intended improvement is: no author-written Rust parser or scanner C for +the admitted shared IR, explicit syntax/field/identity contracts, and one +language declaration producing both pure Rust and native C products. Publish +comparative results only after matching source coverage and output obligations. + +| Gate | Evidence required before claiming an advantage | +|------+------------------------------------------------| +| Author effort | Count handwritten files/lines and commands from empty external repo to working Rust consumer | +| Granularity | Nested word/substitution/redirect field queries and edits without source-string reparsing | +| Portability | Scheme/Rust/C artifact and field equality over accepted, rejected and incomplete corpus | +| Incrementality | Fresh-equivalent forward/inverse edits including quoted markers, nesting and Unicode; changed work counters | +| Performance | Matched cold build, binary size, startup, full parse/edit p50/p95, allocation and peak memory | +| Product isolation | One selected language links no GQL/Cypher/Bash sibling artifact; two languages retain distinct identities | +| Ergonomics | Clean Cargo consumer without Gerbil/Node; native C/Rust facade with explicit lifecycle and error behavior | + +Upstream reference interfaces: +- https://tree-sitter.github.io/tree-sitter/cli/init.html +- https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners.html +- https://github.com/tree-sitter/tree-sitter-bash/blob/master/grammar.js + +* Ordered implementation and closure + +1. Add a backend capability preflight over Grammar/Parser/Scanner IR and make + the standalone external records package the minimal authoring/product gate. + Reuse the existing Scheme-owned generator; do not misuse the Rust + =gerbil-parser-build-support= test-macro crate as a second generator. +2. Generate package wrappers, receipts and typed syntax/field catalogs from + that admitted artifact. Qualify atomic generation and a clean external + Cargo consumer before adding Bash-specific syntax complexity. +3. Port contextual/deferred state to Rust from the normalized scanner contract; + compare state traces, source-exhaustive events and edit histories against + Scheme. No handwritten downstream scanner is required for admitted modes. +4. Port Bash source corpus to =deflanguage= with the node/field contract above. + Recovery, incomplete syntax and nested word structure are required admission + work, not a later documentation promise. +5. Add language-bound native C AOT packaging and length-bearing handle APIs. + Qualify startup/shutdown, ownership and a real Rust FFI consumer on Linux and + macOS. Keep pure Rust and native-runtime distribution dependencies explicit. +6. Run an independent language repository containing only author-owned inputs, + generate both selected products, test parity and inspect dependency closure. + Only then publish comparative Tree-sitter metrics and Bash support claims. + +This order is one downstream-language product track. Completing packaging alone +or preserving the historical source parser does not close Bash/Rust support. + +* Bash source integration and verification + +At the human's explicit direction, the existing Bash branch was integrated into +PR 13 rather than kept as an unsubmitted experiment. Its original head is +preserved at =codex/bash-source-prototype-before-integration=. The sole Bash +feature commit was rebased onto the current integration branch, then merged +with an actual two-parent merge commit. Older core changes in its historical +ancestry were not replayed over current PR optimizations. + +Merge head: =c20dae08c313a38d894af8be9a7db89d79dbb8a0=. +Ten affected native modules compiled with two workers in 40.10 seconds. A +90-second parallel test budget and five-second real-output watchdog admitted +all four selected modules: 16 Bash cases, three public-language cases, five +style cases and two GQL/Cypher consumer compatibility cases. This consumer test +selection retains the source overlay; it is not an installed-only distribution +gate. All selected modules exited successfully with final GC and harness +receipts. No full Bash language conformance, Rust AOT parity or standalone C +language product is claimed. The preserved source implementation is now +available for users and for the canonical-IR port. + +Verification is owned by =languages/bash/parser-test.ss= and the shared native Scheme +harness. Language/FFI performance admission uses the ASP =benchmark.ss= +contracts. The Bash corpus and implementation live in =languages/bash=; +there is no document-side parser audit, snapshot format or analysis pipeline. + +#+begin_src sh +gerbil env gxi build-test-driver.ss compile +GERBIL_PATH="$PWD/.gerbil" GERBIL_LOADPATH="$PWD/.gerbil/lib:$PWD" \ + GAMBOPT=max-heap=1G,debug=q \ + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log .gerbil/parser-bash-source.log \ + --require 'TEST-SUITE-OK modules=1' --require '^OK$' \ + -- gxi t/fixtures/tla-sany-differential/test-suite.ss languages/bash/parser-test.ss +#+end_src + +* Shared scanner / contextual AOT / language C ABI implementation + +The public =rust-runtime-support= facade now exports +=generate-rust-scanner-module= and +=generate-contextual-language-rust-runtime-module=. The latter takes the existing +LanguageGrammar descriptor and its Scheme-compiled contextual product. It +emits ordinary Rust data: LanguageSpec, shared ScannerSpec, LR state positions +and ContextualParserSpec. =parse_contextual(&CONTEXTUAL, source)= uses the LR +state's position, retaining lookahead across reductions and every trivia byte. +The generator admits the Scheme product against its parser machine before +writing output; Rust binds the scanner's base grammar identity to LanguageSpec. +Generated identities assume a trusted AOT producer, as existing LR tables do. + +Rust executes the closed =contextual-scanner-opcodes.v1= contract: longest +match/rank, balanced nested words, quote removal, UTF-8 ranges and FIFO deferred +markers. Successful admission and mode/dispatch indexes are cached once per +immutable static ScannerSpec identity, matching the LR validation boundary. +Contextual product admission and terminal/lexical catalogs are also indexed +once per static product. Pulling each token uses indexed lookups instead of +scanning both full catalogs; first-declared duplicate lookup semantics remain. +The caches retain no input text, checkpoint or marker obligations. Each source +executor shares its read-only plan through Arc; checkpoints still reject another +executor even when both share that plan. Dispatch admission checks duplicate +keys by indexed insertion rather than scanning all preceding cells. Immutable +checkpoints share persistent two-stack FIFO storage; enqueue and checkpoint +clone are constant work. Rear reversal is amortized along one linear scan; +replaying old branches can repeat reversals. Iterative nesting and iterative +unique-tail release avoid recursive stack growth. Contextual GLR forks fail +explicitly, matching the current Scheme contextual admission boundary. + +The independent Records fixture compares canonical Scheme/contextual events, +canonical/contextual source-indexed trees, and a generated language C entry. Scanner +traces come from Scheme execution of the same IR. Committed generated Rust +fixtures belong to the Cargo package, and native CI regenerates/compares all +three products. No downstream Rust scanner or core builtin dispatch change is +required. These bounded fixtures do not establish complete Bash AOT support. + +The =language-abi-support= facade exports =generate-language-pack=, +=register-language!= and =release-language!=. The generator takes +module/export identities and creates a normal Scheme native module plus a C +header with =PREFIX_language_create(void)=. Each successful create returns a +new opaque uint64 handle; zero means admission failure. Native package builders +compile the generated entry with =-I/path/to/gerbil-parser/include=. ELF clients +link the generic ABI object and selected language entry object explicitly. +Language/compiler dependencies remain selected by the downstream descriptor. + +Module namespaces follow their source paths: =src/ffi/language-abi.ss= +provides the language boundary, and =include/gerbil-parser/language.h= names +its C contract. Version numbers belong to contract data returned by +=gerbil_parser_language_abi_version()=, not module paths or C identifiers. +The public result type is =gerbil_parser_result=; this rename introduces no +alias for the removed versioned names. + +=include/gerbil-parser/language.h= is language independent. It accepts an +explicit source pointer/length, preserves embedded NUL, rejects noncanonical +UTF-8 and enforces 64 MiB before copying. Successful parsing transports the +existing GPA1 event format, with syntax acceptance in its header. Results are +C-owned and replaced/released explicitly. Handles never reuse released numeric +identities. The runtime owner OS thread is pinned at native module initialization; +foreign pthread calls fail before entering Gambit. Scheme logical-thread checks +remain in the handle owner. This API requires an initialized embedding runtime +and loaded libraries; startup/shutdown, unloading and a safe Rust FFI owner are +not established by the native child exit test. Private generated callbacks use +Gambit ___U64/___S32 declarations, while public headers use uint64_t/int32_t. +Their widths agree without assuming that the platform's C typedef spellings +are compatible (unsigned long and unsigned long long differ on Linux LP64). + +Performance is measured by =t/benchmarks/language-abi/benchmark.ss= through +=t/language-abi-benchmark-test.ss=. The ASP test framework is outside the +native C consumer's dependency closure. The fixture's contextual product is +compiled during macro expansion, then embedded as immutable data; runtime +language creation prepares its scanner/LR binding once per handle. A repeated +C call does not spawn a process, recreate a runtime or compile a grammar. +=src/runtime/parser-ir-data.ss= owns runtime-only IR access. Parser, recovery +and incremental modules no longer import =src/compiler/parser-ir= for that +accessor, removing a runtime edge to the LR construction dependency closure. +Report startup, handle creation and warm callback batches separately. A process +wall time that includes module loading is never a per-call throughput metric. +The 100-call UTF-8 workload uses 40 samples with an unchanged 100 ms p95 ceiling +and a 20 ms target. Each attempt preserves paired process CPU/GC counters and +inner C-batch wall time. benchmark-run/result returns the exact ASP-admitted +attempt, so a separately ranked CPU percentile cannot be substituted for the +wall p95 observation. Progress output remains inside the ASP timed thunk and +outside the inner C-batch timer; both boundaries are explicit. Allocation +counters are raw process observations, not exclusive parser allocation claims. +Passing the ceiling alone does not establish the target or an advantage over +another parser. + +Remaining product work includes generic generated Cargo/native package manifests, +atomic publication across products, standalone runtime lifecycle and Rust FFI +ownership, typed accessors, contextual GLR branch state, and the canonical Bash +syntax/field port. There is no comparative Tree-sitter performance claim. + +* Parser-test declarations + +Import =:gerbil-parser/language-test-support= in =parser-test.ss=. Define one +exported suite with =deflanguage-parser-tests= and the language's POO loader: + +#+begin_src scheme +(deflanguage-parser-tests example-parser-test "Example language" + (loader example-language) + (fixtures "inherited fixture values") + (accepted "assignment" "x = 1" (nodes Assignment) (tokens identifier number)) + (accepted-many "literal forms" '("1" "2")) + (rejected "incomplete assignment" "x =" (diagnostics)) + (identity "identity" (language "example") (contract "example.v1")) + (property "projection value" + (bindings (actual (example-project (parse-example "x = 1")))) + (equal (cdr (assq 'name actual)) "x"))) +#+end_src + +| Clause | Engine-owned obligation | +|---+---| +| =accepted= / =rejected= | Parse once through the Loader =.parse= slot; valid artifact, status, exact source and source digest | +| =accepted-many= / =rejected-many= | A nonempty list of source strings; the same invariant checks for each | +| =fixtures= | Invoke the Loader's named =fixtures= test service, retain inherited corpus values, verify count, validity, source and digest | +| =identity= | Equality for declared =schema=, =language=, =version= and =contract= fields | +| =ast= | Exact engine AST grammar with mandatory fields, token text and byte spans | +| =property= | Explicit sequential bindings and at least one equality assertion for extension-specific results | + +Artifact expectations are =(nodes Kind ...)=, =(without-nodes Kind ...)=, +=(tokens kind ...)=, =(counts (Kind nonnegative-count) ...)=, =(diagnostics)=, +=(artifact-ref key expected)= and +=(projection project (field expected) ...)=. Node/token names and projection +keys are literal symbols; counts are nonnegative integer declarations. Ordinary +source expressions remain values, so generated corpora do not need user-owned +iteration/assertion code. Property bindings support application projection and +receipt APIs; the compiler accepts no arbitrary test-case statement body. + +The macro validates the effective Loader through =LanguageLoaderContract=; +POO inheritance/fixture override uses the existing Loader slots. The macro does +not create a second Loader type, upgrade =language-entry.v2=, or grant Rust/C +capabilities. Common tests require no author-written lambdas. A multivalue +receipt API may use a property binding to collect its returned values explicitly. + +Bash, Arithmetic, HL7, FHIRPath, GQL and openCypher have migrated to this syntax, +as has the independent Records consumer. TLA+ layout uses the same syntax in +its canonical =parser-test.ss= entry. Bash's two source-contract/checkpoint controls +remain under =t/= with their engine owner and all original assertions. HCL's +strategy/parity suite now uses closed source generators and named engine routes. +TLA+ core, layout, SANY candidate and closure now share the canonical +=parser-test.ss= entry: 57 declaration rows replace the three separate raw +suites while retaining their syntax, corpus, proof, native and rejection obligations. + +** Bound IR, structure and engine execution controls + +The same declaration accepts the following advanced obligations: + +| Clause | Constraint and check | +|---+---| +| =parser-ir= | Named rule count, root, materialization, conflict policy and case policy | +| =bound-ir= / =bound-rules= | Binding count, exact source metadata, digest prefixes, structured binding identity and ordered references | +| =lr= | Exact state/fork counts, absent nullable-prefix helpers and initial shift actions | +| =lr-receipt= | Execute the declared machine/source with a positive branch budget; check remaining tokens and named branch/completion fields | +| =parallel= | A positive worker count and nonempty sources; a barrier coordinates real parser calls; all workers join and each result equals its own checked fresh source | +| =incremental= | Replace the first declared nonempty text; compute UTF-8 byte start/deletion length; compare incremental artifact to a checked fresh parse and assert receipt fields | +| =recovery-rejected= | Require a valid, lossless rejected artifact and assert recovery receipt fields | + +Additional artifact expectations are =(root Kind)=, +=(diagnostics nonnegative-count)=, =(subtree pattern)=, +=(without-subtree pattern)= and =(field-counts Kind field (count ...))=. +Subtree patterns have the form =(node Kind (lexemes text ...) (children pattern ...))=. +The children clause selects direct nodes of the declared kinds through transparent +fields, then requires the exact selected count and order. Lexeme requirements +inspect that node's descendants. These are partial semantic queries; use =ast= +for an exact complete tree with mandatory fields, bytes and token text. +Field counts select matching nodes in preorder and count direct named fields. + +#+begin_src scheme +(accepted "conjunction precedence" source + (subtree (node ValueExpression (lexemes "AND") + (children (node ValueExpression (lexemes ">")) + (node ValueExpression (lexemes "=")))))) +(bound-rules "source identity" parser-bound-ir + (program (source path "grammar-source/input.bnf") + (source-prefix replacementExpressionDigest "sha256:"))) +(lr "static conflict controls" parser-ir + (state-count 1317) (fork-count 321) + (empty-nullable-prefixes "$boolean factor") (initial-shifts "RETURN")) +(parallel "independent requests" 9 sources) +(incremental "Unicode byte edit" source + (replace "λ中" "μ文") (schema "gerbil-parser.incremental-receipt.v1")) +#+end_src + +These examples describe checks, not extra backend capabilities. Incremental +and recovery clauses derive their machine from an admitted Grammar IR Loader; +a source-only Loader rejects those requests. Parallel workers propagate parser +exceptions after joining all results. No import starts workers. Loader schema +remains v2 and parser/scanner/Bound IR schemas remain unchanged. + +The =fixtures= row may add =(status accepted)= or =(status rejected)= to +require a uniform corpus expectation as well as checking every fixture +against its declared artifact status. Mixed corpora omit this constraint. + +** Generated native strategy qualification + +=parser-test.ss= may declare =installed=, =strategy-parity= and =fixture-parity= +rows. The engine owns lexical execution, route selection, indexed checkpoint +reference execution, exact artifact comparisons and fast/fallback coverage. +It has no language import or language-name dispatch. A lexical-only row uses +=(routes)=; a parser parity row names at least two distinct routes, or declares +a required acceptance outcome. Coverage reuses the single lexical result. + +#+begin_src scheme +(deflanguage-parser-tests sample-parser-test "native strategy qualification" + (loader sample-language) + (installed "generated reduction step is installed" step) + (strategy-parity "large source retains the indexed artifact" + (indexed 1024 (concat "key" index " = 1\n")) + (routes entry source-default source-raw source-generic indexed) + (accepted #t)) + (fixture-parity "official accepted corpus" accepted + (routes entry source-default indexed) (accepted #t))) +#+end_src + +The closed source collection algebra is =sources=, =indexed=, =seeded= and +=append=. Indexed rows produce one concatenated source; seeded rows produce +an ordered collection using the exact LCG constants 1103515245, 12345 and +2147483648. Text templates support string literals, =index=, =concat=, +=choice=, =character=, =integer=, =if-even=, =repeat=, scoped =bind= and +=when-equal=. Bounds, names, duplicate bindings, unknown templates, routes and +options are validated during expansion. No callback or free-form generator +body is admitted. Each seeded collection resets its state once, retaining +depth-first draw order across the full collection. + +| Route | Engine execution | +|---+---| +| =entry= | Effective Loader parse method | +| =indexed= | Ranked scanning, indexed LR checkpoint driver and canonical artifact builder | +| =source-default= | Installed generated source recognizer's default route | +| =source-ranked= | Generated recursive recognizer with fast lexing disabled | +| =source-raw= | Generated recursive recognizer with canonical shortcut disabled | +| =source-generic= | Generated recursive recognizer with simple-source shortcuts disabled | + +The last four routes require an optional =native-test-profile= POO value on +the Loader. Admission accepts only the =recursive-source= profile, checks its +exact machine digest, requires source identity with the installed recognizer, +and requires a declared lexer procedure. HCL's existing generated recognizer +implements this parameter contract. The engine does not infer these parameters +for arbitrary source backends: an absent or incompatible profile is unsupported. +The lexer remains an explicitly declared generated product; this slot does not +prove closed Scanner IR ownership or Rust/C portability. The profile is an +inherited value, so compatible downstream overrides remain possible and are +validated against their effective descriptor. No runtime test import is added +to the production Loader, and no per-parse contract check is introduced. + +HCL's 27 former raw test cases now occupy 29 declaration rows: its numeric +boundary and generated-number obligations were separated, and its Loader +identity has its own row. The five seeds, all generated source sequences, +fast/fallback thresholds, all large-source shapes, exact artifacts, 15 official +fixtures and typed rejection controls remain executable. Independently retained +port-writing references under =t/= compare all 1,024 generated sources exactly. +Generated HCL scanner/recognizer ownership is now in =src/compiler/=. The +recursive source provider uses the common build protocol with an explicit closed +lexical/grammar admission corridor. + +** Descriptor-bound model receipts and fixture selection + +The test declaration now supports =fixture-group= for accepted/rejected members +of the effective inherited Loader corpus, =accepted-fixture= for an indexed +source, and =fixture-catalog= for pinned counts and first-source digest. Fixture +groups use a validated POO override and reuse the checked artifacts rather than +parsing the group again. =syntax-kind= checks exact grammar metadata; =fields= +and =token-counts= constrain emitted artifact structure. + +=model-receipt= takes literal model/source/config data, positive workers and a +closed =(stdout text exit-code)= or =(unresolved)= tool fixture. Filename +components, tool exit bounds and receipt keys are checked during expansion. +The engine owns temporary files, process execution, stdout quoting and cleanup. +=(nonzero exit-status)= tests the existing raw Gerbil wait status without +assuming it equals a shell exit code. These fixtures qualify receipt admission +and failure handling; they do not establish actual TLC model checking. + +Production TLC qualification lives in =src/language/tlc.ss= and parses through +the admitted descriptor without importing builtin languages. The parser entry +exports all scope wrappers from its canonical =parser.ss=; the old +=qualification.ss= facade is removed. The existing receipt and Loader +schema versions are unchanged. + +=native-entry= compares Loader parsing, descriptor parsing and the actual native +binary status. =portable-rejected= exercises the real Rust producer rejection +and exact error. SANY proof policy remains unsupported by standalone Rust AOT; +these checks preserve that boundary rather than grant a portable capability. +All language directories now have three Scheme interfaces. TLA+ scanner/proof +execution belongs to admitted structured POO engine recipes. + +** Generic POO fused reduction build strategies + +=language-build-support.ss= exposes =FusedReductionStrategy.= and its contract, +=make-fused-reduction-strategy=, =declare-language-build-strategy= and +=emit-language-build-strategy=. HCL and Arithmetic both declare the same +engine-owned capability on their actual public Loader: + +#+begin_src scheme +(slots build-strategies: + (list (declare-language-build-strategy 'fused-reductions + (make-fused-reduction-strategy my-language-grammar)))) +#+end_src + +=LanguageLoader.= defaults =build-strategies= to an empty list. Named entries +are checked for unique names, a valid strategy contract and exact identity +with the Loader's effective descriptor. Inheriting a strategy bound to another +descriptor rejects during admission. This remains =language-entry.v2=. +Building through =emit-language-build-strategy= selects only a declared entry; +it invokes the validated engine emitter rather than a caller-supplied callback. +Parser dispatch does not emit code or validate build strategies per source. + +The strategy itself is a POO prototype with these effective slots: + +| Slot | Admission or extension rule | +|---+---| +| =descriptor= | An admitted language grammar; its Parser IR must be the exact machine IR | +| =digest= | Derived internally from the descriptor machine; explicit stale overrides reject | +| =step-name=, =event-name=, =digest-name= | Distinct valid Scheme binding names; reserved/imported dependencies cannot be shadowed; override through POO slots | +| =metadata= | POO object for downstream build annotations | +| =.emit= | Common inherited method dispatches to the admitted engine provider | + +#+begin_src scheme +(def MyReductions. + (.o (:: self FusedReductionStrategy.) + step-name: 'reduce-step + event-name: 'reduce-events + digest-name: 'reduce-digest + metadata: (.o consumer: 'my-project))) +;; No user-written thunk is needed for a slot value. +(declare-language-build-strategy 'fused-reductions + (make-fused-reduction-strategy my-language-grammar MyReductions.)) +#+end_src + +=src/compiler/fused-reduction.ss= has no builtin language imports or +language-name dispatch. It validates every production ID and the closed +=pass=/=concat= plus =field=/=alias= action algebra before producing any output. +The same structured generator emits recognition and event program reductions. +LR transition lookup, precedence, lexical modes and rejection remain owned by +the existing runtime. The HCL generated product now lives at +=src/compiler/hcl-reductions.ss=; the former language product and benchmark +producer were removed. The generic =scripts/generate-reductions.ss= CLI accepts +any module exporting one language descriptor and verifies deterministic output. + +This capability produces Scheme fused LR reductions. It does not confer Rust +or C portability, or generalize HCL's recursive lexical/source corridor. +=src/compiler/recursive-source.ss= and =src/compiler/hcl-source.ss= belong to +the engine; =RecursiveSourceStrategy.= admits the reviewed HCL corridor. + +** Common POO build capability protocol + +=BuildStrategy.= and =BuildStrategyContract= now own descriptor/IR identity, +exact digest, POO metadata, registered provider and truthful =kind=/=format= +slots. =FusedReductionStrategy.= and =RustRuntimeStrategy.= inherit this common +prototype and apply their own additional constraints. =LanguageLoaderContract= +validates the common protocol, and =emit-language-build-strategy= invokes +=emit-build-strategy=. Neither common dispatcher branches on language or strategy +kind. The Loader imports no concrete generation backend; convenience constructors +are provided by =language-build-support.ss=. + +HCL and Arithmetic both declare the two real engine providers: + +#+begin_src scheme +(slots build-strategies: + (list (declare-language-build-strategy 'fused-reductions + (make-fused-reduction-strategy my-language-grammar)) + (declare-language-build-strategy 'rust-runtime + (make-rust-runtime-strategy my-language-grammar)))) +#+end_src + +A downstream name is independent of provider kind. POO prototypes may inherit +metadata and supported recipe slots; =make-bound-build-strategy= binds the +actual descriptor and validates effective inherited values. Each concrete +constructor additionally checks its own strategy contract, so a Rust constructor +cannot be supplied a fused reduction prototype. + +| Layer | Authority | +|---+---| +| Language Loader | Named declarations, uniqueness and exact descriptor binding | +| Common POO contract | Source/machine IR identity, digest, metadata and registered target identity | +| Engine provider | Specific closed recipe admission and canonical producer | +| Output consumer | Actual compilation/execution qualification for its backend | + +Engine extensions register an opaque provider with a name, format identifier, +admission predicate and emitter through =declare-build-strategy-provider= in +=src/compiler/build-strategy.ss=. They inherit =BuildStrategy.= and add their +recipe constraints. Adding such a provider requires no edit to the Loader +registry or common emitter. Registration is an engine extension boundary, not a +language-authored arbitrary writer slot. Plain POO objects posing as providers, +forged target identities and stale digests reject. The common build entry uses +the registered provider even when a recipe overrides its =.emit= method. + +Emitters must materialize/validate output before writing to the caller's port. +The fused provider retains its closed LR action algebra; the Rust provider calls +the existing =language-rust-runtime-module-source= producer unchanged, preserving +Scanner IR and portable-policy rejection. Both providers are build operations; +repeated source parsing invokes neither provider admission nor generation. +A format identifier does not grant a new portable runtime capability. No C +provider is implemented by this change, and Scheme-only parser policies remain +unsupported by Rust. + +** Header-derived delimiter rules shared by Scheme and Rust + +The closed LexicalExpr constructors =(header-delimiter prefix count index)= and +=(header-data prefix count stops)= replace language-owned scanner callbacks. +For example HL7 declares =(header-delimiter "MSH" 5 0)= for its field separator +and =(header-data "MSH" 5 "\r\n")= for data. An independent header-record +language declares prefix ="記"=, count =2= and stops =";"=, through the same +=deflanguage= and POO =deflanguage-parser-loader= boundary. + +Admission requires a nonempty string prefix, an exact delimiter count from 1 +through 32, and a valid zero-based delimiter index or string of stop characters. +The source must start with the prefix followed by exactly the declared number +of distinct, non-alphabetic, non-numeric delimiter characters. The count defines +the header span; subsequent source may immediately follow it. Data consumes a +nonempty run excluding those delimiters and explicit stops; spaces are preserved. +Rules return no match on malformed or incomplete headers. There are no callback, +state, language-name or version-name slots in this lexical recipe. + +Scheme execution compares source characters directly. Rust borrows the UTF-8 +header slice and returns byte endpoints; neither implementation allocates a +separator list or hash table. Header validation is repeated for each lexical +rule invocation; this change does not add a per-source prepared-header cache or +claim a measured runtime speedup. The bounded count keeps that work explicit. +Unicode endpoint and selected alphabetic/numeric rejection cases are tested; +these receipts are not exhaustive Unicode-table equivalence evidence. + +The language keeps ordinary metadata/fixture extension slots on its POO Loader. +The inherited binding enforces descriptor identity. Extending an already bound +metadata object preserves its required identity slots, whereas replacing it +with an unbound metadata object rejects. The Loader schema remains v2. + +The Rust producer emits corresponding =LexicalExpr::HeaderDelimiter= and +=LexicalExpr::HeaderData= values from canonical Scheme IR. Two complete generated +language specs compile and execute with Rust: the real HL7 fixture and +custom message delimiters, and the independent Unicode header-record language. +CI regenerates both formatted Rust source products and compares them exactly. +=languages/hl7/scanner.ss= is deleted; no import alias or fallback scanner remains. +HL7's projection is a closed =RecordProjection.= recipe in =grammar.ss=. +The engine validates ownership, evaluates selectors and publishes identity. + +** Complete authoring convergence + +All eight packs now contain exactly =grammar.ss=, =parser.ss= and +=parser-test.ss=. The Scheme topology gate enforces the file and declaration +boundary. Bash source execution uses the common =SourceStrategy.= protocol; +line sources demonstrate an independent provider. TLA+ source metadata, fixtures +and three grammar scopes share =grammar.ss= and canonical parser entry. Closed +structured scanner and proof policy recipes are verified with an independent +checklist language. HL7 and a Unicode record share =RecordProjection.=. Shared +=GraphSyntax*= objects produce source accepted by both GQL and openCypher. + +Build input is resolved by =src/language/module-input.ss=. Multi-scope modules +require an explicit =#export-name= selector; the generator CLIs and build C ABI +use that same constrained resolution. Loader v2 and existing ABI versions remain +unchanged. The old module paths are removed, without import aliases. + +This closes authoring ownership. Shell source execution and structured TLA+ +policies still require equivalent standalone Rust IR consumers before they may +advertise portable support. Recursive HCL lowering admits its exact reviewed +corridor; universal recursive code generation is not implied by common dispatch. + +** Closed structured lexical fragments + +A language declares local spelling with =deftext-profile= and composes fragments +using =(ref name)=. TLA+ now uses the same shared text-profile interpreter and Rust +emitter as FHIRPath and HL7. The new closed operations are: + +| Form | Constraint | +|---+---| +| =(run-containing class required minimum maximum)= | Consume a maximal bounded run; at least one consumed character must match =required=. Classification and recognition share one traversal. | +| =(ends-in class body)= | Accept the complete positive-width =body= only if its last character matches =class=. | +| =(ends-not-in class body)= | Accept the complete positive-width =body= only if its last character does not match =class=. | +| =(alphabetic)= | The pinned Scheme alphabetic class, generated as a Rust table; no host Unicode version substitution. | + +Bounds, character classes, recursion depth and nullable roots are admitted by +the engine. Classes and spellings are canonical data, included in grammar +identity and emitted to Rust. No callback name or lambda appears in these local +lexical declarations. Module context now uses closed =module-text= data and +request-owned source indexes. Proof policy continues through its separate POO +contract and remains a portable execution obligation. The language-entry schema +remains v2. + +** Request-owned header capture preparation + +Closed =header-delimiter= and =header-data= rules lower to a deduplicated capture +plan owned by the compiled parser machine. A Scheme parse prepares each distinct +header once, including invalid results. The dynamic scope binds both that plan +and the exact source object, restores outer scopes on failure, and gives parallel +requests independent captures. Ordinary lexing, directed parsing, lossless +failure materialization and incremental resume share this preparation. Empty +plans keep the ordinary lexer path. No mutable cross-request cache is retained. + +Rust prepares the same distinct captures once per lexical request and borrows +the delimiter spans. The prepared matcher owns its source borrow, so callers +cannot pair it with another input. Nested choices use the same context. Header +alphabetic/numeric rejection uses the frozen Scheme character classes. +These changes retain the lexical IR and language-entry schema; they change the +execution plan, not grammar identity. The standalone native producer includes +=src/runtime/lexical-source= before its consumers. + +This implements prepared header captures. TLA+ nesting preparation, shell +recognition and portable proof-policy execution remain separate obligations. + +** Request-owned module framing + +The =module-text= opcode contains seven bounded strings: repeated opening border, +header word, repeated closing border, nested comment opening/closing strings, +line-comment prefix and the extra name character. Admission rejects malformed +profiles before preparing source state. The compiler emits the same values to +Rust =ModuleTextProfile=; effective values participate in the grammar digest. + +Scheme and Rust prepare source-owned header intervals and depth transitions. +Repeated border runs occupy one interval, including all admissible suffix starts. +Depth preparation skips quoted strings, nested comments and line comments. Module +text stopping preserves raw header recognition, including headers within text; +this is intentionally distinct from the depth walk. Binary queries replace +rescanning the whole source prefix for each token. Invalid/unterminated inputs +retain their original framing behavior. There is no mutable cross-request cache. + +Alphabetic, numeric and whitespace classes come from the pinned Scheme producer. +Rust uses generated tables and byte boundaries; Scheme uses character boundaries. +A generated standalone parser checks exact endpoints, lossless accepted modules, +nesting and rejection. Independent UNIT framing and a numeric extra name character +exercise values beyond the TLA+ spelling. Long border runs have a bounded-record +control. Full SANY proof policy portability remains separate from framing. + +** Source descriptors use the same native result product + +An admitted source descriptor now binds through =bind-language-abi= and +=register-language!=. Its =ResultProfile.= supplies the native syntax, +field and terminal domains; grammar descriptors continue to derive these from +the grammar algebra. Source descriptors reject a contextual parser product +before binding. The GPA1 buffer, native descriptor v1 and language handle ABI +v2 are unchanged. Bash and an independent POO line language use the actual +public C descriptor/parse/release functions, including Unicode source, invalid +UTF-8 rejection, result reuse and released-handle rejection. Syntax rejection +is carried by the existing GPA1 status and admitted fallback terminal. + +Standalone Rust Word/Binding and Command/Source recognition now execute shared +closed profiles and match Scheme controls, as described in +[[file:0008-extensible-language-packs.org][the shared engine contract]]. This native C product still executes the Scheme +Source engine. Full Rust ParseArtifact identity, diagnostics and native binding +remain separate from recognition/event parity. Construction field sets constrain +publication but do not yet express field cardinality. + +** Scheme Source edit sessions + +Applications may keep a Source session without adding scanner code to a language +repository. =parse-source-language/session= takes the same source descriptor on +every edit and returns an opaque session whose artifact uses the normal Source +publication contract: + +#+begin_src scheme +(import (only-in :gerbil-parser/src/language/source + parse-source-language/session source-language-session-artifact + source-language-session-scanned-token-count + source-language-session-reused-token-count)) +(def initial (parse-source-language/session bash-source-language "cat < canonical IR -> +compiler publication -> prepared machine -> request execution -> artifact. +Shared engine mechanisms belong beneath this sequence. Language authors do not +configure matcher caches, generic registries, opcode lowering or worker lifetimes. + +The [[file:../research/shared-ranked-scanner-execution.org][integrated execution study]] now supplies one decision index for +lexical specialization, prepared representation, casefold complete requests and +execution evidence. These four responsibility modules follow this same sequence. +They preserve every historical experiment and negative receipt, while separating +retained mechanisms from withdrawn prototypes and unclosed qualification gates. + +The [[file:../research/compiled-language-byte-publication.org][compiled language publication study]] +extends this sequence through immutable sidecars and linked AOT admission. +It removes whole-text intermediates at the compiler publisher; +content addresses and compressed output are checked against the original text +specification. Runtime admission retains its existing reader and failure order +after the experimental byte reader produced mixed complete-path CPU results. +This is separate from request artifact publication and from retained-heap or +full parser performance claims. + +| Design obligation | Implementation owner | Executable control | Scope and open boundary | +|-------------------+----------------------+--------------------+-------------------------| +| Compiled language transport preserves the ordered IR contract | [[file:../../src/compiler/language-artifact.ss][Immutable publisher]] shares owned UTF-8 bytes; [[file:../../src/runtime/language-artifact.ss][filesystem/AOT loader]] retains complete text decoding | [[file:../../t/language-artifact-test.ss][Original-text parity, encoding, framing and concurrent publication controls]]; [[file:../../t/benchmarks/language-artifact/benchmark.ss][complete roundtrip and admission benchmark]] | Cold compression, immutable publication and full datum admission remain; byte-reader candidate withdrawn; allocation, CPU and platform receipts belong to the linked study | +| One effective grammar and selected source origins | [[file:../../src/compiler/normalize.ss][normalization]] and [[file:../../src/compiler/lr.ss][resolved LR analysis]] | [[file:../../t/resolved-grammar-test.ss][resolved grammar controls]] | Finite LR entry admission and rejected LR action occurrence diagnostics exist; admitted fork and runtime origin catalogs remain open | +| Ordered declaration origins survive complete binding publication | Normalization, syntax-origin collection and [[file:../../src/compiler/bound-ir.ss][Bound IR owner]] use invocation-owned construction and source selection | [[file:../../t/bound-ir-index-test.ss][source selection and large catalog controls]], composition/declaration/ABI consumers and [[file:../../t/benchmarks/bound-source-catalog/benchmark.ss][complete sidecar benchmark]] | [[file:../research/bound-declaration-publication.org][Publication realization]] removes variadic catalog flattening and temporary construction spines while preserving IDs, digests and forward-reference admission; performance populations and platform CI remain separate | +| Canonical digest input retains the declaration contract under physical sink changes | Same Bound IR owner uses the official writer with explicit UTF-8/LF byte output and existing native SHA-256 | Independent original text digests for Unicode declarations and every identity; complete sidecar comparison and final-input component controls | [[file:../research/bound-declaration-publication.org::*Canonical digest input follows the same publication owner][Digest-input study]] separates allocated flow, writer/encoding/hash costs and negative CPU evidence; bytes remain materialized, and no grammar cache or author option is introduced | +| Backend/capability selection at preparation | [[file:../../src/runtime/lr-parser.ss][LR preparation]] | [[file:../../t/lr-semantic-preparation-test.ss][semantic preparation controls]] | Private CLOS methods select executors; this does not supply an open generic extension protocol | +| Canonical operand actions lower once beneath all author interfaces | [[file:../../src/runtime/reduction-plan.ss][Prepared semantic kernels]] select two backend factories, preserve ordered chains and specialize empty/unary execution; [[file:../../src/compiler/fused-reduction.ss][shared generated expressions]] and Rust reduction apply the nullable-origin contract | [[file:../../t/lr-stack-reduction-test.ss][Independent expected trees and request-boundary reuse]], [[file:../../t/fused-reduction-test.ss][generated recognition/event controls]] and [[file:../../rust/gerbil-parser-runtime/tests/unit/nullable_boundary.rs][Rust boundary control]]; [[file:../../t/benchmarks/lr-request-stack/benchmark.ss][cold and complete-artifact comparisons]]; CI runs the shared public request matrix | [[file:../research/lr-semantic-kernel-staging.org][Staging study]] records 93 Scheme and 146 Rust cases, the rejected per-operand generic selection and mixed CPU observations; stable latency improvement is unproved; exact-revision CI is a separate gate | +| CPU evidence conserves complete parser work and includes natural GC | [[file:../../t/benchmarks/parser-stage-cost/benchmark.ss][Shared sampler]] owns complementary sub-batches, complete-result validation and raw CPU/GC receipts | [[file:../../t/benchmark-test.ss][Independent work and leg-position invariants]] plus actual complete-request counts; native compilation and nine source-owned cases pass | [[file:../research/lr-semantic-kernel-staging.org::*Complementary CPU observations][Control rejection and corrected observations]] retain twenty disjoint groups, four nominal quanta per variant, all samples and unchanged admission limits; fresh complete-request CPU qualification is distinct | +| Canonical semantic names have one admission contract | [[file:../../src/compiler/lr.ss][canonical operand predicate]], LR preparation, fused Scheme and Rust publication | [[file:../../t/lr-stack-reduction-test.ss][malformed and complete-product controls]] and [[file:../../t/rust-runtime-generation-test.ss][Rust publication controls]] | Symbol names and proper field/alias chains are required even in unused productions; 22 local cases and ten-language ABI byte parity passed; [[file:../research/prepared-lr-action-plans.org::*Canonical action admission across product owners][research witness, falsifiers and receipts]] | +| Complete production semantics precede indexing and execution | Same canonical owner and standalone LR lowering; compiler/runtime share layout classification; complete rule references resolve through the existing production-owner index | Production numbering/width/precedence/operand controls, both semantic backends, five construction routes, fused empty-output rejection and Rust absent/existing file controls | [[file:../research/prepared-lr-action-plans.org::*Closed rule references across canonical IR consumers][Closed reference contract]] follows the historical 32-case production qualification; forward references and declared dead cycles remain legal; productivity and grammar/table equivalence are distinct | +| Physical LR tables and layout capability are admitted before execution and publication | [[file:../../src/compiler/lr-table.ss][shared table contract]] consumed by compiler, runtime, fused Scheme and Rust; canonical terminal catalog, declared LHS symbols and reachable EOF acceptance share this boundary | [[file:../../t/lr-index-test.ss][domain, vocabulary, graph, layout and identity controls]] and [[file:../../t/rust-runtime-generation-test.ss][atomic publication rejection]] | [[file:../research/lr-table-admission.org][Table study]]; historical 39-case and ten-language receipts remain separate from the production-relative extension; grammar/table equivalence, balanced markers and guard fallback correctness remain open; disconnected post-conflict states are retained | +| Prepared layout capability controls request execution and continuation admission | [[file:../../src/runtime/lr-parser.ss][deterministic/GLR selectors and checkpoint requirement]], [[file:../../src/runtime/parser.ss][source scope and shared contextual preparation]] and [[file:../../src/runtime/layout.ss][shared casefold lookup]] | [[file:../../t/layout-boundary-test.ss][ambient-context, streaming, branch, restoration, Unicode guard and continuation controls]]; [[file:../../t/contextual-parser-test.ss][raw/prepared publication ownership]] | [[file:../research/lr-table-admission.org::*Checkpoint and contextual execution admission][Continuation contract]] extends the [[file:../research/lr-table-admission.org::*Prepared capability and request-local layout execution][request study]]; layout continuations are rejected before construction, full layout source parsing retains GLR; layout continuation capture, contextual GLR, arbitrary balance and fallback equivalence remain distinct | +| Request source coordinates have bounded physical storage | [[file:../../src/runtime/layout.ss][column builder and private reads]] select packed u32 storage under a proven column bound and stage primitive writes with hygienic syntax | [[file:../../t/layout-column-storage-test.ss][per-byte, Unicode, tab, CRLF and exact integer controls]]; [[file:../../t/benchmarks/layout-columns/benchmark.ss][shared preparation and complete GLR product benchmark]] | [[file:../research/lr-table-admission.org::*Compact request-owned source coordinates][Coordinate study]]; constant-time query and complete artifacts are retained; the slower direct character traversal is rejected; current CPU/allocation observations do not imply language-wide latency or retained-heap savings | +| Rust target capacities follow canonical Scheme admission | [[file:../../src/compiler/rust-runtime.ss][Rust publication entry]] admits u16/u32 domains, i32 production precedence and absence of layout capability before source allocation; [[file:../../rust/gerbil-parser-runtime/src/engine/parser.rs][shared reduction]] checks i64 accumulated scores before mutation and propagates overflow globally through GLR | [[file:../../t/rust-runtime-generation-test.ss][publication controls]], [[file:../../rust/gerbil-parser-runtime/tests/unit/selective_glr.rs][cumulative ordering and atomic failure controls]], plus internal GLR branch-order controls | [[file:../research/lr-table-admission.org::*Checked cumulative precedence and global failure][Checked numeric contract]]; 145 runtime cases pass, with prior 69 Scheme cases and ten-language ABI parity; numeric overflow handling is closed within the declared target domain; arbitrary-precision target equivalence remains outside this contract | +| Static matcher configuration shared across lexical modes | [[file:../../src/runtime/contextual-scanner.ss][contextual preparation]] | [[file:../../t/contextual-scanner-test.ss][rank/action/state controls]] | Region/guard sharing exists; primitive closures and literal source-size strategy retain separate admission | +| Nested regions retain request-owned state | [[file:../../src/runtime/region-scanner.ss][region executor]] | [[file:../../t/region-scanner-test.ss][region controls]] and [[file:../../t/contextual-deferred-scanner-test.ss][deferred controls]] | Prepared opener data is shared; counters, frames and delimiter obligations belong to requests | +| Contextual program and source publication have explicit owners | [[file:../../src/language/source-strategy.ss][source strategy]] | [[file:../../t/source-strategy-test.ss][strategy controls]] and [[file:../../t/source-scanner-test.ss][source scanner controls]] | These focused controls do not establish general cross-edit Rust subtree reuse | +| Ranked competition preserves winner semantics | [[file:../../src/compiler/lexical-lexer.ss][lexical lowering]] | [[file:../../t/lexical-first-character-test.ss][competition controls]] | Original fold retained; hygienic loop candidate withdrawn after complete-request admission failed | +| Generated entries declare engine-owned representation obligations | [[file:../../src/compiler/machine.ss][machine backend admission]] | [[file:../../t/language-surface-test.ss][backend contract and atomicity controls]] | Captured prepared representation and generated entry products are distinct; queries do not prove arbitrary procedure bodies | +| Preparation cost remains visible | [[file:../../t/benchmarks/contextual-scanner/matched-scale.ss][contextual benchmark]] and [[file:../../t/benchmarks/regular-mode/request-cost.ss][regular benchmark]] | Native prepared products checked after measurement | Contextual preparation controls are added in this slice; historical paired receipts remain separate from fresh runs | +| Artifact publication cost follows shared engine ownership | [[file:../../src/runtime/artifact.ss][canonical artifact owner]] and [[file:../../t/benchmarks/parser-stage-cost/benchmark.ss][generic stage benchmark]] | [[file:../../t/artifact-publication-cost-test.ss][publication observation contracts]] and existing event/source controls | Encoding, event publication and identity intervals are separate; supplied products do not imply executed construction stages | +| Private recognition variants close their subclass protocol | [[file:../../src/runtime/recognition.ss][recognition value owner]]; Gerbil selects checked direct access for final records | [[file:../../t/source-recognition-test.ss][extension, wrong-kind access, mutation and relocation controls]] and [[file:../../t/benchmarks/lr-request-stack/benchmark.ss][independent LR and publisher comparison]] | [[file:../research/semantic-reduction-cost.org::*Closed private recognition variants][Compiler mechanism and measured scope]]: prepared recognition plus publication medians improve locally; GQL dispersion is retained, scanning and end-to-end latency are unqualified | +| Certified windows and rejected products construct final event containers directly | Same artifact owner uses the official list builder and a private group-comparison cursor | [[file:../../t/artifact-window-publication-test.ss][large complete-product, Unicode and assignment controls]] plus the existing incremental consumer suite; [[file:../../t/benchmarks/artifact-window/benchmark.ss][complete publication benchmark]] | [[file:../research/artifact-publication-cost-boundaries.org::*Direct certified-window and rejected-product publication][Publication realization]] removes temporary flattened groups, reversed windows, replacement event lists and rejected-product ID lists; source identity, event schema and certificate obligations remain; local qualification is separate from platform CI | +| Canonical event containers have an independent executable control | Same shared benchmark and official =with-list-builder= in the artifact owner | Fresh-container/shared-payload controls, full artifact equality and source admission | Copy allocation is not a retained-heap census; variable counter medians cannot establish traversal waste | +| Shared sampling does not depend on a language workload module | [[file:../../t/benchmarks/parser-stage-cost/benchmark.ss][sampling owner]] with direct GQL, actor and LR callers | Existing population/GC controls, language-independent batch guards and native dependency plan | [[file:../research/shared-parser-cost-sampling.org][Research contract]]; owner migration is not runtime performance admission | + +The first row's semantic and provenance limits are detailed in +[[file:../research/dsl-engine-contracts.org][the DSL contract audit]]. This table records responsibility rather than +introducing another authoring DSL or a compatibility route. + +The production-relative LR extension follows one implementation boundary through +compiler admission, runtime preparation, fused strategy and Rust publication. +The [[file:../research/lr-table-admission.org::*Production-relative admission closure][complete consumer matrix]] passes 72 cases, including ten-language ABI parity +and absent/existing output-file rejection. Its graph condition is reachable EOF +acceptance, not removal of disconnected states; actual GQL conflict resolution +falsified that stronger proposal. The canonical terminal catalog and official +Gerbil queue are reused. All claims remain admission correctness; preparation +overhead, full grammar/table equivalence and exact-head platform CI are distinct. + +The subsequent canonical-reference closure uses the existing production-owner +index before analysis, preparation and publication. Its baseline missing-owner +witness was admitted; the final native build and 117-case, thirteen-suite +consumer matrix reject it before execution or output while preserving forward +references and declared unused cycles. The +[[file:../research/prepared-lr-action-plans.org::*Closed rule references across canonical IR consumers][integrated production study]] records the Scheme witness, two design +questions, the additional project policy beyond Knuth's alphabet definition, +and exact qualification receipts. This closes rule-owner resolution, not finite +derivation or grammar/table equivalence, and makes no performance claim. + +The runtime-capability follow-up reproduces ambient layout leakage on the same +plain runtime and tokens, then changes all three LR action selectors to use the +runtime's prepared capability. The tokenized source owner binds request-local +columns and empty initial frames; the layout consumer reuses the existing +prepared casefold index. The three runtime owners and focused module compile +natively in 70.397 seconds; fourteen suites pass 131 cases in 26.984 seconds, +including complete artifacts, independent GLR, generated products, source-origin +controls and ten-language ABI bytes. The +[[file:../research/lr-table-admission.org::*Prepared capability and request-local layout execution][integrated capability study]] records the matched failing baseline, +Scheme scope witness, standard/source correspondence and qualification limits. +No user-facing switches or language-specific adapters are added. Tested request +isolation is closed; general layout balance, fallback equivalence and fresh +exact-head CI remain separate obligations. + +The continuation follow-up closes the same capability contract at checkpoint +creation and contextual preparation. Raw contextual execution now delegates to +the owned prepared plan, including publication identity ownership. The parent +control admits layout objects without captured layout state; the repaired +seventeen-suite native matrix passes 167 cases in 23.001 seconds. Full layout +source requests still produce complete GLR artifacts and no reusable snapshots. +The [[file:../research/lr-table-admission.org::*Checkpoint and contextual execution admission][integrated continuation study]] records the Scheme witness, design questions, +failed artifact-admission attempts and final receipts. Layout continuation +capture and contextual GLR remain unimplemented; no throughput or CPU improvement +is inferred, and exact-head platform checks remain distinct. + +* Scheme witness: declaration, preparation and repeated requests + +#+begin_src scheme +;; Engine-owned lowering of an admitted declaration. No new author slot. +(def (prepare-rule-matcher declaration cache) + (or (table-ref cache declaration #f) + (let* ((expression (lower-rule-matcher declaration)) + (plan (make-contextual-matcher-plan + expression (prepare-matcher expression)))) + (table-set! cache declaration plan) + plan))) + +;; Rule policy is separate from executable configuration. The actual local +;; macro is defined inside decode-rule, where these metadata bindings exist. +(defrule (finish expression executor) + (make-runtime-scan-rule name mode form expression executor rank action)) +#+end_src + +** Question: why is this design preferable? + +S-expression declarations permit structural equality and engine-owned matching. +Hygienic =defrule= expansion retains lexical rule policy without allocating a +per-rule constructor callback. POO composition selects declared values and +origins; private CLOS preparation methods select backend/capability combinations; +named algorithms implement their execution. Each mechanism has a concrete +responsibility. Authors retain ordinary values unless they need actual extension. + +This is deliberately selective: the preparation cache shares expensive immutable +region/guard configuration. It does not intern request state or all Scheme +closures. Rejected broad-cache and multiple-value experiments are retained in +[[file:../research/scheme-reduction-abstractions.org::*Contextual matcher preparation across modes and source sizes][the preparation study]]. + +** Question: how is the claim scientifically tested? + +The falsifiers are changed rule rank/action outcomes, state leaking between +requests, changed Unicode byte boundaries, or preparation cost moved into scan +timing. Shared guard controls exercise independent modes and requests on short +and long sources. Existing region/deferred/source suites exercise the adjacent +semantic obligations. Preparation controls independently declare expected words, +check token kinds, complete lexemes and final UTF-8 byte coverage after timing. + +The historical paired region configuration reduces admitted preparation allocation +by approximately 45%. Identifier CPU results are mixed; neither that allocation +result nor the papers prove whole-parser speedup. The papers motivate staging +under explicit assumptions; semantic controls and matched measurements decide +which lowering is retained in this engine. + +* Reproducible preparation controls + +The existing benchmark exports =benchmark-matcher-preparation=. It exercises +one-mode identifier, sixteen-mode identifier, a 128-literal catalog, a region +configuration and a guarded region configuration. The region source includes +Unicode quotes and nested interpolation. Defaults are eleven samples and three +preparations per sample; both counts must be positive exact integers. + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/contextual-scanner/matched-scale) +(benchmark-matcher-preparation 11 3) +#+end_src + +Run this expression through the repository's configured Gerbil environment. +For native cost evidence, compile the benchmark module through the existing +compiler owner and load its native product. Source interpretation is a distinct +control. No new executable, process wrapper or baseline executor is introduced. +Compare revisions through the same caller, toolchain, configuration and sample +order, with explicit loaded-product identities. Each measured interval contains +only plan preparation: IR construction, source encoding and returned-product +validation are excluded. The final plan in each batch is validated; the output +does not assert that every intermediate plan was scanned. + +The output records CPU milliseconds per preparation, GC count and admitted +allocated bytes per preparation. Collection or a negative raw delta makes the +allocation value =#f=, not zero. A GC is forced before each interval; this isolated +preparation control does not measure natural application GC, amortized request +latency, scan throughput or complete-language parsing. Existing =scan=, +=prepare-scan= and =plan-batch= entry modes retain those distinct boundaries. + +* Qualification of the preparation-control slice + +The benchmark module compiles natively with the installed Gerbil/Gambit and GCC +toolchain in 16.912 seconds, exit 0 and final =OK=. Its native caller completes +in 12.033 seconds, exit 0, =MATCHER-PREPARATION-OK= and final =OK=. All fifteen +preparation samples validate their returned products, including Unicode region +and guarded-region sources. Existing =scan=, =prepare-scan= and =plan-batch= +modes each retain two reported samples and complete =SCANNER-SCALE-OK=. +Both gates retain the five-second idle limit and report real compiler/module +progress. This is focused native caller qualification over reused dependency +products, not a fresh complete-package build or a new paired speedup result. + +The modularization check compares all 2,311 original lines with their corresponding +overview/module ranges; every historical section is retained exactly. All 65 +local file links in the seven touched audit/research documents resolve. This +check does not qualify external paper URLs or local links in unrelated documents. + +* Remaining gates and next work + +** Integrated study qualification + +The ranked execution study previously accumulated 1,302 lines across lexical +lowering, representation ownership, experiments and receipt admission. The new +overview maps each obligation to its existing engine owner and executable +consumer, with a single retained/withdrawn decision table, Scheme witness and +two design questions. Four responsibility modules retain all 1,296 historical +body lines exactly; Scheme reconstruction reproduces the complete original +text from the published modules, including ordering and whitespace. Their +sizes are 590, 242, 245 and 243 lines. No runtime source or default changes in +this reorganization. + +Incoming audit links now name the actual module and original heading. The +Scheme qualifier checks local files and heading targets across this audit, +the execution overview/modules and the parent scanner study. The first temporary +restructuring attempt was rejected by historical equality after an incorrect +output-port call; the first link-qualifier attempt failed import admission for +an unavailable library. The corrected qualifier uses the project's existing +core-backed string search utility. Neither failed attempt is a passing receipt. + +This closes research organization and link integrity. It does not close the +remaining engine obligations listed in the implementation table or convert +historical allocation observations into CPU benefits. At =c12dcb0=, Rust CI +passed while Linux Scheme was running and macOS Scheme queued. Fresh exact-head +platform results remain separate from document and local native qualification. + +** Engine qualification sequence + +The [[file:../research/dense-lr-item-space.org][dense item-space implementation]] addresses the suffix study's remaining +production-count times maximum-width padding. One private layout and logical +coordinate operations now serve all LR construction routes and conflict-origin +publication. Explicit epsilon/production-boundary controls, independent suffix +scans and complete compiler-product benchmarks own its qualification. This is +a compiler representation change; complete-request latency remains a separate +measurement obligation. Existing CI jobs and required stages are unchanged. + +Rejected LR action provenance is now implemented in the existing lowering and +table-publication owners. The [[file:../research/lr-conflict-source-origins.org][focused origin module]] records the Scheme witness, +two design questions, selected POO source contract and qualification boundary. +Local qualification passed all 41 source GxTest cases against the final native +producer (37 existing semantic controls and four new origin controls). The +producer and focused test module also completed native compilation with the +unchanged five-second real-output idle gate. Both the build module catalog and +the conformance composition suite own the new module. The subsequent exact +=11b2cf1= [[https://github.com/tao3k/gerbil-parser/actions/runs/38019193501][platform CI run]] passed Linux Scheme, macOS Scheme and Rust AOT. + +1. Keep exact-head Linux, macOS and Rust AOT CI separate from local native + overlays. The earlier =f3dff04= macOS idle failure remains negative history; + the completed =11b2cf1= platform run qualifies that later revision. This + publication attribution slice requires fresh exact-head CI. +2. Prioritize producer construction and publication traversals using the + [[file:../research/lr-request-cost-boundaries.org][request cost study]] and + [[file:../research/shared-source-projection-publication.org][source publication study]]. A new optimization needs complete + artifact/field/span parity and a matched phase control. Rejected continuation, + projection callback and dense-prefix candidates remain rejected. + +The [[file:../research/artifact-publication-cost-boundaries.org][publication attribution implementation]] now localizes the latter boundary: +29 source cases passed against the native producer, and twelve native profiles +across three language families at two scales completed 240 full-product checks. +Observed event-publication allocation exceeds hash/header allocation in these +workloads. This is a next-work localization receipt, not a performance benefit +admission or an assertion that required canonical event storage is redundant. + +The subsequent [[file:../research/artifact-publication-cost-boundaries.org::*Executable event-storage control][event-storage implementation]] supplies that independent control and replaces +the engine's duplicated private queue with Gerbil's official =with-list-builder=. +It applies to both recognition and event-program publication. The canonical +event constructors, author interfaces and artifact schema are unchanged. +The initial =map vector-copy= control exposed variable FHIRPath allocation +counters despite passing the existing no-GC filter; those observations are +retained as negative evidence against deriving a precise storage budget from +one median. This refactoring is an abstraction/qualification change, not an +admitted complete-request performance improvement. + +Final native qualification rebuilds artifact publication, its shared benchmark +and focused test in 41.776 s; all 31 source cases pass in 12.914 s. Twelve native +cross-language profiles complete 240 observed requests and 240 copies in +33.982 s, with full equality/source checks and final process receipts. Under +the official builder, admitted copy allocation P50/P95 agree in all twelve +profiles and closely track event-publication allocation. The small difference +does not establish recoverable storage; recognition/reduction cost separation +is now the next optimization obligation. The research module retains exact +population counts, rejected prototype and receipt paths. The verified =a1df0ca= +CI run has passed Rust AOT; Linux/macOS Scheme were still in progress. Fresh +CI is required for this subsequent source slice. + +The subsequent [[file:../research/shared-parser-cost-sampling.org][shared sampling migration]] closes the cross-language dependency on GQL's +workload module. GQL stages, actors and LR comparisons import the shared sampler +directly; GQL sampling aliases are removed. Native qualification builds the +owner and five consumers/controls in 76.380 s, then rebuilds existing missing +and stale diagnostic dependencies. Both rejected attempts remain recorded. +The final source run passes 30 cases from the staged test content in 42.303 s. +Twelve cross-language paired summaries complete in 44.542 s with a stable +identical-executor control and no admitted complete-request benefit. Three +actor configurations complete in 21.921 s and retain process-only allocation +ownership. Source semantics and default representations remain unchanged. +The verified preceding =ecebab9= CI has passed Rust AOT; Linux is running and +macOS queued. This migration requires fresh exact-head platform CI. + +The shared scanner study is modularized into +[[file:../research/shared-scanner-staging-and-fusion.org][one overview and four responsibility modules]]. Its original experiments, +failed runs, revision identities and negative results are preserved. This audit +is the implementation entry; those modules supply detailed arguments and receipts. + +* Complete-request cost admission correction + +The [[file:../research/ranked-scanner-specialization.org::*Cross-language audit of callback removal][cross-language callback audit]] withdraws the private ranked loop macro. +Reproducible allocation reduction did not establish a complete-request CPU +benefit on GQL, FHIRPath and Arithmetic. The original fold is restored; the +independent order and Unicode test remains. Historical component receipts are +retained with their evidence boundary. + +The existing LR request benchmark now exports =benchmark-lr-request-backends=. +It checks twenty-four prepared/public routes against one canonical recognition +artifact, distinguishes requested preference from prepared admission, and records +untimed actual source stages plus generated execution capabilities. GQL rejects +the event executor, FHIRPath admits it, and Arithmetic's public request uses its +generated recognition drive. Enabling an event preference alone therefore does +not certify event execution of the complete public request. + +That audit identifies an explicit semantic representation contract for generated +execution as the next obligation. The following section implements the contract; +matched complete-request cost admission remains separate. No default backend is changed on the strength +of prepared allocation or the single-order representation diagnostic. Bash's +contextual route remains independently qualified by its own source products; +these LR controls do not substitute for that qualification. + + +The [[file:../research/parser-execution-evidence.org::*Executed interleaved backend qualification][executed paired qualification]] now replaces the single-order diagnostic as +performance evidence. Twenty ABBA/BAAB groups per workload measure 19,200 native +calls in 53.291 seconds, following ten passed measurement controls. Small +prepared event workloads meet the observed-regression criterion. The large GQL +identical-executor control fails, so the executable gate rejects complete-request +benefit admission. Semantic pass and performance rejection are distinct outputs; +no backend default changes on this receipt. +* Installed representation admission + +=parser-machine-backend-representation= reports the captured prepared runtime or +an installed drive/source/step/event-step product. The engine owns this closed +domain; language entries retain their existing backend declarations. Missing +optional entries return =#f=; malformed machine/kind queries reject. A retained +event machine continues reporting event-program after the dynamic preference +changes. Its generated drive may still report recognition. + +The contract exposed a concrete admission gap: a recognition =step= could be +installed on an event machine. Both the direct machine installer and the +declarative batch precheck now reject that mismatch. A batch with a valid drive +followed by such a step leaves the drive uninstalled. This preserves all-or-nothing +admission and avoids interpreting recognition values with an event reducer. + +The cross-language benchmark reports these contracts alongside actual untimed +source stages. Availability and declared representation remain capabilities; +a source specialization can decline. The existing canonical artifact equality +controls qualify known generated implementations, not arbitrary procedure bodies. +The [[file:../research/prepared-parser-representation.org::*Engine-owned semantic representation of installed entries][research argument]] records the Scheme witness, both design questions and limits. + +The shared request executor also reports the actual generated-drive or prepared- +checkpoint branch, rather than relying on the inclusive source-recognition stage. +CI-owned source-contract tests cover generated execution under both preferences +and ordinary accepted/rejected/empty requests. The benchmark requires the expected +branch to complete before admitting its cost samples. Nested child intervals are +not additive to their inclusive source-recognition parent. + +** Qualification of representation and actual-route admission + +Native rebuilding covers the current lexical producer, machine, interface suite +and caller (109.692 s); the final installation constraint rebuilds machine and +interface controls (32.273 s); the actual-branch owner and caller rebuild in +31.856 s. Every compile exits 0 with final =OK= under the five-second idle gate. +Other dependency products are reused; this is not complete-package qualification. + +The final native interface/backend suite passes all nine cases in 10.707 s. +Source evaluation over the native producer passes all nine cost-observer and +language-source-contract cases in 13.255 s. The final native backend caller +qualifies twenty-four prepared/public routes across GQL, FHIRPath and Arithmetic, +two source scales and two preferences, in 15.774 s. Each timed route retains +canonical artifact equality; each public route has an untimed completed-branch +witness. The caller uses two samples of two calls as an execution gate, not a +matched performance comparison. Final markers and process exits propagate. + +No default executor changes and no speedup claim follows from these receipts. +The remaining obligation is measured complete-request cost admission for any +future representation change. Exact pushed-head platform and Rust AOT CI remain +separate from these local controls. + +After integrating =0a1b326=, the shared sampler and current caller rebuild +natively in 51.951 s. The updated caller retains the twenty-group ABBA/BAAB +minimum and runs two calls per timed batch: twelve paired workload summaries, +covering the twenty-four prepared/public representation routes, complete in +24.615 s with exit 0, =LR-REQUEST-BACKENDS-OK= and final =OK=. Every source +witness asserts the completed engine branch and every batch validates its product. +The identical-executor GQL control is admitted stable; complete-request benefit +admission is =#f=. This small-batch rerun qualifies the merged execution and +admission path, not a portable CPU benefit. The earlier two-sample caller receipts +remain historical execution controls; they do not satisfy the current paired +performance protocol. + + +* Shared literal IR and casefold lookup + +Bounded and ranked literal preparation and case-insensitive action lookup now +share the literal trie builder. The prepared vocabulary is shared across LR +states; the original state row remains the authority for entry admission. +ASCII early misses avoid whole-string uppercase allocation, while Unicode +conversion retains its original owner. No language declaration changes. + +The [[file:../research/casefold-request-qualification.org::*Shared literal vocabulary and casefold lookup qualification][native qualification]] separates complete-request allocation evidence from +component CPU evidence. The =11b2cf1= complete matrix has 19,200 calls and 844 admitted +allocation intervals. Latest =9374421= graph qualification passes 44 semantic +cases and repeats the paired component benefit. GQL saves 2,944 or 24,448 bytes per request; FHIRPath and +Arithmetic allocation controls are unchanged. Alternating lookup measurements +admit early-miss benefit for long identifiers, but whole-parser CPU acceleration +and the Bash/tree-sitter throughput gap remain unproved. Withdrawn prototypes +and compiler qualification failures remain in the research history. + + +* Complete-request casefold allocation proof + +The [[file:../research/casefold-request-qualification.org::*Complete casefold request evidence][complete request matrix]] executes 19,200 calls on one production graph, with +only the casefold module pair withdrawn in the reference. Scheme validates 916 +exact no-GC allocation intervals and identical source/prepared saving deltas. +Long-identifier requests reduce prepared allocation by 37.62--38.29% and full +source allocation by 16.86--16.95%. Arithmetic allocation stays unchanged. +Corrupted counters, forged CPU summaries and incomplete receipts are rejected. +CPU benefit admission fails the unchanged control gate; no complete-parser CPU +acceleration or Bash/tree-sitter throughput conclusion is made. diff --git a/docs/research/README.org b/docs/research/README.org new file mode 100644 index 00000000..8d6a5b89 --- /dev/null +++ b/docs/research/README.org @@ -0,0 +1,158 @@ +#+title: Parser research + +Research documents explain algorithm choices and their relation to the public +Grammar/Parser/Scanner IR. Executable correctness belongs in Scheme test suites; +performance admission belongs in the ASP =benchmark.ss= contracts. + +Use the repository's native harness and existing =Justfile= entries. Keep +workloads, samples and thresholds in their Scheme owners. Research documentation +has no generated result snapshots, separate JSON report schemas, source-rewriting +instrumentation or runtime-binding replacement wrappers. + +* Reproduction and evidence + +Run commands from the repository root with the declared Gerbil dependencies +installed. Git documents name repository source owners and executable controls; +machine-local paths and temporary logs are not reproducible evidence. Historical +local timings below are observations, not independently replayable receipts or +current performance guarantees. Reproduce measurements with the committed +benchmark workload and its sampling/admission policy on the revision under test. + +The continuation and contextual ownership controls use the normal source test +entry after native package compilation: + +#+begin_src sh +gerbil compile -O +gerbil env gerbil test -v 5 t/layout-boundary-test.ss t/contextual-parser-test.ss t/lr-fragment-transfer-test.ss +#+end_src + +Require successful process exit and the test harness's final =OK=; a printed +intermediate marker alone is insufficient. The shared bounded CI execution, +native build configuration and platform checks are defined in +[[file:../../.github/workflows/ci.yml][the committed CI workflow]]. For complete GQL stage measurements use the +existing Scheme qualification entry: + +#+begin_src sh +gerbil env gxi qualify.ss local --suite gql-profile --gerbil-path .gerbil +#+end_src + +Workloads, semantic controls, batch counts and measurement admission belong to +[[file:../../t/gql/benchmark-profile-test.ss][the profile suite]] and its imported Scheme benchmark owners. The observed +execution route and admitted CPU/allocation samples determine what a result can +claim; elapsed build time and an older CI run cannot qualify the current parser. + +* Research map + +[[file:capture-prefix-four-layer-optimization.org][Recognition capture through four layers]] connects parsing papers, explicit +work counts, generic LR implementation and complete 64-record CPU controls. + + +[[file:history-commit-model-implementation.org][History commit across the three layers]] records the same-owner callback +re-entry gap, formal commit obligations and the actual Scheme repair. + + +[[file:../architecture/0019-model-responsibilities.org][Lean/Quint responsibility boundary]] separates semantic proofs, lifecycle +checks, native conformance and matched performance, and derives explicit reuse +and amortization questions from the parsing papers. + + +[[file:module-loading-reuse.org][SDK module loading reuse]] documents source-content invalidation, official +failure/reload semantics, test preparation and Scheme CPU controls. + + +The [[file:parser-evidence-chain.org][model and implementation evidence chain]] connects Lean, Quint, +native Scheme ASCENT review and source-bound Python observations. It preserves +missing implementation/performance obligations and records negative controls. + +The [[file:parser-engine-formal-model.org][Lean semantics and operation model]] connects typed preparation, +reverse-stack reduction and counted AST sequence materialization. It distinguishes +kernel proofs from Scheme conformance and measured performance. + +The paper/framework synthesis remains in +[[file:scheme-parser-performance-framework.org][scheme-parser-performance-framework.org]]. + +The [[file:../audit/engine-design-realization.org][design-to-code audit]] connects the active research decisions to +implementation owners, executable controls and remaining qualification gates. +Use it to choose the next coherent implementation task. + +For the current Gerbil source study and the ordered runtime work, read +[[file:shared-scanner-staging-and-fusion.org][shared engine specialization, source spans and fusion]]. It separates +implemented preparation from proposed fusion and preserves negative receipts. + +For current compiler-source work, read +[[file:engine-reduction-generation.org][shared engine reduction generation]]. It +traces canonical rules into both semantic backends and separates implementation +changes from language benchmark evidence. + +The [[file:lr-semantic-kernel-staging.org][LR semantic kernel study]] applies specialization to admitted operand +actions, retains request-local offsets and constructors, and qualifies empty, +unary and wide reductions against an independent tree oracle. It records +preparation regressions as well as complete-product benchmark results. + +Start authoring research with the +[[file:scheme-deflanguage-design-synthesis.org][Scheme parser DSL design overview]]. It owns the three user interfaces, +shared objective, evidence scope, decisions and closure sequence. Detailed design +is modularized by responsibility: + +- [[file:scheme-dsl-author-interface.org][Complete author DSL interfaces and package examples]] +- [[file:scheme-contextual-parser-ir.org][Coordinated contextual IR migration and qualification obligations]] +- [[file:scheme-bash-source-frontend.org][Bash source frontend gap and complete-package compiler obligations]] +- [[file:scheme-dsl-engine-admission.org][Engine admission, provenance and artifact publication]] +- [[file:scheme-build-ownership.org][Native image storage, compiler ownership and platform qualification]] +- [[file:scheme-dsl-semantic-contracts.org][Semantic contracts and composition evolution]] +- [[file:scheme-dsl-complexity-ownership.org][Complexity ownership and author change tasks]] +- [[file:scheme-dsl-research-basis.org][Primary research basis and design method]] + +These modules share the overview's single architecture decision and package-level +research sequence. The focused documents below +provide supporting experiments rather than competing author interfaces. + +Language authoring research, annotated papers and proposed experiments are in +[[file:scheme-language-pack-design.org][Scheme language pack design]]. The document distinguishes the current +authoring interfaces from unimplemented research proposals. + +The deeper [[file:scheme-language-pack-semantics.org][binding, field semantics and author task study]] develops namespace +policies, conservative count equations and concrete HCL/arithmetic examples. + +The executable [[file:scheme-macro-binding-experiments.org][Gerbil macro binding experiments]] compare preserved definition +references, caller parameters and contextual reconstruction across modules. + +The local [[file:scheme-poo-language-components.org][Scheme syntax and POO component study]] connects existing grammar roles +and explicit composition with expression/list vocabulary and compiler phases. + +The local [[file:scheme-rule-context-design.org][rule-context declaration study]] checks how a rule can supply its own +identity once, with imported macro renaming and actual parser-event controls. + +The local [[file:scheme-list-role-composition.org][multi-rule list and POO composition study]] checks transparent per-item +fields, deterministic helper identities and an explicit expansion-phase provider. + +The local [[file:scheme-component-provenance.org][component provenance and admission study]] reuses existing Bound IR +inputs and checks which identities change when origins or composition metadata change. + +The local [[file:scheme-composition-history.org][override and removal history study]] distinguishes effective declaration +origins from retained contributions, with parser/binding admission controls. + +The local [[file:scheme-source-origin-contract.org][reader source-origin study]] distinguishes macro definition and component +invocation, with exact token-coordinate controls and retained timeout receipts. + +The local [[file:scheme-declaration-diagnostics.org][declaration diagnostics study]] checks typed parameter errors, real-reader +coordinates and explicit unknown origins for source-free syntax. + +The local [[file:scheme-nested-declaration-context.org][nested declaration-context study]] checks source-preserving wrappers and +renamed imports, while distinguishing parameter location from outer author context. + +The local [[file:scheme-syntax-parameter-context.org][official syntax-parameter context study]] checks delegated author context, +scope isolation and declaration-versus-expression boundaries with real POO roles. + +The current [[file:dsl-engine-contracts.org][DSL and engine contract audit]] connects choice semantics, +resolved grammar analysis, Source continuations and scanner state to AOT/FFI +admission, with an ordered implementation and qualification plan. + +The focused [[file:scheme-source-edit-sessions.org][Source edit-session study]] defines the shared Scheme/Rust +lexical reuse boundary, owned history and complete-recognition controls. + +* LR request model foundation + +[[file:lr-request-model-foundation.org][Research and current-code basis for complete-request modeling]] defines +input-scope stability, lifecycle invalidation and request-wide break-even before +selecting the next engine optimization. diff --git a/docs/research/artifact-publication-cost-boundaries.org b/docs/research/artifact-publication-cost-boundaries.org new file mode 100644 index 00000000..7af6270f --- /dev/null +++ b/docs/research/artifact-publication-cost-boundaries.org @@ -0,0 +1,406 @@ +#+title: Canonical artifact publication cost and ownership +#+startup: overview + +Part of the [[file:shared-scanner-staging-and-fusion.org][shared engine study]] and its +[[file:shared-source-projection-publication.org][source publication argument]]. Implementation baseline: PR #13 =11b2cf1= +plus this source slice. This module connects an open cost-attribution obligation +to the existing artifact owner, source tests and cross-language measurements. + +* Research question + +Which work inside =artifact-materialization= should be optimized first: source +encoding, canonical event publication, or source identity construction? An +inclusive materialization interval could not distinguish those operations. +This prevents using the research proposal to choose a measured implementation +boundary. It is not evidence that hashing or Gambit GC is defective. + +[[https://arxiv.org/html/2304.05276#S2.SS8][flap Section 2.8]] excludes user semantic actions from its allocation-free +generated execution claim; Section 5.5 distinguishes recognizer pseudocode from +constructing ASTs or other semantic values. Our inference is that the required +lossless artifact must be measured explicitly before applying staging or fusion +claims. This attribution change implements that measurement obligation, not +flap's DGNF normalization, fusion algorithm or linear-time theorem. + +* Engine-owned phase contract + +The existing =with-parser-cost-stage= macro now marks three boundaries inside +=src/runtime/artifact.ss=. It retains direct expression evaluation when the +observer is disabled; parameter lookup still has a cost. No language slot, +callback protocol or author configuration is added. + +| Stage | Work inside the interval | When absent | +|-------+--------------------------+-------------| +| =artifact-source-encoding= | Convert the source string to UTF-8 bytes | The caller already supplied bytes | +| =artifact-event-publication= | Walk recognition/event programs or resolve a committed raw tape into canonical event vectors and list cells; rejected products publish flat token events | The caller already supplied canonical events, or an uninstrumented specialized constructor owns them | +| =artifact-source-identity= | Hash supplied source bytes and construct the canonical artifact header | Publication failed before reaching identity construction | + +The raw-tape resolver is a private named function. Its committed prefix, +trivia resolution, event IDs and completeness checks are unchanged. Identity +construction executes after that function returns, outside the event interval. +Supplying events does not imply that event construction ran in this constructor. +The existing source-validation interval remains a separate caller obligation. +Incremental event rewriting and generated direct sinks are not fully attributed +by this slice; an absent phase is not a zero-cost phase. + +The generic stage benchmark now reports =request-samples=, invocation =samples=, +=completed-samples= and =incomplete-samples=. A recovered failure can belong to a +complete rejected artifact while its failed inner interval remains incomplete. +Repeated stage invocations remain separate observations. CPU quantiles use the +reported invocation population; allocation quantiles retain only zero-GC, +nonnegative counter intervals. These populations must not be interchanged. + +* Scheme witness and design questions + +#+begin_src scheme +;; Engine code; grammar.ss retains syntax and rules only. +(let ((events + (with-parser-cost-stage 'artifact-event-publication + (raw-event-tape-events tokens tape committed-count trivia? byte-length)))) + ;; The canonical identity owner reuses bytes and observes its own hash/header. + (artifact grammar-digest source 'accepted events '() source-bytes)) +#+end_src + +** Question: why does this design improve the DSL boundary? + +Cost ownership follows engine operations. Recognition, event programs and raw +tapes publish through shared canonical contracts, so authors do not need to +write per-language profiling adapters or expose transient event machinery in +=grammar.ss=. =parser.ss= keeps metadata and parser adaptation; =parser-test.ss= +keeps language behavior and workload selection. Scheme macros express the +optional observation at the owner while preserving its normal result. + +** Question: why is the design scientifically testable? + +An observed constructor must equal its unobserved complete artifact. Source +admission checks Unicode byte coverage and identities independently. Supplied +bytes/events must not generate fictitious construction observations. A failed +tape must preserve its original error, publish =completed=false= for the failed +interval, and never report a later identity stage. The observer's dynamic scope +must be restored. These are executable obligations, not a speed claim inferred +from the presence of a macro or a paper result. + +* Native and source qualification (2026-10-10) + +The artifact producer compiled natively with optimize/static options; the +benchmark module and focused test module also compiled. The first aggregate +build rejected an unsupported =count= identifier after producer compilation +(=23.727s=). The supported =foldl= summary then compiled; that run rejected an +extra closing parenthesis in the new test (=12.298s=). The corrected focused +test compiled in =15.120s=, exit zero and final =OK=. The final aggregate native +build of all three modules completed in =67.122s=, exit zero, no supervision +reason or missing receipt, and final =OK=. Neither rejected run is a successful +aggregate build receipt. + +Source GxTest against the native producer passed all =29= cases in =16.234s=, +exit zero, four =MODULE-OK= markers and final =OK=. These cover the new five-case +publication contract, existing observer behavior, persistent event programs +(including deep publication and relocation), and public source contracts. +The focused test is registered in the native conformance publication group and +its build module catalog. No separate qualification executable is required. +The benchmark helper is explicitly included in that native catalog. The +repository's test ownership plan passed all twelve controls in =1.230s=, +exit zero and final =OK=, including the actual repository catalog/import/suite +coverage check. Receipt: unarchived local execution. + +Raw receipts: unarchived local execution. + +* Cross-language localization receipt + +The existing =measure-parser-stages= benchmark executed GQL RETURN lists, +FHIRPath additions and Arithmetic additions at =64= and =512= operands, under +both requested semantic preferences. Each of the =12= profiles has =20= requests. +Every request equals a full unobserved reference and passes source admission +outside the measured request. The native run completed in =11.401s=, exit zero, +=PUBLICATION-STAGES-OK= and final =OK=. All commands retained the five-second +real-output idle gate; no synthetic output or relaxed idle budget was used. + +The following =512=-operand rows use requested preference =#f=. The requested +preference is not proof of the selected representation or executed backend. + +| Family | Event publication CPU P50 ms | Identity CPU P50 ms | Event publication allocation P50 bytes | Identity allocation P50 bytes | Admitted event / identity allocation intervals | +|--------+------------------------------+---------------------+----------------------------------------+-------------------------------+-----------------------------------------------| +| GQL RETURN | 0.4280 | 0.0080 | 1168768 | 2960 | 20 / 20 | +| FHIRPath addition | 0.3785 | 0.0040 | 1790976 | 2960 | 20 / 20 | +| Arithmetic addition | 0.2335 | 0.0030 | 1048896 | 2960 | 19 / 20 | + +All publication and identity phases completed in all 240 measured requests. +The requested =#t= profiles retain the same admitted allocation medians for +these two phases. The Arithmetic =#f= publication allocation population excludes +one GC interval. Natural GC remains in CPU observations. Nested intervals are +inclusive, observer overhead is included in the complete request, and medians +must not be added to form a cost decomposition. These are localization results, +not paired uninstrumented latency comparisons or a backend preference decision. + +Receipt: unarchived local execution. +The Scheme population checker admitted all twelve summaries and retained the +requested preferences separately. There is no runtime speedup claim. + +* Consequence for the next optimization + +In these measured workloads, event publication allocates far more than source +identity construction. That interval includes required canonical vectors/list +cells and transient traversal storage; the measurements do not separate those +components or justify eliminating the public artifact. A subsequent producer/ +consumer change must account for retained versus transient storage, preserve +full event/field/span/diagnostic equality and qualify complete requests. Hash +caching, dropping source admission or reviving a rejected local iterator does +not follow from this evidence. Exact-head platform CI remains a separate gate. + +* Direct certified-window and rejected-product publication + +The shared artifact owner now applies the official Gerbil list-builder idiom +to certified incremental windows and flat rejected token products. This is an +implementation of the publication ownership obligation above. It does not alter +the grammar DSL, recognition algorithm, edit certificate or public event schema. + +The old window publisher flattened assigned token groups for comparison, +reversed the old window to find its end, constructed replacement event lists, +then copied those lists into the result during a right fold. The new publisher +compares groups with a private tail-recursive cursor, borrows their token lists +through a vector index, and emits replacement vectors directly into the final +list. It visits the original event stream in source order. Unchanged event +vectors remain shared; fresh list cells belong only to this publication request. +The old artifact and certified token groups are never mutated. Rejected token +publication similarly assigns contiguous IDs while emitting, without an =iota= +list or a preliminary token count traversal. + +This is our engineering inference from the ownership measurements, not an +implementation of [[https://arxiv.org/html/2304.05276#S2.SS8][flap's staging theorem]]. Sections 2.8 and 5.5 distinguish +generated recognition from semantic value construction. Canonical event output +still requires allocation. Our transformation removes temporary containers at +that semantic boundary; it does not establish allocation-free parsing or +lexer/parser fusion. Gerbil's =std/list/list-builder.ss= supplies the hygienic +=emit= syntax; its mutation stays inside newly allocated queue cells. + +#+begin_src scheme +;; Private shared-engine publication, not a new author callback or POO slot. +(with-list-builder (emit) + (for-each + (lambda (source-token) + (emit (make-token-event next-id source-token)) + (set! next-id (+ next-id 1))) + certified-token-group)) +#+end_src + +** Question: why does this improve the DSL and Scheme boundary? + +The language author still defines syntax in =grammar.ss=, metadata/adaptation in +=parser.ss= and behavior in =parser-test.ss=. Publication complexity is removed +at its shared owner. Hygienic syntax expresses direct construction while ordinary +named functions express group admission. No language-specific allocator, public +multiple-dispatch protocol or additional user configuration is required for this +closed representation operation. + +** Question: why is the transformation scientifically testable? + +[[file:../../t/artifact-window-publication-test.ss][Independent source controls]] compare expansion, contraction, equal-width and +Unicode edits with freshly constructed complete artifacts. Empty assignment +groups preserve event order; wrong assignments are rejected. A 20,000-token +window checks full product equality, contiguous IDs, source admission, shared +node vectors and independent list ownership. The rejected-product control +checks the same large source and final token ID. Existing incremental consumer +tests retain certificate, field, span, fallback and reuse receipt obligations. + +[[file:../../t/benchmarks/artifact-window/benchmark.ss][The permanent benchmark]] measures complete window and rejected publication, +including source encoding and identity construction. Inputs, prepared groups, +reference artifacts and independent source validation are outside timing. +Each measured result must equal its complete reference. Allocation admission, +natural GC and CPU populations use the existing shared sampler. Reproduction: + +#+begin_src sh +gerbil compile -O +gerbil env gerbil test -v 5 t/artifact-window-publication-test.ss t/artifact-publication-cost-test.ss t/recovery-incremental-test.ss +gerbil compile -O t/benchmarks/artifact-window/benchmark.ss +gerbil env gxi -e '(import :gerbil-parser/t/benchmarks/artifact-window/benchmark) (benchmark-artifact-window)' +#+end_src + +Nine local native consumer modules passed all 99 cases in 34.572 seconds, +exit zero, no watchdog reason or missing receipt, and final =OK=. This reused +native graph includes cross-language ABI, event-program, contextual, layout and +incremental controls. It is distinct from clean-checkout platform CI. A temporary +comparison manifest initially selected the current producer for its supposed +baseline; that build is excluded from matched performance evidence. The original +Git source must be independently compiled before a baseline can be admitted. + +The first isolated original-producer compilation stopped under the unchanged +five-second idle watchdog. A sequential retry after other native compilations +finished completed successfully without changing optimization or supervision +limits. Against that qualified original producer, the 20,000-group warmup failed +with Gambit's =Number of arguments exceeds implementation limit=: admission +called =(apply append assigned)= with one positional argument per group. This +is a bounded call-arity defect in our engine algorithm, not a GC diagnosis. The +new cursor passes the same large complete-product control. The rejected run +is not assigned latency or allocation quantiles. Matched cost comparison uses +the separately declared 1,000-token input on which both producers can complete; +this does not erase the large-window negative control or alter a latency gate. + +The sequential A/B/B/A comparison used that independently compiled original +producer and the new producer, with the same dependency graph, native compiler +options and permanent benchmark. Each component used 20 samples of five complete +publications. Both complete reference products passed; every block exited zero +with final =OK=. All 20 allocation samples in each component were admitted: + +| Producer block | Window bytes/call | Window CPU P50, five calls (ms) | Rejected bytes/call | Rejected CPU P50, five calls (ms) | +|----------------+-------------------+---------------------------------+---------------------+-----------------------------------| +| Original A | 487515.2 | 2.075 | 212019.2 | 0.128 | +| Direct B | 247515.2 | 1.946 | 164099.2 | 0.112 | +| Direct B | 247515.2 | 2.813 | 164099.2 | 0.113 | +| Original A | 487515.2 | 2.087 | 212019.2 | 0.128 | + +The repeated allocation reductions are 240,000 bytes/call (49.2%) for the +window and 47,920 bytes/call (22.6%) for the rejected product on this input. +Window CPU is inconsistent, so these observations admit no CPU improvement. +They measure complete publication, including identity construction, but do not +measure complete parsing, live retained heap, actor throughput or every language. +The large production-only run also completed with 20 samples of five calls; +its allocation populations admitted 17 window and 18 rejected samples. Its +measurements are not paired with the original producer, which failed warmup. + +The previous =8a206b5= platform CI exposed a separate ownership omission: +=t/layout-column-storage-test= was in the compilation catalog but absent from +=t/conformance-main.ss=. Both Scheme platforms rejected it with =missing +conformance test import=; Rust AOT passed. The native suite entry now imports and +registers that suite and this window suite. The same import/export ownership +check passed locally for both modules in 9.785 seconds, exit zero and final =OK=. +That check fixes the observed cause; new-revision platform completion remains +an independent gate. + +* Executable event-storage control + +The shared benchmark now implements =measure-artifact-event-storage=. This +closes the next attribution obligation at the same publication boundary. It +copies an already source-admitted artifact's event list into fresh list cells +and shallow-copied vectors, preserving event order, vector widths and every +payload. IDs, symbols, byte offsets and token lexemes remain shared values. +The control uses the existing canonical representation; it introduces no new +parser backend, language configuration or production storage protocol. + +The installed Gerbil runtime uses =vector-copy= in =raw-table-copy= in +=src/gerbil/runtime/table.ss=. Its =src/std/list/list-builder.ss= supplies +=with-list-builder=, and =src/std/vector/vector.ss= composes that macro with +vector iteration. The engine's private =collect-canonical-events= now uses +that same official builder in place of its hand-written queue function. Both +its recognition and event-program sinks retain their existing event constructors +and source-order authority. The independent copy control uses the same builder +with =for-each= and native shallow copying. This is a source-grounded reduction +in duplicated infrastructure, not an inference that a higher-order function or +unchecked mutation always improves parser performance. Builder mutation is +confined to fresh private list cells. + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark + :gerbil-parser/languages/arithmetic/parser + (only-in :std/list/list-builder with-list-builder)) + +(let* ((source "1 + 2 + 3") + (artifact (parse-arithmetic source))) + ;; Complete requests localize the publication interval. + (measure-parser-stages parse-arithmetic source 20) + ;; Independent control isolates construction of the existing containers. + (measure-artifact-event-storage artifact source 20)) + +;; Shared benchmark implementation: native shallow copies preserve payloads. +(def (copy-canonical-event-storage events) + (with-list-builder (emit!) + (for-each (lambda (event) (emit! (vector-copy event))) events))) +#+end_src + +** Question: why does this improve the DSL boundary? + +The engine's canonical event representation supplies the control without a +second language-specific walker. Authors continue declaring syntax/rules and +public parser metadata; they do not need to describe memory layouts or provide +profiling sinks. Scheme procedure composition expresses the copying operation +directly, while the engine remains the authority for publication and source +admission. Generic dispatch would add no useful type decision to this homogeneous +list of canonical vectors. + +** Question: why is the control scientifically testable? + +Each copied event vector and list cell must be distinct, each payload must retain +its identity, and mutating a copied vector must leave the original vector intact. +The benchmark reconstructs the complete artifact header around the copied events +outside the measured interval, checks complete equality, then independently +checks source admission. This preserves accepted/rejected status, diagnostics, +fields, Unicode byte coverage and both source/grammar identities. Empty products +and invalid inputs have explicit controls. Allocation observations retain the +same no-GC, nonnegative-counter admission as the stage benchmark. + +This is an empirical container-construction control, not a retained-heap census, +an allocation lower-bound theorem, or an alternative parser. Copy-loop and +statistics overhead remain in the interval. Lexeme payloads and artifact-header +construction are outside it. The summary reports event count and total vector +slots alongside CPU/allocation populations; no fixed Gambit object-size formula +is assumed. Subtracting independent medians would not establish recoverable +allocation or a complete-request speedup. A large gap motivates a traversal +experiment with matched complete-request controls; it does not admit that change. + +** Rejected attribution shortcut + +The initial =map vector-copy= prototype compiled natively in =27.682s= and +passed all =31= source cases in =16.822s=. Twelve profiles with twenty copies +each completed in =9.062s=, with full-product equality and source admission. +At 512 operands, GQL's copy allocation median was =1167728= bytes against +=1168768= publication bytes; Arithmetic's was =1047856= against =1048896=. +FHIRPath's copy counter instead varied: its requested =#f= median was =872448= +and P95 =1789936=, despite all twenty intervals passing the no-GC/nonnegative +counter filter. The requested =#t= median was =872440= with the same P95. + +This is negative evidence for treating the prototype's median as a precise +storage budget or the remainder as traversal waste. No allocation saving is +admitted from these values. The installed allocation counter's raw delta and +occupied-object terms are not an object-lifetime census. The builder rerun +checks the construction protocol directly; it does not assert a cause for the +prototype's counter variation or weaken the allocation admission filter. +Receipts: unarchived local execution. + +** Official-builder qualification and decision + +Source baseline: =a1df0ca= plus this slice. The artifact producer, shared +benchmark and focused test module compiled natively in =41.776s=. All =31= +source cases passed against that producer in =12.914s=, including the existing +deep event-program and relocation controls. Both commands exited zero with +final =OK=, no supervision reason and no missing receipt. Existing native +conformance ownership includes the updated test and benchmark modules. + +The twelve cross-language profiles completed =240= observed complete requests +and =240= event copies in =33.982s=, exit zero, =EVENT-STORAGE-CONTROLS-OK= and +final =OK=. Every product passed equality and source admission. The following +512-operand rows use requested preference =#f=; the =#t= allocation medians +match, without proving which representation executed from preference alone. + +| Family | Events | Vector slots | Publication allocation P50 bytes | Copy allocation P50 / P95 bytes | Admitted copy intervals | +|--------+--------+--------------+----------------------------------+---------------------------------+-------------------------| +| GQL RETURN | 8737 | 38022 | 1168768 | 1167808 / 1167808 | 20 | +| FHIRPath addition | 14369 | 54382 | 1790976 | 1790016 / 1790016 | 20 | +| Arithmetic addition | 8185 | 32738 | 1048896 | 1047936 / 1047936 | 20 | + +The builder control's admitted allocation P50 and P95 agree in all twelve +profiles. The 64-operand FHIRPath requested =#t= control excludes one GC interval +and retains nineteen allocation observations; CPU still includes that sample. +All other copy profiles retain twenty. A separate Scheme report checks each +publication population completes twenty intervals and each copy population +contains twenty samples, with its actual allocation population reported. +An initial temporary report rejected an incorrect summary field index; the +corrected reader completes with =EVENT-STORAGE-REPORT-OK profiles=12=. + +Required container construction closely tracks publication allocation in these +workloads. The experiment provides no evidence of a large removable traversal +allocation budget. The small difference is not admitted as recoverable storage; +the observed publication interval includes observer overhead and the copy +interval has its own construction/statistics overhead. The official-builder +migration preserves the reported publication allocation medians. It is not a +complete-request speedup admission, and the two run elapsed times are not a +performance comparison. + +The next engine experiment should therefore separate source recognition and +semantic reduction costs, which remain larger request allocations in these +profiles, instead of treating canonical publication vectors as disposable. +Changing the public representation would require a separately reviewed semantic +contract and Rust consumer qualification. No such change is implemented here. +This updates the preceding localization decision with the missing independent +control rather than appending an unrelated language-specific optimization. + +Final receipts: unarchived local execution. The five-second real-progress +idle gate remains unchanged. Exact-head CI is separate: at verification, +=a1df0ca= Rust AOT passed while Linux/macOS Scheme jobs remained in progress. diff --git a/docs/research/bound-declaration-publication.org b/docs/research/bound-declaration-publication.org new file mode 100644 index 00000000..f65c1693 --- /dev/null +++ b/docs/research/bound-declaration-publication.org @@ -0,0 +1,319 @@ +#+title: Ordered declaration origins and complete Bound IR publication +#+startup: overview + +* One compilation responsibility + +This implements the publication part of the +[[file:scheme-component-provenance.org][component provenance study]] and +[[file:scheme-source-origin-contract.org][reader-origin contract]]. The original producer is =5bc7d2f=; +the integrated parent =4319c94= retains its source and adds shared CPU sampling. +The language author interfaces remain =grammar.ss=, =parser.ss= and +=parser-test.ss=. No new author composition clause, callback or IR schema is +introduced. + +The user-selected +[[https://github.com/tao3k/gerbil-ascent/blob/f84155662b57ec8034bb60f0d14968715ad15cce/docs/scheme-coding-style.org][Scheme Coding Style]] guides ownership, representation, admission and evidence. +The relevant upstream mechanisms are Gerbil =2591dcd='s +=std/list/list-builder.ss= and =gerbil/expander/stx.ss=. The former owns fresh +queue cells through hygienic syntax; the latter provides =stx-for-each= and +defines =syntax->list= as =stx-map values=. Source syntax can therefore be +visited directly without first copying an identity-mapped list. + +* Paper, contract and implementation + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Ryan Culpepper and Matthias Felleisen, Fortifying Macros, ICFP 2010]] argues for +checking macro uses and explaining errors at the author's language boundary. +Our inference is that optimization must preserve the selected declaration, +reader syntax and contextual source record through lowering. The paper does +not prescribe our indexes, canonical digests or list representation. This +change does not implement Racket's =syntax-parse= system. + +| Owner | Implemented responsibility | Preserved contract | +|-------+----------------------------+--------------------| +| [[file:../../src/compiler/normalize.ss][POO normalization]] | One ordered composition collector; direct field-origin emission | Parent order, repeated inherited occurrences, role order, cycle rejection and selected source ownership | +| [[file:../../src/compiler/language-artifact.ss][Declaration expansion]] | =stx-for-each= emits field origins directly | Reader syntax identity, field order, scope and malformed-row diagnostics | +| [[file:../../src/compiler/bound-ir.ss][Bound IR publication]] | Invocation-owned source selection; direct section, field and reference construction | First source entries, qualified field names, present false values, declaration/reference order, IDs and canonical digests | + +All binding IDs are collected before references are validated, so forward +references remain legal and unresolved references fail before publication. +The required reference list still exists: it is canonical digest input. The +temporary flattened binding catalog, per-kind field batches, reversed bound +sections and variadic reference flattening are removed. Published source records +and declaration payloads are borrowed exactly as before; queue mutation is +confined to new list cells. + +Source selection previously repeated linear =assoc= searches for every binding. +One private index per known source namespace now retains original association +entries. This preserves the first row and a present =#f= value; qualified field +keys use structural equality. The index belongs to one binding invocation and +is neither published nor retained by a grammar or parser. With bounded key +shapes and normal hash behavior, source indexing and lookup cost is expected +to grow with source records plus bindings, rather than their product. This +algorithmic analysis is separate from measured complete-publication CPU. + +#+begin_src scheme +;; Private publication extent: data order and complete closure share one owner. +(with-list-builder (emit) + (for-each + (lambda (section) + (for-each + (lambda (binding) + (table-set! binding-ids (bound-grammar-ir-ref binding 'bindingId) #t) + (for-each emit (bound-grammar-ir-ref binding 'references))) + (cdr section))) + sections)) +;; Validate references after every binding ID is present, before publishing IR. +#+end_src + +** Question: why does this improve the Scheme DSL boundary? + +Author declarations retain their vocabulary and source syntax. POO composition +selects the effective declaration; the compiler owns its transient construction +and lookup costs. Hygienic construction syntax expresses private list ownership, +=syntax-case= retains identifier/context-sensitive admission, and named functions +express closed algorithms. Adding a public generic protocol would not help this +fixed publication contract. + +** Question: why is this scientifically testable? + +[[file:../../t/bound-ir-index-test.ss][Independent controls]] check first-entry selection, empty first sections, +present false values, qualified fields, defaults, duplicate declaration rejection +and unresolved references. The 20,000-binding case preserves full declaration +order without a variadic call. The original producer rejects that same input +with Gambit's =Number of arguments exceeds implementation limit= at reference +flattening. This is an engine algorithm defect, not a GC diagnosis or a timing +sample. Existing composition, declaration, LR-origin and language ABI consumers +check the entire compiler chain. + +[[file:../../t/benchmarks/bound-source-catalog/benchmark.ss][The permanent benchmark]] measures complete binding, closure checks and all +canonical digest construction. Its original constructor is an experiment input, +never a production fallback. Every original/candidate product must equal the +entire current sidecar, including source records and digests. Catalog construction +is outside timing. Existing shared samplers retain natural GC, admitted allocation +populations and alternating CPU order. These are binding-publication measurements, +not complete grammar compilation or parsing throughput. + +* Reproduction + +#+begin_src sh +gerbil compile -O +gerbil env gerbil test -v 5 t/bound-ir-index-test.ss t/grammar-composition-contract-test.ss t/grammar-composition-lowering-test.ss t/grammar-composition-execution-test.ss t/language-artifact-test.ss +gerbil compile -O t/fixtures/bound-source-catalog.ss t/benchmarks/bound-source-catalog/benchmark.ss +gerbil env gxi -e '(import :gerbil-parser/t/benchmarks/bound-source-catalog/benchmark) (benchmark-bound-source-catalog 20 5 512)' +#+end_src + +The isolated comparison reconstructs only the original owner, with a different +module basename and the same dependencies; reference code is not committed: + +#+begin_src sh +mkdir -p .cache/bound-source-reference/src/compiler +cp gerbil.pkg .cache/bound-source-reference/gerbil.pkg +git show 5bc7d2f:src/compiler/bound-ir.ss > .cache/bound-source-reference/src/compiler/bound-ir-reference.ss +gerbil compile -O .cache/bound-source-reference/src/compiler/bound-ir-reference.ss +gerbil env gxi -e '(import (prefix-in :gerbil-parser/src/compiler/bound-ir-reference original-) :gerbil-parser/t/benchmarks/bound-source-catalog/benchmark) (compare-bound-source-catalog original-bind-grammar-ir 20 2 512)' +#+end_src + +* Qualification and negative evidence + +The completed normalization, syntax-origin and Bound IR chain passed 12 native +consumer modules and all 136 cases in 47.121 seconds, exit zero, final =OK=, +no supervision reason or missing receipt. The original large-catalog control +exited one with the call-arity error; it is not a successful performance receipt. +Optimized native compilation of the two source-origin owners completed in +56.481 seconds. Local reuse of native artifacts remains distinct from clean +platform CI. + +An initial CPU experiment used ten-call batches. It completed some component +measurements and two paired populations, then stopped under the unchanged +five-second idle watchdog during the larger catalog. That aggregate run is +rejected. Two-call batches retain the same complete inputs, twenty alternating +groups and the sampler's one-millisecond resolution requirement; they allow +genuine group receipts within the original progress budget. Allocation intervals +that contain GC are excluded rather than replaced with plausible estimates. + +The completed two-call CPU comparison used 20 alternating groups per size, +preserved every complete sidecar and passed the one-millisecond batch-resolution +check. It exited zero in 32.655 seconds with final =OK=: + +| Syntax kinds (two bindings each) | Ratio P10 | Ratio P50 | Ratio P90 | +|--------------------------------+-----------+-----------+-----------| +| 16 | 0.718 | 1.019 | 1.169 | +| 128 | 0.526 | 1.062 | 1.773 | +| 512 | 0.635 | 0.810 | 1.149 | + +Ratios are candidate/original user+system CPU including natural GC. The upper +quantiles and small/medium medians reject a stable speedup claim. The incomplete +ten-call experiment also produced inconsistent ratios: its 16-kind median was +1.413 and its 128-kind median 1.030. Neither population can be discarded merely +because a later median looks better. + +Its component allocation population admitted 19 samples on each 16-kind side, +with medians 1,794,904 and 1,712,315.2 bytes/call. This modest observation is not +a large-catalog claim: the original 128-kind population admitted zero samples, +the candidate admitted six, and both 512-kind populations admitted zero. Missing +allocation evidence remains missing. Canonical serialization, digest construction +and retained object effects still need attribution before another performance +claim; no private index result proves full grammar compilation speed. + +The saved native comparison entry was then invoked with the original constructor, +20 groups, two calls and 16 kinds. It preserved all complete sidecars and exited +zero in 28.583 seconds with =BOUND-SOURCE-COMPARISON-OK= and final =OK=. Its CPU +ratios were 0.746 / 0.985 / 1.646 (P10/P50/P90), again rejecting stable speedup. +The integrated parent =4319c94= completed Ubuntu Scheme, macOS Scheme and Rust +AOT CI successfully. Those parent checks do not qualify the new source slice; +its pushed revision still needs its own clean-checkout platform results. + +The publication revision =4e6095c= subsequently completed all three checks in +[[https://github.com/tao3k/gerbil-parser/actions/runs/38039773306][its exact-head CI run]]: Ubuntu Scheme, macOS Scheme and Rust AOT runtime. +This closes platform qualification for that revision, independently of the +following digest-input experiment. + +* Canonical digest input follows the same publication owner + +The next implementation compares against =4e6095c=. That producer writes each +declaration, each reference catalog, the grammar and the final section catalog +to a character string, then allocates UTF-8 bytes before SHA-256. The full +sections include already published source records and per-binding identities; +optimizing only source lookup does not remove this traversal or encoding cost. + +The candidate keeps the canonical Scheme =write= operation and the existing +SHA-256 implementation. A private named function writes directly to a fresh +UTF-8 byte port. It removes the intermediate character string and its subsequent +encoding traversal; it still materializes bytes and performs digest computation. +It is not incremental hashing, serializer replacement or memoization. Each call +owns its port and bytes; no mutable grammar cache or new author option exists. + +#+begin_src scheme +(def (canonical-digest value) + (sha256-bytes + (call-with-output-u8vector '(char-encoding: UTF-8 eol-encoding: lf) + (lambda (port) (write value port))))) +#+end_src + +The [[https://gambitscheme.org/4.9.7/manual/][Gambit manual, byte-port settings and u8vector ports]] specifies character +and line-ending conversion at byte ports. Both settings are explicit: implicit +encoding escaped non-ASCII symbols and strings in our independent local probe, +and the documented line-ending default depends on the operating system. LF +preserves the character writer's newline bytes. Gerbil =2591dcd='s +=std/text/markup/sxml/tal/syntax.ss= uses =call-with-output-u8vector= with =write=; +=std/crypto/digest.ss='s generated =sha256= consumes bytes through its native +digest context. These mechanisms supply the implementation, not a new emitter +or foreign hashing protocol. + +** Question: why is this preferable at the Scheme DSL boundary? + +The existing declaration, POO-selected origins, lineage and reference structure +remain digest inputs. The compiler chooses the physical sink beneath the DSL; +authors retain the three interfaces without encoding switches. The named +function owns a closed algorithm. A generic dispatch or macro would add no +binding or extension responsibility here. The public canonical-text operation +continues to produce text for consumers that actually request it. + +The Fortifying Macros correspondence remains preservation of the author's +validated declaration and context across compiler layers. Direct byte output +is an engineering inference, not a technique proved or prescribed by that paper. + +** Question: how is equivalence and cost scientifically tested? + +The independent control retains the original character writer and UTF-8 text +hashing. It reconstructs every declaration digest and checks all per-binding +reference digests, final reference digest, grammar digest and identity digest. +Non-ASCII declaration and field names, Unicode origins, escaped quotes/newlines +and byte-vector lineage are included. A default byte port would fail this +control. Existing source selection, closure, large catalog and language ABI +consumers remain necessary beside it. + +The existing complete-sidecar comparator accepts the isolated =4e6095c= +constructor and checks every timed product against the candidate. The added +=benchmark-bound-identity-input= controls final-section text writing, UTF-8 +encoding, direct byte writing and SHA-256 separately. Inputs are prepared before +timing; every result is checked afterward. Their independently sampled +percentiles must not be summed or represented as fractions of a complete +publication interval. They exclude per-binding serialization, source indexing +and reference admission; complete-publication comparison includes those costs. + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/bound-source-catalog/benchmark) +(benchmark-bound-identity-input 20 5 16) +;; Use the original constructor reproduction above with revision 4e6095c. +(compare-bound-source-catalog original-bind-grammar-ir 20 2 512) +#+end_src + +The fixture and benchmark use their existing source-owned compilation command. +No additional test executable, scheduler, runtime fallback or schema version is +introduced. CPU, admitted allocation samples, natural collections and exact-head +platform qualification remain separate evidence obligations. + +** Native cost observations + +The complete-publication component comparison used 16 syntax kinds, 20 samples +and five calls per sample. Both constructors returned identical full sidecars. +Nineteen allocation samples were admitted on each side; one natural-collection +interval on each side was excluded from allocation attribution. Median allocated +bytes per publication were 1,712,392.0 for =4e6095c= and 1,328,073.6 for direct +byte output, a 22.4% reduction in this population. This is allocation flow, not +retained heap or language request memory. + +The same run's 20 alternating two-call CPU groups gave candidate/original ratios +0.705 / 1.046 / 1.370 at P10/P50/P90. That population does not admit a CPU +speedup. The run exited zero with both complete-component markers and final +=OK= in 14.898 seconds. + +The final-section controls used the same 16-kind input, 20 samples and five +calls. All 20 allocation samples were admitted in each component. The existing +sampler checks full returned text, bytes or digest after every timed batch. + +| Isolated component | CPU P50 microseconds/call | Allocated bytes/call | +|--------------------+---------------------------+----------------------| +| Character writer | 542.2 | 385,315.2 | +| UTF-8 encoding of supplied text | 35.0 | 19,691.2 | +| Direct UTF-8 byte writer | 433.6 | 189,899.2 | +| SHA-256 of supplied bytes | 8.8 | 947.2 | + +These observations locate substantial work in canonical writing for this final +catalog, rather than native hashing. They do not establish a universal ranking, +complete publisher phase fractions or parse-request latency. Component samples +were sequential; their CPU percentiles are not a paired significance test. +The text writer's wall P95 batch was 61.107 ms with 4.654 ms CPU and no GC; +that delay cannot be labeled collection overhead. The controls exited zero in +12.864 seconds with =BOUND-IDENTITY-INPUT-BENCHMARK-OK= and final =OK=. + +The initial native test compilation failed because an experiment used an +unimported =append-map=. The corrected control uses the official list builder. +This failed attempt exited nonzero with missing final =OK=; it is not included +in successful receipts. Before explicit LF was added, the twelve-suite consumer +matrix passed 137 cases in 50.983 seconds. Final-source qualification follows +the explicit character and line-ending settings and must be recorded separately. + +The completed size matrix used 20 alternating groups, two calls per side and +the unchanged minimum CPU-duration admission threshold. It preserved all full +sidecars and exited zero in 65.586 seconds with final =OK=. + +| Syntax kinds | CPU ratio P10 | CPU ratio P50 | CPU ratio P90 | +|--------------+---------------+---------------+---------------| +| 16 | 0.838 | 1.025 | 1.403 | +| 128 | 0.612 | 0.875 | 1.718 | +| 512 | 0.882 | 1.031 | 1.431 | + +The median favors neither implementation consistently across sizes; every upper +decile ratio exceeds one. Direct output is retained for removal of the measured +intermediate allocation and explicit encoding ownership, not as a demonstrated +CPU or latency win. Large-catalog allocation admission, retained heap and +whole-language parsing remain unmeasured by this matrix. + +The final UTF-8/LF producer and saved component benchmark compile natively in +23.568 seconds, exit zero and emit final =OK=. The twelve-suite consumer matrix +then passes 137 cases in 59.006 seconds, exit zero, with no missing markers and +final =OK=. It includes the independent original-text identities, 20,000-kind +publication, composition/origin consumers, language surface and entry contracts, +ten-language ABI alignment and runtime/contextual parsing. Modular documentation +qualification preserves the 1,302-line historical study and resolves 134 local +links and 21 heading references. These are local receipts; the following pushed +revision requires its own completed platform checks. + +Revision =39fd28f0006b9ad116c94b06c7aabaf41e45277f= subsequently completes +all three [[https://github.com/tao3k/gerbil-parser/actions/runs/38045035142][exact-head platform jobs]]: +Rust AOT runtime and Ubuntu/macOS Scheme tests with language packs. +The [[file:compiled-language-byte-publication.org][compiled language transport study]] +continues the same ownership sequence through immutable publication and runtime +admission, retaining the original reader after a negative byte-port experiment. diff --git a/docs/research/capture-fold-cost-model.org b/docs/research/capture-fold-cost-model.org new file mode 100644 index 00000000..4bd4d10c --- /dev/null +++ b/docs/research/capture-fold-cost-model.org @@ -0,0 +1,802 @@ +#+title: Capture fold costs, terminal observations and complete-request qualification + +* Starting point + +This study starts at =459ec36e1961f3d8d95ce4ebec13843d164a4bdf=, including +the earlier selector/goto/owned-stack optimization and the rejected scanner +experiments. The sole PR is #13. This is a new candidate against that exact +starting point; it cannot inherit an earlier CPU gain. + +* Paper and current-code model + +The representation and ordered-computation discussion in +[[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]] motivates eliminating intermediate traversal. +The context-maintenance costs in +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] require paying for retained +capture and reuse data. Neither source proves an optimization in this engine. + +The [[file:singleton-scanner-model-first.org][preceding cost diagnosis]] found 322 fragments, 256 terminal leaves and eight +sparse snapshots of semantic depth five in the actual 64-record request. Its +captured-LR samples included substantial natural GC. These observations identify +a paid path; they do not establish that object records alone explain allocation +counters or that increasing a heap limit removes the work. + +The existing capture reduction restores source-order children and counts in one +reverse-stack pass. It then traverses children again to build the terminal yield. +Metadata accessors separately classify ordinary tokens, fragments and views. +The first hypothesis fused terminal-yield construction into the existing pass; +its measured rejection led to demand-driven derived-index construction. +The nullable path, sparse snapshot policy, source grammar and published event +representation remain the same. The later shared-context hypothesis changes the +private fragment representation and therefore requires its own model and controls. + +* Four-layer obligations + +| Layer | Obligation | Boundary | +|-------+------------+----------| +| Papers and current code | Locate redundant capture work and retain reuse obligations | Motivation is not measured benefit | +| [[file:../../formal/ParserEngine/Capture.lean][Lean]] | Exact terminal/list/boundary observations, prepared-row ownership and explicit cost charges | No native heap or GC refinement | +| [[file:../../formal/quint/YieldMemo.qnt][Quint memo publication]], [[file:../../formal/quint/CaptureContext.qnt][context replacement]] and existing sparse-capture protocol | Complete observer-visible publication, old-node retention and live kind/executor/target checks | Finite protocols are not Scheme refinement | +| Scheme controls and Benchmark | Frozen-oracle slots, ordered yields, exact frontiers, full artifact and source validity | CPU and allocation require actual measurements | + +=yield_join_order= preserves the ordered terminal sequence. The left and right +fold lemmas imply =fused_yield_observation=. A relocated subtree remains an opaque +piece; the transformation neither distributes nor reconstructs a view. Binary +association is private metadata. Transfer's token comparisons and successful +terminal/branch visit counts are preserved; the number of diagnostic visits +before an early mismatch may depend on the changed binary association. + +=CaptureCost= accounts separately for transitions, allocated work, retained data +and collection. =capture_cost_requires_all_charges= needs premises for each +charge; fewer logical classifications alone does not prove lower total cost. +The native allocation counter is not an inventory of retained Scheme objects. +The fragment/children/terminal tree and snapshot results remain fully paid. + +* Rejected fusion and rejected premise + +The first candidate fused capture's reverse-prefix pass and yield construction, +with a continuation macro for all piece metadata. All 198 frozen variants and 51 +semantic Cases passed. Its complete-request CPU ratios were: + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before reference control | 0.950364 | 1.018470 | 1.026300 | +| Candidate/reference | 0.951798 | 0.977695 | 1.029913 | +| After reference control | 0.944417 | 1.021300 | 1.093129 | + +The unchanged gate rejected it in 54.864556 seconds. A subsequent no-GC captured +LR replay still allocated exactly 65,851,728 counter bytes per 256 calls, the +same as the earlier observation. Faster no-GC samples did not remove the +allocation or establish the complete-request gain. The implementation was +withdrawn; reassociation remains a modeled hypothesis, not shipped code. + +An attempted dead-signature premise was invalidated by the actual +=boundary-matches?= consumer in [[file:../../src/runtime/reuse.ss][source reuse]]. Captured lookahead signatures +certify the exit lexical boundary independently of the consumed terminal yield. +Their removal was withdrawn before qualification. The retained +=changed_boundary_rejected= model explicitly rejects changed kind/lexeme +signatures. No missing signature is treated as permission to transfer a fragment. + +* Demand-driven terminal index + +[[file:../../src/runtime/lr-parser/recognition.ss][The capture owner]] already retains a complete immutable grammar topology, +span/count metadata, semantic values and exact lookahead. A terminal-yield tree +is a derived index used by transfer admission. The new candidate keeps a pending +cache in the same physical slot and constructs that index only upon a transfer +query. Nullable productions retain their known empty index. No capture signature, +source-boundary check, fragment production or public event is discarded. + +The first query uses an explicit postorder work list so deep left recursion does +not create a recursive Scheme call chain. Each fragment publishes only a complete +index. Children retain their original source-order left fold and exact binary +association. Relocation wrappers retain their original nesting. Subsequent queries +reuse the cached tree. Grammar topology and relocation metadata are engine-owned +immutable inputs; changing them behind the index invalidates the producer premise. +Token leaves remain the original objects, as in the eager index. Semantic observer +updates do not alter that grammar topology. + +=delayed_yield_first_exact= and =delayed_yield_warm_exact= bind the memo to its +immutable producer. =fresh_yield_allocations_eliminated= applies only when there +is no index demand. =forced_yield_allocations_retained= keeps all construction +charges in the forced case. [[file:../../formal/quint/YieldMemo.qnt][YieldMemo]] covers fresh capture without derived work, +cold/warm queries, invalid topology and rejected partial publication. It passed +four directed histories and 1,000 seeded invariant traces. + +The lazy-only candidate passed all 198 frozen structural cases in 9.347719 +seconds and all 51 semantic Cases in 52.939281 seconds. The historical +[[file:../../t/benchmarks/lr-request-stack/capture-reference.ss][capture oracle]] is unchanged; controls force the candidate's derived index +before comparing the complete representation, including binary association and +exact remaining stack-tail identity. + +Its complete-request gate was still rejected: before-control P10/P50/P90 +=0.949026/0.990672/1.062209=; candidate =0.932465/0.968496/0.982648=; +after-control =0.949487/0.991530/1.110078=. The candidate P90 and after-control +upper limit both failed. The receipt completed in 63.331801 seconds. This is +not admitted CPU improvement. + +The combined candidate stages only span/count classification through +=with-capture-bounds=; =capture_bounds_exact= proves those fields independently +of the derived yield. It must not accidentally force the deferred index while +classifying pieces. A full forced-request diagnostic also includes both fresh +capture and first-demand index construction inside its clock. Deferred work is +not silently excluded from that scope. + +* CI source-first native admission + +At =459ec36=, both semantic lanes failed after the worker controls: the full +runner explicitly sets =GERBIL_LOADPATH= to the source directory. Its direct +FFI-helper import attempted to interpret =##c-declare=. A matched local +source-first entry reproduced the error in 3.608364 seconds. Preloading alone +still failed; a late import alone reached successful tests but left the exit +name unbound. Neither intermediate run is accepted. + +The final entry first admits the compiled helper, evaluates its public import +and binds the returned native procedure explicitly before defining the harness. +The source-first eight-Case control then passed in 12.982582 seconds, exit zero. +A new source-first worker control joins the seven existing lifecycle controls; +all eight passed. The repair was pushed separately as =95a1ca3=. Exact-head CI +is a separate gate. No compiler, heap limit, test budget or idle fence changed. + +* Reproduction and unchanged CPU gate + +#+begin_src sh +just formal +just formal-protocol +just prepare-capture-reference 459ec36e1961f3d8d95ce4ebec13843d164a4bdf .data/capture-lazy/reference selector-input +just benchmark-capture-prefix .data/capture-lazy/reference/lib selector-input +just diagnose-capture-cost +#+end_src + +The reference builds eleven frozen source owners: ten native owners and the +interface-only LR facade. The incremental session and reuse consumers are rebuilt +against each representation; scanner, record grammar, SDK and GCC are shared. Native image +selection and full artifact/frontier fingerprints must agree before timing. +The CPU gate still includes scanner, LR capture, publication, full artifact +equality and source validation for fresh 64-record ASCII sessions. Independent +recognition projection remains a preflight control. It uses 122,880 timed +requests, 256 startup requests per worker, natural GC, a 90-second total fence, +a five-second idle fence, stable 0.90..1.10 reference controls and candidate +P90 below 0.95. No timing threshold or workload is changed for this candidate. + +* Combined candidate qualification + +The combined lazy-index and span-consumer candidate passed all 198 frozen +variants in 7.424127 seconds and 52 semantic Cases in 14.812942 seconds. +Its reference controls were stable, but the original CPU gate still rejected: + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before reference control | 0.975575 | 1.022228 | 1.080627 | +| Candidate/reference | 0.889660 | 0.939904 | 0.980126 | +| After reference control | 0.964964 | 0.978874 | 1.080939 | + +The run completed in 42.698281 seconds. All five candidate groups were below +1.0, but candidate P90 failed the unchanged 0.95 criterion. This is retained as +a rejected admission result, not promoted from its favorable median. + +The final span consumer uses the SDK's checked =using= class binding, as in +=std/struct/pqueue=. It admits the private final fragment once for its three +slot reads; view relocation keeps the checked generic path. This does not change +the logical =capture_bounds_exact= contract or introduce unchecked module scope. + + +* Checked binding and allocation evidence + +The checked-binding trial retained a rejected candidate P90 of 0.973094; its +following control phase did not finish within the original 90-second fence. +A subsequent complete run produced candidate P10/P50/P90 of +0.940524/0.962280/0.962433 and following-control P90 of 1.101305. Both runs fail +the unchanged gate. Three unrelated busy processes had already exited before +an authorized temporary suspension could occur; no process was suspended. + +The recorded captured-LR replay reduced admitted no-GC allocation-counter bytes +per request from 257233.3125 to 240913.3125: a reduction of 16320, consistent +with eliminating 255 eagerly built terminal branches. This counter is not a +measurement of retained object size. The whole-request and forced-index replay +batches encountered collection and did not admit equivalent no-GC allocation +values. Their stage timings are diagnostic scopes, not matched speedup evidence. +The added =whole-forced-yield= stage pays for a fresh complete request and first +terminal-index construction and validation inside its clock. + +* Shared transition context + +The next hypothesis separates request payload from repeated transition metadata. +The initial immutable five-slot context retained runtime, executor, production, entry state +and exit state. A fragment retains nine payload slots. The runtime owns a +production/entry directory capped at 65536 row-entry cells, with rows prepared once at the canonical capture driver entrance. +Executor and exit-target checks remain live; replacement publishes a new context +and cannot change contexts already retained by old fragments. Cloned runtimes +start with independent directories; oversized tables use uncached contexts. + +The Lean payload reconstruction theorem preserves arbitrary consumers, while +its slot-cost condition explicitly charges contexts and directory cells. A +smaller fragment alone does not establish lower total allocation. The Quint +protocol exercises repeated acquisition, executor/target replacement, old-node +retention and rejection of stale context reuse. Native controls passed 198 +frozen capture variants and 54 semantic Cases. The historical oracle reduction +body remains unchanged; a test-only constructor adapter maps its logical fields +to the new representation. The first shared-context run had stable reference controls, but candidate +P10/P50/P90 0.919248/0.952465/0.975666 failed admission in 48.851939 seconds. +The next hypothesis hoists directory admission to the canonical driver entrance. +A private runtime/production row supplies coordinates. Lean requires that +producer premise and retains live executor/target checks for each reduction. +Direct calls retain checked coordinate admission. Uncaptured execution does not prepare rows. Custom captured execution uses the +checked per-reduction path and can prepare a directory on first demand. Frozen controls also compare this prepared-row path. +CPU admission remains a separate obligation. + +The terminal index relies on engine-owned immutable capture topology. It +publishes only a complete derived index. Concurrent first queries may construct +equivalent indexes more than once; the model does not claim a lock-free +exactly-once allocation guarantee. + +* CI preparation and progress ingestion + +At published head =95a1ca37b675a1507439ce04844f7de325487af2= the native-exit +source-first repair passed both FFI lanes. Ubuntu semantics instead exhausted +its idle fence in the 20000-binding catalog Case; macOS semantics exhausted it +in GQL generation. These failures are retained and not relabeled as passes. + +Empty reference catalogs now reuse a content-derived digest, returning a fresh +string copy to every caller. The regression checks the independent text writer +and mutation isolation. This eliminates repeated hashing and port preparation +without exposing shared mutable digest strings. + +A single console logger drains at most 128 real messages per flush. Owner pipe +readers can ingest progress without waiting for the console mutex; fatal +watchdog receipts still flush before native exit. Queueing does not create +progress, extend deadlines or change the official test verbosity. The mailbox +can retain output while the console is slower; bounded drain batches do not +constitute a bounded queue. A four-owner local batch covering both failing +Cases passed in 34.711484 seconds with the original idle fence. Exact-head +remote CI remains required after publication. + + +* Driver rows and fixed protocol: retained negative CPU evidence + +The prepared-row trial had reference-control P10/P90 of 0.942814/1.053177 before +and 0.902893/1.056581 afterwards. Candidate P10/P50/P90 was +0.934300/0.954982/0.984585. Admission failed in 35.081605 seconds. + +The subsequent fixed-argument capture protocol and SDK checked context binding +had before-control P10/P90 0.949444/1.048368 and after-control +0.905698/1.052772. Candidate P10/P50/P90 was +0.913759/0.942463/0.965977. Admission failed in 33.723967 seconds. Its complete +reference controls were stable; neither its median nor the separate allocation +result overrides the rejected P90. + +Both diagnostic versions admitted a no-GC captured-LR counter delta of +56398160 per 256 requests, or 220305.3125 per request. Against the starting +257233.3125 this is 36928 fewer counter bytes, approximately 14.36 percent. +The slot model alone does not predict that native counter difference. Collection +and full-request CPU remain distinct obligations. The fixed protocol passed +198 frozen variants; the full seven-owner regression passed in 8.891532 seconds +after rebuilding an existing stale diagnostic interface. The failed preparation +with an unbound profiling function was not counted as a passing regression. + +* Unary child storage hypothesis + +The actual request has 193 unary reductions. Each eagerly allocates a one-cell +children list even when fresh recognition only reads the child's bounds/count. +The engine-owned child is a token vector or final fragment/view, distinguishable +from an already materialized Scheme list. The next representation stores that +child directly in the existing slot. First observation publishes a stable +single-element list; empty and wider lists retain their representation. + +Lean establishes exact list observation, repeat observation and explicit cold +allocation charges. The same complete-publication premise as =YieldMemo.qnt= +applies: no partial list is published. Concurrent first observers can construct +equivalent complete lists; there is no exactly-once allocation claim. Original +token identities, ordered children and the persistent stack suffix must survive. +Frozen controls materialize both observations before comparing private records; +the historical reference reduction body is unchanged. Full forced-yield diagnosis +must pay this child-list construction together with terminal-index construction. + + +* Unary and boundary CPU receipts + +The unary representation passed 198 frozen combinations and 79 semantic Cases. +Its first full CPU trial passed in 43.906424 seconds: + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before | 0.949968 | 0.988071 | 1.051620 | +| Candidate | 0.912493 | 0.927410 | 0.940669 | +| After | 0.910725 | 1.003436 | 1.044409 | + +An independent repeat had before P10/P90 0.947636/1.051921 and after +0.903179/1.056393. Candidate P10/P50/P90 was +0.915714/0.940165/0.972373, so its P90 failed. The first pass establishes a +qualified local observation, not stable repeated admission. + +The next representation adds captured lookahead kind to the immutable context +and retains the captured lexeme reference in the fragment. The kind, executor +and target remain live cache checks. A boundary query constructs and retains its +original kind/lexeme pair; it never reads a subsequently mutated token vector. +Malformed live kind/lexeme values fail admission before a fragment is published. +Each caller still gets the fragment's stable pair, and boundary checks remain +mandatory. The actual 322-node record tree retained eleven distinct contexts. +The revised slot inequality charges six context slots and nine fragment slots; +it does not count the old lookahead pair as a free object. + +The first trial of this representation had candidate P10/P50/P90 +0.910560/0.921237/0.973357 and an invalid following-control P10 of 0.893484. +It was rejected in 32.883676 seconds. Two following trials were declared in +advance, with both required to pass: + +| Trial | Phase | P10 | P50 | P90 | +|-------+-------+-----+-----+-----| +| 1 | Before | 0.953359 | 1.002471 | 1.048658 | +| 1 | Candidate | 0.921758 | 0.923183 | 0.945555 | +| 1 | After | 0.903032 | 1.001349 | 1.057075 | +| 2 | Before | 0.947887 | 0.973606 | 1.034462 | +| 2 | Candidate | 0.905670 | 0.914851 | 1.000445 | +| 2 | After | 0.893892 | 1.025717 | 1.055345 | + +Trial 1 passed in 31.496684 seconds; trial 2 failed the candidate criterion and +following-control floor. The declared two-trial qualification failed. No +outlier is removed and the passing run does not erase either rejection. The +subsequent malformed-value guard is included in the final candidate and must be +qualified in the independent runner; these local timings predate that guard. + +The dedicated Ubuntu =capture= CPU lane compiles current and frozen owners with +the same SDK compiler options, selects one allowed processor and performs the +original gate twice. Its baseline is the study's =459ec36=, not the older +selector-input baseline. Dispatch =CI= with =cpu-scope=capture= to select that +lane; the default =all= preserves every existing CPU lane. Both native image +inventories are printed before timing. The immutable runner receipt is the separate acceptance gate. + +* CI isolation follow-up + +At =6b36512= Ubuntu's catalog still exceeded its idle fence while other owners +ran; macOS's hot-path receipt measured 1.602186 seconds against its unchanged +900ms ceiling. The test graph was co-scheduling CPU/GC admission with unrelated +owners. A declared =test-worker-affinity= value of =exclusive= now defers those +resource owners until every parallel owner joins. Ordinary runtime affinity +retains Worker-zero execution. No new interpreter process is needed. + +The new control places the exclusive owner first in the input list and requires +both parallel owners to have completed. All nine lifecycle controls passed. The +four-owner failure-coverage batch passed in 25.346152 seconds, with the original +20 sampler groups, 256 calls per leg, 20 hot-path samples and 900ms ceiling. +An attempted two-group sampler control was rejected by its existing minimum and +withdrawn; the failed preparation is not reported as a pass. =734015f= publishes +the declaration/scheduler change. Exact-head CI can still reveal further work. + +* Diagnostic demand states and remaining model-directed work + +The topology preflight observes child lists, warming its own private caches. +Fresh captured-LR samples now compare against an independently fresh reference, +while the preflight retains independent semantic projection and mode validation. +Forcing the timed result merely to match a warm private cache would change the +scope, so it is not used. The rejected warm/cold comparison run is retained as +a diagnostic failure. =whole-forced-yield= now pays for child-list and boundary +materialization as well as terminal-index construction and full validation. +Earlier terminal-only forced-scope medians cannot be compared with this scope. + +The bounded dense context directory is charged separately from row-entry cells. +Oversized grammars use exact uncached contexts; their context count can approach +the fragment count, violating the modeled slot break-even condition. Sparse +bounded interning is a model-directed next hypothesis, not an established gain. +Cold transfer must pay all deferred work. Neither fresh-request receipts nor +finite models establish an incremental-transfer speedup across languages. + + +* Complete resource ownership and final allocation receipt + +The =734015f= Ubuntu run failed in =t/gql/runtime-benchmark-test.ss= admission. +Pending exclusive owners were not accepted as passing after that failure. The complete +audit identified ten modules calling ASP benchmark admission. Each now declares +boolean =test-worker-exclusive?= independently of symbolic =test-worker-affinity=. +This includes the ABI benchmark, which retains its runtime affinity. Worker zero +is the same caller thread during parallel and exclusive phases. An official +=std/sync/barrier= acknowledges the preceding console drain before timed owners +start; it does not emit progress or acknowledge individual lines. + +Eleven native lifecycle controls cover shared preparation, source-first loading, +parallel completion before exclusive execution, cross-phase runtime affinity, +invalid exclusivity type, failure propagation, silence, and duplicate ownership. +The known-failure five-owner batch, including GQL runtime admission, passed in +25.352558 seconds. The invalid-metadata Fixture's initial syntax error was fixed +before accepting its intended rejection receipt. The final native LR regression and runner-hosted CPU qualification are separate +acceptance gates. + +The final captured-LR diagnostic admitted a no-GC allocation-counter delta of +50393424 per 256 fresh requests: 196849.3125 per request versus the starting +257233.3125. This is 60384 fewer counter bytes, or approximately 23.47 percent. +The complete seven-scope diagnostic passed in 14.306226 seconds. This includes +a forced-observation scope, but is not a matched whole-request CPU comparison. +Runtime preparation and directory retention remain explicit separate costs. +Cached executor references can retain their closure environments until cache +replacement or runtime release; bounded cell count alone is not a bound on all +transitively retained memory. + + +* Final validation protocol + +The guarded representation passed 81 native Cases in 19.670243 seconds and +198 frozen combinations in 16.927932 seconds. Final admission rejects malformed +kind/lexeme values before publishing a context or fragment. Its rejection control +also checks that a fresh runtime's context directory remains pending. Both kind +and lexeme tag collisions are covered. A source-consistency run repeats these +controls after the compiler entry used by the Ubuntu CPU lane. + +Test declarations impose independent typed constraints: symbolic thread affinity +and boolean resource exclusivity. An exclusive ABI owner retains its affinity. +There is no reserved affinity symbol encoding resource ownership. The public +preparation guide describes those slots; eleven lifecycle controls qualify their +native behavior. These controls are not a Lean proof of Scheme scheduling. + +Runner logs bind source revision, both native inventories, the full artifact and +frontier fingerprints, all timing groups and the original decision. Consult the +=CPU capture= job for the dispatched revision. A local passing observation and +allocation-counter reduction cannot substitute for its two successful trials. + + +The final source-consistency regression passed 81 Cases in 19.230311 seconds and +198 frozen combinations in 7.837042 seconds. These are correctness receipts; +the independent Ubuntu CPU gate still determines performance acceptance. + +The final allocation replay passed in 10.163615 seconds after the admission-order +change. Three no-GC captured-LR samples independently reported the same +50393424 counter-byte delta per 256 requests; the 23.47 percent counter reduction +therefore survives the final source-consistency check. + + +* Independent Ubuntu result and prepared capture transport + +The isolated =9200000= Ubuntu trial at +[[https://github.com/tao3k/gerbil-parser/actions/runs/38751817673/job/116487759485][CPU capture]] retained stable reference controls but failed the original gate: + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before | 0.950996 | 0.999049 | 1.051565 | +| Candidate | 0.923874 | 0.929773 | 0.951840 | +| After | 0.978136 | 1.001067 | 1.048638 | + +It completed in 35.855021 seconds. The 0.951840 ratio is not rounded into a pass; +the second trial did not run after the first failed. Ubuntu semantics, FFI and +generation passed separately. The diagnostic matrix exposed an older observer +still interpreting the scalar selector input as a list. Its wrapper now accepts +the token or false directly; complete-product equality remains mandatory. A new +native regression requires real token and EOF observations. The original complete +matrix, with 20 samples/groups, passed locally in 29.403846 seconds. + +The next cost hypothesis changes input transport rather than the terminal +observations. An immutable four-slot =lr-capture-plan= retains runtime, +production, width and bounded row storage. The driver owns one plan per +production and passes that capability into capture. The capture call drops from +ten arguments to eight; its context lookup drops from eight arguments to five. +Width is read from the plan instead of resolving runtime metadata and indexing +the width vector again on every reduction. Uncaptured requests prepare no plans. +Oversized row grids still produce exact uncached descriptors; plan objects and +the directory remain charged preparation/retention costs. + +The grammar table and widths are immutable producer inputs for a prepared +runtime, unlike caller-owned mutable token vectors. Native controls check every +plan's runtime identity, production ordinal and width against those inputs. +Cloned runtimes start with their own pending plan directory. Entry ownership and +live executor/target/kind checks remain required. Cache-free plans are compared +with the same frozen oracle, not admitted through a compatibility mode. + +=prepared_capture_prefix_exact= requires the producer's width equality before +preserving children and suffix. =prepared_capture_arguments_saved= counts logical +call words, not native stack allocation or CPU. The revised slot break-even +condition also charges four slots for every prepared plan. Quint adds a wrong-width +negative history; all six directed histories and 1000 seeded invariant traces +passed. The native batch passed 90 Cases in 19.647147 seconds and 198 frozen +combinations in 18.372885 seconds. CPU qualification remains a separate gate. + + +The first eight-argument trial had candidate P10/P50/P90 +0.918514/0.922365/0.945011, but following-control P10 was 0.896861. The gate +rejected it in 31.930211 seconds. The passing candidate percentile does not +override the invalid control. The next staging hypothesis uses the SDK's checked +=definline= contract for the same capture body, preserving validation and ordered +state effects while exposing it to the driver compiler. The existing +=fused_driver_order= obligation retains callback/error state; native controls and +complete-request measurements remain necessary. No workload or gate is changed. + + +The checked inline variant passed the same 90 native Cases in 20.580977 seconds +and 198 frozen combinations in 13.986023 seconds. Its local CPU ratios were +before 0.966366/0.998073/1.046040, candidate 0.871885/0.925930/0.949430, +and after 0.899453/0.989380/1.115945. The unchanged gate rejected the following +control in 35.762327 seconds. The candidate percentile alone is not accepted. +The observed 64-record root has a false executor, confirming that it uses the +canonical LR path; no assumption about arbitrary custom executors is needed. +Independent Ubuntu qualification must evaluate the plan/staging revision. + + +** Canonical goto ownership and custom executor isolation + +Ubuntu run =38760634859= at =a6b6b38= passed generation, corpus, diagnostics, +semantics and FFI. Capture CPU qualification failed: candidate P10/P50/P90 was +0.921400/0.936945/0.959359, and following-control P90 was 1.101842. The +49.830187-second rejection is retained; the second trial did not execute. +Neither the candidate threshold nor the reference stability gate was met. + +The next model separates canonical goto ownership from a caller-supplied executor. +For a prepared immutable grammar, runtime, production and entry determine the +canonical target. Only canonical transitions may publish in a prepared plan's +context row. Under these producer premises, =canonical_context_admission= proves +that the remaining live kind comparison admits the exact six-field context. +Quint checks eight directed histories and 1000 seeded invariant traces, including +custom executor isolation and rejection of a changed canonical target premise. +These checks establish an abstract admission protocol, not a native CPU result. + +The engine now keeps custom executor contexts uncached. They retain their actual +executor and target, cannot replace a canonical row entry, and do not retain a +borrowed callback globally through that row. Canonical hits compare the live +kind; immutable prepared goto ownership supplies the target. Live symbolic-kind +and string-lexeme validation still precedes publication. Each boundary snapshot +retains the observed lexeme, including when the caller later mutates its token. + +The unused diagnostic cache wrapper is removed. The historical oracle constructs +its private context directly, independently of the candidate cache. Synthetic +capture geometry uses the same custom executor sentinel in both oracle and +candidate; actual parser requests exercise canonical goto caching. A native +control obtains a real prepared goto entry, checks canonical reuse before and +after custom calls with changed targets, and checks runtime isolation. Oversized +plans remain uncached. The original 64-record CPU gate and all budgets are kept. + + +Native qualification passed 90 Cases across eight modules in 7.520843 seconds +and all 198 geometry combinations in 9.524587 seconds. An initial control failed +because it assumed augmented production zero had a goto; the corrected control +selects a real nonempty prepared column. That failed run remains retained. + +Two consecutive unchanged complete-request CPU trials passed: + +| Trial | Reference before P10/P50/P90 | Candidate P10/P50/P90 | Reference after P10/P50/P90 | Seconds | +|-------+-----------------------------+----------------------+----------------------------+---------| +| First | 0.947586 / 0.999273 / 1.055947 | 0.914600 / 0.923168 / 0.944132 | 0.904643 / 1.001789 / 1.050335 | 30.102411 | +| Repeat | 0.957353 / 1.000648 / 1.024456 | 0.915619 / 0.918091 / 0.940139 | 0.950028 / 0.998975 / 1.055404 | 30.129078 | + +The complete 64-record ASCII request has local median CPU reductions of 7.68% +and 8.19% against the frozen =459ec36= reference, with valid reference controls. +These compare the accumulated capture implementation against the frozen baseline; +they do not isolate the incremental contribution of canonical cache admission. +The scope includes scanning, LR, capture, publication and full artifact/source +validation. Both trials retain the original sample counts, natural GC and budgets. +Independent exact-head Ubuntu qualification remains required. Custom executor +correctness is covered, but its CPU performance is not qualified by this workload; +its descriptors deliberately allocate independently rather than retaining callbacks +in the canonical cache. + +The seven-scope allocation/GC diagnostic passed in 10.667238 seconds. Every whole +request sample in this run collected, so the diagnostic correctly reports no +admissible allocation-byte estimate for those samples. No additional allocation +reduction is claimed from removing the redundant canonical checks. The earlier +23.47% no-GC allocation reduction and the matched CPU evidence remain separate. + + +** Preserve custom reuse in disjoint bounded cache partitions + +The uncached custom branch above introduced a potential allocation regression for +repeated custom transitions. It is superseded by two disjoint halves of each +prepared row. Canonical entries keep their original index and live-kind check; +custom entries use the second half and retain executor, target and kind checks. +The original total 65536-cell cap now charges both halves before allocation. +Oversized grammars retain the exact uncached fallback; the four-slot plan and +six-slot context do not grow. Cached callback retention remains bounded by row +replacement and runtime release, as described earlier; it is not eliminated. + +Lean proves that the halves cannot alias and every custom index is within its +row. Existing =cached_context_admission= supplies the custom equality obligation; +=canonical_context_admission= supplies the canonical obligation. Quint's nine +directed histories and 1000 seeded invariant traces passed, including repeated +custom reuse across canonical calls. Native tests additionally check identity +reuse, different custom targets, canonical isolation and the aggregate cell cap. +The earlier CPU trials describe the preceding uncached-custom revision. Both +original complete-request trials must be repeated for this final partitioned +revision before claiming its CPU qualification. + + +Final partition qualification passed the same 90 native Cases in 22.842109 +seconds and all 198 frozen combinations in 11.942724 seconds. The two predeclared +complete-request CPU trials both passed without changing their gates: + +| Trial | Reference before P10/P50/P90 | Candidate P10/P50/P90 | Reference after P10/P50/P90 | Seconds | +|-------+-----------------------------+----------------------+----------------------------+---------| +| First | 0.952443 / 1.000426 / 1.051520 | 0.912720 / 0.915341 / 0.938827 | 0.901624 / 1.000147 / 1.052192 | 30.073934 | +| Second | 0.938748 / 1.000852 / 1.034124 | 0.912651 / 0.918500 / 0.942580 | 0.909949 / 1.007365 / 1.060078 | 30.111638 | + +This final partitioned implementation reduces local median complete-request CPU +by 8.47% and 8.15% against the frozen =459ec36= reference, including scanning, +LR, capture, publication and validation. Candidate P90 is below 0.95 and both +reference controls are valid in each trial. The accumulated implementation is the +comparison subject; the cache partition change alone has not been isolated. +The uncached-custom intermediate results above remain historical evidence. +Independent exact-head Ubuntu qualification is still a separate pending gate. + + +The custom half validates an exact nonnegative entry below the state count +before adding the partition offset. Without this guard, a negative entry could +land in a canonical cell even though its shifted index is in bounds. This is the +native producer check for the Lean bounded-entry premise. Native controls reject +negative, upper-bound and fractional entries before verifying that the canonical +cell and previously returned custom contexts retain their identities. + + +The guarded revision passed 90 native Cases in 19.168094 seconds and all 198 +frozen combinations in 8.117394 seconds. Its first complete-request CPU trial +passed in 29.981083 seconds: before P10/P50/P90 was +0.948623/1.003199/1.053786, candidate was 0.907454/0.911699/0.937180, +and after was 0.908831/1.004442/1.050407. This is an admitted local median +CPU reduction of 8.83% against the frozen complete-request reference. + +The identical second trial was rejected. Before was +0.950133/1.002896/1.052643, candidate was 0.913246/0.916092/0.941963, +and after was 0.899192/1.001904/1.047376. Following-control P10 is below +0.90; it is not rounded into a pass. The candidate percentile does not override +that invalid control. Consequently the guarded final revision has one admitted +local complete-request trial, not two consecutively admitted trials. Independent +Ubuntu qualification must decide the remaining gate. No local outlier is dropped +and no workload, compiler, sample count, budget or threshold is changed. + + +Independent Ubuntu run =38766566952= at =a2937dc= rejected the first CPU trial +in 49.947361 seconds. Reference before P10/P50/P90 was +0.967146/1.008946/1.066613, candidate was 0.910949/0.924071/0.950277, +and after was 0.980494/1.002177/1.048464. Both controls are valid; candidate +P90 exceeds 0.95. The second trial did not execute. This is an unresolved CPU +qualification failure, even though its median improves. No rounding, workload +change or discarded percentile can close that gate. + + +* Owned packed children: reject a space leak before accepting a CPU result + +The paper/code review distinguishes representation specialization from retention. +Sperber and Thiemann's sections 3 and 4 specialize static grammar work; Wagner +and Graham's sections 3 and 7 make persistent tree representation and sequence +costs part of the incremental design. Our inference is that a child container +must account for everything it retains, including links outside its own span. +Neither paper establishes native allocation or CPU improvement for this engine. + +A borrowed persistent-stack prototype passed 91 native Cases and 198 frozen +combinations. Its two local complete-request trials passed, with candidate +P10/P50/P90 of 0.881510/0.906677/0.933381 and +0.889410/0.920909/0.941449. Nevertheless it is rejected: a fragment can retain +an unrelated stack suffix, and later reuse can keep an obsolete prefix alive. +The =PrefixChildren= model makes that retention premise explicit; its algebraic +correctness does not discharge the resource obligation. No borrowed-stack code +is retained in the engine. Initial fixture failures (a missing semantic alias and +an outer unary wrapper assumption) remain recorded rather than being counted as +native acceptance. + +The admitted representation instead uses a private one-slot POO record +=lr-capture-children= holding an owned source-order array. Widths four and above +copy only their child references into that array during the existing bounds/count +traversal. Widths two and three keep eager lists; the existing unary and nullable +paths are unchanged. First observation converts the array to a stable cached +list. Token identity, source order, offsets, boundary signatures and yield-fold +association are preserved. No stack link or unrelated suffix is stored. The +threshold selects a generic representation, not a language or input size. + +Lean's =packed_children_exact= and =packed_children_warm= establish exact first +and repeated observation. =packed_children_exclude_suffix= establishes the +prefix-only snapshot. =packed_children_no_demand= charges both descriptor and +array; =packed_children_first_demand= charges the later list too. Object counts +do not prove byte, GC, or CPU costs. The existing Quint memo protocol checks +publication/generation ownership; its finite traces are not native refinement. + +Native qualification passed 91 Cases in 14.368916 seconds and all 198 frozen +combinations in 11.236343 seconds. The new regression mutates the original stack +and suffix links after capture and retains exact children and suffix identity. +Cross-request ownership and stable first-observation identity are also checked. +Three organization Cases passed in 1.622812 seconds, including the descriptor's +exact single-array slot. The full unchanged 20-by-20 LR diagnostic matrix passed +in 28.956321 seconds. Lean and source-input binding controls passed. + +** Allocation scopes and first-demand cost + +Three no-collection captured-LR samples each used 49,344,848 bytes per 256 requests, +compared with the preceding 50,393,424: 4096 fewer bytes per recorded request, +or 2.08%. Four counter controls retained identical totals: recorded scan +38,814,032; uncaptured LR 22,237,520; publication 22,749,520; validation +10,920,656. This is a captured-LR allocation result, not a complete-request byte +reduction or an isolated CPU speedup. + +The seven-scope diagnostic passed in 10.869075 seconds and still charges first +child-list/boundary/yield demand in its forced-request scope. Three added +allocation-only controls each observed zero collections and 133,591,760 bytes +per 256 complete requests. They explicitly collect before the counter window +and report no CPU result. This is an absolute allocation observation without a +matched full-request byte baseline. The separate CPU gate retains natural GC; +no collection is moved outside its clock. + +** Complete-request CPU qualification + +The first trial was rejected in 32.937010 seconds: before P10/P50/P90 was +0.958691/1.001726/1.060223, candidate was 0.892406/0.914911/0.916655, +and after was 0.898375/1.004041/1.051651. Its invalid following control is +retained despite the candidate percentile. The planned second trial passed in +32.603685 seconds: before was 0.955781/1.003807/1.051668, candidate was +0.910066/0.915611/0.922253, and after was +0.906807/1.001665/1.052147. It admits an 8.44% local median CPU reduction +for the complete 64-record ASCII request against frozen =459ec36=, with valid +reference controls and natural GC included. + +This is one admitted final-representation trial, not two consecutive admitted +trials. It covers the accumulated capture implementation and does not isolate +the packed array's incremental CPU contribution. Independent Ubuntu qualification +remains required; the earlier stable-control Ubuntu P90 failure is not superseded +by local measurements alone. + + +* Checked width countdown and rejected static staging + +** Independent Ubuntu result for the owned representation + +Exact-head manual run =38773702045= at =48607f3= rejected the complete-request +CPU trial despite valid controls. Before P10/P50/P90 was +0.950547/0.997357/1.056113; production was +0.926694/0.933375/0.951428; after was +0.976627/1.001183/1.050118. The production P90 exceeds the unchanged strict +0.95 ceiling. The following trial did not run. The preceding local admission +is therefore not independent Ubuntu CPU closure. + +** A semantically valid specialization that increased allocation + +Following the static-grammar specialization argument in +[[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]], an experimental macro expanded widths two, three +and four into direct frame observations. Lean proves exact source order, +suffix preservation and metadata observation order for those bounded shapes. +The native experiment passed 92 Cases and all 198 metadata combinations. +However, its complete-request CPU P10/P50/P90 was +0.943139/0.948627/0.972223 with valid before and after controls. Two +no-collection captured-LR samples allocated 58,782,032 bytes per 256 requests, +compared with 49,344,848 for the owned-array implementation: approximately +19.13% more. The experiment failed both the CPU gate and its allocation goal. +Its mutable local joins may affect representation, but no profile establishes +boxing as the root cause. The macro and unrolled runtime branches were removed; +the semantic theorems and negative results remain research evidence. + +** Producer-checked countdown domain + +The retained candidate validates every capture width as a nonnegative fixnum +before publishing the runtime's plan directory. The width slot carries that +same type. Checked fixnum operations implement only the width countdown and +packed-array index, with one shared decrement per packed child. Lean establishes +that a positive admitted countdown remains bounded and yields a valid array +index. Source positions and token totals remain general integers. No module-wide +unchecked declaration is introduced. This follows the explicit typed-slot and +fixnum style in Gerbil's =std/struct/pqueue.ss= without importing its unchecked +execution policy. + +Negative, fractional, string and oversized widths all fail before the plan cache +is published. The native eight-owner group passed 93 Cases in 12.029587 seconds; +all 198 frozen metadata combinations passed in 9.465403 seconds. + +The first complete-request trial was rejected in 31.683993 seconds. Before was +0.949247/1.024382/1.057254, production was +0.915535/0.919205/0.929363, and after was +0.905631/1.002836/1.109345. Its following control exceeds 1.10 and invalidates +the trial. The planned second trial passed in 31.429802 seconds: before was +0.948452/1.000623/1.055922, production was +0.908671/0.913919/0.927657, and after was +0.906122/0.998743/1.057086. This admits an accumulated local median CPU +reduction of 8.61% against frozen =459ec36= over the unchanged complete +64-record ASCII workload, including natural GC. It does not isolate the +countdown's incremental contribution. The direct predecessor comparison is +reported separately; failed controls and the Ubuntu rejection remain binding. + + +** Direct predecessor control rejects a new countdown speed claim + +A separate native image was compiled from exact predecessor +=48607f3957f8a7b68b891a1f2391fdad3ac93942= with the same selected owner list and +compiler policy. The first build attempt omitted a relative Source support +module; that incomplete image was not measured. The complete Source snapshot +was then materialized and the selected closure rebuilt successfully. + +The unchanged full-request gate, now comparing against this immediate +predecessor, reported before P10/P50/P90 +0.971273/1.000056/1.020767, production +0.974875/1.003406/1.026985, and after +0.970117/0.995106/0.997229. Both controls were valid and the candidate failed +the original CPU ceiling. The point median is 0.34% slower; this single run +establishes no incremental CPU gain. The earlier 8.61% remains an accumulated +comparison to =459ec36=, not a countdown success. The countdown candidate is +not admitted as an optimization. Runtime withdrawal and any subsequent child +representation experiment require separate source and native receipts. diff --git a/docs/research/capture-metadata-model-first.org b/docs/research/capture-metadata-model-first.org new file mode 100644 index 00000000..ec43be22 --- /dev/null +++ b/docs/research/capture-metadata-model-first.org @@ -0,0 +1,162 @@ +#+title: Model-first classification of captured recognition metadata + +* Paper and code basis + +Sperber and Thiemann's [[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]], section 3.1.3, motivates +eliminating redundant constructor/selector work while preserving computations +and their order. It also discusses transport of multiple results; adding a +metadata tuple or callback is not assumed free. Wagner and Graham's +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] motivates preserving reuse context +and charging the checks that support it. Neither paper establishes a speedup +for our representation; the following obligations come from current code. + +At baseline =54746a3b1617639fac05805f6ac9aae4a1861555=, +[[file:../../src/runtime/lr-parser/recognition.ss][the recognition owner]] reads start, end, token count and terminal yield through +four separate piece projections during unary reduction capture. Each projection +classifies the same object as a view, token or fragment. Views recursively +interpret coordinates and terminal-yield relocation. Base pieces have direct +fields. The caller invokes no semantic callback between these reads. + +The earlier [[file:lr-request-model-foundation.org][request scope model]] rejects unproven cross-reduction metadata +caching because token vectors and lexemes are mutable. This proposal retains +no metadata beyond the current synchronous capture operation. Source hashing, +FFI event coding, scanner policy, LR state/action selection and artifact +validation are unchanged. + +* Model completed before implementation + +[[file:../../formal/ParserEngine/Capture.lean][Lean Capture]] adds explicit token, fragment and nested-view representations. +Tokens have count one; fragment metadata includes empty terminal yields. Start +and end use integer coordinates, and terminal yield is opaque. Relocation is +an arbitrary pure function: no associativity, canonicalization or rearrangement +is assumed for nested views. + +=classified_metadata_exact= proves the proposed base classification equals all +four existing projections. =classified_consumer_exact= transfers that equality +to an arbitrary pure consumer. The Scheme owner must establish valid piece +representations, stable local fields and no intervening effects; the theorem +does not verify mutable object races or the compiler. It does not permit token +metadata to outlive its existing scope or change observable child identity. + +For =b= ordinary unary children and =v= view unary children, count complete +piece-kind decisions: + +#+begin_example +Existing: 4 * (b + v) +Candidate: b + 5 * v +Break-even: v <= 3 * b +#+end_example + +A view retains the old projections and pays one additional outer decision. +The model therefore includes a view-only regression example. These are logical +classification counts, excluding accessor validation, scalar loads, native call +cost, allocation, GC and all other request work. A whole-request CPU claim must +include those costs and cannot follow from this count alone. + +[[file:../../formal/quint/CapturePlan.qnt][Quint CapturePlan]] retains ownership of the immutable capture epoch and +publication lifecycle. New unary and nullable invalidation histories ensure +that a local classification change does not justify publication after mutation. +It does not duplicate the pure metadata algorithm or become a parser oracle. +Existing request ownership, cancellation and snapshot models retain their own +scopes. Lean and the locked Quint package passed before the runtime change. + +* Rejected engine experiment + +The tested unary branch classified a base token or fragment once, then passes its +existing scalar fields directly to the unchanged fragment constructor. Local +Gerbil =defrule= syntax keeps that constructor in one source location without +a new tuple, multiple-value transport, runtime callback or strategy protocol. +For views, the existing four projections preserve exact recursive behavior. +The zero-width and wider reduction paths remain unchanged. + +This was a generic engine transformation for captured unary productions. +No language fixture or grammar is altered. The local syntax lives inside the +existing capture definition, so the historical module-organization gate still +compares every other runtime execution form unchanged. + +* Controls before timing + +[[file:../../t/benchmarks/capture-prefix/controls.ss][Capture controls]] compare complete transparent fragments against the frozen +independent reference across 198 combinations: 11 productions, nine token/ +fragment/empty-fragment representations including nested views and positive/ +negative deltas, and EOF/Unicode lookahead. They check exact yield association, +source-order children and physical suffix sharing. The controls passed with +exit zero in 7.605 seconds under the existing 90/5-second watchdog. + +The initial expanded control script had an unmatched parenthesis and failed +compilation before running assertions. It was corrected before any CPU run; +that setup failure is not a semantic or performance result. + +* Actual complete-request distribution + +[[file:../../t/benchmarks/capture-prefix/worker.ss][The persistent worker]] counts its real captured tree outside all timed +intervals. The fresh 64-record ASCII source has 322 reductions: + +| Width | Reductions | +|-------+------------| +| 0 | 1 | +| 1 | 193 | +| 2 | 64 | +| 4 | 64 | + +Among unary children there are 128 tokens, 65 fragments and no views. The model +therefore predicts 772 versus 193 root piece-kind decisions, a reduction of +579 such decisions. Both native workers must report identical inventories and +complete artifact digest. These counts do not establish CPU percentages and +must not be applied to edited/view-heavy requests. + +* Complete-request CPU admission + +The baseline image is compiled from the exact baseline Git source with the +normal SDK compiler and current package pins. The candidate changes only the +recognition image. The existing two persistent workers verify distinct selected +recognition-image hashes and identical shared checkpoint, state, parser, session +and grammar images. No runtime function binding is replaced. + +Each timed call constructs a fresh capture-enabled incremental session for 64 +ASCII records, covering scanning, LR execution, capture and artifact publication. +It compares the complete artifact and validates source binding on every call. +Independent recognition replay is checked before timing. Expected artifact: + +#+begin_example +sha256:c42a4b2281ad7f1921dcd426606db0a2ccb5cbd9925078e67cd479aea27bf24c +#+end_example + +Process user-plus-system CPU is measured inside each worker. Startup, protocol +IPC and driver CPU are excluded. Natural GC, equal host QoS, before/after +reference controls, all groups and the existing 90-second total / 5-second +silence bounds remain. Five groups per phase each contain 32 alternating legs +of 256 calls: 122,880 timed complete requests, plus two 256-request warmups. +The minimum/maximum control ratios must lie in =0.90..1.10=, and candidate P90 +must be below =0.95=. No outliers may be discarded. + +* Reproduction + +#+begin_src sh +just formal +just formal-protocol +just prepare-capture-reference 54746a3b1617639fac05805f6ac9aae4a1861555 .data/capture-metadata/reference +just benchmark-capture-prefix .data/capture-metadata/reference/lib +#+end_src + +This qualifies fresh capture-enabled Scheme requests only. Edited requests, +view-heavy workloads, other grammars, capture-disabled parsing, host RPC and +Rust/Tokio consumer behavior need separate evidence. + +* Negative complete-request result + +All 198 semantic controls passed. The complete-request experiment completed +122,880 timed requests but failed the CPU gate: + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before reference/reference | 0.933886 | 1.003465 | 1.056129 | +| Candidate/reference | 0.961911 | 1.006100 | 1.014672 | +| After reference/reference | 0.980577 | 1.008979 | 1.051860 | + +The controls were stable, but candidate P90 exceeded =0.95= and the median did +not improve. The supervisor exited 70 at 48.082 seconds with error-output and +no anchored success receipt. The engine experiment was reverted. The model, +expanded controls and this negative result remain useful evidence that fewer +logical classifications alone did not yield complete-request CPU improvement. +No runtime optimization is claimed or retained from this experiment. diff --git a/docs/research/capture-prefix-four-layer-optimization.org b/docs/research/capture-prefix-four-layer-optimization.org new file mode 100644 index 00000000..509e4b98 --- /dev/null +++ b/docs/research/capture-prefix-four-layer-optimization.org @@ -0,0 +1,242 @@ +#+title: Paper, model, engine and Benchmark: recognition capture + +* Required optimization chain + +An optimization has four distinct owners: + +| Layer | Required output | Capture case | +|-------+-----------------+--------------| +| Research paper | Algorithmic argument, assumptions and applicability | Static LR structure and incremental node/context capture | +| Model | Semantic equivalence, lifecycle constraints and a cost hypothesis | Prefix order/tail/count preservation; capture publication guard; fewer list visits | +| Engine implementation | A reusable generic operation | Fused capture plus nullable/unary specialization in the LR engine | +| Scheme Benchmark | Actual complete-workload comparison, including controls | Two native images, persistent Scheme owners and complete 64-record captured requests | + +Lean and Quint remain separate within the model layer. A theorem without a +measurable hypothesis, a lifecycle test without a runtime owner, or a faster +helper without complete-request evidence does not close this chain. + +* Papers and the concrete hypothesis + +Sperber and Thiemann's +[[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]] specializes grammar-static LR decisions and uses +binding-time improvements and list-loop unrolling. Applied here, reduction width +is grammar data: empty and unary captures need not execute a general split/fold. +This is an application of that staging principle, not the paper's parser generator +or a reproduction of its speed measurements. + +Wagner and Graham's +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] requires retained structure and +context for incremental operation and discusses compact syntax and node reuse. +Our capture retains the current engine's metadata contract. Optimizing its list +walks does not establish the paper's incremental complexity bound, maximal node +reuse, or balancing theorem. + +Source inspection found =take=, =reverse=, =drop= and =last=, followed by separate +token-count and terminal-yield folds over the same reduction prefix. The initial +hypothesis was to fuse the prefix split and count accumulation while preserving +exact yield association. Full-request measurements then identified unary captures +as the dominant case: 193 of 322 reductions on the 64-record workload. + +* Model-derived work reduction + +[[file:../../formal/ParserEngine/Capture.lean][Lean Capture]] proves that the fused prefix equals =reverse(take width stack)= +and retains =drop width stack=. It proves nullable/unary identities and that +reversing the input does not change additive token counts. Terminal-yield +construction retains the original source-order left fold, rather than assuming +that arbitrary yield representations have equivalent association. + +The explicit partial operation model counts prefix/list inspections: + +#+begin_example +Former: 5 * width + (width - 1) +Fused: width == 1 ? 1 : 2 * width +#+end_example + +States traversal, nested metadata getters, constructor fields, type checks, +branch selection, publication, hashing and GC are outside that count. They remain +charged by the complete-request Benchmark. Therefore fewer modeled visits alone +do not establish a CPU improvement. + +Actual capture widths are =((0 . 1) (1 . 193) (2 . 64) (4 . 64))=. Applying this +model yields 3,141 former versus 961 candidate list inspections. That 69.40% +modeled decrease is neither an instruction count nor an end-to-end speed claim. + +[[file:../../formal/quint/CapturePlan.qnt][Quint CapturePlan]] owns extent/lifecycle safety: prefix work completes +before yield publication; nullable/unary extents have their own admitted costs; +invalidation withdraws output. Its directed incomplete-publication mutation is +rejected. Lean owns list semantics; Quint does not duplicate the LR algorithm. + +* Generic engine implementation and semantic controls + +[[file:../../src/runtime/lr-parser/recognition.ss][capture-reduction]] is shared LR execution infrastructure. No language +pack, scanner DSL or public Loader schema changes. Empty captures bind their +nullable anchor directly. Unary captures read the single child and its yield. +Wider captures construct source-order children and sum token counts in one prefix +pass, share the suffix, then perform the existing yield fold. + +[[file:../../t/benchmarks/capture-prefix/controls.ss][Direct Scheme controls]] compare all capture fields against a +[[file:../../t/benchmarks/lr-request-stack/capture-reference.ss][frozen former implementation]] for all 11 record-grammar productions, including +nullable/unary widths, Unicode leaves and relocated values. They check exact +terminal-yield shape, source order and physical suffix identity. The complete +request also checks a projected artifact from the captured tree before timing. + +This control is a comparison, not runtime binding replacement. The production +module contains one implementation and no compatibility path. + +* Complete-request CPU protocol + +[[file:../../t/benchmarks/capture-prefix/benchmark.ss][The Scheme Benchmark]] starts exactly two persistent +[[file:../../t/benchmarks/capture-prefix/worker.ss][workers]]. Each admits one real native capture image. Both reuse the same +SDK and native state/checkpoint/parser/session/grammar images, verified by loaded +file hashes. Each capture image must be native, loaded from its selected root, +and different from the other's image. Artifact identity and capture-width +inventory must match. Startup and image hashing are outside request timing. + +Every timed call creates a fresh captured session for 64 ASCII assignments, +=a=1\n= repeated 64 times: scanning, LR execution, recognition capture and +ParseArtifact publication. Exact expected artifact equality and full source +validation are also charged. Process user plus system CPU, including natural GC, +is read inside the worker. No source parser, child process or Scheme runtime is +started per request. IPC and orchestration CPU are excluded: this qualifies the +Scheme engine request, not Tokio/FFI or whole-system dispatch. + +Warm both images with 256 requests. Before and after controls compare the frozen +worker with itself. Production compares the candidate with that frozen worker. +Each phase has five groups of 32 alternating ABBA/BAAB legs, 256 complete calls +per leg. All 122,880 timed requests contribute. With five groups, P10 and P90 are +the minimum and maximum: every control group must stay within =0.90..1.10=, and +every production group must be below =0.95=. No outlier is dropped. + +Darwin workers use the same checked =QOS_CLASS_USER_INTERACTIVE= setting from the +[[file:../../t/support/qualification/benchmark-qos.ss][test-only scheduling owner]]. Linux CI inherits the existing processor-affinity +policy. Production scheduling is unchanged. Total timeout is 90 seconds and +silence timeout is 5 seconds; only actual completed batches report progress. + +* Reproduction + +Build the current production package and native test support through their normal +owners. The frozen reference is Git source, compiled with the normal SDK compiler; +its package pins must equal the current package pins. + +#+begin_src sh +just prepare-capture-reference 76d6cc6c946cb8ca7692d8911e43801b951e340f +just benchmark-capture-prefix +just formal +just formal-protocol +#+end_src + +CI preparation compiles direct controls once; both platform semantics lanes run +them. A separate manual Ubuntu CPU lane prepares the frozen image and runs the +complete Benchmark. Source-bound model observations include the actual engine, +control and benchmark owners. The full native evidence-chain review remains a +separate gate; these observations do not imply it has completed. + +* Retained negative observations + +| Experiment | Before P10/P90 | Production P90 | After P10/P90 | Admission | +|------------+----------------+----------------+---------------+-----------| +| Generic fusion, 20 small groups | 0.852836 / 1.006797 | 0.963061 | 0.990926 / 1.169986 | Rejected | +| Generic fusion, five larger groups | 0.895597 / 1.028437 | 0.961984 | 0.971090 / 1.040409 | Rejected | +| Specialized, native-image hashes checked | 1.001281 / 1.070468 | 0.985365 | 0.961232 / 1.041222 | Rejected: CPU | +| Specialized, repeat before QoS normalization | 0.966944 / 1.065490 | 0.953879 | 0.959947 / 1.044087 | Rejected: CPU | + +The specialized run before image-hash enforcement passed with production +P10/P50/P90 =0.923492/0.925716/0.943256= and controls +=0.956106/1.040820= before, =0.964671/1.047191= after. Its 5.67% P90 CPU reduction +is a local observation. The later rejected repeats above prevent treating it as +a stable host-independent guarantee. Final qualification must use the image and +host-admission controls, rather than promote that earlier result. + +The initial production build also caught an unrelated integration error from the +previous commit: an interpreted history-control script was discovered as a library +module. It is now explicitly assigned to the test owner in =build.ss=. The normal +SDK/GCC package build then completed successfully in 84.268 seconds. + +* Final admitted complete-request observation (2026-10-14) + +With native image hashes and both Darwin QoS admissions verified, the final run +completed in 49.788 seconds with process exit zero and the exact anchored +=CAPTURE-COMPLETE-CPU-PROOF-OK= marker. All 122,880 timed full requests contribute. + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before frozen/frozen | 0.956280 | 1.018125 | 1.051384 | +| Candidate/frozen | 0.914970 | 0.938014 | 0.945449 | +| After frozen/frozen | 0.952151 | 0.958334 | 1.045281 | + +The worst production group used 5.45% less worker CPU than the frozen native +engine. This qualifies complete fresh captured Scheme requests for the 64-record +ASCII grammar on this host under the declared conditions. It does not qualify +ordinary capture-disabled parsing, Unicode, other grammars, incremental edit +speed, Tokio/FFI, or whole-system CPU including the Benchmark transport. +Those are separate follow-up workloads, not implied benefits. + +The test-only QoS integration initially failed to compile because its foreign +binding was not declared to the Scheme expander. The protocol then rejected +initialization before any timed work; no CPU result was admitted. Declaring that +binding and requiring successful compilation before execution resolved it. + +* Next optimization questions from this model + +1. Retain lookahead metadata once per input extent rather than per reduction. + Require a model of input identity/immutability and measure the added guard cost. +2. Investigate balancing terminal-yield concatenation for deep lists. Preserve + source order, span relocation and transfer admission; do not infer the paper's + optimal incremental bound from list fusion alone. +3. Move static capture shapes into prepared execution plans if this removes + further runtime dispatch. Charge preparation and binding validation using the + existing amortization theorem, then test full requests with and without capture. + +Each candidate must state its paper argument, new model obligation, shared engine +owner and complete Benchmark workload before implementation is promoted. Negative +CPU results can reject a candidate even when its logical work count decreases. + +* Final correctness qualification + +The normal package build, final 42-job Lean build and all nine Quint modules +passed. Quint totals 66 directed tests and eight sampled invariant runs. Direct +capture controls exercised all 11 productions with exact frozen parity. Existing +LR fragment-transfer and recognition-snapshot Suites passed 29 cases with final +=OK= and process exit zero in 48.722 seconds. These cover transfer, relocation, +nullable anchors and snapshot projection beyond the one measured grammar. + +The engine operation assumes an admitted nonnegative width that fits the semantic +and recognition stacks; validating arbitrary malformed LR tables is an upstream +contract. Lean's list equation is more general than the Scheme operation's error +behavior on an invalid stack, so no total error-equivalence claim is made. + +* Historical organization gate + +The Linux semantic lane at =857e6e6= rejected the changed capture body because +the runtime organization gate still required every form to equal the pre-split +source. That historical source remains pinned. The gate now permits exactly +one =capture-reduction= definition, verifies the frozen semantic control equals +the historical body after renaming, and compares all other forms unchanged. +The independent capture controls compare the evolved body against that frozen +control across every production, relocation, empty and Unicode lookahead, +exact children, terminal-yield association, and physical suffix sharing. +This bounds the admitted algorithm delta without replacing historical receipts. + +Local repair qualification passed all three organization cases in 2.955 seconds +and all 44 capture combinations (11 productions times four input variants). +No new CPU gain is claimed by these gate repairs; the preceding complete-request +CPU result retains its original measurement scope. + +* Cold admission qualification repair + +At =70844f3=, macOS capture controls exceeded the 5-second silence budget +before producing any receipt. The CI process result was =elapsed 5.514 seconds, +exit 137, reason idle-timeout= with the success marker missing. This differs +from a failed assertion or a verified-complete child stalled during shutdown. + +The controls now export =verify-capture-prefix!=, and CI explicitly invokes it +after the existing official-module preloader admits the compiled closure. +Dependency starts/completions report real work; there is no periodic heartbeat. +All 44 comparisons still run before the anchored success receipt. Only after +that call returns does the existing disposable-child exit interface run. +Exceptions stop this sequence and remain nonzero qualification failures. +The 90-second total and 5-second silence budgets are unchanged. + +Local execution of the exact CI command passed in 5.826 seconds with exit zero. +The original direct script entry still invokes the same verification procedure. +This repairs test admission and completion, with no new parser CPU claim. diff --git a/docs/research/casefold-request-qualification.org b/docs/research/casefold-request-qualification.org new file mode 100644 index 00000000..36304387 --- /dev/null +++ b/docs/research/casefold-request-qualification.org @@ -0,0 +1,327 @@ +#+title: Casefold vocabulary and complete request qualification +#+startup: overview + +Part of [[file:shared-ranked-scanner-execution.org][the integrated execution study]]. +Historical revisions, failed experiments and receipt limits retain their original scope. + +* Shared literal vocabulary and casefold lookup qualification + +A row-without-literals guard alone changes none of the measured allocation +values in the initial twelve-route child. The next candidate reuses the shared +literal trie representation for case-insensitive LR lookup. One prepared +vocabulary is shared by all action rows; the original row index still admits +only the current state's entry and preserves first duplicate semantics. +ASCII prefixes are converted while traversing the trie, avoiding a new whole +uppercase string. Non-ASCII input delegates to the original whole-string +Unicode conversion. The exact literal lookup retains priority over normalized +lookup and token-kind fallback. There is no language hint or loader option. + +The trie builder also owns bounded and ranked literal preparation. Existing +ranked matching, precedence, declaration order and mode admission stay in their +original execution loops. Independent action-index tests compare the new +casefold operation with the original whole-string conversion across empty keys, +prefixes, list/table boundaries, duplicate entries, row-specific misses and +Unicode mappings. The following native qualification bounds the performance claims. + + +Native construction initially fails the five-second idle fence twice, at +54.246 and 60.068 seconds. The runtime object completes when selected real GCC +pass products are observed, but the later macro-phase compilation fails that +fence at 73.113 seconds. Gerbil's compiler driver deliberately drops ordinary +GSC C options for the macro phase (=gsc-cc-options= with =phi?=), so runtime +instrumentation does not cover that phase. Compiling the same generated macro +source with the SDK GSC and selected GCC diagnostic products completes in +14.996 seconds. Compiler and optimization flags remain unchanged apart from +diagnostic output. An all-pass diagnostic attempt writes about 679 MB and fails +the total fence at 127.608 seconds; it is not a successful build receipt. + + +** Positive allocation evidence and bounded CPU scope + +At source graph =11b2cf1=, seven modules pass 35 semantic cases with the new +native objects in 30.247 seconds. Four native matrix children complete in +57.506, 72.633, 89.158 and 68.538 seconds with exit zero and final workload +markers. Candidate/reference/reference/candidate order executes 19,200 timed +calls. Each child verifies the LR and action-index object identities. Both +module prefixes change together; the lexical and shared scanner products stay +fixed. The candidate uses =o15=/=o3= and the baseline uses =o6=/=o2=. + +A Scheme checker validates all 844 admitted raw no-GC allocation intervals, +repeated variant values and unchanged-language controls. The exact batch +counter deltas establish the following saving per request: + +| GQL units | Boundary | Reference bytes | Candidate bytes | Saved bytes | Reduction | +|-----------+----------+-----------------+-----------------+-------------+-----------| +| 64 | Prepared | 230928.8 | 227984.8 | 2944 | 1.27% | +| 64 | Source | 542916.0 | 539972.0 | 2944 | 0.54% | +| 512 | Prepared | 1894000.8 | 1869552.8 | 24448 | 1.29% | +| 512 | Source | 4330076.0 | 4305628.0 | 24448 | 0.56% | + +All FHIRPath and Arithmetic allocation values are unchanged at both sizes and +boundaries. CPU is not admitted as a complete-parser benefit: the mean of child +medians includes a 16.03% increase for small GQL source execution, while the +unchanged large prepared Arithmetic control improves 17.93%. These descriptive +values expose drift; neither is a causal optimization result. + +The native =benchmark-lr-casefold= component uses all 2,834 actual GQL action +rows. It checks original entry identity against whole-string conversion before +measurement. Each row is queried with a non-keyword identifier, using suffixes +of 32 or 256 characters (complete identifiers of 43 or 267 characters). Twenty +ABBA/BAAB groups per width execute 9,068,800 timed lookups in total. Initial GC +occurs once per width; natural collections remain in CPU measurements. +The completed child takes 25.289 seconds, with exit zero and =LR-CASEFOLD-OK=. + +| Identifier characters | CPU ratio P10 | CPU ratio P50 | CPU ratio P90 | Lookup benefit | +|-----------------------+---------------+---------------+---------------+----------------| +| 43 | 0.1185 | 0.2213 | 0.2723 | Admitted | +| 267 | 0.0377 | 0.0405 | 0.0524 | Admitted | + +The empirical P90 ratio is below the existing 0.95 margin for both component +workloads. This establishes early-miss lookup benefit on this native SDK and +actual row vocabulary. It does not describe the live state frequency of a full +request, keyword hits, Unicode-heavy input, portable throughput or the Bash +comparison with tree-sitter. The complete-request allocation saving and the +component CPU benefit are separate positive results. + + +** Latest PR graph requalification + +The optimization is rebased onto =9374421=, including canonical production, +LR table and layout-capability admission. The component imports the shared +=parser-stage-cost= sampler directly; no compatibility alias or duplicate +sampler is introduced. Its native rebuild completes in 23.190 seconds. +Canonical authority and consumer rebuilds complete before execution; the final +LR runtime rebuild completes in 25.108 seconds with GCC and the original +qualification fences. Seven source modules now pass 44 cases in 16.802 seconds, +including the new producer admission tests and the casefold/scanner controls. + +A second native paired component child completes in 23.553 seconds with exit +zero and =LR-CASEFOLD-OK=. The lookup-benefit decision repeats on the latest graph: + +| Identifier characters | CPU ratio P10 | CPU ratio P50 | CPU ratio P90 | Lookup benefit | +|-----------------------+---------------+---------------+---------------+----------------| +| 43 | 0.1210 | 0.2436 | 0.2952 | Admitted | +| 267 | 0.0365 | 0.0421 | 0.0513 | Admitted | + +The complete-request allocation table remains explicitly attributed to +=11b2cf1=. Later artifact-publication changes have their own cost owner and are +not covered by those historical absolute allocation values. Latest-graph +semantic and component qualification does not silently relabel that matrix. + + +* Complete casefold request evidence + +The request benchmark now exports =benchmark-lr-casefold-requests=. It uses +actual =RETURN= requests at 64 and 256 identifiers, with ASCII padding of 32 or +256 characters. Each identifier has the form =identifier__=; +its complete length is 45--47 or 269--271 characters. Short GQL identifiers and +an independently generated Arithmetic entry are controls. Every batch validates +complete products; untimed independent GLR, canonical artifacts and actual +source-route witnesses remain the admission authority. + +** Matched production graph and exact counters + +The production graph is =f75aa18= with the new benchmark caller. The reference +withdraws only the LR/action-index casefold changes using those two modules' +=9374421= source. Shared scanner preparation, canonical/layout admission, +artifact publication and the new sampler remain fixed. Candidate native LR/index +objects are =o17=/=o3=; the rebuilt reference objects are =o18=/=o4=. The caller +verifies these identities before timing. Source and static products are restored +from checked backups after reference construction; no alternate engine remains +in the repository. + +Candidate/reference/reference/candidate children each execute twelve stages, +twenty samples and twenty calls: 19,200 timed calls in total. They complete in +51.673, 54.037, 38.715 and 59.651 seconds, with exit zero and final markers. +The 3 GiB heap, GCC SDK, 120-second total fence and five-second idle fence stay +fixed. Warmup and initial GC follow the shared sampler; natural GC stays in CPU +observations. The benchmark caller compiles in 22.194 seconds. Reference index +and LR compilation take 9.153 and 38.688 seconds respectively. + +=casefold-evidence.ss= checks every raw sample count and reconstructs CPU P50 +through the shared percentile owner. It admits allocation only in complete +no-GC intervals, compares exact integer batch counters across repeated children, +and requires source/prepared saving equality. All 916 admitted intervals pass: + +| Padding | Identifiers | Prepared reduction | Source reduction | Saved bytes per request | +|---------+-------------+--------------------+------------------+-------------------------| +| 32 | 64 | 9.87% | 4.24% | 24960 | +| 32 | 256 | 9.80% | 4.30% | 101184 | +| 256 | 64 | 38.29% | 16.95% | 141440 | +| 256 | 256 | 37.62% | 16.86% | 561728 | + +Short GQL requests save 2,944 bytes at both boundaries. Arithmetic allocation +is unchanged at both boundaries. These are complete-request allocation results, +not extrapolation from the earlier lookup component. The equality of source and +prepared byte deltas isolates the same eliminated normalized strings across +both routes. + +** CPU rejection and verifier negative controls + +CPU benefit admission requires both unchanged Arithmetic controls to have child +median ratios within 0.95--1.05 and an observed ratio range containing one. +A stage must also have its slower candidate child median below 0.95 times the +faster reference child median. All timed CPU samples, including GC, participate. +This is a conservative repeated-run gate, not a statistical confidence interval. + +The unchanged prepared Arithmetic ratio is 0.9057; its unchanged source ratio +is 1.1933. The control gate therefore fails. No complete-parser CPU benefit is +admitted. Descriptive ratios of child medians for the larger identifier padding +are 0.6851/0.6823 at prepared boundaries and 0.8485/0.8749 at source boundaries. +Short prepared GQL has ratio 1.0687. None of these ratios is promoted to causal +CPU evidence while the controls drift. The smaller-padding, larger source +request also has ratio 0.9856, preserving a useful near-zero result. + +The first verifier attempt rejects a fractional allocation summary: converting +raw counters to exact integers produces an exact rational quotient, while the +sampler publishes a flonum. Summary parity now explicitly compares the sampler's +flonum representation; exact integers still own all saving calculations. No +tolerance or admission threshold is relaxed. The corrected verifier completes +in 15.208 seconds with =CASEFOLD-REQUEST-EVIDENCE-OK=. + +Three controlled corrupted receipts are rejected: an allocation counter changed +by eight bytes, a CPU summary changed by one millisecond, and a missing completed +workload marker. The negative-control child completes in 7.132 seconds with exit +zero and =CASEFOLD-EVIDENCE-NEGATIVE-CONTROL-OK=. No forged positive receipt is +stored in the repository. Validation and replay remain Scheme-owned. + +** Delivery boundary + +The =f75aa18= Ubuntu Scheme/language-pack job and Rust AOT job pass. Its macOS job +is cancelled after the PR advances. These checks qualify that head only. The +later Rust target-capacity admission at =8a60fb8= is retained during rebase; it +does not alter the Scheme execution or sampling owners measured here. New-head +CI is a separate delivery gate from these local cost observations. + +* State-initial admission experiment: withdrawn + +A proposed dispatch refinement added a 26-bit uppercase ASCII initial mask to +prepared LR action rows. It rejected an ASCII letter before walking the shared +literal trie when that state's vocabulary could not admit that initial. Empty +strings, punctuation and non-ASCII initials retained the existing path; a later +Unicode character still used whole-string conversion when traversal reached it. +The implementation followed the SDK's =fxarithmetic-shift=/=fxand= idiom, also +used by its vector utilities. No language-specific rule was introduced. + +The 44-case, seven-module semantic gate passed in 16.829 seconds. The independent +whole-string oracle now includes =ZÉ=, =+É=, =zé=, =xé= and =+é= to exercise row +admission before a later Unicode character, as well as punctuation and existing +cross-state, duplicate and entry-identity controls. + +** Complete-request decision + +The same twelve complete-request stages ran candidate/reference/reference/ +candidate, twenty samples of twenty calls each: 19,200 timed calls. Both variants +used the =9184d01= Scheme graph; only the action-index object changed (=o4= mask, +=o3= original). LR =o17=, lexical =o6= and the native benchmark caller remained +fixed. Children completed in 19.216, 17.019, 16.916 and 17.628 seconds. Compiler, +heap and qualification fences stayed unchanged. The concurrently published +=ad8c6a3= change affects Rust execution and documentation, not this Scheme graph. + +The shared Scheme verifier now exposes +=compare-casefold-request-cpu-receipts= for dispatch-only experiments. It uses +the same raw-sample reconstruction and CPU gate, while requiring every repeated +complete-request allocation counter to remain exactly unchanged. The verifier +admitted 920 no-GC intervals and completed in 12.143 seconds. Arithmetic control +ratios were 0.9712 prepared and 1.0460 source, satisfying the existing control +gate. No stage satisfied the repeated CPU benefit gate. + +| Request | Units | Prepared median ratio | Source median ratio | +|---------+-------+-----------------------+---------------------| +| Padding 32 | 64 | 1.1469 | 1.0672 | +| Padding 32 | 256 | 0.9829 | 1.0430 | +| Padding 256 | 64 | 1.0576 | 1.1062 | +| Padding 256 | 256 | 0.9439 | 1.0760 | +| Short GQL control | 64 | 1.0248 | 1.0640 | + +Ratios divide the sum of two candidate child CPU medians by the sum of two +reference medians. They are descriptive, not confidence intervals. The smaller +prepared request is slower in both candidate children than both references; +the apparent 0.9439 result does not repeat across children. This experiment does +not justify publication. The production mask and its native/static products +are removed. The original shared trie remains, with its previously established +complete-request allocation savings. The additional Unicode oracle cases and +allocation-preserving CPU verifier remain as reusable qualification tools. + + +* Guarded CPU controls and lookup workload coverage + +The current =benchmark-lr-casefold= replaces the earlier count-only component +with complete vectors of original action entries. Before measurement it checks +entry identity for every state. The reference now skips rows without literals +before calling =string-upcase=, matching the production guard. Historical early +miss ratios above therefore do not qualify this stronger reference. + +The existing Scheme batch sampler measures process user plus system CPU from +Gambit process-statistics slots 0 and 1. ASP's nearest-rank percentile helper is +reused. The ASP micro-kernel source uses =cpu-time= and a minimum batch duration; +its baseline-subtracted operation budget is not substituted for this paired +parser comparison. Natural GC remains part of measured CPU. Wall time and +allocation observations are retained but cannot satisfy the CPU gate. + +Each workload has twenty ABBA/BAAB groups, bracketed by two twenty-group +identical-reference controls. Both control P10/P90 ranges must stay within +0.90..1.10; production P90 must be below 0.95. Every batch must resolve at least +one millisecond of CPU. These decisions are reported separately from successful +completion and semantic equality. A false admission is not a correctness failure. + +The native twenty-iteration matrix completes in 68.574 seconds, exit zero, +with =LR-CASEFOLD-CPU-CONTROLS-OK=. It executes 68,016,000 timed state lookups, +including controls, across 2,834 actual GQL rows. + +| Workload | Production/reference CPU P50 | CPU P90 | Stable controls | CPU benefit admitted | +|----------+------------------------------+---------+-----------------+----------------------| +| Early miss, 43 characters | 0.2116 | 0.2742 | No | No | +| Early miss, 267 characters | 0.0414 | 0.0501 | No | No | +| Keyword hit, match | 2.1993 | 2.7419 | No | No | +| Late miss, matching | 1.5871 | 1.9155 | No | No | +| Unicode fallback, eclair with accented initial | 1.0594 | 1.4663 | No | No | + +These observations expose a workload boundary missing from the old long-miss +component. Uniform traversal of every state is a diagnostic distribution, not +the state frequencies of complete requests. Short hit and late-miss ratios +must not be hidden by long identifier results or described as portable causal +regression percentages while the identical-operation controls drift. + +A focused repeat selects =early-miss-43= with 200 iterations using the same +entry oracle, guard, order, native objects and admission thresholds. It completes +in 87.556 seconds, exit zero, with the same completion marker. Its 136,032,000 +timed lookups produce CPU P50 0.1902 and P90 0.2076. Before-control P10/P90 is +0.8781/1.0963; after-control is 0.8039/1.1555. Increasing the batch does not make +these controls pass, so the repeat also refuses CPU admission. The 120-second +total and five-second real-progress watchdog are unchanged. + +Receipt stems are =parser-cpu-controls-a= and +=parser-cpu-controls-short-large=. Raw paired batches retain CPU, wall and GC +observations; no frozen engine or JSON result snapshot is checked in. The +historical matrix preceded the final single-entry-point consolidation and the +one-millisecond admission check; the larger repeat uses the consolidated body. +Final source compilation and focused semantic qualification are recorded below. + +The next engine decision must address the crossover between bounded keyword +lookup and early rejection of long identifiers. A dispatch change needs matched +complete-request CPU qualification as well as hit, late-miss and Unicode +controls. This benchmark change ships no new runtime optimization and does not +claim that the Bash/tree-sitter throughput gap has closed. + +** Final source and rebased capability qualification + +The change is rebased onto =36c9f84=, the prepared grammar layout-capability +fix. GCC recompiles =layout.ss=, =lr-parser.ss=, =parser.ss= and the final +benchmark in 40.719 seconds, exit zero. This is build duration, not a runtime +CPU result. Focused source tests pass twelve LR index cases and nine layout +boundary cases in 13.905 seconds, with =TEST-SUITE-OK=, final =OK= and qualified +exit zero. + +The final keyword-only benchmark completes in 42.172 seconds, exit zero, after +6,801,600 timed lookups. The complete entry oracle passes; every timed batch +resolves the one-millisecond CPU floor. Production/reference CPU P50 is 1.8592 +and P90 is 2.4120. Before-control P10/P90 is 0.5971/1.4207 and after-control is +0.4931/1.4899. Both control and benefit admission are false. This checks the +final executable admission path without promoting successful completion to a +performance pass. Its receipt stem is =parser-cpu-controls-final-keyword=. + +Across the three children, 210,849,600 timed state lookups complete. These are +component observations, not that many full parser requests. No compiler, +timeout, heap budget, runtime implementation or expected semantic value is +changed to obtain a pass. diff --git a/docs/research/compact-recognition-children.org b/docs/research/compact-recognition-children.org new file mode 100644 index 00000000..f78c2afe --- /dev/null +++ b/docs/research/compact-recognition-children.org @@ -0,0 +1,179 @@ +#+title: Compact recognition children and constructor continuations + +* One implementation and qualification unit + +This study follows [[file:lr-owned-stack.org][request-owned LR execution]] and +[[file:lr-unary-chain.org][the rejected unary-chain study]]. It concerns the engine's logical child +representation, construction and publication. Language grammars, parser schemas, +canonical field/alias declarations and the flat artifact protocol retain their +existing responsibilities. The implementation remains experimental until the +original complete-request CPU admission passes. + +* Evidence before choosing the representation + +At the pushed =b5e1895= reference, a fresh complete GQL request performs 279 +non-EOF action selections over 37 distinct lookaheads. Every observed selection +has a literal-capable row. The grammar has 2,834 states, 1,477 goto columns and +21,630 goto edges. Dense column spans would require 1,326,241 slots. These work +counts establish possible redundancy and storage cost, not CPU attribution. + +The complete-source continuation sample described in the unary-chain study +places the driver, streaming source and event publisher ahead of alias and +sequence materialization frames. Inclusive samples overlap. The long RETURN +request executes 464 alias actions, 462 with singleton bounds, but only one +wide materialization. Repeated flattening is therefore insufficient as the +primary explanation for its cost. + +** Rejected request classification + +A request-local vocabulary classifier retains exact and casefold keys once per +lookahead, returning each state's original action entry. Its exploratory +complete-request preflight does not establish the required benefit. Benchmark +compilation overlaps part of that first preflight, so it is not an uncontended +admission series. Its source, complete native closure and raw output remain in +ignored =.data/parser-lookahead-vocabulary-study/=. The classifier is removed +from production; no cache or dense action matrix is retained. + +** Constructor transport and direct row selection + +The frozen reference's Gambit C contains =___BEGIN_ALLOC_VALUES(2UL)=, +value-count checks and endpoint extraction in constructor paths. The first +constructor-continuation candidate removes all 18 emitted allocation sites +from that reducer translation unit. This is a site count, not 18 allocations +per request. The representative GQL preflight allocates 6,336 fewer bytes per +request, while complete CPU still misses the required margin. + +Combining that candidate with direct row selection removes per-state selector +and token-lookup closures. Four independent native blocks retain the original +40 samples of 1,000 requests for every stage. Admission rejects the identical +SDK control's repeat-band separation. Descriptive conservative target ratios +also fail to establish a five-percent benefit: representative GQL prepared/full +ratios are 1.05530/1.00405; long RETURN ratios are 1.00818/1.08956. These are +negative observations, not admitted stage results. Direct row selection is +removed. The raw series and native ASCENT rejection remain in ignored +=.data/parser-row-construction-study/=. + +An earlier first reference launch reaches the original 120-second total budget +before producing request samples. Its timeout is retained separately from the +subsequent completed series. The cause of the startup variation is not proven. + +* Representation hypothesis and private contract + +A logical child is =(optional field, recognition value)=. Absence of field +metadata requires neither a new value nor a record containing =#f= and that +same value. The compact representation stores an unfielded token/node/fragment/ +position view/event value directly. Explicit metadata uses one final +=labelled-recognition-child= record. Field names are never inferred from token +or node kinds. + +The logical predicate and checked projections accept this finite representation +family. The first-class constructor and the staged constructor rule share one +algorithm. Internal users migrate directly; an old unfielded record protocol is +not maintained. Only a metadata record has a mutable field slot. A caller that +needs a field constructs a labelled child rather than mutating a bare value. +Node/fragment storage, ordered child sequences, token identity and the external +flat artifact format remain separate obligations. + +#+begin_src scheme +;; An absent metadata slot expands to its existing value. +(construct-recognition-child #f token) + +;; A declared field owns the only extra metadata record. +(construct-recognition-child 'value token) + +;; One checked variant elimination supplies both projections to the consumer. +(with-recognition-child child (field value) + (emit-value! value field delta translated? #f)) + +;; Constructor continuation binds scalar endpoints before constructing a node. +(with-recognition-value-bounds value (start end) + (make-recognition-node kind start end children)) +#+end_src + +Normal token/node/fragment/event bounds reach the consumer without a temporary +multiple-value object. Position views retain the original recursive arithmetic +association; their uncommon path may still use the external values protocol. +The ordinary first-class =recognition-value-bounds= operation is preserved. + +** Question: why does this exploit Scheme and Gerbil's strengths? + +The =#f= pattern is visible to =defrules= before execution. The compiler sees an +existing value instead of a constructor call. =defrule= inserts the actual +consumer at the finite variant boundary, sharing a checked algorithm between +first-class projections, semantic constructors and publication. This removes +intermediate representation and call boundaries without adding author-facing +configuration. Existing POO multiple dispatch still chooses the semantic +backend/action factories at preparation; it does not redispatch per child. + +Gerbil =2591dcd= =std/io/bio/input.ss= uses =defrules __bio-read= to place +operation bodies inside a finite input protocol. Its compiler's +=optimize-top.ss= retains an expression when no wrapper is required. Those are +source organization and staging patterns, not inherited speed claims. +[[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][Sperber and Thiemann, The Essence of LR Parsing]] specializes grammar-static work +away from dynamic execution. Here the static absent-field declaration removes +metadata construction; this study does not reproduce their parser generator. + +** Question: why is this design scientifically justified? + +=ChildStorage= in the Lean model distinguishes a bare tree from a named tree. +=compact_child_roundtrip= proves that decoding an encoded logical child retains +its exact optional label and tree. The endpoint-continuation theorem retains +empty, singleton and ordered wider-input boundaries. These are logical claims +under the model's finite tree/natural-coordinate assumptions, not proofs of +Gambit allocation, signed relocation arithmetic, FFI behavior or CPU benefit. + +Native Scheme tests separately check payload identity, single evaluation, +checked invalid input, metadata mutation and event-lowering snapshots, scalar +bounds and relocation. Independent GLR publication checks the complete product. +Actual request counters and matched native measurements are separate evidence. + +* Qualification boundary + +The existing evidence command includes ten Scheme modules, extending its prior +scope with source-recognition and event-program ownership tests. Representative +request diagnostics execute outside every timed interval in that same process. +Python runs through the locked =uv= project and transports source/product hashes; +Scheme recomputes statistical admission and native ASCENT joins the receipts. +No CI lane, custom build scheduler, timeout increase or instrumentation callback +is added to the timed executor. + +All four target stages remain mandatory: representative GQL and long RETURN, +each with prepared LR and complete source requests. The conservative candidate +repeat bound must be below 0.95. FHIRPath/arithmetic controls, P95 limits and the +identical SDK encoder's ratio and repeat-band overlap remain unchanged. Failing +or censored trajectories are retained; lower allocation cannot substitute for +complete CPU admission. + +Qualification results belong below only after their source and actually loaded +native modules are bound. Native module binding follows Gerbil's loader: select +the last contiguous =.oN= before the first missing version. Repository notes use +relative receipt owners, never machine-specific absolute logs. + +* Rejected compact-child qualification + +The source-bound =parser-compact-dispatch-study= retained seven production +owners and both complete native closures. Ten Scheme modules passed 108 Cases. +Four independent 40-by-1000 measurements completed after retaining the first +reference startup timeout without request samples. The unchanged admission +rejected the SDK repeat-band control; native ASCENT recorded the missing +=record-cpu-improvement= and denied execution authorization. + +The following conservative CPU ratios are descriptive, not admitted benefits: + +| Request | Prepared LR | Complete source | +|---------+-------------+-----------------| +| Representative GQL | 1.31077 | 1.26984 | +| Long RETURN | 1.01503 | 0.98638 | +| FHIRPath 64 | 1.00089 | 1.17551 | +| Arithmetic 1 | 0.98865 | 1.03060 | + +The candidate fails the original target margins independently of the rejected +control, and FHIRPath's complete call exceeds the non-regression bound. The +production representation and candidate-only tests were restored. Frozen +candidate source, native images, failed trajectories and ASCENT receipts remain +in the ignored study. Lower allocation is not sufficient evidence to replace +the production representation. + +Module-level native sampling then redirected investigation toward LR dispatch, +checked token calls and publication. See +[[file:lr-request-dispatch-cost.org][request dispatch and official inline contracts]]. diff --git a/docs/research/compiled-language-byte-publication.org b/docs/research/compiled-language-byte-publication.org new file mode 100644 index 00000000..22fe42a7 --- /dev/null +++ b/docs/research/compiled-language-byte-publication.org @@ -0,0 +1,208 @@ +#+title: Compiled language byte publication and admission +#+startup: overview + +[[file:../audit/engine-design-realization.org][Engine realization audit]] | +[[file:bound-declaration-publication.org][Declaration publication study]] + +* One responsibility boundary + +This realization follows canonical IR from immutable compiler publication +through filesystem or linked AOT admission. Source baseline is +=39fd28f0006b9ad116c94b06c7aabaf41e45277f= on PR #13. It extends the +declaration identity study across producer and consumer ownership. It concerns +compiled language IR, rather than request-owned ParseArtifact publication in +[[file:artifact-publication-cost-boundaries.org][the request artifact study]]. + +Authors continue to supply =grammar.ss=, =parser.ss= and =parser-test.ss=. +No encoding, cache, reader or publication option is added to =deflanguage=. +The ordered datum and content-addressed locator are observable contracts; +the intermediate character string is private storage. + +| Stage | Owner | Complete obligation | +|-------+-------+---------------------| +| Canonical write | [[file:../../src/compiler/language-artifact.ss][Compiler publisher]] | Existing =write= representation, explicit UTF-8 and LF | +| Address and compression | Same publisher | Hash those bytes; compress once at level 9 for all missing roots | +| Immutable installation | Same publisher | Verify decompressed bytes before atomic rename; verify competing target | +| Digest admission | [[file:../../src/runtime/language-artifact.ss][Runtime loader]] | Validate locator and authenticate decompressed bytes | +| Datum admission | Existing loader for files and AOT images | Complete UTF-8 decoding, exactly one datum and expected schema | + +The publisher owns its fresh byte vector through digest, compression and +verification. Warm publication verifies sidecars without recompression. +The runtime retains complete UTF-8 decoding and its scoped character input port. +Compression, runtime decoding and reader allocations remain. Text cache receipts +keep their character writer because they have text consumers. Their identities +and generators are outside this change. + +* Official mechanisms and paper transfer + +The [[https://gambitscheme.org/4.9.7/manual/][Gambit manual, I/O and ports]] supplies +byte-vector ports and encoding settings. Explicit output =UTF-8= and =lf= make +canonical writing an engine obligation. Strict byte input was studied as a +separate candidate and withdrawn after complete-path CPU admission. Official +availability is not sufficient evidence that an alternative port improves this +engine's workload. + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Culpepper and Felleisen, +Fortifying Macros]] informs preservation of declared invariants at an +implementation boundary. Transferring this discipline to generated IR transport +is a design inference. The paper establishes neither byte-port equivalence nor +concurrency safety or engine performance; independent controls provide bounded +observations. No parser or macro theorem is claimed for this change. + +The user-selected [[https://github.com/tao3k/gerbil-ascent/blob/f84155662b57ec8034bb60f0d14968715ad15cce/docs/coding-style/representations-and-results.org][Scheme representation guidance]] +also separates ordered wire formats from private storage. These are closed +named algorithms with no competing extensible implementations requiring public +multiple dispatch. + +* Scheme witness and the two design questions + +#+begin_src scheme +;; Compiler-owned bytes feed the address and compression. +(call-with-output-u8vector '(char-encoding: UTF-8 eol-encoding: lf) + (lambda (port) (write canonical-ir port))) + +;; Existing runtime admission retains complete decoding and datum framing. +(let (text (utf8->string decompressed-bytes)) + (admit-digest decompressed-bytes locator) + (call-with-input-string text + (lambda (port) + (let ((value (read port)) (trailing (read port))) + (unless (eof-object? trailing) (error "trailing datum")) + (admit-schema value expected-schema))))) +#+end_src + +Question 1: Why does this improve the Scheme DSL design? + +The S-expression remains the ordered semantic product. Gerbil macro expansion +and POO composition determine it before publication. Ordinary Scheme =write= +and =read= carry it across storage without another serialization DSL. Engine +owners choose physical ports; authors declare syntax, metadata and observable +parser behavior at the existing interfaces. + +Question 2: What makes the design scientific and falsifiable? + +An independent character writer specifies historical bytes. Tests compare the +complete digest, locator, compressed output and admitted datum against that +specification. Malformed comments must fail even with matching digests. +Benchmarks compare complete products from saved original publisher and loader +against current owners. Representation alone cannot establish CPU, retained +heap or full parser latency improvements. + +* Admission order and negative controls + +The retained runtime keeps its existing admission order: complete UTF-8 decoding, +digest, framing and schema. The byte-port experiment proposed identity before +interpretation and tested that changed error priority. It was withdrawn with +the reader candidate; no alternate admission path remains. Complete decoding +still rejects malformed input in ignored trailing comments. A BOM is not +silently stripped: the current reader treats +it as additional framing and rejects the artifact. CR/LF inside strings retains +the historical datum value. + +[[file:../../t/language-artifact-test.ss][Admission controls]] cover Unicode and +supplementary characters, independent compressed-byte parity, warm republishing, +both storage paths, digest/framing/schema rejection, corrupt targets, missing +roots and twelve concurrent publishers. New controls reject illegal leading +bytes, overlong sequences and incomplete multibyte tails in trailing comments. +The preliminary port probe compared these outcomes with =utf8->string= followed +by the character reader. + +* Measurement contract + +[[file:../../t/benchmarks/language-artifact/benchmark.ss][The source-owned benchmark]] +constructs inputs using the original character writer before timing. Warm +roundtrips include canonical writing, hashing, immutable file verification and +filesystem admission. Embedded admission includes image unpacking, +decompression, hashing, complete reading and schema admission. Every result +equals the complete expected product; immutable input is checked again after +measurement. These overlapping intervals cannot be summed into phase fractions. + +Original owner procedures are experiment inputs, never engine fallback routes. +The shared sampler owns allocation population and paired CPU admission. Natural +GC remains enabled; unresolved CPU batches fail admission. Allocation samples +with unavailable attribution are excluded, never replaced with zero. Cold +compression and atomic installation are semantic +controls, not a claimed speedup from a warm benchmark. + +After the repository's normal source build, the retained component control is +available through the existing Gerbil tools: + +#+begin_src sh +gerbil env gxc -O t/benchmarks/language-artifact/benchmark.ss +gerbil env gxi -e '(import :gerbil-parser/t/benchmarks/language-artifact/benchmark) + (benchmark-language-artifact 20 5 128) + (benchmark-language-artifact 20 5 512)' +#+end_src + +For the matched original/current experiment, take the compiler and runtime +=language-artifact.ss= owners from the declared source baseline, compile them +under distinct experiment-only module names, and pass their exported publisher, +roots loader and embedded loader to =compare-language-artifact=. Keep all other +dependencies, port settings and native compiler flags equal. Use 20 groups, +20 calls per batch and the same 128/512-row fixtures. Baseline module renaming +belongs to the isolated experiment; no such modules are delivered by the engine. + +* Withdrawn runtime reader experiment + +The native prototype used an explicit UTF-8/LF input byte port with strict +encoding errors. Its first consumer matrix passed 140 cases in 46.939 seconds. +This includes the prototype's digest-first error priority, which is not a +retained contract. The first benchmark attempt rejected an embedded CPU batch +below the unchanged one-millisecond resolution requirement; it exited nonzero +and is not a completed CPU receipt. Twenty calls per batch then produced 20 +alternating groups per interval and scale, with natural GC and full equality. +That complete experiment exited zero in 37.795 seconds. + +| Rows | Roundtrip CPU ratio P10 / P50 / P90 | Embedded CPU ratio P10 / P50 / P90 | +|------+------------------------------------+-----------------------------------| +| 128 | 0.513 / 0.925 / 1.843 | 0.349 / 1.141 / 3.540 | +| 512 | 0.990 / 1.154 / 1.315 | 0.549 / 0.774 / 2.109 | + +Ratios are candidate/original. The medians disagree across sizes and intervals, +and every upper decile exceeds one. The runtime candidate is withdrawn rather +than admitted on allocation arguments alone. The final compiler-only publisher +requires its own consumer and matched performance receipts below. + +* Retained publisher qualification + +The final compiler-only experiment passes complete product and immutable-file +checks, exits zero in 70.345 seconds and emits final =OK=. Component populations +use 20 samples and five calls per sample. These are admitted allocation samples, +not a retained-heap census or all-process allocation across natural collections. + +| Rows | Roundtrip original bytes/call | Roundtrip current bytes/call | Admitted original/current samples | Reduction | +|------+-------------------------------+------------------------------+-----------------------------------+-----------| +| 128 | 817,632 | 664,720 | 18 / 18 | 18.7% | +| 512 | 2,817,176 | 2,203,464 | 14 / 13 | 21.8% | + +Embedded admission is a separate control with the restored original reader: +128-row allocation is exactly 408,080 bytes/call on both sides (20 samples +each). For 512 rows it is 1,479,673.6 versus 1,479,664 (17 samples each), +which supplies no meaningful allocation improvement claim for that owner. +The roundtrip reduction follows compiler publication; no runtime optimization +is retained. + +Paired controls use 20 alternating groups and twenty calls per batch, retaining +the one-millisecond CPU resolution requirement and natural GC. + +| Rows | Roundtrip CPU ratio P10 / P50 / P90 | Unchanged-reader control P10 / P50 / P90 | +|------+------------------------------------+-----------------------------------------| +| 128 | 0.469 / 0.892 / 1.964 | 0.218 / 0.868 / 1.307 | +| 512 | 0.652 / 0.952 / 1.307 | 0.521 / 1.101 / 1.908 | + +The roundtrip median is lower at both sizes, but upper deciles exceed one and +even the unchanged-reader control varies substantially. This is not a stable +CPU speedup claim. Sequential roundtrip wall P95 per call is 3.553 versus +3.975 ms at 128 rows and 31.480 versus 14.678 ms at 512 rows; these disagree +across sizes and do not establish a parser P95 improvement. No percentile +sum, GC attribution from wall delay, cold-publication latency or whole-language +parse performance is inferred from these observations. + +The final twelve-module native consumer matrix passes 139 cases in 55.490 +seconds, exits zero and has complete case/suite/module/harness markers and +final =OK=. It includes composition and selected origins, declaration binding, +immutable publication and caches, both runtime loaders, language surface and +entry contracts, ten-language ABI alignment, ordinary and contextual parsing. +The final publisher, restored runtime, tests and benchmark compile natively +and exit zero. These local receipts do not substitute for completed exact-head +Ubuntu/macOS Scheme and Rust AOT CI after publication. diff --git a/docs/research/dense-lr-item-space.org b/docs/research/dense-lr-item-space.org new file mode 100644 index 00000000..9348cd7a --- /dev/null +++ b/docs/research/dense-lr-item-space.org @@ -0,0 +1,196 @@ +#+title: Dense dotted-position ownership across LR construction + +* Research obligation and scope + +The [[file:dsl-effective-grammar-admission.org::*LR suffix facts: compositional reverse evaluation][prepared suffix study]] removes repeated RHS suffix scans but explicitly leaves +production-count times maximum-width padding unresolved. This implementation +closes that representation obligation in the compiler, before LR(0), lookahead, +canonical LR(1), partition refinement and conflict-source publication diverge. +It changes no language declaration, request executor, ABI or CI job. + +The staging argument in the parent study connects grammar-only facts to +Krishnaswami and Yallop's static Null/First analysis and Sperber and Thiemann's +Scheme LR specialization. Those papers motivate preparation ownership. They do +not prove this particular representation correct or supply its measured speedup. +The argument below is a separate bijection over the admitted production table. + +* Representation and invariant + +For production i with RHS width w_i, define o_0 = 0 and +o_(i+1) = o_i + w_i + 1. The dotted position (i,d), where 0 <= d <= w_i, +has core ID o_i + d. The domain size M is sum_i(w_i + 1), rather than +P times (1 + max_i w_i). Every production owns one completed position, including +epsilon productions. There are no holes or overlapping production intervals. + +The private =lr-item-layout= record owns terminal count, production offsets and +a dense inverse owner catalog. Prepared owner lookup is constant time; dot +lookup subtracts that owner's offset. Storage and preparation are O(P + M). +This does not make arbitrary big-integer bitset operations constant time. +It bounds the core-ID domain on which those operations run. + +Order by production then dot is preserved. Advancing a noncompleted core by one +stays inside its production. Completion, FIRST suffix lookup, prediction, +action masks and shift-origin diagnostics consume the same layout functions. +Public production IDs, state discovery order and rule origins are unchanged. +There is one representation; no padded-layout compatibility branch remains. + +#+begin_src scheme +;; Independent fixture: widths 1, 3, 0, 1, including augmented start. +;; Every completed item is retained, and the epsilon interval has width one. +;; Expected IDs are explicit rather than reconstructed through the decoder. +'((0 0) (0 1) (1 0) (1 1) (1 2) (1 3) (2 0) (3 0) (3 1)) + +;; Engine consumers inspect logical coordinates, never numeric packing details. +(core-item-production-id item layout) +(core-item-dot item layout) +(make-core-item production-id dot layout) +#+end_src + +** Question: why does this exploit Scheme beyond S-expression notation? + +The canonical grammar remains inspectable data. A named preparation algorithm +derives one immutable layout; a private named record makes ownership explicit; +dense vectors implement its finite domain. Consumers share named coordinate +operations rather than copying arithmetic conventions. POO continues to own +extensible grammar composition and hygienic DSL rules own declaration lowering. +This fixed numeric representation needs neither a public extension slot nor +generic dispatch in bitset traversal. Authors express rules, not allocation +strides or packed-item decoding. + +** Question: how is the design scientifically justified? + +Disjoint intervals establish injectivity, every valid dot lies in an interval, +and their concatenation covers exactly [0,M). Ordering and successor preservation +establish the representation obligations used by existing LR algorithms. +Independent coordinate and suffix controls falsify boundary errors; canonical +reference and origin tests falsify altered construction and diagnostics. +Complete original/candidate LRSpecs test publication equivalence separately +from CPU and allocation observations. A reduced domain is a deterministic +storage result, not evidence of complete-request latency improvement. + +* Executable coverage and reproduction + +- [[file:../../src/compiler/lr-automaton.ss][Layout owner]] and all packed-coordinate consumers use the new representation. +- [[file:../../t/resolved-grammar-test.ss][Resolved controls]] check explicit epsilon/unary/wide coordinates and 325 real + positions instead of 8,515 padded positions; independent forward FIRST/nullable + scans check every suffix of the skew-width fixture. +- [[file:../../t/canonical-lr1-reference-test.ss][Canonical reference controls]] retain their independent construction oracle. +- [[file:../../t/lr-conflict-origins-test.ss][Conflict origin controls]] check public logical dots and selected source paths. +- [[file:../../t/benchmarks/lr1-partition/item-space.ss][Construction benchmark]] checks complete LRSpecs at three scales across LALR, + canonical LR(1) and follow-partition LR(1). CPU includes natural in-call GC; + allocation is admitted only by the existing shared counter contract. + +Run the normal package build and the existing conformance build, which owns +the shared construction fixture, then source-owned tests. The benchmark entry is: + +#+begin_src sh +gerbil env gxi t/benchmarks/lr1-partition/item-space.ss 10 +#+end_src + +For a matched comparison, run the same benchmark source and fixture against +separate original/candidate compiler products from the same toolchain, and +compare every =ITEM-SPACE-PRODUCT= datum before interpreting samples. Do not +combine old and new compiler modules in one dependency graph. Normal CI keeps +its existing semantic and generated-product gates; this research benchmark +does not add a required PR stage. + +* Qualification boundary + +The changed compiler producers were compiled natively. The five original +producers were rebuilt from the parent commit with the same toolchain and +compiler flags. Focused source GxTest qualification passed 49 cases covering +resolved grammar admission, conflict origins and Scheme style contracts in +43.714 seconds, with exit zero and no supervisor failure. Tests exposed and +removed a duplicate padded-layout constructor in backward follow seeds and an +old stride assumption in the independent partition oracle. + +Four sequential original/candidate/candidate/original runs passed the unchanged +120-second total and five-second real-output idle limits in 11.221, 9.906, +9.727 and 12.709 seconds. Each run checked ten measured compilations per workload +against its complete warmup LRSpec. All nine published products were identical +across all four runs. The fixture scales have 85, 325 and 1,285 actual positions, +compared with 595, 8,515 and 132,355 padded positions respectively. + +These are descriptive local CPU medians, including natural GC, over twenty +observations per side. They are not confidence-qualified speedup claims. + +| Short rules / wide RHS | Route | Original CPU ms | Dense CPU ms | +|------------------------+-------+-----------------+--------------| +| 32 / 16 | LALR | 0.1215 | 0.1835 | +| 32 / 16 | Canonical LR(1) | 0.1020 | 0.2085 | +| 32 / 16 | Follow partition | 0.2070 | 0.1285 | +| 128 / 64 | LALR | 0.9780 | 0.4540 | +| 128 / 64 | Canonical LR(1) | 0.4510 | 0.3155 | +| 128 / 64 | Follow partition | 0.9890 | 0.3565 | +| 512 / 256 | LALR | 21.2215 | 1.8265 | +| 512 / 256 | Canonical LR(1) | 2.1455 | 1.5990 | +| 512 / 256 | Follow partition | 2.6025 | 1.7285 | + +At the largest scale, admitted allocation medians were 19,047,904 to 2,328,248 +bytes for LALR (16/20 admitted original samples and 20/20 candidate samples), +8,749,368 to 3,496,640 for canonical LR(1) (10/20 on each side), and 9,262,480 +to 4,009,624 for follow partition (5/20 and 6/20). The shared allocation contract +rejects GC-contaminated or invalid deltas; its subsets are separate from CPU +samples. The small-grammar regressions remain visible in this historical run. +Inverse-owner preparation and accessor costs were hypotheses, not measured +causes of those single-call observations. The phase and batch controls below +test that attribution separately. + +Negative results remain outside admission. Initial untraced imports and an +unchanged follow-automaton GCC rebuild exceeded the idle bound. The full source +canonical matrix reached its total bound during the mixed-context compression +case; partial cases do not qualify that matrix. Attempts to include actual GQL +construction exceeded the idle bound and supply no admitted language timing. +The focused local tests used retained runtime dependencies and qualify compiler +behavior only. Fresh complete semantic and generated-product qualification +belongs to the existing exact-head CI gates. No deadline or CI stage was added. + +* Small-grammar attribution controls + +The [[file:../../t/benchmarks/lr1-partition/item-space.ss][same benchmark owner]] now separates layout preparation from layout plus +symbol-catalog construction. Four sequential original/dense/dense/original runs +passed the existing 120-second total and five-second idle bounds in 20.551, +13.611, 9.004 and 16.121 seconds. Each phase has forty observations per side, +with 2,000 preparations per observation. Symbol projection is checked outside +the timed interval; CPU includes natural GC. + +| Fixture | Phase | Original CPU us/call | Dense CPU us/call | +|---------+-------+----------------------+-------------------| +| 32 / 16 | Layout | 0.80525 | 0.76300 | +| 32 / 16 | Layout and symbol catalog | 1.76075 | 1.46150 | +| 128 / 64 | Layout | 2.42900 | 2.70225 | +| 128 / 64 | Layout and symbol catalog | 24.27075 | 4.94850 | + +At 32 / 16, admitted layout allocation rises from 1,952.616 to 2,272.616 bytes +per call (36/40 samples on each side), while layout plus catalog falls from +6,760.616 to 3,648.616 (34/40 and 36/40). These process-counter observations +include the equal per-call measurement loop overhead. They are not retained +heap sizes. At 128 / 64, all original catalog allocation observations include +GC and are rejected; no allocation comparison is admitted for that phase. + +The small complete-compiler control additionally repeats one hundred +compilations per measured batch. Four original/dense/dense/original runs passed +in 29.520, 20.926, 17.284 and 15.138 seconds. All three complete LRSpecs match +across all four runs. Each route has forty batches per side; every batch exceeds +one millisecond of process CPU. Reported CPU is per compilation. + +| Route, 32 / 16 | Original CPU ms | Dense CPU ms | +|----------------+-----------------+--------------| +| LALR | 0.152220 | 0.127615 | +| Canonical LR(1) | 0.148555 | 0.130990 | +| Follow partition | 0.157000 | 0.125810 | + +The earlier regression does not reproduce in these resolved batches. Preparation +does not exhibit the earlier 62--107 microsecond growth either. This rejects +inverse-owner initialization as a sufficient explanation. It does not establish +which combination of timing resolution, heap state, host contention and compiler +optimization context produced the earlier single-call medians. + +These diagnostics reuse the qualified native producer graphs. The original +producers were rebuilt against common helper interfaces; the first four dense +producers retain their earlier compilation context. Matching products does not +remove that optimization-context confound. Fresh dense rebuild attempts exceeded +the idle bound during optimizer admission and GCC compilation, respectively; +partial products were excluded. The batched medians are descriptive diagnostics, +not a newly admitted causal speedup. A fresh common-context comparison remains +required before attributing a small-fixture timing difference to representation. diff --git a/docs/research/dsl-effective-grammar-admission.org b/docs/research/dsl-effective-grammar-admission.org new file mode 100644 index 00000000..ac202883 --- /dev/null +++ b/docs/research/dsl-effective-grammar-admission.org @@ -0,0 +1,2182 @@ +#+title: Effective grammar admission and source origins +#+date: 2026-10-09 + +Part of [[file:dsl-engine-contracts.org][the integrated DSL and engine study]]. +Existing dated receipts retain their original revision and scope. + +* Current status at de8bef5 + +The historical design below motivated work that is partly implemented. Use +[[file:dsl-engine-contracts.org::*Current implementation map][the current implementation map]] and the owning implementation sections before +turning an old proposal into a new task. + +- Selected repetition occurrences and finite nullable reference origins are + implemented; see [[file:dsl-source-execution.org::*Implementation: origins and Source continuation closure][origin and continuation closure]]. +- Source/Command suspension and iterative result destruction are implemented in + =command_source/execution.rs= and =result_projection.rs=. Their live-frame + budget is not a complete byte/work budget. +- Scheme scanner session reuse and LR fragment transfer have separate admission + controls; see [[file:dsl-lr-transfer.org][the LR transfer contract]]. General Rust Source subtree reuse + and full mapped artifact publication remain separate work. +- [[*Dependency-driven nullable closure (2026-10-09)][Dependency-driven nullable closure]] is implemented in the existing LR owner. + Productivity, general target choice proofs and helper/conflict origin + serialization remain design obligations. + +* Historical study: admission, suspension and reuse + +** Historical revision and existing mechanisms + +This study inspected =2577253f96b2b5993706c939dbe5088ae0d56a3a=. +The following observations and proposals describe that revision. Several have +since been implemented, as recorded above; this section is retained as the +design rationale rather than the current task list. + +| Existing mechanism | Exact owner | Design consequence | +|--------------------+-------------+--------------------| +| Effective row owner, composition operation and contribution history | =src/compiler/normalize.ss=, =effective-grammar-row= | Extend expression origins from these selected rows; retain override history without admitting removed declarations | +| Immutable source state with owner, source, character/byte offsets and context | =src/runtime/source-scanner.ss= | Reuse requires edit mapping and semantic context comparison; a checkpoint API already exists | +| Command program counter, node frames and repeat/balance tasks | =rust/gerbil-parser-runtime/src/engine/command_program.rs= | Extend the existing VM suspension boundary; do not introduce an independent command interpreter | +| Synchronous Command/List host calls and depth guard | =rust/gerbil-parser-runtime/src/engine/command_source.rs= | Explicit Command tasks alone do not remove Source recursion | +| Explicit event traversal, private recursively owned recognition children | =command_source.rs=, =result_projection.rs= | Deep-input qualification must include destruction and error cleanup, not only recognition | + +The last observation is a potential lifecycle risk inferred from the recursive +ownership shape. No stack-overflow reproduction or failure depth is established +here. =PreparedResultProfile::check_node= checks only the immediate child's +owner/source/span, so it is not a demonstrated recursive validation bottleneck. + +** One effective grammar, several target obligations + +The typed algebraic parsing paper's Section 3 defines separate conditions for +alternative apartness and sequence separability. Its concatenation facts require +those preconditions; they cannot be copied as universally valid CFG equations. +Our design inference is to keep common facts separate from a predictive backend's +proof obligations. A legal LR grammar may fail the predictive proof. + +The existing nullable/FIRST analysis remains the common authority. A proposed +analysis result also records productivity, reference components, declaration +origins and conservative scanner effects. These are private records computed by +named engine functions. An unknown effect is explicit, not an empty effect set. + +#+begin_example +Effective declarations + selected origin rows + -> common resolved facts + -> LR admission: supported constructs, conflicts and policy + -> predictive admission: supported constructs + determinism proof + -> scanner/Command admission: progress, state and result obligations + -> Scheme / Rust emission from the admitted target plan +#+end_example + +A failed predictive proof rejects that requested target with an author-local +reason; it does not change choice semantics, silently select another backend or +reject an otherwise valid LR target. Conflict witnesses and proof failures must +identify the target and relevant expression occurrence. + +Productivity is distinct from nullability: =A -> A= has no finite terminal +word, whereas =A -> A a | b= is productive left recursion. Productivity may be +computed monotonically with terminals/epsilon productive, choice as any child, +and sequence as all children, with repeat/optional admitting epsilon. Admission +must separately reject repeat of a nullable operand. This is a proposed analysis, +not an additional implemented fixed point. Unreachable rule policy must remain +explicit; it is not a reason to label every reference cycle invalid. + +POO override invalidation follows semantic dependencies. Editing only release +metadata must not reconstruct LR states; replacing a rule or scanner policy +invalidates its dependent analysis and target admission. Cached facts must bind +the effective grammar and policy identity. A semantic digest must include all +recognition-affecting declarations; a hash alone does not replace equality or +resolve ownership. No new public schema version follows from this proposal. + +** Preserve expression occurrences through transformations + +Current implementation: [[file:lr-conflict-source-origins.org][rejected LR conflict source origins]] implements helper +occurrence paths and selected POO source evidence without changing canonical +productions. The following proposal retains its historical transformation scope. + +Langcc's Section 2.3 explains delayed intermediate reductions in terms of the +surrounding continuation. Section 2.4 bounds XLR execution by simultaneous +branches; unbounded branching loses its stated bound. We infer that helper +transforms need both origin records and measured conflict witnesses here; adopting +a numeric branch cap does not prove that our admitted grammar belongs to XLR. + +Start from =effective-grammar-row= and assign each expression occurrence an +internal identity: selected rule, child path, selected contribution and available +reader coordinates. Equal expressions at two paths remain distinct occurrences. +Generated repetition helpers point back to that identity and their lowering +operation. This avoids exposing generated helper names in DSL diagnostics. + +The proposed next origin test extends the current owner/operand diagnostic: +base role -> nullable override -> inherited repeated reference. Require the +selected override and reference path in the diagnostic; restoring a consuming +override must produce a fresh admitted plan. Source coordinates unavailable in +current syntax data remain unknown rather than fabricated. + +** Resume Source through the existing Command VM + +=execute_initial= currently holds local frames/tasks while =step= calls +=host.command()= or =host.list()= synchronously. Both Source host methods increment +=Cursor.depth= and re-enter Source parsing. The architectural change is to retain +the VM state across these calls and return a typed request to the outer engine. + +Proposed private request/result vocabulary: + +#+begin_example +Command execution -> Completed(node) | NeedCompound(field) | NeedList(field, until, allow_empty) +Outer engine -> push Source continuation + suspended Command execution +Source completion -> resume Command execution with exactly one owned result +Failure -> unwind all suspended state and pending result ownership +#+end_example + +The actual names and representation remain open. Store program counters, repeat +progress checkpoints, ordered node frames and the target field in private typed +records. Command call/node/repeat/balance already have task semantics and should +retain them. Word/Region need suspension only if their actual call graph also +re-enters Source; do not turn every leaf operation into a scheduler request. + +Source continuations own pipeline/and-or/list assembly and the stop trigger. +A single request-local cursor owns token position, byte position, pending heredoc +FIFO and links. Preserve today's deferred-body behavior before considering +scoped queues. Parent result publication happens once after child completion; +trivia and significant ranges retain source order. No state is shared mutably +between independent parses. + +The scheduler must account for the entire live request: pending tasks, retained +frames/captures, deferred markers, consumed input and produced output. Removing +=256= alone is insufficient. Check actual storage/work limits with structured +failure; do not add a new user-written host callback or scheduler slot. + +Private result storage may require an arena or iterative destruction. Decide +using a deep lifecycle test before changing the representation. A deep parse that +succeeds but overflows during artifact release is not a successful migration. + +** Incremental reuse needs boundary validation + +Wagner and Graham's Sections 4–5 reuse shiftable subtrees while validating their +right edge against lookahead. A parser-state equality check is sufficient but +can be overly restrictive. Their sequence bounds also exclude unbounded context +effects. The design inference is to separate scanner restart checkpoints from +parser subtree admission, and to test context-sensitive sequences explicitly. + +Tree-sitter's scanner contract stores the complete lexical state so edits and +ambiguities can restore it. The corresponding engine proposal must include +pending deferred markers and parser-induced context changes, not only the +scanner's mode. Existing immutable Scheme checkpoints are the starting point. + +An edit must map both byte and character offsets and preserve UTF-8 boundaries. +A candidate retains old source ownership until an explicit mapped result is +constructed for the new source. Reusing an old =ProjectedNode= directly violates +its current source-identity contract. Reuse diagnostics should distinguish +changed content, scanner context mismatch, parser boundary failure and unsupported +effects. Do not expose those internal states as required language author data. + +** Historical implementation gates derived from this study + +| Order | Work | Semantic gate | +|-------+------+---------------| +| 1 | Extend selected row origins to expression occurrences | Nullable override diagnostic names selected contribution/path; removed declaration cannot own the error | +| 2 | Suspend existing VM at Command/List requests | Existing Source corpus and exact diagnostics unchanged; mixed nesting exceeds former 256 boundary | +| 3 | Qualify the whole deep-input lifecycle | Recognition, events, artifact publication/release and failure cleanup all complete without host recursion failure | +| 4 | Account for request resources | Limit failure releases frames and markers; independent parse still succeeds; semantic gates keep their existing deadlines | +| 5 | Prototype scanner restart + subtree boundary admission | Full reparse parity for delimiter, quote, heredoc, multibyte and context-changing edits | + +Ordinary correctness cases assert deterministic semantic properties and ownership. +ASP performance cases measure complete calls, allocations, peak live storage and +work; include both nesting and wide lists. No speedup follows from this design +alone. Language packs continue using their existing grammar/parser/test interfaces +and POO value extension points; scheduler and analysis implementation stay inside +the engine. + + + +* Paper transfer: syntax contracts and resolved grammar contracts + +Reviewed primary sources on 2026-10-09: + +- [[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Culpepper and Felleisen, Fortifying Macros (ICFP 2010)]], sections 3--4: + named syntax classes combine patterns, attributes and side conditions with + contextual diagnostics. Transfer the separation between declaration structure + and semantic obligations. Gerbil =defrules=/=defsyntax= remain our substrate; + this is not a claim that Racket syntax classes are implemented in Gerbil. +- [[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][Krishnaswami and Yallop, A Typed, Algebraic Approach to Parsing (PLDI 2019)]], + sections 3 and 6: grammar facts justify a restricted predictive parser, and + staging specializes static grammar information. Transfer analysis before + execution and separation of static grammar from dynamic input. Their LL(1) + restrictions and linear-time theorem do not cover every grammar or backend here. + +The repository connection is one contract chain: + +| Layer | Existing owner | Evidence or remaining obligation | +|-------+----------------+----------------------------------| +| Hygienic vocabulary | =src/language/grammar.ss=, =defgrammar-syntax= | Generated local vocabulary is a =defrules= expansion; no new operation keywords | +| Declaration admission | =src/language/grammar.ss=, =src/compiler/language-expander.ss= | Check syntax/rules and retain declaration-local diagnostics | +| Effective composition | =src/compiler/normalize.ss= | Analyze selected rule values and origins after composition | +| Resolved grammar | =src/compiler/lr.ss= | Existing nullable fixed point and original repetition obligations | +| Target admission | =src/compiler/recursive-source.ss= and LR compiler | Restricted HCL admission remains explicit; general choice-equivalence proof remains open | +| Publication cost | =src/ffi/language-artifact-codec.ss= | Complete payload equality remains independent of recognition strategy | + +Macro validation cannot establish resolved nullability from declaration shape +alone. A POO override can change a referenced rule from consuming to empty; +=t/resolved-grammar-test.ss= tests that effective value and retains source-owner +diagnostics. Conversely, target restrictions do not justify rejecting productive +left recursion from an LR-capable grammar. These are distinct admission stages +within one pipeline, rather than additional author configuration. + +Next executable closure is the same call/array package across vocabulary expansion, +composition, source diagnostics and backend admission. Keep identical fields, +ranges and rejection expectations. The published sources establish the design +method; actual controls establish what this implementation supports. + +* Staging must preserve one semantic producer + +[[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][Krishnaswami and Yallop, section 6]] separates static grammar/environment +from the dynamic input and specializes an intermediate representation. +[[https://www.s48.org/essence/][Sperber and Thiemann's Essence]] applies partial evaluation to a general +LR parser; its author-maintained [[https://github.com/mikesperber/essence][Scheme implementation]] connects the generator +to the TOPLAS 2000 paper. These sources justify studying specialization of one +semantic producer, rather than maintaining unrelated parser implementations. +Our engine has no corresponding general specialization-correctness proof. + +The present implementation boundary is concrete: + +| Static after descriptor admission | Dynamic for every parse | +|-----------------------------------+-------------------------| +| Effective syntax/rules and selected origins | Source bytes and admission outcome | +| Grammar identity and result catalogs | Source digest, node IDs and byte spans | +| Syntax-kind, terminal and field indexes | Ordered event traversal and result lifetime | +| Target strategy and its admission restrictions | Scanner/recognition state and natural GC | + +The encoder consumes canonical events using already indexed descriptor symbols. +Removing a temporary row-index list changes traversal storage, not declaration +meaning or accepted language. It needs byte-level output controls and measured +complete calls; it does not require an author-facing allocation option. + +#+begin_src scheme +;; Question 1: Why is this preferable for the author DSL? +;; Row numbering is private publication state; syntax/rules stay unchanged. +(let loop ((remaining events) (row 0)) + (unless (null? remaining) + (write-event! language payload row (car remaining)) + (loop (cdr remaining) (+ row 1)))) +;; Question 2: What justifies this implementation change? +;; The same ordered events and descriptor indexes must yield identical bytes. +(check native-fresh-output => canonical-output) +(check native-session-output => canonical-output) +;; Allocation and wall/GC receipts qualify different performance claims. +#+end_src + +These are internal-engine expressions and test expectations, not a fourth +required language interface. Current measurement and FFI lifetime controls are +owned by [[file:source-edit-cost-contract.org][the complete Source edit cost contract]]. A Scheme heap object borrowed +by C cannot be treated as a stable address through an allocating callback. +This obligation belongs to the transport owner and remains independent of +macro hygiene, grammar recognition and target admission. + +* Dependency-driven nullable closure (2026-10-09) + +The selected grammar still flows through the existing =compute-nullable= owner +in =src/compiler/lr.ss=. This change replaces repeated complete production scans +with occurrence obligations; it adds no author slots, schemas or backend-specific +analysis. Resolved repetition diagnostics and their selected POO source map keep +using this same completed nullable index. + +Krishnaswami and Yallop's [[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][A Typed, Algebraic Approach to Parsing]], sections 3, 5.2 +and 6, motivates deriving static grammar facts before parser specialization. +Their restricted predictive grammar result does not establish a complexity bound +for our LR parser. The worklist below is our implementation of the existing +nullable equations, rather than a claimed reproduction of their parser. + +** Invariants and work bound + +Each terminal-free production owns a private =nullable-clause= containing its +lhs and unresolved occurrence count. Register one reverse edge for each rhs +nonterminal occurrence, including duplicates. An empty production publishes its +lhs immediately. A terminal anywhere in a production blocks that entire clause; +it cannot become nullable through its prefix. Marked operands are unwrapped by +the existing =base-symbol= operation. + +After all clauses are registered, drain newly published names. Each name is +published at most once. Each reverse edge decrements its clause once; reaching +zero publishes its lhs. Multiple alternatives can publish the same name only +once. Pure cycles and references without a definition remain false. A cycle +with an empty alternative propagates normally. + +Soundness follows by induction on publication: an empty clause is nullable, and +a discharged clause has only nullable rhs operands. Completeness follows at +worklist exhaustion: every clause whose rhs operands are nullable has reached +zero, so no additional nullable lhs can be derived. With expected constant-time +table operations, graph construction and propagation cost O(P + R), for P +productions and R rhs occurrences, with O(P + R) auxiliary storage. In contrast, +a reverse-ordered chain required repeated full production scans. + +The public list still follows first lhs occurrence in production order; the +lookup table contains exactly those names. These are nullable facts, not +productivity facts. Nonempty-language analysis and FIRST remain separate +obligations, and the author grammar/tree is not rewritten by this change. + +** Executable evidence + +[[file:../../t/resolved-grammar-test.ss][Resolved grammar controls]] compare the new result against an independent +repeated-scan oracle for 512 combinations of three-rule cyclic grammars. Other +cases cover marked operands, duplicate occurrences, alternative productions, +terminal blockers, undefined references, first-occurrence ordering and a reverse +chain of 4096 names. +Existing selected-override/source-origin diagnostics remain checked. Together +with the canonical LR reference Suite, 38 Cases passed on the pinned Scheme SDK +after explicit GCC module compilation. + +The real TLA+ layout grammar retains exact complete LRSpec equality against the +pre-change snapshot: 40 rules, 280 lowered productions and 390 states. Ten +complete construction samples in each local run had CPU medians 15.068 ms before +and 14.868 ms after, with no in-call GC. This narrow difference establishes no +end-to-end speedup. The retained improvement is dependency-bounded analysis, +with semantic equality controls; it does not imply faster Rust AOT execution, +FFI transport or whole-language parsing. + +* Productivity: finite derivation and LR entry admission + +[[file:dsl-productive-entry.org][Productive LR entry admission]] owns the implementation, fixed-point argument, +author policy, selected-source diagnostics, semantic controls and reported +complete-construction qualification. The requested entry must have a finite +derivation; productive recursion and unused dead rules retain their independent +target obligations. + +* FIRST propagation: contract and implementation + +The current =compute-first= owner remains in =src/compiler/lr.ss=. Its consumer +is LR lookahead/state construction and target analysis. A dependency candidate +must register rhs nonterminals only through the nullable prefix, include the +first terminal/nonnullable operand and stop afterward. Cyclic prefixes require +least-fixed-point propagation. Nullable membership must be complete before these +edges are constructed; no incremental partial index may escape admission. + +FIRST's result contract includes canonical terminal order from the production +terminal catalog, not the order in which worklist facts arrive. Duplicate edges, +marked symbols, nullable cycles, nonnullable blockers and dead tails need matched +controls. Multiple terminal facts can update the same nonterminal, so nullable's +one-publication bound does not automatically apply to FIRST. Measure dependency +construction, mask propagation and result materialization together, then compare +complete LRSpec and real full-construction cost at the same compilation tier. +The implementation below applies this contract to the existing owner. It changes +analysis evaluation and storage; it introduces no public DSL vocabulary or +choice/admission policy. Productivity still requires its separate semantic +consumer and policy decision before it can change target admission. + +** Implementation and fixed-point argument (2026-10-09) + +The baseline is =2feaa26727dddc379a066c6b991cd1a48c9f5504=. The implementation +assigns dense indices in first-lhs order and stores FIRST/pending masks and +reverse targets in private vectors. A source/target edge is registered once, +even when several productions or rhs occurrences induce it. References without +a defining lhs retain the existing empty FIRST lookup behavior; their admission +belongs to the existing reference validator. + +Build every nullable-prefix dependency before draining work. Direct terminals +seed lhs masks. A newly admitted bit enters that nonterminal's pending mask; an +empty-to-nonempty pending transition queues the index once. Popping an index +clears its pending mask and propagates only that delta to its reverse targets. +Facts arriving while queued are coalesced. A self cycle has no seed of its own. + +Each seed is justified by a terminal with a nullable preceding prefix. Each edge +is justified by a nonterminal at such a prefix. Propagation is therefore sound +for the existing FIRST equations. At queue exhaustion, every admitted terminal +has reached every dependent lhs, establishing the same least fixed point as +repeated scans. Termination follows from finite monotone terminal membership; +this is not a proof that every grammar derives a completed terminal word. + +Materialize public rows only afterward using the existing terminal catalog and +first-lhs name order. Queue order and delta arrival order cannot change canonical +rows. Every lookup row is derived from that same completed mask. Epsilon remains +a nullable fact rather than a FIRST terminal. + +For N nonterminals, T terminal identities and E unique prefix dependencies, +each index can publish at most T nonempty batches and each edge transmits a +particular terminal bit at most once. Graph discovery and deduplication visit the +prefix occurrences; propagation work is bounded by those fact/edge updates. +Account separately for arbitrary-width mask operations and ordered result +materialization: neither nullable's one-publication bound nor a constant-time +bignum assumption applies. Dependency storage is O(N + E), alongside masks, +catalogs and the required public output rows. + +** Controls and matched cost evidence + +[[file:../../t/resolved-grammar-test.ss][The existing resolved grammar Suite]] compares an independent list/set scanning +oracle with the new completed FIRST rows and table. It includes 512 cyclic +combinations, eight actual LR fixture families, nullable and nonnullable prefix +blockers, duplicate edges, marked terminals/nonterminals, undefined references, +canonical ordering, a 128-terminal cycle and a 4096-name reverse chain. Its dead +tail control keeps FIRST separate from productivity. The canonical LR reference +Suite checks the existing construction and recognition routes. + +Local native A/B compiled the baseline scanning function and candidate with the +same pinned Scheme SDK, GCC and optimization settings. Both returned complete +FIRST rows and lookup membership; every measured call was compared for equality. +Five samples per workload alternated execution order in one process, with explicit +GC before each call excluded from CPU timing. These measurements include graph +construction, propagation and output materialization; they exclude imports and +nullable analysis, which is held identical for the pair. + +| Workload | Baseline scan CPU median (ms) | Delta CPU median (ms) | +|----------+-------------------------------+----------------------| +| Reverse chain 256 | 3.953 | 0.154 | +| Reverse chain 512 | 16.729 | 0.307 | +| Reverse chain 1024 | 61.224 | 0.565 | +| Reverse chain 2048 | 234.479 | 1.159 | +| Real TLA+ layout FIRST | 0.379 | 0.328 | + +These are analysis-call results on one local host, not Rust/FFI/parser throughput +or an end-to-end latency promise. Complete TLA+ LRSpec exactly matches the saved +baseline. Twenty complete default LALR construction samples gave CPU medians +14.989 ms before and 14.789 ms after with no in-call GC. An initial table-based +candidate instead measured 15.306 ms; it was replaced by the dense-vector variant. +Retain this negative measurement: the narrow real full-call difference supports +no stable end-to-end speedup claim. + +Final local qualification after rebasing onto =344826a=: 44 source-mode resolved +grammar/canonical LR Cases and 361 compiled semantic Cases across 23 groups, plus +five independent research entries, completed with status zero and final receipts. +The first compiled preparation exposed a missing static =lr1-construction= fixture +dependency. It is now declared once in the existing conformance module catalog; +final Scheme-owned preparation completed in 75.76 seconds with the same total +and idle deadlines. No timeout or compiler backend was changed. Delivered-head +Ubuntu/Rust CI remains a separate receipt. + +* LR suffix facts: compositional reverse evaluation + +The source baseline is =0255378568984f3f87e1f17fbf8425e30b0c2891=. +=make-core-lookahead-catalog= in =src/compiler/lr-lookahead.ss= previously +scanned =sequence-first= and =sequence-nullable?= separately for the suffix +following every nonterminal occurrence. An n-item nullable rhs therefore repeated +quadratic operand traversal, even though the catalog itself was built only once. +Both LALR propagation and canonical LR(1) closure consume this catalog. + +** Paper transfer and semantic boundary + +[[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][Krishnaswami and Yallop, Figure 2 and section 3]] give compositional Null/First +transfer for concatenation. We apply the familiar nullable-prefix recurrence to +our existing syntactic CFG FIRST facts. The paper's complete type algebra, +FLast/separability obligations and predictive-parser theorem do not certify our +LR construction. In particular, our FIRST facts are not a productivity proof: +a terminal followed by a dead nonterminal can retain that leading terminal. +We preserve this behavior and the separate completion admission contract. + +Let F[i] and N[i] summarize the rhs suffix beginning at position i. For the +empty suffix F[n] = 0 and N[n] = true. With completed per-symbol facts H[i] +and Q[i], the recurrence is: + +#+begin_example +F[i] = H[i] union (if Q[i] then F[i+1] else 0) +N[i] = Q[i] and N[i+1] +#+end_example + +A terminal has a singleton H and false Q; a missing reference has empty H and +false Q. Before incorporating position i, the builder publishes F[i+1] and +N[i+1] at its core-item slot only if that item is a nonterminal. Completed, +terminal and padding slots retain the previous zero/false values. The existing +core-symbol catalog has already unwrapped operand marks, so aliases/fields do +not change this analysis. An empty production makes no publications. + +Induction backwards from n shows that each published suffix agrees with the +independent forward scans. Every rhs position is visited once; nonterminal FIRST +lists are converted to masks once per distinct referenced name. This removes +quadratic suffix traversal. It does not remove the existing padded vector +allocation, whose size is production-count times layout-width. Big-integer mask +operations also depend on terminal count; no constant-time bitset assumption or +whole-parser linear bound is claimed. The analysis adds no language-author slot, +loader schema, public grammar rewrite or admission restriction. + +The [[file:dense-lr-item-space.org][dense dotted-position implementation]] now addresses that retained padding +obligation across all construction and diagnostic consumers. Its bijection, +independent controls and compiler-cost evidence are separate from this suffix +analysis and from complete-request latency admission. + +** Semantic and matched native controls + +The resolved grammar suite compares both entire catalog vectors against +forward =sequence-first=/=sequence-nullable?= scans. Controls include five real +LR fixture families, marked terminals/nonterminals, empty productions, nullable +and nonnullable heads, empty FIRST, undefined references, duplicate operands and +a 256-item marked nullable rhs. Full canonical/partition/parse controls remain. +The focused source gate passed 54 Cases (37 resolved grammar and 17 canonical +reference), final =TEST-SUITE-OK= and =OK=, exit 0, in 18.537 seconds after the Case moved into this compiled-entry-owned +suite. The earlier 24.554-second source receipt overlapped baseline compilation; +neither duration is a semantic-test speed claim. + +Temporary baseline and candidate modules were compiled with the same fixed +Gerbil/GCC SDK, optimize/static settings, and already-completed input facts. +One Scheme process alternated invocation order, forced GC before each call and +compared both result vectors after every pair. CPU timing includes allocation, +mask conversion, traversal and publication; it excludes imports, explicit GC, +input preparation and equality checks. Seven pairs per synthetic length and +21 pairs per real TLA+ workload produced these medians: + +| Workload | Baseline CPU ms | Reverse fold CPU ms | +|----------+-----------------+---------------------| +| 512 nullable rhs operands | 7.557 | 0.042 | +| 1024 nullable rhs operands | 31.219 | 0.055 | +| 2048 nullable rhs operands | 119.182 | 0.088 | +| Real TLA+ complete suffix catalog | 0.246 | 0.209 | +| Real TLA+ complete LALR construction | 14.954 | 14.669 | + +Five fixture families were also built through both LALR and canonical LR(1) +using baseline/candidate compiler modules, with exact complete LRSpec equality. +All 21 paired TLA+ complete constructions retained exact LRSpec equality. +The supervised measurement exited 0 in 13.700 seconds, emitted +=SUFFIX-NATIVE-AB-OK= and passed the unchanged 90-second total/5-second idle +limits. No allocation count, parser throughput or FFI gain was measured. The +small real complete-construction difference is not promoted to a stable +end-to-end improvement. Baseline sources and raw receipts are local temporary +qualification inputs; no JSON snapshots or runtime binding replacement were +introduced. + +** Compiled consumer and remaining remote gate + +The suffix Case belongs to =t/resolved-grammar-test.ss=, which is admitted by the +compiled conformance entry. It is not duplicated in the canonical reference +suite. After the move, native preparation exited 0 in 74.165 seconds under the +unchanged 240-second total/5-second idle limits. The compiled semantic entry +passed 371 Cases across 23 groups, including the new suffix Case. The semantic +process and five independent research processes all exited 0 and emitted the +final =DSL-RESEARCH-CLOSURE-OK= and =OK= markers. + +These are local Darwin/GCC receipts. The baseline's Ubuntu run +[[https://github.com/tao3k/gerbil-parser/actions/runs/37891042027][37891042027]] failed with semantic child status 139 (=signal-exit=); its last +Case output was =all parser tests avoid display assertions=. The negative +conformance controls behaved as expected. There is no backtrace establishing +the signal's cause, and local compilation/execution did not reproduce it. +Local passing receipts do not repair or supersede that Ubuntu failure; the +new delivery needs its own exact-head Ubuntu result before remote closure. + +* Unified suffix facts across LR consumers + +This follow-up starts from =47fd87003f4bd3298fe9d472c076f25ab6e02815=. +The previous lookahead-specific directory had a useful reverse fold, but FOLLOW +propagation, initial backward partitioning and follow-core metadata still scanned +suffix lists independently. They need facts at different positions, not different +FIRST/nullable meanings. The engine now owns =make-core-suffix-catalog= in +=src/compiler/lr-lookahead.ss= and removes =make-core-lookahead-catalog= directly. +There is no compatibility mode or new language-author declaration. + +** Position contract and compositional argument + +The producer receives the completed FIRST and nullable tables, one terminal +index and the existing core-symbol/layout catalog. Slot i stores facts for the +suffix beginning at dotted position i. Every valid position is published, +including terminal positions and the completed core: the empty suffix is +FIRST=0 and nullable=true. Padding remains zero/false. The backwards recurrence +in the preceding section is unchanged; induction now covers all valid slots. +Marked operands still use the unwrapped core symbol. Undefined references +retain empty FIRST and false nullable; FIRST does not establish productivity. + +| Consumer | Fact position | +|----------+---------------| +| LALR closure propagation | core + 1, after the predicted nonterminal | +| Canonical LR(1) closure | core + 1 | +| Forward reachable FOLLOW edges | core + 1 | +| Initial backward compatibility partition | core, including completed cores | +| Follow-core NFA metadata | core + 1 | + +All five consumers read the same fact meanings. Static masks are read once per +occurrence rather than re-converted for every child alternative in the FOLLOW +edge construction. Existing follow/state/terminal order and conflict policies +remain unchanged. Each catalog invocation still allocates its own padded +vectors; this change introduces neither global memoization nor whole-build +catalog identity/reuse. Its allocation bound remains production-count times +layout-width, and large-mask arithmetic depends on terminal count. + +The source-test oracle independently scans each complete suffix, including empty +productions, terminal heads and completed positions, and compares both entire +vectors. The 54 focused Cases passed with final =TEST-SUITE-OK= and =OK=, actual +exit 0, in 20.581 seconds. Existing independent forward-follow, initial-partition, +canonical/partitioned/follow construction and accepted/rejected parse controls +remain in the canonical reference suite. There is one suffix Case, owned by the +resolved grammar suite and shared by source and compiled conformance. + +** Complete outputs and matched cost + +Five baseline compiler modules were compiled from the exact starting revision +under a separate temporary package, using the same Gerbil/GCC SDK, optimize and +static settings as the candidate. They bind all five LR construction routes to +the baseline implementations. One Scheme process alternated baseline/candidate +order, forced GC before each invocation and checked exact output afterward. +The admitted run had no overlapping compilation or diagnostic process. + +Five grammar families matched complete LRSpec through all five construction +modes (25 pairs), and matched all initial backward partition/candidate/terminal +outputs (5 pairs). In addition, 21 paired real TLA+ LALR constructions and 21 +paired mixed-context follow-partition constructions matched full LRSpec. The +FOLLOW measurements include terminal catalog, core catalog, suffix facts, edge +construction, propagation and result publication, with completed input +FIRST/nullable preparation outside timing. Complete-construction measurements +include all compiler analyses and publication. CPU medians are: + +| Workload | Pairs | Baseline ms | Shared suffix ms | +|----------+-------+-------------+------------------| +| FOLLOW: 512 nullable operands | 7 | 15.099 | 0.077 | +| FOLLOW: 1024 nullable operands | 7 | 60.110 | 0.116 | +| FOLLOW: 2048 nullable operands | 7 | 238.279 | 0.202 | +| Real TLA+ complete FOLLOW analysis | 21 | 1.596 | 0.800 | +| Real TLA+ complete LALR construction | 21 | 14.882 | 14.993 | +| Mixed-context complete follow-partition construction | 21 | 1.475 | 1.469 | + +The complete LALR median increased slightly in this local run; no whole-parser, +whole-construction, FFI or Rust throughput improvement follows from the analysis +speedup. Allocation and GC counts were not measured. The admitted process +emitted =UNIFIED-SUFFIX-NATIVE-AB-OK= and exited 0 in 15.216 seconds under the +unchanged 90-second total/5-second idle limits. Imports, explicit GC and equality +checks are outside the timed intervals. + +** Retained failures and remaining algorithmic/remote work + +Earlier A/B attempts hit the five-second idle limit before main or at the full +TLA+ follow-partition construction. None of their costs is admitted. Tracing +all real interface imports resolved the preparation observability gap without +changing the deadline. A direct, owned baseline diagnostic was sampled and +interrupted; it is not a successful qualification. Its sample showed numerical +operations, GC and the baseline follow-automaton module executing, rather than +waiting for a process. A separate temporary diagnostic baseline, never used in +the admitted A/B, then recorded metadata/seed completion (2240 cores) and +backward refinement completion (16 rounds). It hit the unchanged idle limit in +the following determinization phase, exit 70/CLI 124 in 9.230 seconds. This +locates a separate existing full-TLA follow-determinization cost; it does not +prove the cause of state growth or repair it. The candidate's correctness and +cost claims above do not cover completion of that workload. + +The previous revision's Ubuntu run +[[https://github.com/tao3k/gerbil-parser/actions/runs/37892498572][37892498572]] again exited semantic status 139 at the last output +=all parser tests avoid display assertions=. The earlier =8c3364a= Ubuntu +semantic step passed before cancellation; later test-object compiler option +changes are a concrete comparison boundary, not an established crash cause. +The workflow now requests a bounded GDB replay and all-thread backtrace only +when the Ubuntu semantic gate has failed. The original gate remains failed and +the replay retains 90-second total/5-second idle limits. Local configuration +checks confirm its condition and options; execution/backtrace evidence still +requires the delivered Ubuntu run. No compiler switch, widened deadline or +success substitution is introduced. + +** Local compiled consumer receipt + +After all three engine consumers and the shared suffix Case were migrated, +conformance preparation exited 0 in 108.297 seconds under the unchanged +240-second total/5-second idle limits. The compiled semantic entry passed 371 +Cases across 23 groups, including the full-position suffix Case. It and all five +independent research entries exited 0, with =DSL-RESEARCH-CLOSURE-OK= and =OK=. +This local Darwin/GCC result does not resolve the repeated Ubuntu signal or the +full TLA+ FOLLOW determinization workload. The delivered head needs its own +Ubuntu result; the failure-only GDB step is prepared, not locally executed. + +* SCC epsilon-closure experiment: rejected for complete construction + +** Paper transfer and invariant + +[[https://www.cs.cmu.edu/~cdm/resources/Tarjan1972-sccs.pdf][Tarjan, Depth-First Search and Linear Graph Algorithms (1972), section 4]] +provides the DFS numbering, active stack and lowlink SCC algorithm. The proposed +engine transfer was an explicit DFS stack, followed by completed reachability +masks in sink-first component order. For a component C, its closure is its member +mask union the closures of all outgoing components. For a seed set S, closure(S) +is the union of its component closures. A seed already contained in the result +can be skipped, since its own closure is contained there too. + +This is an engine implementation experiment, not a new grammar declaration or +POO slot. Tarjan's linear graph traversal bound does not make all reachability +masks linear: bigint operations, component closure materialization and retained +mask size remain additional costs. It also does not remove subset-construction +state growth. No parser runtime, FFI or Rust throughput follows from this study. + +** Semantic and complete-cost controls + +Baseline was exact revision =07a1f9c=, compiled under a separate temporary package +with the same Gerbil/GCC SDK, optimization and static settings. An independent +BFS oracle matched all 512 directed three-node graphs and all eight initial sets +(4096 comparisons), plus empty graph, 4096-node chain and 4096-node cycle controls. +The initial candidate passed 55 focused Cases, final =TEST-SUITE-OK= and =OK=, +exit 0 in 19.811 seconds. Both subsequent complete-output runs matched five +grammar families through all five LR modes: 25 complete LRSpec pairs each. + +The final experiment completed sink leaves without DFS frames, used direct +single-seed queries and skipped seeds already reached. One exclusive Scheme +process alternated old/new order, forced GC before each call and checked exact +results outside timed intervals. Closure timings include preparation and queries +for every singleton seed, not just a warm lookup. Complete LR timings include +all compiler analyses and publication. No concurrent build or test was running. + +| Workload | Pairs | Baseline CPU ms | SCC CPU ms | +|----------+-------+-----------------+------------| +| 128-node chain, all seeds | 7 | 1.419 | 0.094 | +| 128-node cycle, all seeds | 7 | 2.416 | 0.076 | +| 512-node chain, all seeds | 7 | 42.379 | 0.330 | +| 512-node cycle, all seeds | 7 | 78.736 | 0.279 | +| 1024-node chain, all seeds | 7 | 243.973 | 0.744 | +| 1024-node cycle, all seeds | 7 | 446.493 | 0.635 | +| Mixed-context complete FOLLOW, 8 regions | 21 | 1.444 | 1.502 | +| Mixed-context complete FOLLOW, 24 regions | 21 | 4.097 | 4.214 | +| Mixed-context complete FOLLOW, 48 regions | 21 | 9.935 | 10.171 | +| Distinct-exit recursive grammar, 8 rules | 7 | 0.149 | 0.133 | +| Distinct-exit recursive grammar, 16 rules | 7 | 0.230 | 0.233 | +| Distinct-exit recursive grammar, 32 rules | 7 | 0.433 | 0.422 | + +These are local CPU medians, without allocation or GC-count measurements. Final +process emitted =SCC-NATIVE-AB-OK= and exited 0 in 12.353 seconds under unchanged +120-second total/5-second idle limits. Earlier candidates also increased all +three mixed-context complete medians. The final small recursive-grammar results +are mixed and too small to justify a general complete-construction speedup. + +A separate actual TLA+ FOLLOW construction still hit the five-second idle limit, +exit 70/CLI 124 in 9.427 seconds, with its required completion marker absent. +That probe is a failure, not an admitted cost. Its last real trace was completed +LR(0) lookahead propagation; it does not localize the candidate's later stall. +The earlier baseline phase diagnostic located determinization after 16 backward +refinement rounds. Neither observation proves repeated epsilon BFS is the +principal cost of that workload. + +** Decision and next experiment + +The SCC implementation and its temporary semantic Case were removed before +publication. Complete mixed-context construction regressed approximately 2–4% +even after the structural fast paths. Synthetic closure gains cannot substitute +for improvement in the consumer. The delivered engine retains the previous +closure algorithm and test ownership; no unused SCC API or compatibility mode +is added. + +The next experiment needs actual graph/block counts after forward refinement, +SCC distribution, determinized state/transition counts and complete phase costs +on the failing TLA+ workload. Instrumentation must report completed work and +retain the five-second no-progress gate. It must distinguish refinement cost, +closure work and subset state growth before selecting another algorithm. A +future demand-driven component cache requires exact complete-output controls, +preparation-inclusive cost and memory accounting; no size threshold is justified +by this rejected experiment. + +* Ubuntu compiled AST contract: explicit traversal frames + +The failure-only GDB replay of =07a1f9c= in +[[https://github.com/tao3k/gerbil-parser/actions/runs/37894856785/job/113703791578][Ubuntu job 113703791578]] +caught SIGSEGV at =___H___kernel= followed by hundreds of identical frames in +=___H_gerbil_2d_parser____t____test_2d_style_2d_contract_2d_test=, immediately after +=all parser tests avoid display assertions=. This identifies the failing process +and repeated module stack; it does not establish a Gerbil toolchain defect or +source line. The replay exited 255 and the original semantic gate remained failed. + +The retained code change replaces recursive AST conjunctions with an explicit +pending-frame list. Each frame carries the form and its lexical owned-port set. +Quoted data remains inert, captured fixture writes retain their owned port, +forbidden calls still reject, and shadowing lambdas remove inherited authority. +Capture arguments and handler expressions are queued individually, so a bare +=quote= argument/body cannot suppress checking a later forbidden expression. +There is no compiler change, deadline extension or skipped source inspection. + +One additional Case checks 20000 nested wrappers around a forbidden call, +20000 sibling assertions, deeply nested legal fixture writes and shadowed-port +rejection. Existing contract Cases retain quoted negative fixtures and Rust text +assembly restrictions. The same Case is owned by the source Suite and compiled +semantic entry. Ubuntu confirmation belongs to the new delivered head; a local +Darwin pass cannot by itself close the observed Linux crash. + +** C host tail-call contract and object-cache identity + +The Ubuntu compiler log identifies GCC 13.3, =-D___TRUST_C_TCO= and a final =-O0= +for the AST contract object. In the pinned SDK's =gambit.h=, trusted host exits +use mandatory =musttail= only when the C compiler advertises it. Otherwise they +use an ordinary return-call, while =___USE_PROPER_TAIL_CALL_EXCLUSIVELY_TO_EXIT_HOST= +removes stack-trip recovery. [[https://gcc.gnu.org/gcc-15/changes.html][GCC's release notes]] record mandatory statement +tail calls as a GCC 15 addition. A local GCC 13 =-O0 -foptimize-sibling-calls= +control still generated a call and return, rather than a tail branch. This +explains why merely moving the explicit flag after =-O0= would not prove safety. + +The retained test-object policy preserves the SDK optimizer and trusted tail-host +contract, and selects =-U___SINGLE_HOST -D___MULTIPLE_HOSTS=. The pinned header +supports this module-local selection explicitly. Each Scheme procedure has its +own C host, so GCC no longer optimizes one giant expanded test-module function. +Production objects keep their SDK single-host mode. This changes compilation +granularity without changing the compiler, heap limit, public DSL or test suite +ownership. Complete semantic execution and build progress are required controls; +header-size equality alone is insufficient. + +Two rejected host-policy experiments are retained here. Removing +=___TRUST_C_TCO= selected stack-trip recovery and required +=___unwind_c_stack_to_trampoline=, absent from this immutable SDK. Native linking +failed (qualification exit 70 in 141.017 seconds). A per-test centralized +trampoline instead preserved local state sizes but the complete executable had +no semantic progress and hit the unchanged five-second idle gate. That binary +is rejected. The GCC 13 header control had matched global-state size 4064, +processor-state size 960, OS offset 648 and memory offset 688 on Darwin ARM64; +this layout receipt did not prove compatibility with the SDK's execution +strategy. Neither rejected option is part of the retained code. + +Test objects now retain an adjacent Scheme =.options.sexp= identity containing the +GSC path and effective per-object C options. Missing or changed identity forces +recompilation even if source timestamps are unchanged. The identity is published +only after successful compilation. Source timestamp checks remain; preparation +continues to own immutable SDK/input identity. This prevents the new tail-host +policy from silently reusing previously compiled unsafe test objects. + +The first centralized preparation linked successfully but rejected its final +snapshot: new option records had been classified as dependency inputs. That +attempt exited 70 in 80.054 seconds and is not an admitted preparation. The final +classification excludes only generated C/object/option products beneath the +library's =static/= directory. Independent cache controls publish a new option +record during preparation, prove the next call reuses the result, then change +the dependency interface and require rebuilding. Negative controls keep SCM, +SSI, records outside =static/= and =static-extra/= siblings in the input domain. +Failed object builds invalidate the previous option record before compilation, +so a partially overwritten object cannot retain its earlier success identity. + +** Retained implementation receipt + +After the per-procedure host policy and option-record classification stabilized, +complete preparation exited 0 in 128.332 seconds under unchanged 240-second +total/5-second idle limits. The compiled semantic entry passed 372 Cases across +23 groups, including the 20000-depth AST controls. It and all five independent +research entries exited 0, with =DSL-RESEARCH-CLOSURE-OK= and =OK=. This is local +Darwin/GCC execution evidence; Linux closure requires the delivered Ubuntu run. +The rejected SCC experiment supplies no claimed compiler/runtime speedup. +The following identical full preparation emitted =CONFORMANCE-CACHE-HIT= and +exited 0 in 16.838 seconds, without recompilation. Its content inventory and +hashing are included in that wall time; this is cache correctness evidence, +not a parser throughput measurement. + +* Static labeled-successor facts in FOLLOW subset construction + +** Paper and workload diagnosis + +[[https://arxiv.org/pdf/cs/0612120][Ranzato and Tapparo, Generalizing the Paige-Tarjan Algorithm by Abstract Interpretation]] +(section 1, pages 1–2) separates a stable partition's correctness from the +algorithm that reaches it. The transfer here treats the completed FOLLOW +quotient and its labeled successor relation as immutable facts during subset +construction. The existing reverse/predecessor refinement remains unchanged; +no logarithmic partition-refinement bound is claimed for this implementation. + +An isolated baseline copied from =f6dfc3b= recorded completed work on the actual +TLA+ grammar. It finished 16 backward refinement rounds (8888 NFA blocks), 23 +forward rounds (3984 quotient blocks), and 8149 DFA states. It exited 0 with the +required full-construction marker in 16.438 seconds. A separate exclusive CPU +diagnostic reported NFA edges 121.001 ms, forward refinement 648.036 ms and DFA +construction 6974.526 ms; that process exited 0 in 18.647 seconds. These phase +costs include diagnostic output and are target-selection evidence, not matched +speedup measurements. Output-item publication is outside the DFA phase cost. + +The prior idle-limit failures did not prove a process wait or an infinite +computation. Real completed-state output let this workload finish under the +unchanged 90-second total/5-second idle limits. State growth and its cost remain. +The retained engine trace uses the existing =GERBIL_PARSER_LR_TRACE= switch: +completed backward/forward partitions, NFA edges, every 128 processed DFA states, +and every 128 published state-item rows. A quiet computation still fails the +original idle gate; there is no timer-based output or heartbeat. + +** Indexed successor accumulation and invariant + +After the forward quotient is complete, the engine indexes each shift-bearing +block's label once. Let M contain precisely the blocks whose shift target mask +is nonzero. For a DFA state S and label a, the original target is: + +=epsilon-closure(union { shift(q) | q in S, label(q) = a })=. + +Intersecting S with M before enumeration gives the same union, since blocks +outside M contribute zero. A dense local vector accumulates target masks by +indexed label; the first visit to a zero slot records the label for this state. +Only touched slots are cleared before processing the next state. Blocks are +visited in ascending original index order, and first-seen labels retain their +order. The engine therefore interns successor states in the original order. + +The full epsilon-closed state S remains the state identity and the input to +output-item publication. Completed/nonshifting items, lookahead masks and reduce +behavior remain present. The epsilon-closure algorithm, its cache admission, +partition boundaries, conflict policy and output materialization are unchanged. +Each complete compiler invocation owns its scratch row; concurrent invocations +share no mutable row. This introduces no language-author metadata or POO slot. + +Static label hashing and core/symbol lookup happen once per shift-bearing NFA +block. The hot state loop scans only those blocks and uses vector accumulation; +it no longer creates a symbol hash table per state or hashes each visited block's +symbol. Extra storage is one block-label vector, one shift-source mask, one +label catalog and one target row. Bigint operations and subset state growth +retain their existing costs; allocation and GC counts require separate evidence. + +** Complete outputs and matched CPU cost + +An independent baseline package retained the original algorithm with the same +completed-work trace as the candidate. Both used the same Gerbil/GCC SDK, +optimization and static settings. One exclusive Scheme process alternated call +order, forced GC before each call and compared entire LRSpec afterward. Imports, +explicit GC and comparisons are outside the timed intervals. Index construction, +all compiler analyses, in-call collections and full publication are inside. +Trace settings were identical for baseline/candidate; these are traced-call costs. + +Five grammar families matched all five LR modes (25 exact complete-output +pairs). The actual TLA+ calls additionally matched all 8149 states and the entire +LRSpec, not merely state counts. CPU medians were: + +| Complete construction workload | Pairs | Baseline ms | Indexed shift ms | +|--------------------------------+-------+-------------+------------------| +| Mixed FOLLOW, 8 regions | 21 | 1.637 | 1.633 | +| Mixed FOLLOW, 24 regions | 21 | 4.442 | 4.357 | +| Mixed FOLLOW, 48 regions | 21 | 10.581 | 10.344 | +| Actual TLA+ default LALR | 21 | 14.292 | 14.292 | +| Actual TLA+ FOLLOW | 3 | 9505.538 | 8617.710 | + +The three full TLA+ FOLLOW pairs were 9685.312→8807.003, +9505.538→7556.410 and 8957.293→8617.710 ms. Every pair improved in this run; +the median decreased approximately 9.3%. Three pairs and their variation limit +the claim to this local completed-construction workload. No FFI, parser-runtime, +Rust throughput or whole-CI gain is inferred. The complete A/B emitted +=SHIFT-INDEX-NATIVE-AB-OK= and exited 0 in 76.405 seconds under unchanged +90-second total/5-second idle limits. The default LALR path was unchanged. + +** Reproducible Scheme benchmark owner + +The existing complete TLA+ construction benchmark is now named +=t/benchmarks/lr1-partition/construction.ss=. It retains default LALR selection +and accepts a second argument =follow-partition-lr1=, with a positive sample +count first. It checks every sample against a complete reference LRSpec and +reports CPU, in-call collection count and collection CPU separately. Both paths +reuse the same Scheme benchmark owner; timings remain outside semantic Cases. +For the expensive FOLLOW workload use the existing trace switch and the normal +qualification gate, for example one sample plus reference construction. The +benchmark's per-call collection observations are distinct from matched A/B cost. + +** Ubuntu production-object reuse correction + +The =f6dfc3b= Ubuntu job failed before semantic execution, during reusable library +object compilation: idle-timeout, process exit 70, elapsed 20.093 seconds. Its +four unfinished C objects were Cypher grammar, GQL grammar, HCL reductions and +arithmetic drive. GQL, Cypher and HCL also had SCM translation in that log, so +the four unfinished objects cannot all be attributed to option records. Requiring +records for unchanged production policy adds recompilation whenever their C and +object timestamps were otherwise reusable. That extra path is a cache-scope +regression; cold translation/optimization cost remains a separate gate. This +failure occurred before the semantic entry and supplies no semantic-crash result. + +Effective option records now enforce the changed per-procedure test-host policy. +Production objects retain their prior timestamp reuse, while complete preparation +continues to certify source/SDK/compiler/environment identity. A real library +preparation control moved aside arithmetic drive's generated option record. The +corrected builder emitted =COMPILED-LIBRARY-OBJECTS-READY=, exited 0 in 13.419 +seconds, and emitted no arithmetic-drive recompilation. Production cold-build +behavior is not qualified by this reuse control; the new Ubuntu run is required. + +The existing independent resolved/canonical source Suites passed 54 Cases with +=TEST-SUITE-OK= and =OK=, exit 0 in 16.875 seconds. They retain exhaustive +accepted/rejected string controls and exact canonical/partitioned parse products; +this change introduces no duplicate semantic Suite or timing loop in a Case. + +The reproducible FOLLOW benchmark (one sample plus full reference) completed +8149 states with full LRSpec equality: CPU 8869.908 ms, 84 in-call collections, +collection CPU 4843.012 ms. It exited 0 in 25.866 seconds under the same +90-second total/5-second idle gate. This single-call collection observation +suggests allocation as a next target; it does not compare allocation before and +after. An earlier invocation completed child output but its required-marker +argument contained an invalid regular expression, so controller admission failed; +that invocation is not a passed qualification. + +The next algorithm experiment should measure indexed-set traversal allocation: +current bit clearing creates intermediate bigints while repeatedly visiting wide +closed-state masks. A shrinking-mask or limb cursor needs sparse/dense/high-index +oracle controls, exact complete artifacts across LR modes, preparation-inclusive +cost and paired in-call allocation/collection observations. Small fixnum sets +need their own complete-cost control. That prospective cursor remains unimplemented +in this revision; the retained optimization is static labeled-successor indexing. + +** Final local consumer receipt + +After the engine index, trace, benchmark rename and cache scope stabilized, +complete preparation exited 0 in 85.116 seconds under unchanged 240-second +total/5-second idle limits. The compiled semantic entry passed 372 Cases across +23 groups; it and all five independent research entries exited 0 with +=DSL-RESEARCH-CLOSURE-OK= and =OK=. The renamed benchmark's omitted-mode branch +also completed default LALR, 390 states and full-reference equality, exit 0 in +5.164 seconds. That mode check supplies no additional matched performance claim. + +A reproducible expensive-workload command uses one sample plus reference: + +#+begin_src sh +GERBIL_PARSER_LR_TRACE=1 gerbil env gxi qualify.ss run \ + --timeout 90 --idle-timeout 5 \ + --log .data/tla-follow-construction.log \ + --require 'summary workload tla-layout construction follow-partition-lr1' -- \ + gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (call-with-compiled-interface-trace (lambda () (load "t/benchmarks/lr1-partition/construction.ss") (eval (quote (main "1" "follow-partition-lr1")))))' +#+end_src + +Native logs are runtime products. No JSON snapshot or additional report wrapper +is required. Local Darwin/GCC results remain distinct from Ubuntu cold-object +compilation and delivered-head CI. The unchanged five-second progress gate and +SDK/compiler identity must accompany those remote results. +An identical subsequent preparation emitted =CONFORMANCE-CACHE-HIT= and exited +0 in 11.902 seconds, with no recompilation. Inventory hashing is included in +that cache wall time. + + +* Immutable indexed-set traversal with local fixnum fields (2026-10-09) + +** Research boundary and design + +Chambi, Lemire, Kaser and Godin, /Better bitmap performance with Roaring +bitmaps/, [[https://arxiv.org/pdf/1402.6407][arXiv:1402.6407v10]], Sections 1--2, motivate partitioning a +bitmap's range and choosing work according to local density. Their Roaring +representation combines sparse arrays and dense bitmap containers. This engine +change adopts the bounded-field traversal idea; it does not implement Roaring, +change the integer set representation, or inherit the paper's measured speedups. + +The [[https://gambitscheme.org/4.9.7/manual/#Bitwise_002doperations-on-exact-integers][Gambit integer-operation contract]] defines =extract-bit-field= as a fixed-width +field shifted to index zero, and =first-set-bit= as the lowest present index. +The engine derives field width from =integer-length(##greatest-fixnum)= rather +than assuming a 64-bit host. Every extracted nonnegative field fits a positive +fixnum. Per-bit subtraction and clearing operate on that field; field extraction +may still allocate internally. The original bigint remains immutable and remains +the identity used in state interning, union, difference and equality. + +The previous traversal clears the lowest present bit by subtracting one and +ANDing the complete remaining integer. Dense wide masks repeat bigint arithmetic +for every callback. The new private traversal starts at the field containing +the first bit and visits fields in increasing order until the last bit. Within +each field it emits the lowest present index plus the field offset. Disjoint +fields and ascending local indices preserve each callback exactly once in its +original order. No mutable global cursor, list conversion, POO slot or author +API is introduced. Nested traversals have independent local state; callback +exceptions propagate without admitting later callbacks. + +A naive field-only experiment substantially regressed the two-bit mask spanning +4096 positions: 1000 traversals, median 0.762 -> 4.323 ms. It passed full artifact +equality but is rejected as the final algorithm. Empty fields count as work. +The retained algorithm selects direct clearing when cardinality is smaller than +the number of fields between the first and last bit. Fixnum masks retain the +original loop without offset arithmetic. This density choice uses input facts, +not a grammar filename, language count, runtime cache or benchmark threshold. +No asymptotic claim ignores field extraction, bigint cost or cardinality setup. + +** Independent controls and complete-call admission + +One Case in the existing canonical LR Suite compares callback lists to independent +per-position membership: all masks below 4096, fixnum and field boundaries, +dense prefixes, sparse high indices through 4096, both direct and field nested +traversals, and immediate callback-failure propagation. The Suite continues to +own its existing canonical/direct parse controls; no duplicate Suite is added. + +The comparison baseline freezes =f4606c3= sources for =funcs=, =lr=, +=lr-automaton=, =lr-lookahead=, =lr-partition=, =lr-conflict-candidates=, +=lr-follow-automaton= and =lr-compiler= under a separate temporary module package. +All indexed-set consumers, including FIRST materialization in =lr=, therefore +use the original traversal on the baseline side. Shared input Grammar/Parser IR +and language fixtures retain their original identities. Both sides are compiled +with identical optimized/static options through the fixed Gerbil/GCC SDK. + +Complete calls alternate baseline/candidate order with explicit GC before each +call. Equality compares the entire LRSpec outside timing. Measurements include +analysis, traversal selection, field extraction, in-call GC and publication. +The existing completed-work LR trace is enabled identically on both sides. +Five grammar families across five construction modes provide 25 complete-artifact +pairs, followed by 21 mixed-family pairs at each size, 21 actual TLA+ LALR pairs +and three actual TLA+ FOLLOW pairs. CPU and in-call GC observations describe +local traced construction, not parser runtime, FFI, Rust AOT or CI throughput. + +The final traversal microcontrol uses 11 batches for each mask. CPU medians: + +| Mask | Traversals per batch | Original ms | Retained ms | +|------+----------------------+-------------+-------------| +| fixnum 1023 | 10000 | 5.018 | 5.047 | +| dense 4096-bit prefix | 100 | 254.531 | 19.310 | +| two bits at 0 and 4096 | 1000 | 0.751 | 0.850 | +| singleton at 65536 | 1000 | 6.152 | 2.936 | +| 16-bit cluster starting at 4080 | 1000 | 5.152 | 1.042 | + +The sparse guard retains a measured selection cost (~0.099 ms per 1000 calls); +it bounds the empty-gap regression rather than claiming zero overhead. Small +fixnum cost is nearly unchanged in this control. Full construction costs are the +admission criterion. Raw temporary native logs retain rejected experiments and +final receipts; no JSON snapshot or extra report supervisor is added. + + +** Retained complete-construction receipt + +The final density-selected traversal passed all 25 full LRSpec mode/family pairs +and the complete paired calls under unchanged 90-second total/5-second idle +limits. Controller exit 0, elapsed 57.949 seconds, required +=CURSOR-NATIVE-AB-OK= present. Complete CPU medians: + +| Complete workload | Pairs | Original ms | Retained ms | +|-------------------+-------+-------------+-------------| +| mixed FOLLOW, 8 regions | 21 | 1.568 | 1.526 | +| mixed FOLLOW, 24 regions | 21 | 4.339 | 4.260 | +| mixed FOLLOW, 48 regions | 21 | 10.363 | 10.062 | +| actual TLA+ default LALR | 21 | 14.899 | 14.251 | +| actual TLA+ FOLLOW | 3 | 8879.933 | 3517.695 | + +The three actual FOLLOW pairs are 9153.577 -> 3517.695, 8879.933 -> 3127.235, +and 6769.970 -> 3819.975 ms. All produce the exact same complete artifact, +8149 states. Baseline variation and only three expensive pairs bound the 60.4% +median reduction to this local traced workload. In-call GC-count medians are +84 -> 26; GC CPU medians are 4842.098 -> 1421.156 ms. These are matched +collection observations, not a direct byte-allocation measurement. Unchanged +production LALR remains 390 states and showed no complete-cost regression here. + +The final traversal microcontrol also exited 0 under 60-second total/5-second +idle limits, elapsed 5.914 seconds. Independent source controls passed 55 Cases +(37 resolved, 18 canonical), =TEST-SUITE-OK=, =OK= and controller exit 0, +elapsed 21.592 seconds. Case count includes the one new shared traversal oracle. +The canonical Suite remains a source-test owner. Compiled conformance +independently admits consumers of the production engine; it does not rerun this +canonical Suite. + +During this experiment, preceding delivered head =f4606c3= completed Ubuntu +Scheme/language and Rust AOT CI successfully (run =37901275708=). Those remote +receipts belong to the preceding head; the density traversal requires its own +post-delivery CI result. GCC and the original progress/deadline policy remain +unchanged. + + +** Compiled consumer closure + +After source and algorithm stabilization, complete preparation exited 0 in +89.723 seconds with unchanged 240-second total/5-second idle limits. The compiled +semantic entry passed 372 Cases across 23 groups. The new traversal oracle is +covered separately by the 55-Case source control above. Semantic, concise runtime, +history, native POO runtime, provenance and +all-language benchmark child entries all exited 0; =DSL-RESEARCH-CLOSURE-OK= +and =OK= were emitted by the existing Scheme process owner. Preparation is kept +outside construction CPU measurements; it is still required for delivery. + +The previously admitted TLA+ +construction benchmark command exercises this engine change without another +benchmark facade or report format. + + +* Delta expansion for shared indexed reachability (2026-10-09) + +** Paper transfer and engine boundary + +Gertjan van Noord, /Treatment of Epsilon Moves in Subset Construction/, +[[https://aclanthology.org/J00-1005.pdf][Computational Linguistics 26(1), 2000]], Section 3.2 and Figure 4, separates +per-state closure memoization from per-subset closure, and computes reachability +by expanding only newly reached states. The paper also identifies the union cost +of memoized singleton closures. This revision retains the existing integrated +closure strategy and cache admission; it does not reintroduce the rejected +whole-graph SCC precomputation or claim that the paper establishes our timings. + +The engine derives a bitset delta implementation of the Figure 4 worklist: +for an expanded node u, compute =delta = edges[u] \ reached=, mark all delta +nodes reached with one union, and enqueue just that delta. The reached set is +reflexive in the initial set and monotonically grows. Every node is queued only +once, so self loops, cycles, multiple initial roots and overlapping neighborhoods +terminate without repeated expansion. When the queue is empty, every reached +node's outgoing row is included, proving closure; each inserted node has an +initial-to-node path, proving no spurious reachability. Bigint difference/union +and queue costs remain part of complete-call measurement. + +=compiler-index-set-reachable= belongs to =src/compiler/funcs.ss=. It accepts +an initial integer mask and a vector of immutable adjacency masks in the same +valid finite index domain. It returns an immutable reached mask and keeps its +queue local to the invocation. This is a named reusable engine algorithm, not a +language-specific closure model or an author-defined scanner. The former private +FOLLOW BFS is removed; both singleton-cache misses and uncached subset closure +use the shared owner. Grammar DSL, POO loader slots, state interning, publication, +conflict policy and the 3500-block singleton-cache admission remain unchanged. + +The old BFS creates a singleton mask and checks membership for every outgoing +edge, including already reached neighbors. The new algorithm performs set +operations once per expanded nonempty row and enumerates only unseen neighbors. +Sinks skip the difference step. This is an implementation transfer of the +worklist invariant; dense and overlapping rows supply the opportunity, while +complete construction controls determine whether the added row operations pay. + +** Independent semantic and cost admission + +One new Case in the existing canonical source Suite compares the engine algorithm +to a list-graph traversal using list membership. All 512 directed three-node +graphs (including self edges) and all eight initial masks give 4096 independent +comparisons. Empty graph, 1024-node chain and 1024-node cycle controls cross +fixnum/field boundaries. The canonical Suite remains a source owner; it is not +also registered in compiled conformance. Together with the existing resolved +Suite, 56 Cases passed with =TEST-SUITE-OK=, =OK= and controller exit 0 in +26.954 seconds under unchanged 120-second total/5-second idle limits. + +The baseline freezes =49e297f= and all eight LR indexed-set modules, including +the previously admitted density-selected traversal. Both sides compile with the +same fixed Gerbil/GCC SDK and optimized/static settings. Complete calls alternate +order with explicit GC before each call; full LRSpec equality remains outside +timing. CPU includes construction setup, closure misses, caching, all analyses, +in-call GC and publication. Real completed-work tracing is enabled equally. +Five grammar families across five LR modes yield 25 exact whole-artifact pairs; +small mixed grammars and default TLA+ LALR each receive 21 pairs, and the real +TLA+ FOLLOW grammar receives seven pairs. This avoids attributing previous +traversal savings to the new closure change. + + +** Complete-construction result + +The paired native process emitted =DELTA-NATIVE-AB-OK= and controller exit 0 +in 66.332 seconds under unchanged 90-second total/5-second idle limits. Every +complete result matches the baseline LRSpec; actual FOLLOW remains 8149 states. + +| Complete workload | Pairs | Baseline CPU ms | Delta CPU ms | +|-------------------+-------+-----------------+--------------| +| mixed FOLLOW, 8 regions | 21 | 1.530 | 1.510 | +| mixed FOLLOW, 24 regions | 21 | 4.163 | 4.143 | +| mixed FOLLOW, 48 regions | 21 | 10.175 | 10.125 | +| actual TLA+ default LALR | 21 | 14.361 | 14.334 | +| actual TLA+ FOLLOW | 7 | 3027.127 | 2250.424 | + +| Actual FOLLOW pair | Baseline CPU ms | Delta CPU ms | Baseline GCs | Delta GCs | +|--------------------+-----------------+--------------+--------------+-----------| +| 0 | 3994.083 | 2339.458 | 48 | 15 | +| 1 | 3027.127 | 1799.456 | 17 | 7 | +| 2 | 2622.948 | 2250.424 | 10 | 13 | +| 3 | 3838.403 | 2121.617 | 27 | 15 | +| 4 | 2769.019 | 2455.940 | 16 | 17 | +| 5 | 3131.278 | 1801.500 | 17 | 7 | +| 6 | 2542.748 | 2250.778 | 10 | 13 | + +All seven complete CPU pairs improve. Median CPU reduction is 25.7%; baseline +variation still limits the claim to this local traced workload. GC-count medians +are 17 -> 13 and GC CPU medians 963.769 -> 750.303 ms, but three candidate +calls collect more often than their paired baseline. Thus neither a universal GC +reduction nor a byte-allocation reduction is established. The SDK byte counter +is not admitted across collections. Parser runtime, FFI and Rust AOT throughput +are outside this construction experiment. + +The retained change is the shared row-delta reachability owner. Further study +should count repeated seed-subset queries and their cache footprint before +adding per-subset memoization from Section 3.2. Existing singleton-cache reuse, +union work, setup and retained masks must remain separate observations. No new +whole-subset cache or SCC implementation is introduced here. + + +** Local compiled closure and delivery boundary + +After algorithm and source controls stabilized, complete preparation exited 0 in +93.767 seconds under unchanged 240-second total/5-second idle limits. Compiled +semantic execution passed 372 Cases across 23 groups. It and the five independent +research entries (concise runtime, history, native POO runtime, provenance and +all-language benchmark) all exited 0; the existing Scheme controller emitted +=DSL-RESEARCH-CLOSURE-OK= and =OK=. The canonical graph oracle remains covered +by the separate source Suite, not counted again in these 372 compiled Cases. + +No JSON snapshot, extra test +supervisor, heap expansion, timeout relaxation or compiler switch is introduced. +The preceding delivered head =49e297f= had Rust CI success while Ubuntu was +still in compiled conformance preparation during local execution. Delivery of +this shared reachability implementation requires its own remote-head checks. + + +* Bounded seed-subset reuse during determinization (2026-10-09) + +** Measured query shape and paper transfer + +Van Noord's [[https://aclanthology.org/J00-1005.pdf][2000 epsilon-move paper]], Section 3.2, memoizes both singleton +closures and their union for a whole seed subset. The engine already caches +singleton closure on large quotients; the missing whole-query reuse is tested +here after the admitted delta expansion and density-selected bitmap traversal. +These earlier improvements remain on both sides of the experiment. + +A temporary compiled diagnostic counted requests after forward refinement: + +| Workload | Quotient nodes | Requests | Unique seeds | Repeated requests | Unique seed bit-length sum | +|----------+----------------+----------+--------------+-------------------+----------------------------| +| mixed FOLLOW, 8 regions | 265 | 158 | 126 | 32 | 15765 | +| mixed FOLLOW, 24 regions | 777 | 462 | 366 | 96 | 132485 | +| mixed FOLLOW, 48 regions | 1545 | 918 | 726 | 192 | 520685 | +| actual TLA+ FOLLOW | 3984 | 347740 | 8149 | 339591 | 22363206 | + +The actual TLA+ workload has 97.7% repeated seed queries. The diagnostic retains +all keys in order to count them, so its 10.442-second controller result is not +an admitted optimization cost. The bit-length sum describes logical mask payload, +not heap bytes; hash entries, object headers and alignment cost extra. + +** Retained cache invariant and bound + +The cache maps an immutable target seed mask to a canonical state ID that has +already been assigned by =intern!=. A hit reuses that ID without recomputing +singleton unions or hashing a newly produced closure into the state index. A +miss retains the original closure/intern path. Because entries are admitted only +after interning, a hit cannot change first-discovery order or skip a queued new +state. Identical seeds in one fixed graph have identical closures; every stored +ID therefore names exactly the original transition target. + +The index lives inside one complete determinization invocation. It is admitted +at the existing large-quotient threshold (3500 nodes). It retains at most one +entry per quotient node and does not evict admitted entries; once full, uncached +seeds still use the original exact computation. Small quotients retain no subset +index. No cache crosses a language, source revision or compiler invocation. + +For a quotient with V nodes, each key contains at most V meaningful bits and +there are at most V keys. Thus additional logical key payload is bounded by +V squared bits plus integer alignment; table/object overhead is not included in +that arithmetic bound. The 3984-node TLA+ bound is 1,984,032 logical bytes before +alignment/headers. Canonical output masks are already owned by the existing +state index/vector; subset values are only state IDs, not duplicate closure +objects. The existing LR trace reports completed cache hits and admitted entries. +No DSL declaration, POO slot, public schema or compatibility route is added. + +** Full artifact and cost control + +The worktree first fast-forwarded to current PR head =7449072=, retaining its +source-priority lowering and native process-replacement changes. Its qualification +boundary was rebuilt before experiments. The baseline freezes all eight LR +indexed-set modules at this head; both sides use the fixed Gerbil/GCC SDK and +identical optimized/static options. Complete calls alternate order, force GC +before timing, and compare the complete LRSpec outside timing. Setup, hash-key +work, cache fill, miss computation, in-call GC and publication are included. +Real completed-work LR tracing is enabled identically. + +Five grammar families across five LR modes supply 25 exact full-artifact pairs. +Every actual TLA+ pair retains the exact complete 8149-state LRSpec. The bounded +index admits 3984 keys and hits 325223 transition queries (93.5% of transition +requests), leaving 22516 exact miss computations. This count, unlike the earlier +unbounded diagnostic, belongs to the retained implementation. + +| Complete workload | Pairs | Baseline CPU ms | Subset CPU ms | +|-------------------+-------+-----------------+---------------| +| mixed FOLLOW, 8 regions | 21 | 1.484 | 1.517 | +| mixed FOLLOW, 24 regions | 21 | 4.109 | 4.108 | +| mixed FOLLOW, 48 regions | 21 | 9.918 | 9.936 | +| actual TLA+ default LALR | 21 | 14.215 | 14.174 | +| actual TLA+ FOLLOW | 7 | 2432.837 | 1806.007 | + +| Actual FOLLOW pair | Baseline CPU ms | Subset CPU ms | Baseline GCs | Subset GCs | +|--------------------+-----------------+---------------+--------------+------------| +| 0 | 2929.288 | 2411.971 | 34 | 17 | +| 1 | 2494.107 | 1801.497 | 18 | 11 | +| 2 | 1933.464 | 2400.943 | 12 | 19 | +| 3 | 2432.837 | 1695.287 | 17 | 9 | +| 4 | 1783.837 | 1768.816 | 7 | 7 | +| 5 | 2696.379 | 1806.007 | 21 | 11 | +| 6 | 1910.477 | 1875.487 | 7 | 7 | + +The actual FOLLOW median decreases 25.8%, but pair 2 regresses and several +pairs increase collection CPU. GC-count medians are 17 -> 11; GC CPU medians +971.720 -> 459.323 ms. These observations establish a bounded local construction +tradeoff, not universal latency, allocation-byte, parser runtime, FFI or Rust +throughput gains. Small complete costs are approximately stable, with the +8-region control increasing 0.033 ms. The process emitted =SUBSET-NATIVE-AB-OK= +and exited 0 in 54.914 seconds under unchanged 90-second total/5-second idle +limits. Prior delivered head =7449072= completed Ubuntu and Rust CI successfully; +the subset-cache delivery needs its own remote-head results. + + +** Final local consumer receipt + +The existing resolved/canonical source owners passed 56 Cases, =TEST-SUITE-OK= +and =OK=, controller exit 0 in 19.796 seconds under unchanged 120-second +total/5-second idle limits. No duplicate Case or Suite is introduced for the +private cache; complete-artifact pairs and real cache-bound traces cover its +large-quotient execution path. + +Complete preparation, including modules affected by the newly synchronized PR +head, exited 0 in 140.339 seconds under unchanged 240-second total/5-second idle +limits. The compiled semantic entry passed 372 Cases across 23 groups. It and +all five independent research entries exited 0, with =DSL-RESEARCH-CLOSURE-OK= +and =OK=. Source Suite counts remain separate from the compiled consumer count. + +Preparation and diagnostic +retention are excluded from the paired construction claim and independently +required for delivery. No JSON snapshot or additional supervisor is introduced. + +* Static labeled predecessor relations: rejected experiment (2026-10-09) + +** Paper transfer and proof obligations + +Ranzato and Tapparo, /Generalizing the Paige-Tarjan Algorithm by Abstract +Interpretation/, distinguishes partition stability from refinement scheduling: +[[https://arxiv.org/pdf/cs/0612120][primary paper]]. FOLLOW's labeled predecessor relation is immutable during +forward refinement; source-to-partition identities change each round. Caching +that relation is only a representation experiment, not the paper's splitter +algorithm or its complexity bound. + +Three Scheme/GCC candidates grouped and sorted labels once and subsequently +projected source indices into current group masks. The first froze every row; +the second froze only initial nonsingleton rows; the third additionally retained +source lists directly instead of copying them into vectors. Initial singleton +elision is sound because refinement only splits. All candidates preserved +first-seen numbering, splitting schedule and full LRSpec equality. Nevertheless, +none earned admission. All implementation and candidate-only test changes were +removed; no helper, cache, POO model or language author API is published. + +** Complete construction receipts, including negative results + +Every candidate used the same native frozen baseline +=abdde8648c5f5704fc66058f27765ccd33f15213=, explicit GC before each timed complete +construction, alternating old/new order and full LRSpec comparison outside +measurement. Preparation and evidence retention are excluded. Each run checks +five grammar families across five modes (25 exact artifacts), then 21 paired +mixed FOLLOW constructions at 8/24/48 regions, 21 actual TLA+ default LALR pairs, +and 7 actual TLA+ FOLLOW pairs. Both sides include identical real stage tracing, +setup, construction, GC and publication; results are local CPU measurements. + +| Candidate | TLA+ FOLLOW old median ms | Candidate median ms | Supervisor result | +|-----------+---------------------------+---------------------+-------------------| +| All rows, vectors | 1782.771 | 1873.932 | exit 0, 50.573 s | +| Initial nonsingletons, vectors | 1825.951 | 1908.429 | exit 0, 54.157 s | +| Initial nonsingletons, lists | 2423.497 | 2101.417 | idle-timeout, child exit 143, controller exit 124 | + +The first two candidates regress approximately 5.1% and 4.5%. Their pairs 4 and +6 increase collections from 10 to 16. This identifies a concrete GC tradeoff, +not a proven complete root cause. Mixed 24/48-region medians for the first are +4.179 -> 4.233 and 9.925 -> 10.075 ms; for the second, 4.362 -> 4.363 and +10.150 -> 10.293 ms. Default LALR is outside the changed FOLLOW path. + +The list representation has a lower single-run median but regresses individual +pairs 2, 4 and 6. More decisively, =ROWS-NATIVE-AB-OK= is followed by +=PROCESS-RESULT elapsed=82.935 exit=143 reason=idle-timeout missing=()=. +The supervisor therefore correctly rejects the run despite the completion +marker. This establishes an uncompleted exit/teardown boundary, not a successful +performance receipt or a proven explanation of the stall. The 90-second total +and 5-second idle limits remain unchanged; there is no compiler switch, timeout +extension or synthetic progress. All three candidates are rejected. + +The experimental resolved/canonical source owners passed 57 Cases (37 + 20), +=TEST-SUITE-OK= and =OK=, controller exit 0 in 26.810 seconds. The candidate Case +independently exhausted 64 labeled relations times 27 partition maps, original +and reversed with duplicate edges (3456 comparisons), plus identities 64/128 and +a changed partition map. Semantic equivalence did not override the failed +performance/exit admission. The Case is removed with its unpublished helpers; +existing source Suite ownership remains unchanged. + +No JSON snapshot or additional +supervisor is introduced. + +** Next structural obligation + +The representation cache does not eliminate the repeated scanning of ambiguous +blocks in each refinement round. A subsequent worklist design must prove that +only groups whose predecessor partition sets changed need to be revisited, +including epsilon label zero, duplicate predecessor groups and transitions that +lose one group identity while acquiring split identities. It must preserve +canonical first-seen final numbering independently of splitter processing order, +compare complete artifacts across all construction modes, and include complete +setup/GC/publication cost. The current synchronous refinement remains the +reference. There is no admitted complexity or end-to-end runtime claim from +this experiment. + +** Restoration and delivery receipt + +After rejecting all candidates, the three source/test files were restored +byte-for-byte from the current head and their native compiler artifacts rebuilt +with the same GCC SDK. The original resolved/canonical source owners pass +56 Cases (37 + 19), =TEST-SUITE-OK= and =OK=; controller exit 0 in 31.882 seconds +under unchanged 120-second total/5-second idle limits. Only this Org research +record is delivered. Previously qualified compiled consumer source is unchanged; +no new production optimization or full consumer requalification is claimed. + +* Dependency-directed synchronous refinement (2026-10-09) + +** Paper transfer and engine contract + +[[https://arxiv.org/pdf/cs/0612120][Ranzato and Tapparo's generalized Paige-Tarjan paper]] distinguishes the +correctness of stable partition refinement from scheduling improvements. The +implementation here retains synchronous rounds and first-seen numbering. It +adds an affected-group frontier; it does not implement smaller-half splitters, +a fully asynchronous worklist, or establish an =O(m log n)= bound. Node sizing, +publication and split-source discovery still scan the node domain per round. + +=compiler-index-partition-refine= in the shared Scheme compiler algorithms +accepts dense initial group IDs, a label/set signature procedure, an exact +successor dependency visitor and a completed-round observer. FOLLOW supplies +shift and epsilon successors. The generic algorithm is language-independent; +no language loader slot, POO model, FFI ABI or author API changes. + +Its obligations are: + +1. The first round marks all groups affected. Signatures are exact sets of + predecessor group identities per label; their equality must be invariant + under bijective renaming of those identities. +2. A round only splits groups. Unsplit source groups map injectively into their + next identities; their renumbering preserves target signature equality. +3. Every source member of a split group visits all successors, including epsilon + label zero and self edges. Every resulting target group is marked affected. + Extra visits from duplicate edges are idempotent. +4. An unmarked group was stable in the prior round and no dependency source + group split. Its membership can therefore be copied without recomputing + signatures. Singleton groups also require no signature. +5. Each node is still published in increasing original index order. The first + encountered child receives the next dense identity. Initial IDs are not + mutated, and the original termination condition is preserved. + +The first child mapping also represents clean-group publication and split +status: -1 unseen, -2 split, otherwise the first child ID. That private numeric +workspace and group sizes are allocated once per invocation and reset per round; +there is no global state or retained signature cache. This avoids three redundant +per-round vectors from the first candidate. + +** Independent executable semantic controls + +The canonical source owner adds one Case with a list-based synchronous oracle. +The oracle recomputes every node's signature and uses an association list for +first-seen groups. Work counts exclude singletons, matching the previous +production optimization. The Case covers all 512 three-node directed graphs +with five initial partitions (2560 controls), 64 two-label cyclic relations with +two initial partitions and duplicate edges (128 controls), an independent +self-loop group beside a propagating chain, and an empty domain. It checks exact +final IDs/count, unchanged input IDs and no increase in signature calls. + +The chain/self-loop control requires exactly 16 candidate signature calls versus +24 existing-engine calls. This demonstrates actual affected-group elision, +separately from singleton elision; it is not a latency claim. + +** Complete cost and live-output sensitivity + +The native GCC baseline is frozen from +=7a91f1dd11d2f8e7eace8a9f90d8921d5b54bc0c= across all eight indexed LR modules. +Both baseline and candidate use matching native compilation options and real +stage tracing. Earlier measurements retained the first full LRSpec during the +second timed construction and used seven alternating actual FOLLOW pairs. The +first candidate's medians were 1782.060 -> 1993.383 ms; the reused-workspace +candidate's were 1819.706 -> 2137.694 ms. Both exit normally, but these are +negative live-output sensitivity receipts, not ignored wins. Retaining an +additional artifact changes collection pressure; the odd sample count also +leaves invocation order unbalanced. These results do not establish a complete +root cause or favorable behavior when previous artifacts remain live. + +For the final complete-construction control, full LRSpec equality is established +outside timing for every workload; timed calls discard results before the next +explicit GC. Actual FOLLOW uses eight alternating pairs, four per order, and +conventional even-sample medians. All setup, construction, GC and publication +remain inside timed calls. This characterizes construction without a live +partner artifact; it does not remove the preceding resource sensitivity. + +| Complete workload | Pairs | Baseline median CPU ms | Frontier median CPU ms | +|-------------------+-------+------------------------+------------------------| +| Mixed FOLLOW 8 regions | 21 | 1.525 | 1.545 | +| Mixed FOLLOW 24 regions | 21 | 4.180 | 4.218 | +| Mixed FOLLOW 48 regions | 21 | 10.112 | 10.119 | +| Actual TLA+ default LALR | 21 | 14.349 | 14.483 | +| Actual TLA+ FOLLOW | 8 | 1925.546 | 1877.562 | + +The actual FOLLOW single-run median decreases 2.49%, but pairs 2 and 3 regress +and several candidate GC counts/CPU times increase. Small FOLLOW controls add +0.007-0.038 ms; default LALR is outside the modified refinement path. The process +checks 25 artifacts across five grammar families and five construction modes, +plus exact results for all cost workloads. It emits =DIRTY-NATIVE-AB-OK= and +exits 0 in 54.939 seconds under unchanged 90-second total/5-second idle limits. +This admits a structurally verified reduction of redundant signature work; +it does not establish stable latency, allocation-byte, retained-artifact memory, +Runtime parsing, FFI or Rust throughput improvement. + +The prior rejected static-row +cache remains removed. No JSON report or additional test supervisor is added. + +** Final consumer and preparation receipts + +The final source controls pass 57 Cases (resolved 37, canonical 20), +=TEST-SUITE-OK= and =OK=; controller exit 0 in 23.461 seconds. The updated +work-count oracle excludes singleton calls and the 16-versus-24 assertions pass. + +The first complete preparation links successfully but then triggers the unchanged +5-second idle guard after the final 1866-product hash report; controller exit 124, +child exit 143, total elapsed 97.556 seconds. Inspection finds both local/store +records and all 1866 copied products written. The hash-to-save/exit path is thus +the evidence boundary; the original save/restore loops report no completed copy +progress. This is a visibility gap, not a demonstrated deadlock or complete +explanation of every historical exit stall. + +The Scheme preparation owner now reports each 64 actually completed copies and +the final partial batch, plus completed local/store record writes. Total entry +counts are computed once per operation; no repeated list-length scan, heartbeat, +new process, compiler switch or widened timeout is introduced. Existing +preparation controls for content invalidation, failed builds, restoration, +corrupt stores, generated caches and object boundaries pass, final =OK= and +controller exit 0 in 1.833 seconds. Saving/restoring real progress markers are +required by the check. + +Complete preparation rerun passes in 82.709 seconds under unchanged 240-second +total/5-second idle limits, including completed 1915-product storage and +=PREPARATION-STORED-RECEIPT=; controller exit 0. The compiled semantic entry passes 372 Cases across 23 groups; it and +all five research entries exit 0, =DSL-RESEARCH-CLOSURE-OK= and =OK=. Source Suite +and compiled consumer counts are distinct. + +* Invocation-owned refinement workspaces (2026-10-09) + +** Resource transfer from the partition refinement design + +The [[https://arxiv.org/pdf/cs/0612120][generalized Paige-Tarjan study]] separates correctness of stable refinement +from algorithmic scheduling improvements. This implementation retains the +previous affected-group schedule, exact signatures, first-seen numbering and +termination. It changes resource ownership within the shared Scheme algorithm, +not splitter selection or its asymptotic time bound. + +Each invocation allocates two ID vectors and two affected-group vectors, alongside +the existing sizes and first-child workspace. Every next ID slot is assigned +before publication; the next affected frontier is cleared before propagation. +Only after all signature reads and successor visits complete are current/next +roles exchanged. Original initial IDs are never overwritten. A signature +callback borrows current IDs during its call; it must not retain that internal +view. The final returned vector survives other invocations, and no buffer is +global or retained by the language descriptor. + +For =r= completed rounds, the previous algorithm allocates =2r + 2= indexed +vectors, including initial dirty flags and size/first-child storage. This version +allocates six, independent of =r=. One-round calls therefore allocate two extra +vectors, two-round calls the same number, and longer calls fewer. Table/signature +allocation, vector reset work and maximum live bytes remain separate costs. +This is a deterministic workspace allocation count, not a total allocation-byte +or latency claim. Language author declarations and ABI contracts are unchanged. + +** Semantic and lifetime controls + +The existing canonical source owner still performs exhaustive graph comparison +with the independent synchronous oracle. Additional assertions require two-round +refinement to return the expected IDs, then a separate invocation to produce a +different partition without modifying either the first returned vector or the +initial vector. Source ownership stays at 57 Cases across the existing two +Suites. They emit =TEST-SUITE-OK= and =OK=, controller exit 0 in 23.024 seconds +under unchanged 120-second total/5-second idle limits. + +** Complete construction controls + +The baseline is frozen from =87c37bf52bc28f4c6da93b42c082463d680286f9= across all +eight indexed LR modules. GCC, native options, 3 GiB heap, tracing and total/idle +limits are matched. Every workload checks exact full LRSpec equality outside +measurement; retained-result pairs additionally compare both timed results. +Each protocol runs 22 alternating pairs for mixed 8/24/48-region FOLLOW and +actual TLA+ default LALR, and eight actual TLA+ FOLLOW pairs, four per order. +Even-sample medians average the two middle values. Construction includes setup, +GC and publication, but excludes imports, explicit preceding GC and comparisons. + +| Lifecycle | Complete workload | Baseline median CPU ms | Workspace median CPU ms | +|-----------+-------------------+------------------------+-------------------------| +| Retain first timed artifact | Mixed FOLLOW 8 | 1.536 | 1.523 | +| Retain first timed artifact | Mixed FOLLOW 24 | 4.157 | 4.157 | +| Retain first timed artifact | Mixed FOLLOW 48 | 10.080 | 10.126 | +| Retain first timed artifact | TLA+ default LALR | 14.495 | 14.304 | +| Retain first timed artifact | TLA+ FOLLOW | 1871.222 | 1766.677 | +| Discard timed artifacts | Mixed FOLLOW 8 | 1.566 | 1.609 | +| Discard timed artifacts | Mixed FOLLOW 24 | 4.311 | 4.296 | +| Discard timed artifacts | Mixed FOLLOW 48 | 10.346 | 10.187 | +| Discard timed artifacts | TLA+ default LALR | 14.736 | 14.663 | +| Discard timed artifacts | TLA+ FOLLOW | 1852.845 | 1885.259 | + +Retained actual FOLLOW medians decrease 5.59%, but all four candidate-second +pairs regress and collections increase in several pairs. Discarded-result +medians increase 1.75%, with four individual regressions. Both protocols emit +=BUFFER-NATIVE-AB-OK= and exit 0 (64.007 and 56.988 seconds) under unchanged +90-second total/5-second idle limits. These results establish no stable latency +win or elimination of retained-output GC sensitivity. Default LALR is outside +the changed FOLLOW path. The admitted implementation bounds multi-round +workspace allocation; neither these controls nor the previous historical batches +support a cumulative runtime/FFI/Rust throughput claim. + +** Repository verification owners + +Git documentation names repository-owned inputs, commands and results. Host +checkout paths and ephemeral raw-log locations belong to local execution +receipts, and are not references readers can reproduce from Git. + +- [[file:../../src/compiler/funcs.ss][Shared indexed refinement algorithms]] +- [[file:../../t/canonical-lr1-reference-test.ss][Exhaustive semantic and lifetime controls]] +- [[file:../../t/benchmarks/lr1-partition/construction.ss][Complete TLA+ construction benchmark]] +- [[file:../../build-conformance.ss][Compiled conformance preparation owner]] +- [[file:../../scripts/preparation-cache.ss][Content admission and actual copy progress]] +- [[file:../../test-processes.ss][Compiled semantic and research process owner]] + +From the repository root with its configured Gerbil environment and compiled +interfaces prepared, use the existing owners: + +#+begin_src sh +mkdir -p .gerbil +gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .gerbil/lr-controls.log --require TEST-SUITE-OK --require '^OK$' -- \ + gxi t/fixtures/tla-sany-differential/test-suite.ss \ + t/resolved-grammar-test.ss t/canonical-lr1-reference-test.ss +GERBIL_PARSER_LR_TRACE=1 gerbil env gxi \ + t/benchmarks/lr1-partition/construction.ss 8 follow-partition-lr1 +gerbil env gxi build-conformance.ss +gerbil env gxi test-processes.ss research .gerbil/research-results +#+end_src + +The construction owner retains its reference artifact to check every sample. +It is an executable single-version consumer, not a substitute for the two +historical frozen-baseline protocols above. + +** Final compiled consumer receipt + +Complete preparation exits 0 in 131.752 seconds under unchanged 240-second +total/5-second idle limits, including the completed 1964-product storage receipt. +The compiled semantic owner passes 372 Cases across 23 groups. It and the five +independent research entries all exit 0, with =DSL-RESEARCH-CLOSURE-OK= and =OK=. +These compiled consumer counts remain separate from the 57-Case source owner. +Preparation time is required qualification, excluded from paired construction +CPU measurements. No native parser runtime or Rust/FFI acceleration is claimed. + +* Group cardinality and propagation frontier experiment (2026-10-09) + +** Paper transfer and candidate proof + +[[https://arxiv.org/pdf/cs/0612120][Ranzato and Tapparo]] separates stable refinement correctness from scheduling +improvements. The existing engine still sizes every current group in a separate +node pass, and scans all nodes when propagating split-source dependencies. +Two candidates were implemented and tested with identical synchronous rounds, +exact signatures, first-seen IDs and termination: + +- Compute next-group cardinalities during ID publication and carry them into + the following round. Initial membership is counted only once. +- Record each affected nonsingleton member in an invocation-owned source vector. + Only those groups can split, so propagation scans that recorded prefix and + visits successors only for members whose old group actually split. + +Candidate one retains separate size/dirty arrays and needs eight indexed +vectors rather than six. Candidate two stores affected status in the sign of +each cardinality: negative affected, positive clean, magnitude exact group size. +All next sizes are counted positively before propagation; marking an already +negative size is idempotent. This returns the workspace to six indexed vectors +and eliminates the separate boolean frontier reset. Zero denotes an unused +slot, never a populated group. Both preserve input and output lifetime contracts. + +For =r= rounds on =n= nodes, the separate membership-sizing pass would drop +from =r*n= node reads to =n= initial reads. Publication still performs =r*n= +node assignments and additionally counts next-group membership, including the +last unused next size map. Propagation would inspect the affected nonsingleton +prefix rather than =n= nodes per continuing round. These are traversal bounds, +not total operation or complexity improvements; synchronous publication remains +linear in every round. Neither candidate implements full Paige-Tarjan scheduling +or establishes its logarithmic bound. + +** Exact artifacts and negative complete costs + +The native baseline is frozen from +=2b86afaed9bf2cf9375aa966c27bedf64696f743= across all eight indexed LR modules. +Each candidate passes the two existing source Suites, 57 Cases, including the +exhaustive synchronous graph oracle, duplicate labels/edges, empty domain, +multiple rounds, preserved input, independent result lifetimes and two new +initially permuted dense-ID controls. Source checks alone do not admit either +candidate's complete cost. + +Native GCC options, tracing, 3 GiB heap and 90-second total/5-second idle limits +are unchanged. Each protocol validates 25 full LRSpec artifacts across five +grammar families and five construction modes, then 22 alternating pairs each +for three mixed FOLLOW sizes and actual TLA+ default LALR, plus eight actual +TLA+ FOLLOW pairs. Both timed results remain live for pair comparison; there +are four pairs per invocation order, and even medians average the middle two +samples. Imports, explicit preceding GC and equality checks are excluded; +setup, construction, in-call GC and publication are included. + +| Candidate | TLA+ FOLLOW baseline CPU median ms | Candidate median ms | Supervisor | +|-----------+------------------------------------+---------------------+------------| +| Separate size/dirty arrays | 1764.333 | 1861.461 | exit 0, 63.748 s | +| Cardinality sign marks affected groups | 1694.290 | 1775.030 | exit 0, 55.423 s | + +Complete medians regress 5.50% and 4.77%. Every candidate-second actual FOLLOW +pair regresses in both batches; GC counts and CPU increase in several pairs. +The first candidate's pairs 2/4/6 increase collections from 10 to 16. The second +candidate still increases them from 9 to 11 in pair 2, 11 to 15 in pair 4 and +8 to 12 in pair 6. This preserves the invocation-order and retained-output +sensitivity rather than claiming its root cause is solved by workspace fusion. +Reducing an internal scan or vector count did not earn complete-cost admission. +Further discarded-result timing is unnecessary once this retained-result +consumer fails to establish an acceptable tradeoff. + +Both candidates are removed, including the cardinality encoding and new source +workspace. The existing six-vector engine remains unchanged. No field, ABI, +language loader slot, schema version, heap limit or compatibility API changes. +The two new permuted-ID controls remain in the existing canonical source Case; +they strengthen a valid baseline contract independently of the rejected algorithm. + +** Next structural obligation + +A stronger scheduler must preserve group identity while processing affected +members, then canonicalize IDs in one final first-seen pass. It must compute the +same coarsest stable partition regardless of worklist order, include self and +epsilon dependencies, and handle predecessor sets containing both retained and +new split identities. It must avoid moving the current full-domain publication +cost into per-split member reconstruction. The existing synchronous algorithm +and independent source oracle remain its references. Complete retained-result +cost, resource ownership and exact LRSpec output remain required controls. +No new production speed or complexity claim is delivered from this experiment. + +** Restoration and retained test receipt + +Production compiler source is restored byte-for-byte from the current head and +the affected native interfaces rebuilt with the same GCC SDK; controller exit 0 +in 9.291 seconds. The resolved/canonical source owners, including both new +permuted-ID controls, pass 57 Cases (37 + 20), =TEST-SUITE-OK= and =OK=; +controller exit 0 in 23.545 seconds under unchanged 120-second total/5-second +idle limits. Only the existing canonical test Case and this Org record change. + +The unchanged compiled consumer is reused: 372 semantic Cases across 23 groups +pass, and its five independent research entries also exit 0. The controller +emits =DSL-RESEARCH-CLOSURE-OK= and =OK=. This is a rerun of the unchanged compiled +production source; no new full preparation receipt or candidate production +optimization is claimed. Repository test/preparation owners are linked in the +preceding verification section; host-specific temporary files are not Git +references. + +* Persistent group identity experiment (2026-10-09) + +** Paper transfer and implementation + +[[https://arxiv.org/pdf/cs/0612120][Ranzato and Tapparo, Introduction, pp. 1-2]] distinguishes correctness of +stable refinement from the scheduling changes that earn its logarithmic bound. +The next candidate implements the preceding structural obligation: retain group +identities and intrusive membership chains across synchronous rounds, process +only queued nonsingleton groups, and canonicalize once at final publication. + +All signatures read the same immutable round IDs. Planned target IDs are +recorded first; member links and IDs are committed only after every signature +read completes. The first child retains the parent identity and other children +receive fresh dense internal IDs. Only successors of members of actually split +source groups are queued, with group-level deduplication. Singleton targets are +excluded. Final publication assigns first-seen IDs without mutating the input. +Signature equality must be invariant under bijective renaming; dependency +visitors must include every shift/epsilon edge, including self edges. + +The candidate uses ten invocation-owned vectors rather than six. Group +cardinalities and membership are initialized once. Per-round sizing and global +publication scans disappear; visited member reconstruction replaces them. +Internal IDs and chains need not follow node order, so final publication is a +separate semantic obligation. The final ID map reuses the member-head workspace. +This is still synchronous signature refinement, not smaller-half splitter +scheduling, and establishes no =O(m log n)= bound. + +** Semantic and complete-cost controls + +All eight LR modules of the native baseline are frozen from +=891364cc476b45bf588b4266d828033c83b1c1e7=. Both sides use the same GCC SDK, +optimization, tracing and 3 GiB heap. The 90-second total and 5-second idle +limits remain unchanged. The existing source owners pass 57 Cases; 25 full +LRSpec comparisons across five grammar families and five construction modes +are exact. Timing includes setup, construction, in-call GC and publication; +imports, explicit preceding collection and artifact equality are excluded. + +Each protocol has 22 alternating pairs for each mixed FOLLOW size (8/24/48) +and actual TLA+ LALR, plus eight actual TLA+ FOLLOW pairs. Even medians average +the two middle samples. The retained protocol holds the first timed result +while constructing the second and checks both timed LRSpecs; the released +protocol checks full artifact equality before timing and discards timed results. + +| Protocol | Workload | Baseline CPU median ms | Candidate CPU median ms | +|----------+----------+------------------------+-------------------------| +| Retained | Mixed FOLLOW 8 | 1.5285 | 1.5220 | +| Retained | Mixed FOLLOW 24 | 4.1270 | 4.1210 | +| Retained | Mixed FOLLOW 48 | 9.9790 | 10.0290 | +| Retained | TLA+ LALR | 14.2050 | 14.1805 | +| Retained | TLA+ FOLLOW | 1712.2985 | 1734.3080 | +| Released | Mixed FOLLOW 8 | 1.5345 | 1.5530 | +| Released | Mixed FOLLOW 24 | 4.1960 | 4.1300 | +| Released | Mixed FOLLOW 48 | 10.0340 | 9.9455 | +| Released | TLA+ LALR | 14.3535 | 14.3350 | +| Released | TLA+ FOLLOW | 1813.1530 | 1786.0880 | + +Controllers exit 0 in 61.229 and 55.125 seconds, respectively, with complete +artifact checks and =STABLE-NATIVE-AB-OK=. Retained TLA+ FOLLOW regresses 1.29%; +released improves 1.49%. Every retained candidate-second pair regresses +(4.19% to 29.87%), while every candidate-first pair improves +(4.01% to 20.18%). Collections in retained pairs 0/2/4 rise from 9/9/12 to +14/14/16; pair 6 has six collections on both sides and still regresses 4.19%. +CPU less in-call GC medians are 1252.8425 -> 1266.9620 ms retained and +1238.4110 -> 1231.1790 ms released. These are diagnostic decompositions, +not independent speed gates or proof of a complete GC root cause. + +The conflicting protocols and order sensitivity do not establish a complete +construction benefit. The candidate is removed; membership chains, appended +internal group IDs and additional workspaces do not enter production. No +runtime/FFI speed or asymptotic complexity gain is claimed. + +** Permanent semantic coverage + +The existing canonical refinement Case gains 512 six-node controls: 64 labelled +cyclic graphs, four initial partitions and both forward/reversed node orders. +They include duplicate edges, conditional self edges and cross-group edges. +The reverse control remaps all edge endpoints and initial IDs, rather than +merely permuting group names. Each checks exact first-seen IDs/group count +against the independent exhaustive synchronous oracle, preserved input and +signature-call count bounded by exhaustive work. These controls apply to the +existing engine independently of the rejected membership-chain implementation. +The Case and Suite counts stay unchanged. + +** Restoration and closure + +Production compiler source is restored byte-for-byte from the baseline head; +its three affected native interfaces rebuild with GCC in 8.951 seconds, exit 0. +The candidate passes the expanded 20-Case canonical Suite in 15.726 seconds. +The restored resolved/canonical owners pass 57 Cases (37 + 20), including all +512 new controls, in 20.867 seconds with =TEST-SUITE-OK= and =OK=, exit 0. +All total/idle budgets remain unchanged. + +The unchanged compiled production consumer is reused: its 372 semantic Cases +across 23 groups and all five independent research entries exit 0. The controller +emits =DSL-RESEARCH-CLOSURE-OK= and =OK=. The new canonical source controls are +excluded from that compiled consumer and are qualified separately above. +No new full preparation or production performance receipt is implied. +Only [[file:../../t/canonical-lr1-reference-test.ss][the canonical source test owner]] and this Org record are delivered. + +The remaining structural obligation is to establish where full construction +CPU/allocation is spent before selecting the next refinement representation. +Removing repeated whole-domain scans alone did not earn admission here. +Group identity, signature representation and member/worklist scheduling must +be measured together, with canonical artifact identity and result lifetimes +preserved. Released and retained costs remain separate evidence; neither +protocol can substitute for the other. + +* Stable FOLLOW edge publication and indexed FIRST reuse (2026-10-09) + +** Ownership and semantic obligations + +The backward refinement owner already computes successor signatures. The old +[[file:../../src/compiler/lr-follow-automaton.ss][FOLLOW construction owner]] then recomputes every lookahead vertex's edge +before determinization, checking that each member agrees. The implemented +handoff publishes the final shift/epsilon vectors from backward refinement and +passes them directly to determinization. These are the same two vectors that +determinization previously allocated, with no new public language slot or ABI. + +A no-split final round preserves block order and IDs. Its active nonsingleton +keys were computed against that round's final index and can be reused. Inactive +and singleton keys may contain earlier round IDs: they are never reused. One +representative is evaluated against the final index for each such stable block. +The stable-partition invariant guarantees that all block members have equal +successor signatures. Reordered production controls and independent canonical +construction qualify this invariant instead of a duplicate production traversal. + +The first native experiment implements only this handoff. On actual TLA+, +=stable-edge-reuse= reports zero reused keys out of 8888 backward blocks: the +last round has no active nonsingleton block. Eliminating the final per-member +pass alone is insufficient to infer meaningful complete cost reduction. +The retained TLA+ FOLLOW median changes 1712.9290 -> 1705.1450 ms (0.45% lower), +with the same invocation-order sensitivity as preceding experiments; its +controller exits 0 in 58.587 seconds. No independent speed claim is admitted +for that variant. + +The next implementation decomposes each vertex's epsilon targets: + +#+begin_example +fixed(core, index) = union over children and FIRST(tail) terminals + of singleton(index[child, terminal]) +epsilon(core, follow, index) = fixed(core, index) + union nullable follow targets +#+end_example + +Set union distributes over the FIRST/follow union and is idempotent, including +when follow is already in FIRST(tail). The fixed portion is translated once per +queried core into block IDs, using one lazy vector local to the current round. +=#f= means uncomputed; an empty set (=0=) is a valid cached result. A new round +gets a fresh vector because splitting can renumber targets. Final publication +uses the same last-round vector; it cannot carry stale targets across rounds or +invocations. Fixed FIRST expansion changes from repeated work per vertex to +once per queried core per round; dynamic nullable follow targets remain exact. +No smaller-half scheduling or logarithmic complexity bound is asserted. + +** Qualification protocol + +The native baseline's eight LR modules are frozen from =891364c= and verified +byte-identical to the production LR modules at the current =fa0f283= head. +GCC optimization, 3 GiB heap, tracing, 90-second total and 5-second idle limits +are unchanged. Complete artifact comparisons cover five grammar families and +five modes. Each cost protocol alternates 22 pairs for mixed FOLLOW sizes +8/24/48 and actual TLA+ LALR, and eight actual TLA+ FOLLOW pairs. Setup, +construction, in-call GC and final publication are included; import, explicit +preceding GC and equality checks are excluded. Retained and released result +lifetimes remain separate controls; even medians average the two middle samples. + +The canonical source owner gains a production-reordering Case across cyclic, +acyclic and nullable-tail families, two sizes and both forward/reversed +production orders. Each compares accepted/rejected parse products with the +independent canonical LR(1) route for valid and invalid token sequences. +Nullable-tail controls exercise fixed FIRST plus dynamic follow targets after +block renumbering. Source totals become 58 Cases (37 resolved + 21 canonical). + +** Complete-cost results with indexed FIRST reuse + +| Protocol | Workload | Baseline CPU median ms | Candidate CPU median ms | +|----------+----------+------------------------+-------------------------| +| Retained | Mixed FOLLOW 8 | 1.5495 | 1.5375 | +| Retained | Mixed FOLLOW 24 | 4.1955 | 4.1920 | +| Retained | Mixed FOLLOW 48 | 10.1790 | 10.1830 | +| Retained | TLA+ LALR | 14.5245 | 14.4460 | +| Retained | TLA+ FOLLOW | 1910.1755 | 1406.1465 | +| Released | Mixed FOLLOW 8 | 1.5245 | 1.4775 | +| Released | Mixed FOLLOW 24 | 4.1410 | 4.1275 | +| Released | Mixed FOLLOW 48 | 9.8985 | 9.9585 | +| Released | TLA+ LALR | 14.1845 | 14.1905 | +| Released | TLA+ FOLLOW | 1762.0145 | 1636.7625 | + +Both protocols pass all 25 family/mode LRSpec comparisons and the additional +complete workload artifact checks, including actual TLA+ FOLLOW. Controllers exit 0 with =EDGE-NATIVE-AB-OK= in 56.020 seconds +retained and 52.101 seconds released. Actual TLA+ FOLLOW complete CPU medians +improve 26.39% and 7.11%, respectively. Every retained pair improves (6.67% to +35.74%). Seven released pairs improve; pair 5 regresses 3.69%, with collections +increasing from 11 to 13 and in-call GC CPU from 479.2500 to 592.9950 ms. +This negative pair is retained rather than presenting a uniform speedup. + +CPU less in-call GC medians change 1236.7045 -> 1100.9600 ms retained and +1245.4975 -> 1094.6945 ms released. Those decompositions support reduced +non-GC construction work, without attributing all complete cost change to one +mechanism. Retained GC counts change from 6 to 5 in baseline-first pairs and +17 to 7 in candidate-first pairs. The invocation-order effect remains visible; +this experiment does not prove a general GC/root-retention solution. + +Mixed FOLLOW 48 increases 0.04% retained and 0.61% released; the smaller +synthetic medians are near unchanged or lower. Default TLA+ LALR bypasses this +FOLLOW change, so its small timing differences are controls rather than an LALR +optimization claim. Admission is limited to this Scheme FOLLOW construction +change and the measured actual TLA+ workload. Runtime parsing, Rust/FFI, +all-grammar acceleration and asymptotic improvements are not established. + +** Final preparation and source receipts + +The final FOLLOW/compiler native interfaces compile with GCC, exit 0 in +9.238 seconds. Final resolved/canonical source qualification, including all +nullable-tail and production-reordering controls, passes 58 Cases (37 + 21) +in 18.343 seconds, =TEST-SUITE-OK= and =OK=, controller exit 0. Independent +full LRSpec checks and paired complete costs are recorded above. + +Full conformance preparation finishes in 106.636 seconds, exit 0, with +=PREPARATION-STORED-RECEIPT= (2013 files) under unchanged 240-second total and +5-second idle limits. Preparation duration is a build receipt, excluded from +construction CPU measurements. Scheme owns the qualification and GCC remains +the compiler. No compatibility mode, heap/deadline increase or language schema +change is introduced. + +The freshly prepared compiled consumer passes 372 semantic Cases across +23 groups. Its five independent research entries also exit 0; all six child +statuses are zero and the controller finishes with =DSL-RESEARCH-CLOSURE-OK= +and =OK=, exit 0. The 58 source Cases remain a separate owner and receipt. +The production change is confined to the FOLLOW compiler; public LRSpec, +language loaders, Runtime and Rust/C ABI are unchanged. + +* Covered-root elimination in memoized epsilon closure (2026-10-09) + +** Paper transfer and shared algorithm candidate + +[[https://aclanthology.org/J00-1005.pdf][Van Noord, Treatment of Epsilon Moves in Subset Construction (2000), +Section 3.2, pp. 66-68]] compares per-state and per-subset closure strategies. +Per-state memoization still pays to union the cached results for each seed; +whole-subset memoization avoids repeated complete queries. The engine already +has both a graph-sized singleton cache and a bounded seed-to-state cache. +This experiment studies redundancy inside a remaining seed-query miss. + +The following additional elimination is our derivation from reflexive +transitive closure, rather than a scheduling theorem stated by that paper: +if =u= is already in a computed closed set =C=, then =closure({u})= is a subset +of =C=. Select one pending seed, union its cached closure into the result, and +subtract that closure from all pending seeds before selecting the next one. +The selected seed is removed by reflexivity, so the loop terminates. Each +omitted seed's entire closure is already covered by transitivity. Therefore +the result equals the union of all singleton seed closures in the old engine. + +The candidate moves the language-specific epsilon memo helper into the shared +indexed-set engine as =compiler-index-set-reachable/memo=. Graph edges are +immutable; the caller owns a graph-sized cache and must never reuse it with a +different graph. Cache entries remain exact singleton closures and only roots +actually queried are populated. Empty seeds return zero. No SCC graph, extra +closure table, unbounded subset retention, public language slot or compatibility +path is added. The existing large-quotient admission cutoff remains unchanged. + +The canonical reachability Case adds cold and warm memo controls for all 512 +directed three-node graphs and all eight seed sets, using its independent list +graph oracle. Populated cache entries are also checked against independent +singleton results. A 1024-node chain/cycle control covers high-bit seeds and +checks that already covered chain roots need no cache entries. Source counts +remain 58 Cases (37 + 21); these controls are added inside the existing Case. + +** Artifact and build ownership + +The frozen baseline contains all eight LR modules from +=d6e6b8a98445c9e7920c029d65f11fe3172556f8=. Each side uses the same GCC SDK, +optimization level and heap. The diagnostic configuration is stated below. Two initial baseline builds fail the +unchanged five-second idle guard while GCC compiles the generated C unit; +GSC has already emitted C and GCC emits no intermediate stdout. Those failed +builds are not qualification receipts. A prematurely started source run also +fails because the new helper interface was not yet compiled, and is excluded. + +GCC's =-fopt-info-optimized= diagnostic emits actual completed optimization +remarks without changing the optimization level. The completed remaining build +exits 0 in 18.418 seconds; the other six frozen baseline modules are rebuilt +with the same diagnostic option, exit 0 in 15.737 seconds. Budgets remain +240 seconds total and five seconds idle. No compiler switch or synthetic +heartbeat is used. An initial native A/B import run also fails the idle guard; +explicit per-module import completion receipts localize and qualify real module +initialization before timing. All imports remain excluded from measurements. + +** First implementation and negative complete cost + +The first implementation selects the root by isolating the least significant +bit with =pending & -pending= and applying =integer-length=. It passes the +58 source Cases in 23.408 seconds, with =TEST-SUITE-OK= and =OK=, exit 0. +The retained native protocol validates 25 family/mode LRSpecs and the five +complete workload artifacts, then alternates 22 pairs for mixed FOLLOW sizes +8/24/48 and actual TLA+ LALR, plus eight actual TLA+ FOLLOW pairs. Construction, +in-call GC and publication are timed; import, explicit GC and equality are not. + +Actual TLA+ FOLLOW CPU medians regress 1515.7385 -> 1855.9905 ms (22.45%). +CPU less in-call GC medians increase 1107.7570 -> 1324.9600 ms. Seven of eight +pairs regress. The controller exits 0 in 57.356 seconds with +=DELTA-NATIVE-AB-OK=; exact artifact equality does not admit the cost. +The dense chain eliminates all covered roots after one query, but that control +cannot establish an actual parser workload improvement. + +The second implementation replaces wide integer root-isolation intermediates +with the existing =first-set-bit= primitive and constructs a singleton only +on a cache miss. It preserves the same pending-set algorithm and tests. +Difference operations and lazy cache population can still offset skipped +queries; the arithmetic rewrite alone is not an admission result. + +** Primitive selection remains negative; exact algorithm extraction + +The second candidate passes the 58 source Cases in 26.310 seconds, exit 0, +and the same exact native LRSpec controls. Retained TLA+ FOLLOW CPU medians +regress 1442.2830 -> 1697.3595 ms (17.69%). CPU less in-call GC medians +increase 1114.8445 -> 1182.6900 ms. Seven of eight pairs regress, despite +removing root-isolation intermediates; the controller exits 0 in 54.214 seconds +with =DELTA-NATIVE-AB-OK=. Both difference-loop candidates are removed. +Released-result timing cannot establish admission once the retained consumer +has already demonstrated this regression, so it is not repeated for them. + +The final production change is instead an exact extraction of the existing +per-seed closure union into the shared indexed-set engine. The function body +matches the previous private =epsilon-closure/memo= body after renaming. +The FOLLOW owner imports it directly; the old private name is removed, with no +wrapper or compatibility path. Field-based/sparse seed enumeration, cache +population, graph admission threshold and cache capacity remain the existing +algorithms. This supplies one generic engine capability for compiler callers, +not a claimed performance improvement or a new POO object exposed to language +authors. The cache's immutable graph and invocation ownership contract is +stated beside its implementation. + +Cold/warm graph controls and exact populated-entry checks remain applicable to +this extracted algorithm. The experimental assertion that covered roots stay +uncached is removed with the rejected scheduling strategy; it is not a semantic +contract. The permanent 1024-node controls retain overlapping, high-bit seed +results, exact root-zero cache content and independent chain/cycle cache owners. + +** Shared extraction qualification boundaries + +The extracted body is checked byte-for-byte against the previous private body +with only its function name replaced (=MEMO-EXTRACTION-BODY-EXACT=). Its three +native interfaces compile, exit 0 in 10.651 seconds. One source run prints all +58 Case successes and =OK= but fails at process completion with idle timeout +in 47.855 seconds; that is not a passing receipt. A final complete source run +passes 58 Cases in 20.509 seconds with =TEST-SUITE-OK=, =OK= and controller exit 0. +Neither timeout nor idle budgets change. + +The attempted complete retained A/B run for the extraction fails its 90-second +total deadline (91.057 seconds including termination), without the final marker. +Six reported TLA+ FOLLOW pairs are an incomplete batch and are not admitted as +a performance result. Concurrent external GCC processes are observed, but that +observation does not prove a complete cause for the timeout or exit-phase stall. +These historical failures remain distinct from later passing semantic receipts. +No new speed gain, stable latency or general timeout fix is claimed for extraction. + +The final native semantic-only protocol passes all 25 family/mode full LRSpec +comparisons and all five complete workload comparisons (including actual TLA+ +LALR/FOLLOW), =MEMO-NATIVE-EXACT-OK=, controller exit 0 in 24.881 seconds. +It deliberately performs no paired timing: this is artifact equality evidence, +not a replacement or relaxed budget for the failed performance protocol. + +Full preparation emits =PREPARATION-STORED-RECEIPT= for 2062 products, but its +controller subsequently exits with idle timeout in 242.276 seconds. A stored +receipt alone does not pass process qualification. The next preparation run +rechecks the same source/toolchain identity and all stored/current product +content hashes, admits =CONFORMANCE-CACHE-HIT= and exits 0 in 11.857 seconds +under the original 240-second total/five-second idle limits. This is valid +content admission of the freshly generated artifacts; the preceding failed +process remains recorded, and no successful full-build duration is claimed. + +The first compiled semantic process fails the five-second idle guard before +any Case output, so its research controller exits 70 and the remaining entries +are not qualified in that attempt. A =DYLD_PRINT_LIBRARIES= diagnostic run then +passes all six entries. An attempted process sample finds that the semantic +process has already completed; no stack-based root cause is established. +The final run removes that diagnostic environment and again passes all six +entries, each exit 0: 372 semantic Cases across 23 groups and five independent +research owners. The controller finishes with =DSL-RESEARCH-CLOSURE-OK= and +=OK=, exit 0. This is ordinary compiled-consumer closure under unchanged budgets, +not proof that all startup/exit timeout causes have been fixed. + +Delivered changes are the direct shared primitive extraction, its FOLLOW +consumer, cold/warm exhaustive graph controls and this Org research record. +Both covered-root difference schedulers are rejected. The preceding indexed +FIRST/stable-edge optimization remains intact; no additional construction speed, +Runtime speed, Rust/C ABI change or language-author configuration is delivered. diff --git a/docs/research/dsl-engine-contracts.org b/docs/research/dsl-engine-contracts.org new file mode 100644 index 00000000..12fb7b10 --- /dev/null +++ b/docs/research/dsl-engine-contracts.org @@ -0,0 +1,175 @@ +#+title: DSL semantics and parser engine contracts +#+date: 2026-10-09 +#+startup: overview + +* Scope and revision + +Current source audit baseline: =a90f23646e4bf43d9d1562d07f45cd3b2fb2674f=, +PR #13, with the productivity entry-admission implementation described below. +Implementation means a mechanism exists in this source revision. Qualification +means the stated command completed with propagated status and its final marker. +Historical receipts retain their own revisions. Neither inspected test code nor +a paper theorem establishes a new execution receipt or a general parser bound. + +The author boundary remains =grammar.ss=, =parser.ss= and =parser-test.ss=. +Syntax expresses language intent; checked POO composition selects effective +values and source origins; engine admission checks the requested target before +lowering and publication. Metadata belongs to the parser entry. Ordinary slot +values need no author-written thunk; engine mechanics remain engine-owned. + +* One paper-to-code argument + +| Primary reading | Transfer to this engine | Evidence and limitation | +|-----------------+-------------------------+-------------------------| +| [[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros]], sections 3--4 and 7.2 | Distinguish declaration shape, resolved obligations and contextual diagnostics | Gerbil macros are the substrate; this does not implement Racket syntax classes | +| [[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][A Typed, Algebraic Approach to Parsing]], sections 3, 5.2 and 6 | Derive static facts before target specialization; retain CFG meaning | Its predictive grammar restrictions and linear parsing theorem do not cover every LR/Source target here | +| [[file:scheme-dsl-research-basis.org::*Abstract interpretation: what can declarations teach the compiler?][Abstract interpretation study]] | Separate fixed-point meaning, convergence and efficient evaluation | Nullable/FIRST/productivity have completed dependency analyses; LR entry admission consumes productivity | +| [[file:dsl-effective-grammar-admission.org::*Preserve expression occurrences through transformations][Practical LR study]] and [[file:lr-conflict-source-origins.org][rejected conflict origins]] | Retain expression occurrences when lowering helpers and investigate measured conflicts | Selected repetition and rejected LR action origins exist; admitted fork and runtime origin catalogs remain open | +| [[file:dsl-lr-transfer.org][Incremental parsing study]] | Admit reuse by source/context and parser boundary obligations | Scheme scanner sessions and LR fragment controls exist; they do not establish general Rust Source subtree reuse | + +The integrated argument is: author declarations select one effective grammar; +analysis establishes facts about that grammar; each target checks its own +obligations; generated execution and artifact publication preserve the selected +meaning. A legal LR grammar need not satisfy a predictive target's proof. Target +failure must report the unsupported obligation rather than change CFG choice +into prioritized choice or silently substitute another backend. + +* Current implementation map + +| Responsibility | Current mechanism and owner | Remaining boundary | +|----------------+-----------------------------+--------------------| +| Author syntax and POO composition | =defgrammar-syntax=, =deflanguage= and =src/compiler/normalize.ss= | Keep future admission and resource mechanics out of language author slots | +| Resolved repetition and finite LR entry admission | =src/compiler/lr.ss= and =t/resolved-grammar-test.ss= | General predictive choice-equivalence proof; productivity is not lexer realizability | +| Nullable/productivity analysis | Same owner; one completion graph with separate domain counters and coalesced fact deltas | FIRST uses its own prefix dependency worklist; no general whole-parser complexity claim | +| Selected origin diagnostics | =expressionOrigin= and finite =nullableReferencePath= | Generated helper/conflict origin serialization and unavailable reader coordinates | +| Source/Command scheduling | =command_program.rs= and =command_source/execution.rs= | Complete byte/allocation/work accounting beyond the live-frame budget | +| Canonical Source publication | =command_source/publication.rs= and Scheme-native fixtures | Rust GPA1 encoding, diagnostic transport and deep live C handle parity | +| Incremental admission | Scheme scanner sessions and LR fragment transfer | General cross-edit Rust Source subtree ownership and mapped publication | +| Test preparation/execution | One compiled semantic dispatch; remaining Suite owners use persistent Scheme Workers | Preserve genuine source and ABI controls; avoid duplicating their Cases | + +Detailed contracts are owned by these documents: + +1. [[file:dsl-recognition-contracts.org][Recognition semantics and target obligations]] +2. [[file:dsl-effective-grammar-admission.org][Effective grammar admission and source origins]] +3. [[file:dsl-source-execution.org][Source execution and artifact ownership]] +4. [[file:dsl-scanner-reuse.org][Scanner contexts and suffix convergence]] +5. [[file:dsl-lr-transfer.org][LR fragment transfer and shared terminal yields]] +6. [[file:dsl-test-preparation.org][Shared test runtime and preparation ownership]] + +* Qualification of the nullable algorithm slice + +=de8bef5= replaces whole-production nullable rescans with occurrence dependencies. +For P productions and R rhs occurrences, expected constant-time table operations +give O(P + R) graph construction and propagation. Each nullable name publishes +once and each registered occurrence is discharged once. The public ordered list, +lookup membership and selected-origin repetition diagnostics retain their contract. + +Local qualification completed 355 compiled semantic Cases across 23 groups and +17 source-mode canonical LR reference Cases. The resolved grammar Suite includes +512 cyclic grammar combinations against an independent repeated-scan oracle, +marked operands, terminal blockers, duplicate occurrences, first-lhs ordering and +a 4096-name reverse chain. Complete TLA+ LRSpec equals the pre-change snapshot. + +Ten real TLA+ complete construction samples had CPU medians 15.068 ms before and +14.868 ms after, with no in-call GC. This difference is insufficient to claim +end-to-end acceleration. It does not measure Rust execution, FFI or incremental +parse latency. Latest delivered-head CI is a separate gate; inspect the PR check +for that exact SHA before treating a local receipt as remote closure. + +* FIRST dependency follow-up + +[[file:dsl-effective-grammar-admission.org::*FIRST propagation: contract and implementation][The FIRST implementation contract]] records the completed nullable-prefix graph, +dense private indices, coalesced deltas, canonical output order, independent +oracles and matched native A/B. This gives the existing LR consumer dependency- +driven analysis without changing target admission. Long reverse-chain behavior +improves structurally; the measured real full-construction difference does not +establish a stable end-to-end gain. The initial slower candidate remains recorded. +After rebasing onto =344826a=, local qualification passed 44 source-mode +resolved/canonical LR Cases, 361 compiled semantic Cases in 23 groups and all +five independent research entries. Exact delivered-head CI remains separate. + +* Productivity entry admission + +[[file:dsl-effective-grammar-admission.org::*Productivity: finite derivation and LR entry admission][The productivity contract]] records the finite-word meaning, occurrence worklist, +selected-rule blocker graph and deliberate LR target policy. An unproductive +selected entry now fails before state construction, across all existing routes +and conflict policies. Epsilon exits and productive LR recursion remain valid; +dead alternatives and unrelated dead rules do not reject a productive entry. +Public LRSpec output for admitted grammars remains unchanged. + +Local qualification completed 52 source Cases, 369 compiled semantic Cases +in 23 groups, all five independent research entries and an exact complete +TLA+ LRSpec comparison. The 15.0955 ms complete construction median after adding +the check is not evidence of acceleration. Exact delivered-head Ubuntu/Rust CI remains a separate receipt. + +* Shared completion analysis + +[[file:dsl-productive-entry.org::*Joint completion solver: one graph, two meanings][The joint solver contract]] records how nullable/productivity share one indexed +occurrence graph while preserving their distinct equations. LR construction now +calls =compute-completion= once. Existing standalone observations project its +completed result. Nullable implication permits two-bit publication; productivity +alone cannot imply nullable. FIRST still consumes the complete nullable table. +Local qualification passed 53 source Cases and 370 compiled semantic Cases in +23 groups plus the five independent research entries. Same-tier native A/B kept +complete LRSpec equality; real TLA+ completion facts measured 0.145 → 0.079 ms. +The full-construction difference (14.715 → 14.525 ms) is too small for a stable +end-to-end gain claim. Exact-head CI is separate. + +* Compositional LR suffix analysis + +[[file:dsl-effective-grammar-admission.org::*Unified suffix facts across LR consumers][The shared suffix contract]] now covers FIRST/nullable for all valid dotted +positions. LALR/canonical closures and FOLLOW metadata read core+1; backward +compatibility partitions read core, including the empty suffix. All consumers +use =make-core-suffix-catalog=; the old lookahead-specific directory is removed. +Independent forward-scan, complete partition and full LRSpec controls cover the +position change. Padding allocation, repeated catalog construction across calls +and large-mask width remain explicit costs. Matched native FOLLOW analysis +improved; full LALR did not show a gain. Full TLA+ FOLLOW determinization and the +Ubuntu semantic signal remain separate unresolved qualification work. + +* Next implementation sequence + +| Order | Concrete code/design work | Consumer and required control | +|-------+---------------------------+-------------------------------| +| 1 | Extend completed finite-derivation facts only for a defined target consumer | LR entry policy is implemented; lexer realizability and derivation certificates need separate contracts before making stronger claims | +| 2 | Extend FIRST/nullable evidence to any proposed target specialization | Keep the implemented completed facts and canonical order; compare full target artifacts and cost before changing admission or emission | +| 3 | Specify target-local CFG choice admission | Keep the reviewed HCL emitter restriction until equivalence is proved; matched token segmentation, common-prefix/nested-node controls, exact output or explicit rejection | +| 4 | Extend Source resource and ABI qualification | Deep success/failure cleanup, independent next request, owned callbacks and diagnostic transport; account for bytes/work separately from frame count | +| 5 | Join mapped scanner and parser subtree reuse at a defined target boundary | Fresh-parse artifact equality on Unicode, quotes, heredocs, delimiter and context edits; preserve new source identity and release ownership | + +Each item must define the existing engine owner, semantic consumer, negative +control and measurement boundary before adding a new abstraction. POO remains +the public extension mechanism where values are extensible; private algorithm +state uses named functions and compact records. No new loader schema or public +compatibility mode follows from this research sequence. + +* Compiled declaration-test host contract + +Declarative syntax/source checks must not turn input AST depth into native call +stack growth. Their engine-owned inspection uses explicit frames with lexical +port authority; language authors add neither traversal nor output exceptions. +Deep rejection, legal fixture capture and shadowed-port controls exercise the +same compiled semantic entry used by CI. + +Large expanded test objects retain the immutable SDK's optimizer and tail-call +contract. Their builder selects Gambit's per-procedure C hosts, preventing one +expanded Suite from becoming a giant C optimization unit. Production objects +retain SDK defaults. Effective object options participate in +cache reuse, with success-only Scheme identity records. Full preparation still +proves the source/SDK closure. See the [[file:dsl-effective-grammar-admission.org][admission study]] for Ubuntu stack evidence, +rejected alternatives and delivered-head qualification limits. + +* Completed FOLLOW graph facts and state-local accumulation + +After quotient refinement, static block labels and shift-source membership are +indexed once by the engine. DFA target rows are local to one construction, +reused across states and cleared only at touched labels. Full closed state +identity and reduce/lookahead publication remain unchanged; first-seen labeled +transition order determines the same state numbering. Language loaders contribute +grammar values through their existing POO contract. + +Performance admission compares the entire target LRSpec and includes indexing, +analyses and publication. Actual completed-state/item output is available through +the existing LR trace switch for long constructions. See the [[file:dsl-effective-grammar-admission.org][admission study]] +for the real TLA+ phase counts, exact-output A/B, cost limits and independent +compiled/source controls. diff --git a/docs/research/dsl-lr-transfer.org b/docs/research/dsl-lr-transfer.org new file mode 100644 index 00000000..e3066861 --- /dev/null +++ b/docs/research/dsl-lr-transfer.org @@ -0,0 +1,167 @@ +#+title: LR fragment transfer and shared terminal yields +#+date: 2026-10-09 + +Part of [[file:dsl-engine-contracts.org][the integrated DSL and engine study]]. +Existing dated receipts retain their original revision and scope. + +* LR fragment transfer admission is a correctness contract + +Here, parser state matching concerns a current LR continuation and an actual +retained grammar nonterminal. It is not support for an obsolete framework or +language interface. The shared engine keeps exact prepared-runtime and reducer +identity, entry-state matching and the admitted nonterminal goto/exit state. +There is no old-architecture adapter or alternate grammar declaration path. + +The source owner still certifies unchanged source and lexical entry/exit +boundaries. The LR injection boundary now independently checks the significant +terminal yield: kinds, lexemes and relocated UTF-8 start/end coordinates must +match retained terminal leaves in order. The first transferred token must also +match the current input token. Equal token count alone is insufficient. +Private relocation views accumulate offsets without mutating retained fragments. +Validation completes before a new checkpoint is constructed; rejection leaves +the original continuation usable. + +#+begin_src scheme +;; Source ownership and lexical boundary checks precede this engine operation. +;; The fragment represents a grammar production, not an AST kind. +(def next + (lr-checkpoint-inject-fragment current fragment byte-delta significant-tokens)) +(lr-checkpoint-resume next) +#+end_src + +Why is this preferable? Language authors retain declarative syntax/rules; the +engine validates the transfer using its own grammar structure and continuation. +They do not write per-language reuse callbacks or repeat parser stack logic. +Why is it justified? A state/goto match certifies the grammar transition, while +exact terminal comparison prevents a caller from attaching a retained semantic +value to different token contents or coordinates. Source and lexical admission +remain separate obligations; this check alone cannot admit an arbitrary edit. + +The validator uses an explicit pending work list over recognition fragments and +views, including nested relocation. Its work is proportional to visited +recognition structure plus terminal comparison; lexeme comparison includes text +length. It adds validation work to each accepted transfer. This is a conservative +engineering guard motivated by the context/reuse distinction in +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Wagner and Graham]], not their optimal reuse algorithm or complexity bound. +Rust currently exposes no LR fragment injection API; this change does not claim +a cross-backend incremental session or AST/CST identity-preserving edit API. + +Local qualification (2026-10-08): the engine-only native transfer suite passed +with final =OK= and zero process status in 4.70 seconds. It compares canonical +ParseArtifact publication against fresh parsing, exercises a multibyte terminal +and nonzero relocation, rejects content/kind/coordinate/count/current-input +mismatches, and reuses the original checkpoint after rejected requests. The +broader local recognition suite could not load stale language interfaces; +complete edit/reuse regression remains an exact-head CI gate, not a local claim. + +* Source scanning sessions consume the convergence proof + +=make-source-scan-session= retains private token and checkpoint sequences. On a +new source it reads the longest common suffix from the scanner-owned snapshots, +scans the prefix from the initial state, and binary-searches old boundaries at +the translated character position. It checks plan identity, the fixed parser +position and full delimiter context before admitting the suffix. It never +restarts from an old matching-prefix checkpoint. + +An opaque =contextual-scan-suffix= binds the independently reached boundaries. +Only checkpoints from that old source at or after the admitted boundary can be +rebound to the new owner; character and UTF-8 byte deltas are distinct. The worker +copies token records and marker text stays shared inside private immutable +checkpoints. Public token snapshots are copied, so consumer mutation cannot +change later reuse. Rebound checkpoints support a second edit without rescanning +the old suffix merely to populate the next cache. + +=parse-source-language/session= integrates this scanner cache into the existing +Source language parser. It owns the input, retains the declaration identity and +runs the original recognition/publication path with the new token stream. Every +published artifact is validated against the declaration and new source identity. +Lexical rejection uses the original parser diagnostic path and retains no cache; +ordinary Source engines call their original declared scanner so projection and +mode handling are preserved; they retain no cache and cannot admit lexical reuse. + +Matched Bash controls now demonstrate one scanned token after a Unicode command +prefix edit with the remaining lexical records reused. Changing an unquoted +here-document marker to a quoted marker scans all seven tokens through the +marker close before reusing the four-token footer. Both artifacts equal complete +fresh parses, and a subsequent edit consumes the rebound checkpoints. Source +mutation, published token mutation and unclosed quote recovery have executable +controls. These counts measure skipped scanner recognition, not preserved token +object identity, parser-state reuse or measured end-to-end latency. + +This implementation retains O(tokens) checkpoint storage, reads source suffixes +linearly and copies/relocates the admitted tail. Recognition and canonical CST +publication still execute fully. Rust/native incremental Source transport and +parser-stack/subtree reuse remain separate implementation work; no ABI, schema +version or language declaration changes are introduced here. + +The previous exact head bc233b8 passed both native CI platforms, but the Rust +ASP gate rejected the inline FIFO test and parent-scope glob import. The test +now lives in =tests/unit/scanner_obligations.rs= with explicit imports. The actual +ASP workspace gate passes against the repaired source. On this macOS host, +relinking that gate with the Nix =cc= failed because =-liconv= was unavailable; +the existing gate binary was executed with its manifest path. The current locked +dependency gate was then rebuilt and passed with the macOS native platform +linker; Scheme compilation continued to use GCC 16. + +After rebasing onto =f21204f=, all 33 focused native cases passed with final +=OK=: Source scanner sessions (10), Source strategies (10) and deferred scanner +admission/rebinding (13). The merged scanner compiled with the pinned Gerbil +SDK and host GCC 16. Rebased Rust runtime passed 138 cases, strict runtime +Clippy passed, and the current locked ASP workspace gate passed after native +platform linking. The public session API takes no caller checkpoints; the +low-level suffix rebind helper requires checkpoints from the recorded old +continuation, which the private Source cache supplies. Full recognition/CST +execution, Rust/native incremental transport and exact-head CI remain distinct +qualification boundaries. + +* Shared terminal yields for LR fragment transfer + +Captured LR fragments now retain a private shared terminal-yield rope. A shift +contributes its existing terminal reference. A reduction concatenates child +yields without flattening descendant tokens; empty and unary grammar wrappers +share the existing yield instead of adding nodes. Relocated children contribute +coordinate views, whose repeated moves collapse their deltas. The original +grammar children, cached semantic value, entry/exit state and lookahead remain +available to the source owner and independent recognition projection. + +The LR transfer validator now walks this yield instead of every retained grammar +production. It still compares each significant terminal's kind, complete lexeme, +relocated UTF-8 start/end, exact count and current first input token. Runtime, +reducer, goto, source ownership and lexical boundary admission remain separate +checks. No digest-only acceptance or grammar/publication schema change is added. + +For an unrelocated yield of T nonempty terminals, the binary representation has +T leaves and T-1 branches. Validation therefore visits 2T-1 nodes regardless of +unary grammar wrapper depth. Coordinate views add explicit visits. The traversal +is iterative; this is not a balanced tree or a logarithmic edit algorithm. +Construction adds one reference per captured grammar fragment and shared branch +records for concatenation, plus relocation views. Capture has extra work and +retained storage; a flat token list is never copied for each ancestor. Ordinary +parses without recognition capture do not construct the yield index. + +The optional =current-lr-transfer-yield-observer= reports the actual visited yield +node count once per validation attempt. It is observational and is disabled by +default; source/grammar certificates do not use its output. The retained fixture +freezes the f21204f grammar-tree traversal as an independent admission control. +A 64-layer grammar with two multibyte-source terminals visits 67 grammar-tree +nodes in that reference and 3 yield nodes in the indexed validator; complete +canonical ParseArtifact publication equals a fresh parse. A 512-terminal +left-recursive control visits 1,023 yield nodes, rejects a changed last terminal, +and reuses the original checkpoint after rejection. Epsilon reduction and a +recaptured relocated child plus a fresh sibling have canonical fresh-parse and +independent projection controls, including another relocation of the new root. + +Local qualification: the pinned Gerbil SDK and GCC 16 compiled the updated LR +engine. All five native transfer cases passed with final =OK= and process status +zero, under the existing 90-second total and 5-second silence gates. These are +structural work counts and semantic receipts, not a measured end-to-end runtime +speedup. Source command/CST subtree reuse, cross-backend incremental transport +and the pushed revision's full CI remain separate qualification boundaries. + +Qualification correction: revision =169caf7= failed the Ubuntu test-style gate +because its ordinary transfer test printed a work-count line. The line was +removed, and the reference count now asserts =67= directly against the yield +count =3=. All five native transfer cases and the test-style contract module +passed locally with final =OK=, zero process status and 13.99 seconds elapsed. +Policy restrictions and timeout/silence limits remain unchanged; latest-head +full CI is still a separate gate. diff --git a/docs/research/dsl-productive-entry.org b/docs/research/dsl-productive-entry.org new file mode 100644 index 00000000..25844196 --- /dev/null +++ b/docs/research/dsl-productive-entry.org @@ -0,0 +1,259 @@ +#+title: Productive LR entry admission +#+date: 2026-10-09 +#+startup: overview + +Part of [[file:dsl-engine-contracts.org][the integrated DSL and engine study]]. +Implementation and reported qualification below belong to =2bfec1f=. + +* Productivity: finite derivation and LR entry admission + +The shared =compute-productive= analysis and =validate-resolved-start= consumer +are implemented in =src/compiler/lr.ss=. LR compilation checks the selected start +before FIRST/state construction. This is an intentional admission change for an +entry with no finite derivation. The domain is the existing +lowered production graph after POO selection. Selected declaration origins must +remain available to the diagnostic consumer. + +For a nonterminal A, =productive(A)= means that A has at least one finite +derivation to terminal symbols, possibly epsilon. The equations are: + +#+begin_example +productive(terminal) = true +productive(empty rhs) = true +productive(rhs sequence) = every rhs operand is productive +productive(A) = some production for A has a productive rhs +nullable(A) implies productive(A) +productive(A) does not imply nullable(A) +#+end_example + +The least fixed point starts with no productive nonterminals. Pure cycles add +no fact. Terminal occurrences satisfy productivity obligations, whereas they +block nullable clauses. Thus the current occurrence propagation is a relevant +implementation pattern, but its terminal policy cannot be reused unchanged. +A private graph-sharing abstraction requires actual consumers and measured +benefit; do not add an author callback or a new language-level analyzer slot. + +Examples establish the necessary distinctions: + +| Effective rules | Nullable | Productive | FIRST alone sufficient? | +|-----------------+----------+------------+-------------------------| +| A -> A | false | false | No finite exit exists | +| A -> epsilon | true | true | Empty FIRST can still have a finite derivation | +| A -> A a or b | false | true | Ordinary productive LR left recursion is valid | +| A -> a B; B -> B | false | false | FIRST can contain a even though no derivation completes | + +** Target policy and paper boundary + +For every existing =compile-lr-spec= route and conflict policy, an unproductive +selected root is an author error: =LR start rule has no finite terminal derivation=. +The lowered grammar is analyzed after POO selection and repetition validation; +no additional author slot or schema version is introduced. A productive root is +not rejected by productivity because of an unreachable dead rule or a dead alternative. +Independent conflict obligations remain: a nullable optional around a pure unit +cycle can still have a reduce/reduce conflict despite its epsilon exit. Neither +the lowered productions nor public LRSpec rows are pruned or changed. Existing +concise-language missing-reference validation still owns that distinct error. + +This target policy is our design choice, not a theorem or requirement from +[[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][Krishnaswami and Yallop, section 3]]. Their algebra includes the empty language +and distinguishes it from the singleton empty word. We preserve that distinction +in the analysis while requiring a usable finite entry for the current LR target. +The paper's predictive typing/linear parsing theorem is not extended to our LR +routes. Terminal productivity is over abstract grammar symbols: it does not +prove lexer realizability, successful precedence resolution, semantic action +termination, tree equivalence or a particular input's acceptance. + +** Occurrence worklist and explanation + +Each lowered production has one remaining count for its nonterminal occurrences. +Terminals satisfy their obligations immediately. A clause with no remaining +nonterminals seeds its lhs, including epsilon clauses. Register every reference +occurrence, including duplicates, before draining work. Each productive name +publishes once and discharges each registered occurrence once. A clause that +reaches zero publishes its lhs; alternative productions share that membership +fact. A cycle without a finite exit and undefined references cannot seed facts. +The result list is materialized in first-lhs order after convergence. + +Soundness follows by induction on publication: seed clauses derive finite words; +a clause whose children are already productive can concatenate their finite +words. Completeness follows by induction on the height of any finite derivation: +its terminal/epsilon leaves seed facts, then its productions' occurrences are +discharged. Expected constant-time table operations give O(P + R) preparation +and propagation for P productions and R rhs occurrences, with O(P + R) storage. +This bound is for grammar analysis, not whole-parser execution. + +The error carries =startOrigin= and a finite =unproductiveRuleBlockers= graph. +Origins use selected rule metadata and expression paths. Every choice alternative +has a blocker; one unproductive operand suffices for a sequence. Reference edges +identify the authored occurrence and target rule. Each rule is expanded once, +so cycles terminate without inventing source coordinates or exposing synthetic +helper names. This graph explains the completed fixed point; a single observed +cycle or bounded failure to enumerate a word is not the proof of unproductivity. + +** Semantic and complete-construction controls + +The existing resolved grammar Suite compares an independent repeated-scan +productivity oracle across 729 three-rule cyclic combinations, eight actual LR +fixture families, alternative exits, duplicate/marked operands and undefined +references. Nullable membership must imply productivity. A 4096-name reverse +chain checks propagation and canonical publication order. Target controls cover +five routes and three conflict policies, finite epsilon/left-recursive exits, +unreachable dead rules, reachable dead alternatives and the terminal-prefix/dead- +tail counterexample. POO override controls retain the selected blocker owner and +restore admission with either a terminal or epsilon exit. Wrapper controls retain +authored paths through alias/repeat1/field helpers; optional/repeat epsilon exits +and an independent reduce/reduce conflict are matched positive/negative controls. + +Local focused qualification completed 52 source Cases (35 resolved grammar and +17 canonical LR), final =OK= and status zero. Formal compiled qualification +completed 369 semantic Cases in 23 groups and all five independent research +entries with final =DSL-RESEARCH-CLOSURE-OK=, =OK= and status zero. The final +Scheme-owned conformance preparation took 88.43 seconds under the unchanged +240-second total and 5-second idle contracts. Complete real TLA+ LRSpec remains +exactly equal to the pre-change snapshot: 40 rules, 280 productions, 390 states. +Twenty complete construction samples measured CPU median 15.0955 ms with no +in-call GC. This is the cost after adding an admission check, not an A/B speedup +claim. Compiler/FFI/runtime and exact delivered-head CI receipts remain separate. + + +* Scheme author example and the two design questions + +#+begin_src scheme +;; Existing grammar.ss syntax: recursion is language intent, not an engine loop. +(deflanguage finite-chain + (syntax + (lexical + (root source-file) + (lex (punctuation Punctuation (literals "a" "b")) + (unknown Unknown (fallback))) + (extras) (keywords) (recoveries) + (conflicts reject) (case-insensitive #f))) + (rules + (source-file (node SourceFile expression)) + (expression (choice (seq expression (literal "a")) (literal "b"))))) + +;; Question 1: Why is this DSL design better? +;; The author writes recursive syntax once. Hygienic Scheme vocabulary and POO +;; selection feed one engine-owned analysis; changing the effective finite exit +;; changes admission without an author-maintained analyzer or callback. +;; Question 2: Why does this DSL design have a scientific basis? +;; Finite derivation has a precise meaning and least-fixed-point equations. +;; Independent oracles, legal LR recursion and selected-origin controls can +;; falsify the implementation. The admission policy is stated separately from +;; the papers' semantic and predictive-parser claims. +;; Limit: these controls establish this contract, not universal DSL superiority. +#+end_src + +* Follow-up checkout verification + +After integrating =2bfec1f=, this checkout rebuilt the LR modules and reference +fixture at the existing optimized compilation tier. The 52 resolved/canonical +LR Cases passed with =HARNESS-OK=, final =OK= and status zero. The Scheme block +above was extracted, imported with the production =deflanguage= macro and +compiled, ending in =PRODUCTIVE-AUTHOR-EXAMPLE-OK=. This verifies its author +syntax; it is not a new tree-equivalence or performance claim. + +* Joint completion solver: one graph, two meanings + +The follow-up source baseline is =2a4175ddea8e6c91f82f0dc91a3f91855aeda29e=. +That revision had separate nullable and productivity occurrence graphs. +=compute-completion= now registers the production/reference graph once and +returns both completed ordered name lists and Boolean membership tables. +=compile-lr-spec= calls it once. Standalone =compute-nullable= and +=compute-productive= observe the corresponding projections; they do not own +separate algorithms or graph construction paths. No language-author slot, public +IR field, schema version or grammar rewrite is added. + +** Semantic domain and efficient evaluation + +For each defined nonterminal, the private two-bit state has only three semantic +values: unknown, productive-only, and nullable-and-productive. A finite empty +word is also a finite terminal word, so nullable admission publishes both bits. +This invariant does not let productivity publish nullable: a terminal-only exit +is a direct counterexample. References without a defining lhs retain obligations +with no publisher; marked operands are interpreted through =base-symbol=. + +A private =completion-clause= holds the lhs index and two remaining occurrence +counts. A terminal disables its nullable counter but satisfies its productive +obligation. Every nonterminal occurrence is counted, including duplicates. +Dense lhs indices, vector dependencies/facts/pending deltas and one worklist +share discovery and scheduling. All dependencies exist before the seed queue +drains. A queued index accumulates new bits; popping clears its pending delta +and decrements only the counters corresponding to those new facts. + +Each domain's equation is unchanged. Soundness of combined nullable publication +follows from nullable implies productive; both derivations concatenate already +justified finite child derivations. Completeness follows from finite derivation +height as in the separate solvers. Queue exhaustion satisfies both equations, +and starting with no facts yields their least solutions. Shared scheduling and +bit coalescing therefore change evaluation work, not the completed facts. +Each name can publish at most two bits and each registered reference occurrence +is visited at most twice. With expected constant-time index operations, graph +preparation/propagation and storage are O(P + R); public list/table publication +is O(N). One graph is shared, not one undifferentiated Boolean fact. + +[[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][Krishnaswami and Yallop, section 3]] supplies the language/fixed-point reading. +The finite joint domain and dependency schedule above are this engine's design; +the paper's predictive grammar typing and linear parsing theorem are not claims +about our LR routes. This refinement does not prove lexical realizability, +semantic-action termination or general runtime speed. + +** Controls and receipt boundary + +The independent repeated-scan nullable/productivity oracles both observe the +joint result across the existing 729 cyclic combinations and eight real LR +fixture families. New controls distinguish productive-before-nullable from +coalesced two-bit publication by reversing seed order. They include duplicate +operands, missing references, terminal-blocked clauses and the empty graph. +Existing finite exits, productive LR recursion, dead-start admission, selected +POO origins and conflict/repetition diagnostics retain their independent controls. + +Local source qualification completed 53 Cases (36 resolved grammar and +17 canonical LR), final =OK= and status zero. Formal preparation completed in +79.21 seconds with unchanged 240-second total/5-second idle contracts. Compiled +qualification completed 370 Cases in 23 groups plus all five independent +research entries, final =DSL-RESEARCH-CLOSURE-OK= and =OK= with all exits zero. +Native cost/equality comparison and delivered-head CI are separate receipts. + +The first standalone cost script emitted 20 samples but failed to exit before +the original idle deadline; its failed receipt is retained and is not a passing +qualification. The paired measurement entry explicitly exits after reporting. +Temporary baseline module preparation also exposed an ambiguous package identity; +explicit =completion-ab= package declarations repaired their generated interfaces +without changing production modules, timeouts or the compiler backend. + +** Matched native cost and complete artifact equality + +The baseline functions and complete LR compiler from =2a4175d= were compiled +into temporary =completion-ab= modules at the same SDK/GCC/optimization tier +as the current engine. The pair imports those native modules without rebinding +production procedures. Samples alternate baseline/candidate order in one +process; explicit GC precedes each timed call and is excluded. Initialization, +graph preparation, propagation and public result materialization are inside the +analysis-call timing. Every pair checks complete ordered names and Boolean +membership. Full construction compares complete LRSpec equality, not a state +count or isolated kernel. Imports, explicit GC and equality checks are outside +the timed region. No compiler ran concurrently during the admitted comparison. + +| Workload | Samples per side | Separate CPU median (ms) | Joint CPU median (ms) | +|----------+------------------+--------------------------+----------------------| +| Epsilon reverse chain 4096 | 7 | 2.341 | 1.334 | +| Terminal reverse chain 4096 | 7 | 1.799 | 1.022 | +| Real TLA+ completion facts | 21 | 0.145 | 0.079 | +| Real TLA+ complete default LALR | 21 | 14.715 | 14.525 | + +The completed supervised pair ended =COMPLETION-NATIVE-AB-OK= and status zero +under the original 90-second total/5-second idle contract (12.98 seconds total). +Direct invocation also exited zero. Earlier supervised attempts emitted the +final equality marker but hit the idle deadline during termination; those failures +are retained, and their underlying termination delay was not stably reproduced. +Do not treat those attempts as passing costs or claim their root cause repaired. +The successful comparison is one local-host analysis-call result. The small +full-construction difference does not establish a stable end-to-end speedup, +parser throughput improvement or FFI/Rust runtime gain. + +The combined LR consumer avoids two graph builds. A standalone projection also +computes the other domain and is not claimed faster than the prior domain-only +routine. Future callers needing both facts must call the joint owner once, +rather than separately request the two projections. Admitted grammar semantics, +diagnostic precedence and public LRSpec output are unchanged. diff --git a/docs/research/dsl-recognition-contracts.org b/docs/research/dsl-recognition-contracts.org new file mode 100644 index 00000000..fbf42da0 --- /dev/null +++ b/docs/research/dsl-recognition-contracts.org @@ -0,0 +1,233 @@ +#+title: Recognition semantics and target obligations +#+date: 2026-10-09 + +Part of [[file:dsl-engine-contracts.org][the integrated DSL and engine study]]. +Existing dated receipts retain their original revision and scope. + +* Scope and evidence + +Repository observations began against commit =f058225a1454732af6144076f5916418bde71153=. +Dated implementation sections distinguish delivered mechanisms and local receipts +from the remaining proposals. Scanner replay and convergence do not establish +parser subtree reuse. No new performance result is claimed. + +Build on [[file:scheme-deflanguage-design-synthesis.org][the vocabulary study]], +[[file:scheme-language-pack-semantics.org][binding and field semantics]] and +[[file:scheme-parser-performance-framework.org][the algorithm qualification framework]]. +This document connects those studies to current backend admission and execution. + +* Repository observations refreshed at de8bef5 + +| Owner | Observed implementation | Remaining design question | +|-------+-------------------------+---------------------------| +| =src/language/entry.ss= | POO Loader values, sealed identity/dispatch slots, effective-object validation | How does extension validity propagate into all generated targets? | +| =src/grammar/algebra.ss= | Local syntax checks reject directly empty repetition bodies | Resolved references are checked by the shared LR nullable owner | +| =src/compiler/lr.ss= | Choice becomes alternative productions; repetition becomes left-recursive helpers; dependency-driven nullable and FIRST analyses exist | Repetition origins exist; productivity and general target proofs remain open | +| =src/compiler/recursive-source.ss= | Choice commits to first successful child; admission pins the reviewed HCL grammar digest and lexical contract | Preserve the restricted admission boundary; prove equivalence before broadening it | +| =src/language/grammar.ss= | Field catalogs are derived through helper references | Derive occurrence constraints without flattening nested node ownership | +| =rust/gerbil-parser-runtime/src/engine/command_source/execution.rs= | Source tasks suspend and resume the Command VM; live-frame budget is 16,384 | Account separately for byte storage, allocations and total work | +| =languages/bash/grammar.ss= | Scanner, region, binding, part, result and command declarations are engine data | Reduce repeated author declarations while retaining semantic control | + +These observations distinguish missing connections from missing algorithms. +In particular, the repository already computes nullable and FIRST globally; +adding another standalone nullable implementation would duplicate authority. +The former Source recursion proposal is now implemented; its historical rationale +below is not evidence of a remaining 256-level Source guard. + +* Choice is part of the language contract + +Tree-sitter's [[https://tree-sitter.github.io/tree-sitter/creating-parsers/2-the-grammar-dsl.html][official grammar DSL]] defines choice independently of argument +order. Ford's [[https://bford.info/pub/lang/peg.pdf][Parsing Expression Grammars]] instead defines prioritized choice. +These are different recognition semantics, even when their surface notation +looks similar. Lexical priority, parse precedence and operational dispatch order +also describe different decisions. + +The current recursive emitter is restricted to a reviewed HCL grammar by exact +digest and lexical-contract admission. The following control therefore concerns +future admission expansion, rather than a demonstrated current public bug: + +#+begin_example +start = sequence(choice(literal("a"), literal("ab")), literal("c")) +input = "abc" +#+end_example + +CFG alternatives admit =abc=. A first-success choice consumes =a= and then +fails at =c=; it does not reconsider =ab= after the enclosing sequence fails. +This is a code-derived counterexample to qualify, not an executed test result. +It should first be tested at the appropriate character or token layer, with +lexical segmentation held identical across backends. Also exercise the helper +reference and enclosing-node versions and compare output fields and ranges. + +Proposed admission rule: a specialized backend implementing first-success +choice may accept CFG choice only when a sound analysis proves its restricted +behavior equivalent, or when it implements the required alternative handling. +An unproved case produces a compile-time diagnostic explaining the unsupported +semantics. Generation must not silently reinterpret the grammar. Existing +Command branch dispatch remains operationally ordered and gets a distinct IR +meaning; it is not automatically an ordered grammar alternative. + +* Analyze the effective grammar before target lowering + +Krishnaswami and Yallop's [[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][A Typed, Algebraic Approach to Parsing]] shows how +compositional Null/First/FLast information can justify a deterministic parser +for a restricted grammar class. The separability conditions matter: its FLast +equations are not valid for arbitrary CFGs. Use its discipline to establish +backend preconditions, rather than require every supported language to be LL(1). + +Proposed pipeline: + +#+begin_example +POO composition -> effective declarations -> resolved expression graph + -> shared semantic analysis -> target admission -> lowering -> execution +#+end_example + +The engine owns analysis; authors own semantic declarations. Internal named +analysis functions compute immutable facts. POO exposes extension values and +policies, not callback protocols for implementing the analysis. + +- Resolve references after inheritance/override/extend, preserving source origin. +- Reuse fixed-point analysis for nullable and FIRST. Validate each original + repeat/repeat1 operand against resolved nullability before helper expansion. +- Record productive versus empty-language rules separately from nullable rules. + Diagnose unproductive cycles without rejecting ordinary LR left recursion. +- Track conservative field occurrences within the nearest node boundary using + the existing count-domain proposal. Keep field membership and cardinality distinct. +- Describe scanner state reads/writes, deferred region obligations and result + ownership as engine effects. Unknown effects prohibit speculative reuse. +- Preserve repeat, separator, delimiter and precedence origins in lowered IR. + Target diagnostics refer to the author's declaration, not only generated helpers. + +A resolved nullable repeat should fail with the reference path that makes it +empty. Runtime zero-width guards remain defensive checks for malformed admitted +programs. They do not replace declaration validation. + +* Grammar transformations and runtime scheduling are separate work + +Zimmerman's [[https://arxiv.org/pdf/2209.08383][Practical LR Parser Generation]] transforms intermediate helper +nonterminals to defer reductions until their surrounding rule is complete. +This suggests retaining EBNF origin and investigating targeted helper transforms +when measured conflicts originate in flattening. Its XLR bounds depend on bounded +branching; the paper does not justify unrestricted backtracking guarantees here. + +The historical execution proposal addressed Source host recursion. It is now +implemented by =SourceTask= and suspended =CommandExecution= records: + +#+begin_example +Source tasks <-> suspended Command VM + Compound/List requests +#+end_example + +The historical task design includes entering a compound, reading a list, assembling +a pipeline, awaiting a child result and completing a node. Each task has a typed +continuation and explicit input/state/output ownership. Nested commands suspend +the parent task instead of calling another host parser recursively. Task records +are private engine records; language declarations do not contain scheduler code. + +Deep alternating compound/list nesting, ordered fields, source ranges, heredoc +FIFO and cleanup have existing controls recorded in [[file:dsl-source-execution.org][the Source implementation +contract]]. Word/Region leaf operations do not automatically become suspension +requests. The remaining resource work must account for bytes/allocations/work +in addition to the current frame limit and retain a structured rejection path. + +* Scanner state is a prerequisite for safe incrementality + +Tree-sitter's [[https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners.html][external scanner contract]] requires complete state serialization +and restoration for edits and ambiguity. It also requires author-maintained C +functions for external tokens. Our design goal is to derive equivalent engine +state operations from declarations for common stateful constructs. + +Proposed shared policies cover quoted regions, balanced delimiters, line markers, +deferred bodies and layout boundaries. The engine derives token/result catalogs +where unambiguous; authors specify exceptional language semantics explicitly. +Demonstrate reuse in at least two languages before adding a public abstraction. + +A reuse candidate must establish grammar identity, source/edit mapping, parser +entry context, scanner entry context, exit-context compatibility and output +ownership. Scanner context includes pending heredoc markers, quote/escape state +and mode, not only an integer mode identifier. Hash equality alone is not proof: +compare canonical state or use interned identities with verified equality. +Checkpoint data must have explicit ownership across Scheme, Rust and C ABI. + +Wagner and Graham's [[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] connects +sequence structure and context matching to incremental cost. Investigate balanced +private repeat storage with unchanged public CST order and field observations. +Their performance bounds rely on structural and grammar assumptions; a balanced +container alone does not establish those bounds for this engine. Delaying helper +reductions and exposing reusable sequence boundaries can conflict, so evaluate +both transformations together before adopting either universally. + +* AOT and FFI must share semantic admission + +Scheme remains the declaration/IR authority. Rust executes admitted immutable +plans; a language-independent C interface manages language, source and artifact +handles. POO extension is resolved before target compilation. No schema version +change is proposed here. + +Target capability validation covers every effective construct. Complete parity +includes acceptance, rejection, diagnostics, ranges, ordered fields, deferred +links, source identity and artifact lifetime; event equality is one component. +The Rust Source implementation does not yet establish complete ParseArtifact/C +ABI parity merely by executing Command profiles. + +Ford's [[https://bford.info/pub/lang/packrat-icfp02.pdf][Packrat Parsing]] motivates eliminating repeated work through memoization +under its assumptions. Warth, Douglass and Millstein's [[https://tinlizzie.org/VPRIPapers/tr2007002_packrat.pdf][left-recursion extension]] +can have superlinear cases. Any memoization proposal here must account for scanner +context and bounded memory; it does not warrant a wholesale PEG conversion or +an unconditional linear-time claim. + +* Implementation follow-up + +The implementation follow-up is rebased onto PR revision =30ac89d=. Resolved +repetition admission now reuses the existing nullable fixed point and visits +original expressions in one postorder pass. It reports the rule, repetition kind +and original operand before LR state construction. Every child is validated even +when the enclosing sequence or choice has an already known boolean result. + +[[file:../../t/resolved-grammar-test.ss][The Scheme semantic controls]] cover nullable references/cycles, wrappers, +effective POO overrides, consuming prefixes, productive left recursion, all LR +construction paths, GLR admission and exact common-prefix fields/ranges. The suite +is included in the existing native =composition= conformance owner. The recursive +strategy control belongs to [[file:../../t/build-strategy-test.ss][the build-strategy suite]]. + +Local GCC/native qualification completed 15 resolved grammar, 31 composition, +5 style and 9 build-strategy cases. Build-strategy tests declare two small +DevelopmentLoaders instead of importing complete language corpus suites; corpus +qualification remains with its language owners. Each module requires propagated +status zero and the final =OK= marker. + +The cold full-package build did not satisfy the local 240-second total / +30-second silence gate. Completing missing native products with the same GCC +compiler enabled focused semantic qualification; it is not evidence that the +cold build gate or runtime performance has improved. Exact-revision CI remains +a separate qualification. + +That historical slice established resolved repetition admission. Subsequent +work implemented selected repetition origins and Source continuations; see +[[file:dsl-source-execution.org][Source execution and artifact ownership]]. =de8bef5= replaces nullable rescans +with occurrence propagation; see [[file:dsl-effective-grammar-admission.org::*Dependency-driven nullable closure (2026-10-09)][the algorithm and qualification contract]]. +General predictive choice equivalence, helper/conflict origin serialization, +diagnostic transport and complete cross-target artifact parity remain open. + +* Current implementation and qualification order + +The current sequence belongs to [[file:dsl-engine-contracts.org::*Next implementation sequence][the integrated implementation plan]]. Keep this +recognition-specific target contract when applying it: + +| Obligation | Current status | Required next evidence | +|------------+----------------+------------------------| +| Resolved repeat/selected origins | Implemented and controlled | Preserve override diagnostics when extending analysis | +| Nullable dependency closure | Implemented at de8bef5 | Preserve completed list/index and exact LRSpec | +| General predictive choice admission | Open; HCL emitter is digest-restricted | Identical lexical segmentation; common-prefix and nested field controls; preserve output or reject compilation | +| Productivity | Open | Distinguish dead cycles from nullable and productive left recursion; specify reachable-rule policy | +| Source continuation scheduling | Implemented | Extend byte/work/resource qualification instead of building a second scheduler | +| Source canonical events | Fixture parity implemented | Diagnostic transport, Rust encoding and deep live C ownership controls | +| Incremental parsing | Scheme scanner sessions and LR fragment controls exist | Prove mapped Source subtree and artifact ownership for a specific target | + +Keep correctness controls in Scheme test owners and runtime measurements in ASP +benchmark owners. Separate build/startup/FFI/scan/parse/artifact costs. Measure +complete calls with matched workloads, work counts, allocation and peak memory; +retain regressions and negative results. No generated JSON research snapshots or +source-rewriting instrumentation are needed. Documentation changes require link +and diff inspection; they do not justify rerunning the long native suites. + + diff --git a/docs/research/dsl-scanner-reuse.org b/docs/research/dsl-scanner-reuse.org new file mode 100644 index 00000000..4964ffbc --- /dev/null +++ b/docs/research/dsl-scanner-reuse.org @@ -0,0 +1,214 @@ +#+title: Scanner contexts and suffix convergence +#+date: 2026-10-09 + +Part of [[file:dsl-engine-contracts.org][the integrated DSL and engine study]]. +Existing dated receipts retain their original revision and scope. + +* Scanner request and checkpoint ownership + +The next incremental boundary is executable ownership, before edit mapping or +subtree admission. A Scheme string is mutable. Previously a scanner retained the +caller's string, canonical receipts exposed the scanner digest and delimiter +strings, and restored obligations retained receipt strings. Mutating an input or +receipt could therefore change a live scanner or a future restored branch. + +The engine now snapshots the source once per request. A private lazy source +identity belongs to that snapshot and is shared by checkpoint publication and +restore. Publishing a receipt copies schema, scanner/source digests, and pending +and active delimiter strings; restoring owns new delimiter strings. The fixed +checkpoint schema and declaration DSL do not change. Language authors add no +scanner callbacks, ownership wrappers, or loader slots. + +This is a semantic precondition for the context reasoning in +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Wagner and Graham, sections 4 and 6]]. Their state-matching condition is sufficient +but not necessary, while conflict resolution requires additional care for subtree +admission. Our exact-source scanner restore remains deliberately narrower than +cross-edit parser reuse. No paper complexity bound is claimed for this change. +The source snapshot costs one O(N) copy per prepared request; hashing is lazy and +occurs once per request instead of once per checkpoint operation. Receipt size +still scales with delimiter obligations, and restore's UTF-8 position check still +examines the prefix. There is no measured end-to-end performance claim. + +[[https://www.cl.cam.ac.uk/~nk480/parsing.pdf][Krishnaswami and Yallop]] use Null/First/FLast information to justify predictive +combinators and staging for their typed grammar class. Those compile-time proofs +cannot substitute for request-owned contextual scanner state, nor be imposed +unchanged on our LR grammar class. The DSL continues to own static declarations; +the engine owns mutable execution, immutable plans, and receipt admission. + +The added semantic controls use both HCL raw and Bash quote-removal delimiter +policies through the same engine. They mutate caller source, receipt identity +strings, queued markers and active markers, then compare canonical states and +exact Unicode body/end tokens after restore. Existing source/machine mismatch +and UTF-8 position controls remain required. Rust already retains an immutable +source borrow and private shared obligations; portable Rust/C checkpoint transport +and cross-edit scanner mapping remain future work. + +Qualification for this ownership slice: pinned-SDK GCC compilation succeeded; +all 23 contextual/deferred scanner cases passed under the unchanged 90-second +batch/five-second silence gate, including both new ownership controls. All 54 +Command Source controls regenerated successfully through the Scheme watchdog; +the formatted Rust source and native binary fixture were byte-identical to the +checked-in products. These are local semantic and target-output receipts, not a +latency result or a portable checkpoint ABI qualification. + +* AOT representation experiment and the remaining edit boundary + +At revision =7c05c47=, Rust and macOS CI passed while Ubuntu failed the unchanged +five-second output-silence gate during reusable native objects. The final active +objects were TLA+ layout/core; core had entered GCC's front end, not Scheme IR +decoding. The compiler and optimization policy remain GCC with the same settings. + +The AOT representation previously wrapped compressed sidecar bytes in Base64 +text. Gambit emitted =___STR8= initializers, whose character-width macro expands +three byte/word alternatives; the runtime then allocated a decoded compressed +buffer before decompression. An intermediate local experiment embedded the original compressed =u8vector= +directly. The final PR implementation at =6c9b513= instead packs network-order +unsigned 64-bit words with byte length and canonical zero tail padding. Both +representations remove the Base64 envelope; the packed word form further limits +C initializer work. The intermediate byte-vector implementation is superseded. Serialized IR, schemas, content hashes, sidecar files, +DSL forms and language-loader metadata remain unchanged. + +A controlled representation probe rewrote only the three embedded payloads in +the actual TLA+ core static Scheme module. The same pinned Gambit and host GCC 16 +preprocessor produced 5,141,913 bytes for the string image and 3,914,507 bytes for +the byte image. Generated C grew from 696,003 to 736,566 bytes because byte +literals use decimal syntax. This is a reduction in macro-expanded material, +not measured runtime latency or proof of the Ubuntu GCC 13 gate. The final decoder removes the Base64 operation but reconstructs compressed bytes +from packed words before decompression. No allocation or latency reduction is +claimed for that decode step. + +The next cross-edit implementation must establish lexical decision validity, +not simply relocate a checkpoint whose prefix text happens to match. A scanner +may have read beyond a token boundary to decide longest match, failed quoted or +balanced alternatives, or the next delimiter. Safe prefix restart needs decision +read-dependency evidence or replay under the recorded parser positions. Suffix +convergence instead requires independently reached states with equal effective +scanner semantics, mode, ordered pending obligations, active delimiter and +expectation, plus equal remaining source and compatible parser context. Neither +criterion is implemented by this representation change; unchanged-source restore +continues to require exact identity. This preserves the scanner-state prerequisite +from the incremental research section without introducing an unsafe resume API. + + +The packed-image constructor now reads complete unsigned words directly from the +original byte input and builds at most one partial tail word. It no longer +allocates and copies a padded byte vector for the whole payload before packing. +The destination word vector and transport framing remain unchanged; the +network-order rule is portable across host endianness. Existing every-tail, +unsigned-max, padding-rejection and Unicode/identity controls apply directly. + +The intermediate local experiment passed 14 artifact cases and 54 regenerated +Source controls. Its broad package build hit the retained 240-second total +budget; native-object completion was then performed separately. Full source-mode +TLA+ qualification still failed before cases at the 90-second admission budget. +These results are not a full language-suite pass for the final packed-word head. + + +The Scheme suite's failure label now gives the total budget precedence when its +bounded read reaches 90 seconds. Previously the read returned =quiet= at that +deadline and incorrectly labeled the total-budget failure as silence. Both +limits and the killing behavior are unchanged; this corrects the diagnostic for +the retained source-mode failure above. + +Final packed-tail qualification: the fixed SDK/GCC engine build passed and all +16 artifact cases passed with final =OK=. The framing corpus additionally covers +4 KiB, 8 KiB and 64 KiB buffer boundaries and neighboring partial tails. Independent +network-order golden words cover unsigned high bits, and mutation of the original +input cannot change the packed image. This removes the constructor's padded +full-buffer copy; it is not a measured parser/FFI runtime speedup. + +On the final packed-tail workspace, all 54 Scheme Source controls regenerated +with identical formatted Rust and native binary fixtures. The integrated packed +image revision =6c9b513= independently passed Ubuntu native and Rust AOT CI in +[[https://github.com/tao3k/gerbil-parser/actions/runs/37727437327/job/113148708306][run 37727437327]]. That baseline qualification does not substitute for exact-head +CI after this constructor and diagnostic change. + +* Independently reached scanner suffix convergence + +The Scheme engine now exports =contextual-scan-state-converged?= and Rust +=ScannerCheckpoint::converges_with=. Two independently reached checkpoints +converge only when they share the admitted immutable scanner plan, mode, +expectation, ordered pending delimiter values and complete active delimiter +(marker, strip-tabs, quoted), and their remaining source is exactly equal. +Scheme compares characters in owned snapshots without copying the suffix; Rust +compares borrowed UTF-8 suffix bytes. Digest equality alone never admits a plan. +Raw Scheme IR requests have a private request identity; cross-request admission +requires the shared prepared plan. No schema version or language DSL changes. + +This is an engine prerequisite for suffix reuse, under an identical future +parser-position schedule. It does not relocate an old checkpoint, prove parser +stack equivalence, or implement incremental Source/CST publication. Existing +restore and step ownership checks still reject foreign sources/requests. The +prefix decision-read-dependency work described above remains open. + +The check is linear in the remaining source and pending marker data; it must not +be presented as a constant-time edit operation. Ordered FIFO comparison ignores +the internal front/back partition and preserves both checkpoints. Matched HCL +raw and Bash shell-quote-removal controls independently scan modified Unicode +prefixes and compare the complete suffix token kinds and relative byte ranges. +Negative controls hold the suffix fixed while changing delimiter properties, +expectation or FIFO order, and reject changed suffixes and foreign plans. +No runtime latency improvement is claimed by this admission primitive. + +Local qualification of this implementation: the pinned Gerbil 0.19 SDK and +host GCC 16 compiled the final scanner module; the existing test-suite gate +passed both scanner modules (12 deferred-delimiter and 12 contextual cases), +with final =OK=. Rust runtime passed 137 cases across two suites, and strict +Clippy for all runtime targets passed. The original 90-second total and +5-second silence test limits remain in force. These local receipts do not +qualify the pushed revision's Ubuntu CI by themselves. + +* Prefix replay admission after suffix edits + +Suffix convergence alone cannot admit a prefix checkpoint on changed source: +longest-match, EOF and quoted-region decisions may have inspected the edited +suffix. The engine now provides =contextual-scanner-replay-prefix= and Rust +=ContextualScanner::replay_prefix=. The caller supplies its recorded parser-position +schedule. Both original and edited sources execute that schedule through the +ordinary scanner, preserving full matcher selection and lookahead. Admission +requires the same prepared plan, exact prefix text, identical token kinds and +UTF-8 ranges, and complete matching endpoint scanner contexts. The original +checkpoint must also be reproduced by the supplied schedule. + +The result belongs to the new scanner. Foreign checkpoint stepping remains +invalid. Empty-prefix admission, pending markers and active heredoc bodies are +covered; changed longest-match/EOF/quote decisions, shortened or edited prefixes, +foreign plans, incomplete schedules and unrecorded external mode changes reject. +Scheme controls exercise HCL raw and Bash shell-quote-removal policies. Rust +executes its existing Scheme-generated specification; no hand-written language +callbacks or DSL fields are added. + +#+begin_src scheme +;; An application records its parser positions while driving the scanner. +;; The engine owns checkpoint validation and creates a state for the new source. +(def next-state + (contextual-scanner-replay-prefix edited-scanner old-checkpoint + recorded-parser-positions)) +(contextual-scanner-step edited-scanner next-state 'command) +#+end_src + +Why is this preferable? A grammar author keeps syntax and rules declarative; +lookahead validation, UTF-8 ranges, deferred obligations and source ownership are +resolved by the shared engine. The caller supplies an existing execution input, +not a second lexical algorithm. Why is it justified? Actual replay checks token +observations and endpoint context on both sources, including lookahead past the +checkpoint boundary. Exact FIFO comparison occurs only at the endpoints, avoiding +an additional sum of queue lengths over all replayed tokens. Cost still includes +all normal matcher work and scanning the prefix twice; no constant-time edit, +parser speedup or whole-tree reuse claim follows. + +This admission primitive is an engineering prerequisite motivated by the +context/reuse distinction in Wagner and Graham's +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]]. It does not implement that paper's +sentential-form parser, balanced sequence transformations or optimal node reuse. +Parser-stack compatibility and Source/CST publication remain separate contracts. + +Local qualification (2026-10-08): the current native scanner dependency closure +compiled under the CI-pinned Gerbil SDK. Both scanner test modules passed +(13 deferred-delimiter and 12 contextual cases), with final =OK= and process +status zero in 3.92 seconds. Rust runtime passed 138 cases; strict runtime +Clippy and the ASP Rust source-policy gate passed. The FIFO convergence test +now resides under =tests/unit= with explicit imports, repairing the policy +failure on revision =bc233b8=. These are local receipts, not qualification of +a subsequent pushed revision. No timeout or silence limit was increased. + diff --git a/docs/research/dsl-source-execution.org b/docs/research/dsl-source-execution.org new file mode 100644 index 00000000..89bd2384 --- /dev/null +++ b/docs/research/dsl-source-execution.org @@ -0,0 +1,198 @@ +#+title: Source execution and artifact ownership +#+date: 2026-10-09 + +Part of [[file:dsl-engine-contracts.org][the integrated DSL and engine study]]. +Existing dated receipts retain their original revision and scope. + +* Current scope at de8bef5 + +The current Rust Source implementation is split across +=rust/gerbil-parser-runtime/src/engine/command_source/mod.rs=, +=execution.rs= and =publication.rs=. =CommandExecution=/=CommandRequest= remain +in =command_program.rs=. Historical receipt counts below apply to their recorded +slices, not to this source audit or a newly executed test run. + +Source continuations, iterative destruction and canonical accepted-event fixture +parity are implemented. Scheme scanner edit sessions and LR fragment transfer +also exist, under [[file:dsl-lr-transfer.org][separate admission contracts]]. Their existence does not establish +general Rust Source subtree reuse, complete diagnostic transport or deep live C +handle parity. The concrete next work is defined in +[[file:dsl-engine-contracts.org::*Next implementation sequence][the integrated implementation sequence]]. + +* Implementation: origins and Source continuation closure + +The implementation extends the selected source map into repetition diagnostics: +rule, expression child path, selected POO contribution, and a finite nullable +reference explanation. It records both the referenced declaration and the +reference occurrence. Reader coordinates remain unknown where unavailable. +A queue visits each referenced rule once; the diagnostic does not expand shared +nullable derivations into an exponential tree. Nullable acceptance remains owned +by the existing fixed point. Generated helper origin serialization and broader +conflict witnesses are still separate work. + +The real declaration compiler binds the diagnostic context only for its compile. +Complete declaration cache hits recheck current repetition admission using the +cached canonical nullable facts before publication. This performs no LR lowering, +fixed point or state reconstruction, and changes no public schema. Tests include +an old complete cache receipt that must reject without invoking either compiler +callback, selected nullable overrides, cyclic exits and a 32-level shared DAG. + +Rust Source now drives the existing Command VM through private continuation +records. Command/List instructions suspend the VM; compound, pipeline, and/or, +list, prefix and assignment continuations resume through one outer request loop. +The synchronous Source host methods and recursion-depth guard have been removed. +Words, result profiles, token ordering and heredoc FIFO remain governed by the +admitted Scheme-generated product. Language author interfaces are unchanged. + +The private resource gate counts pending Source tasks plus VM tasks/node frames, +including growth while a VM runs, with a limit of 16,384 live frames. It returns +=command-source-resource-limit= at an input byte boundary. This is a frame limit, +not a complete byte/allocation/work budget. It does not establish a complexity +bound for the whole parser. + +Private recognition nodes now drain descendants iteratively during destruction. +A Rust control on a 256 KiB thread stack accepts 1,024 alternating group/subshell +levels, constructs events and a lossless event tree, and releases the tree/results. +It also cleans up a completed deep child after a trailing-pipeline failure and +successfully parses an independent next request. A larger input exercises the +resource rejection path; a 6,000-form VM call chain also hits the same budget +without returning to Source. Scheme independently admits the identical deep input +as a valid lossless ParseArtifact. Deep input artifact/C ABI cross-target byte +parity has not been established by these separate controls. + +Local qualification: 133 Rust runtime tests and strict Clippy; 18 resolved grammar, +12 artifact, 31 composition, 5 style, and 9 Source strategy Scheme cases. Scheme +qualification uses the existing native watchdog and GCC. The Source strategy +suite initially hit a loading silence failure because its native product helper +was missing; building that existing conformance helper resolved it without a +backend or timeout change. No end-to-end runtime speedup or cold full-package +build improvement is claimed. Exact rebased-head CI is a separate gate. + +At the time of this receipt, canonical publication, ABI parity and incrementality +were next steps. Subsequent sections record canonical accepted-event parity; +[[file:dsl-lr-transfer.org][the transfer study]] records Scheme scanner sessions and LR fragment admission. +Remaining cross-target obligations include diagnostic transport, Rust GPA1 +encoding, deep live C handle ownership and general Rust Source subtree reuse. + +* Implementation: canonical Source publication and shared lexer preparation + +Rust Source now publishes =CommandEvent= records through one iterative +=CommandParse::walk_events= sink: preorder node IDs, producer-order token IDs, +node boundaries, field boundaries and the complete trivia tape. The existing CST +API projects this same walk rather than maintaining a second traversal. A sink +receives a prefix if later validation fails and must commit only after success. +The result retains the exact source allocation and result-profile identity; +replacing its public root with a tree from equal bytes in another allocation is +rejected before the sink receives anything. + +The Scheme Command Source generator also emits native fixture bytes through +=bind-language-abi= and =abi-parse-binary-payload=, using the public codec +consumed by the C ABI. CI regenerates both products and compares them to the +checked-in fixtures. The 54 controls cover the original declaration and a POO +extension changing the root projection, pipeline role and Unicode form keyword. +Rust compares every accepted canonical record with the decoded Scheme GPA1 +record, including names, IDs, ranges, field scopes and trivia placement. Rejected +controls check acceptance agreement and the Scheme fallback token tape. Native +views borrow the original source slices, reject changed source digests, and +repeated Rust publication starts with independent identifiers. Deep publication +also walks 1,024 alternating compound/subshell levels on a 256 KiB stack. + +This establishes canonical accepted-event parity against native codec bytes for +these controls. It does not implement a Rust GPA1 encoder or qualify live C +handle allocation/release through these fixture tests. GPA1 currently contains +status and events, not diagnostic records; diagnostic content parity and a +language-independent diagnostic transport remain open. No schema increment or +compatibility mode is introduced. + +The previous Ubuntu head failed while compiling the TLA+ core static object: +the GCC output stopped for the existing five-second silence gate. Inspection +found =generated-lexer= expanding the entire mode-plan construction, sharing, +certification and traversal algorithm into each language module. That algorithm +now resides once in =prepare-generated-lexer=; the macro supplies only declared +rule scanners and a delayed certificate factory. Longest-match priority, rule +order, mode-miss fallback, shared-plan identity and lazy certification remain the +same. No backend, optimization level or watchdog deadline is changed. + +In the local TLA+ module, generated Scheme size fell from 254,620 to 156,138 +bytes in the final delayed-factory version. This is a structural code-size +observation, not a runtime speedup or evidence of Ubuntu closure. Gambit's +[[https://www.iro.umontreal.ca/~gambit/doc/gambit.pdf][compiler documentation]] explains that single-host compilation combines a +module's procedures into one C procedure and can make optimization of large +modules expensive. Exact-head Ubuntu CI remains the admission gate for the build +failure; Scheme semantic controls and local GCC compilation are separate evidence. + +Local qualification for this slice: 135 Rust runtime tests, strict Clippy and +formatting; 54 Source/native fixture controls regenerated under the existing +90-second total / five-second silence gate with identical Rust and binary +products; final GCC engine/TLA+ compilation and six lexical-plan Scheme cases. +The added TLA+ case compares every LR mode's shared/independent scan and forced +certificates, plus two accepted and one rejected complete ParseArtifacts. + +A broader source-mode =languages/tla-plus/parser-test.ss= attempt hit the unchanged +watchdog while loading declarations, before its cases ran. That failure is +retained; the direct engine controls do not turn it into a full language-suite +pass. Remote revision =07e1e5c= independently corrects native compiler option +admission and disables GCC macro diagnostic tracking; this slice rebases onto it. +The generated-algorithm consolidation remains a separate structural correction. + +** Source execution and publication ownership + +The canonical Source record implementation exceeded the Rust source-owner +policy at revision e62b77b (1021 lines). The repair separates responsibilities: +command_source/execution.rs owns cursor/task execution and its frame budget; the private +command_source/publication.rs module owns CommandParse, CommandEvent and the +iterative event walk. The command_source/mod.rs facade re-exports the existing public types, preserving +consumer paths. Request construction fields are visible only to the parent +owner; borrowed source, event ordering and byte ranges retain the same contract. +No recursion limit, source rejection rule or policy threshold is weakened. + +The existing runtime crate tests passed after extraction: 135 cases across two +suites, including Scheme Source record comparisons and deep command +controls. The ASP source policy and exact-head native CI remain separate gates. + +Why is this boundary preferable? Execution scheduling can evolve independently +from canonical event publication while both keep one request-owned result. Why +is it justified? The existing event-parity and deep-input controls execute the +same public interface after a responsibility-only extraction. + +The local policy also rejected a sibling command_source.rs/command_source pair +and deep super imports in the first extraction. The completed layout uses one +command_source/mod.rs owner facade, private execution/publication modules and +crate::engine imports. Both findings are repaired rather than suppressed. + +The completed owner layout passed the ASP Rust workspace policy gate (one gate +test, 4.40 seconds, exit zero) and the same 135 runtime cases. The policy's +reported warnings were removed through source structure, without exceptions. + +** Relative module admission and embedded IR identity + +Source-mode test admission now resolves relative symbol imports such as +=./parser= and =../helper= against the importing source. The module identity is +derived from the nearest =gerbil.pkg= package declaration. A compiled interface +uses the existing native dependency loader; a source-only helper follows its +source imports. This closes the previously missed dependency path without a +language-specific loader or a second parser entry point. + +Embedded artifact decoding retains the uncompressed UTF-8 bytes and computes +=sha256-bytes= directly. It no longer encodes the decoded string back into bytes +for identity verification. The largest observed TLA+ contextual IR contains +3,483,189 uncompressed bytes; removing that repeated conversion eliminates one +full-buffer conversion and its temporary allocation. No wall-clock runtime +speedup is claimed. Schema, digest, framing and cache contracts are unchanged. + +Actual decompression and UTF-8 completion stages are observable in the existing +artifact trace. Timeout errors distinguish silence from the total batch budget. +Neither the 90-second batch limit nor five-second silence limit is relaxed. +These traces diagnose real completed work; they are not performance evidence. + +Final local qualification under the pinned Gerbil SDK and GCC: 14 artifact cases +and all 57 TLA+ language cases passed with final =OK=. New controls cover Unicode +identity in sidecar and embedded images, and rejection of five malformed UTF-8 +sequences even when the supplied digest matches the malformed bytes. The earlier +source-mode loading failure remains recorded above; this result closes that +local language-suite failure. Ubuntu CI remains a separate admission gate. + +CI Rust 1.99 Clippy additionally requires an empty-slice assertion to display its +value on failure. The generated native fixture reader now uses =assert_eq!=, +retaining its complete-consumption assertion and all 54 fixture controls. + diff --git a/docs/research/dsl-test-preparation.org b/docs/research/dsl-test-preparation.org new file mode 100644 index 00000000..fbff1983 --- /dev/null +++ b/docs/research/dsl-test-preparation.org @@ -0,0 +1,197 @@ +#+title: Shared test runtime and preparation ownership +#+date: 2026-10-09 + +Part of [[file:dsl-engine-contracts.org][the integrated DSL and engine study]]. +Existing dated receipts retain their original revision and scope. + +* Implementation: single-runtime Scheme test preparation and Workers + +The Scheme Suite entry starts one =gxi=. A single preparation owner admits the +native FFI dependencies and expands each source test module once before any +Suite executes. Its loaded-module and interface caches survive the whole run. +The runtime must finish module initialization before Worker execution; runtime +loading and source expansion are not concurrent test jobs. + +The repository runner accepts an explicit =--workers N= argument before its +source roots and =--remaining-conformance= option. This is repository scheduling, +not a Gerbil configuration: official =gxtest= and =TestConfig= have no Worker +count option or =GERBIL_TEST_CORES= environment variable. Without this argument, +the repository runner queries Gambit's =##core-count= directly, bounded again +by the number of admitted modules. CI uses this same portable default on Linux +and macOS; it does not launch another interpreter or depend on shell =nproc=. The two-Worker negative controls explicitly +request two to exercise their rendezvous contract, not to set CI capacity. + +The runtime shares a 3 GiB heap budget. Loading the full source suite exceeded +the previous 1 GiB budget before execution; this budget covers all loaded modules +and Workers. The inherited POO Case profile uses the same absolute heap +cap because its sampler measures process-wide memory; its growth, sampling, +failure, and duration policies remain inherited unchanged. Each module owns its +=std/test= harness and thread-local test parameters; +shared-runtime barrier controls require two modules to rendezvous in one process. +Modules may export a symbolic =test-worker-affinity= for runtime thread ownership. +Modules may independently export a boolean =test-worker-exclusive?= for CPU/GC +or wall-time admission. All resource-contract owners declare that constraint. +Worker zero remains the caller across parallel and exclusive phases, preserving +ABI thread ownership. An official =std/sync/barrier= drains preceding console +messages before resource admission. The phase barrier creates no test progress. +Declared affine owners execute on the same persistent Worker; independent modules use +the shared queue. The output supervisor never moves a test onto another thread. +Failures stop new admission, join active Workers, and return a failing status. +The original five-second silence and ninety-second module budgets remain; +preparation time is deducted from the execution budget. A timed-out thread ends +the runtime rather than continuing with partially mutated state. + +The conformance build has one content-admission owner. Its S-expression receipt +covers project sources, dependency products/sources, SDK, compiler/environment, +and the native executable plus loadable helpers. SHA-256 and file modes are +checked, failed or input-mutating builds cannot publish a usable receipt, and an +OS file lock releases on exit. Source-only test interfaces removed by +=prepare-gxtest!= are deliberately outside the retained output identity. + +CI restores an isolated conformance product store. Only an exact content match +can copy its declared executable/helper products into the working library; +dependency/package cache restoration remains a separate owner. Scheme controls +exercise unchanged inputs, changed/corrupt/missing products, corrupt receipts, +failed preparation, store recovery/corruption, and mutation during preparation. +These changes remove repeated process startup and unchanged preparation work; +no end-to-end speedup is asserted without a matched complete-call measurement. + +* Implementation: disjoint compiled and source execution ownership + +The compiled conformance semantic entry runs every declared Suite in one process, +checks Suite object identity for repetition, and enforces the original ten-second +budget for each group. The outer supervisor retains its five-second idle fence. +Research commands with process-owning entrypoints remain separate. + +=scripts/test-plan.ss= compares the build catalog, imported Suite names and actual +Suite execution declarations. Missing owners, omitted Suites and repeated execution +fail admission. Following a successful compiled gate, CI passes +=--remaining-conformance= to the source runner. Fully covered modules are excluded before +source preparation; the remaining modules share the existing runtime and Workers. +The default source runner still qualifies every source module when requested. + +GQL support is compiled as one union for local diagnostics, while CI consumes the +products of its prior preparation owners. Missing products fail before execution. +Workspace Rust qualification includes its source-policy integration test, so CI +no longer pays for a separate invocation of that same test. + +Compilation also compares each module's actual phase-zero =-test= exports with +its imported conformance Suites. Public expander metadata is inspected with +runtime evaluation disabled. This catches additional exported Suites even when +all handwritten import declarations are internally consistent. Current-head +source-versus-compiled Case names are compared before admitting the partition. + +Source-only Suite declarations cover genuine source admission boundaries. The +SANY closure Suite needs dynamic module loading for its portable-build rejection +control. It runs in the source runtime; other Suites from the same file run in the +compiled runtime. Declared ownership has a complete union and an empty intersection, +checked against actual exported Suite names during compilation. Source selection +uses the same validated ownership and does not repeat the compiled Suites. + +Python process/compiler controls run once per Scheme CI platform. Ubuntu coverage +is retained there; the Rust-only Ubuntu job no longer repeats the same discovery. +macOS controls remain separate platform evidence. + +The old =t/versioned-language-pack-test.ss= forwarding module is removed. It had +no Cases of its own and re-exported language leaf Suites already discovered from +=languages=. Source admission now requires each test binding to belong to its +module and each Suite object to have one owner. Re-exporting leaf Suites or +admitting one Suite object twice fails instead of multiplying execution costs. + + +* Failure visibility and historical Darwin startup qualification + +Regular run =38760618212= failed its Darwin DSL research semantic process with +an idle timeout before any Suite receipt. The preceding conformance failure +control was expected and its step passed. That negative assertion is not the CI +root cause. Its exact native artifact's ranked subcommand exited successfully in +1.398711 seconds locally, and in 1.395143 seconds with the CI default heap policy. +These historical subcommand diagnostics do not reproduce or repair the full CI +semantic timeout. Full local replay is relocation-sensitive because diagnostic +tests retain the original compiler source locations. It is not counted as current +native semantic acceptance. + +The run published product archives but no qualification-log artifacts. The +artifact uploader defaults omit the hidden =.data= directory. All existing +=.data/**/*.log= uploads now explicitly include hidden files, keeping their +existing file glob and retention. Unexpected process exits also publish status, +reason, acknowledged native phase, elapsed time and the final 8192 characters of +real child output before raising. This adds no heartbeat, extends no deadline, +and does not accept a failed exit. A controlled child exiting 3 retained its +failure status while exposing its native phase and output tail; all four +conformance process controls still passed. The historical Darwin runtime timeout +remains unresolved pending reproducible stack or child-stage evidence. + + +* Large Rust emission owner and the shared VM resource phase + +Runs =38766566952= and =38768502717= passed the compiled semantic entry, then +failed the Source runner at the same Case in =t/rust-runtime-generation-test.ss=: +GQL lexical/selective-GLR Rust table emission produced no new receipt for five +seconds while canonical LR, Source index and other owners ran in the shared VM. +The latter run retained hidden log artifacts, confirming the upload repair. +These are distinct from the earlier historical native-startup timeout. + +The generator owner passed alone locally in 15.246849 seconds, including all +12 Cases and module admission. The observed competing four-owner set also passed +locally in 23.325765 seconds; the remote contention failure was not reproduced +on this host. The candidate scheduling repair declares the existing typed +=test-worker-exclusive?= slot for this large transient allocation owner. It +executes after parallel owners join on the persistent caller Worker. It does not +change the generator, expected Rust output, Case count, five-second idle fence, +or total budget. The same four-owner set passed in 22.245382 seconds with +=TEST-SUITE-EXCLUSIVE modules=1=. This is a scheduling control, not a claimed +wall-clock speedup or final macOS CI acceptance. Formal input binding includes +the newly declared owner; independent macOS qualification remains required. + + +* Canonical LR corpus preparation ownership + +Regular run =38773656015= at PR head =48607f3= exhausted the existing Source +owner budget in =t/canonical-lr1-reference-test.ss= while continuously checking +parse equivalence. Its helper called public =lr-parse= twice per enumerated +word. That entry constructs =lr-prepare= on every call, repeating the same +immutable grammar preparation throughout one sweep. + +Lean's =prepared_corpus_preserves= gives the pure preparation/execution equality; +=corpus_preparations_saved= charges two preparations per sweep instead of two per +word. Native execution must additionally preserve fresh request state and input +ownership. The helper now retains one prepared runtime per parser within the +sweep. Every word still allocates its own tokens and executes the full parser. +There is no global preparation cache or mutable token reuse. A new control +compares prepared and fresh public requests across accepted words, rejected +words, and acceptance after rejection. Its first fixture invocation mistakenly +passed a positional construction argument as a keyword; that failed run is +retained and the call corrected without changing parser expectations. + +The corrected four-owner batch passed in 23.875752 seconds: canonical LR 22 +Cases, Rust syntax 17, Source index 7, and Rust generation 12. This is local +functional/scheduling evidence. It is not a matched Scheme CPU measurement of +the sweep or remote macOS acceptance. + +The separate manual run =38773702045= stopped earlier in the source callback +history control. Its child log contains only =HISTORY-COMMIT-START= before the +five-second idle fence. This establishes a pre-first-module-admission boundary; +it does not identify the exact loader operation or prove that the corpus change +repairs that earlier failure. Both failures retain their original deadlines. + + +** Source-caller import admission before script expansion + +A local replay of the original history command stopped in 5.583568 seconds with +zero child-log bytes. SDK =gerbil/main.ss= documents Gambit's script =##begin= +wrapper, and =runtime/eval.ss= compiles begin bodies before evaluation. A +script's first expression is therefore not a reliable boundary before its +static import expansion. Its runtime preference/preload expressions can be too +late to govern that expansion. This explains a possible cold admission path; +the original remote 21-byte log does not by itself establish its exact stack. + +The CI invocation now uses the existing =preload-test-imports= in a preceding +=-e= expression of the same =gxi=, then executes the original source script. The +shared helper admits the script's declared dependencies through the official +loader with actual module/interface boundary tracing. Every history assertion +and native callback still executes from the same source caller. There is no +extra process, synthetic heartbeat, timeout increase, or conversion of the +control to an AOT caller. The revised invocation passed locally in 22.491491 +seconds with the unchanged five-second idle and 90-second total bounds. This is +local source-caller acceptance; current-head macOS CI remains an independent gate. diff --git a/docs/research/engine-record-access.org b/docs/research/engine-record-access.org new file mode 100644 index 00000000..40d0d8a8 --- /dev/null +++ b/docs/research/engine-record-access.org @@ -0,0 +1,184 @@ +#+title: Closed engine records in LR recognition and AST construction + +* Measured implementation unit + +This study follows [[file:parser-engine-formal-model.org][the reduction and sequence model]] and +[[file:parser-evidence-chain.org][the executable evidence chain]]. The production change declares nineteen +fixed internal records final across the LR runtime, action rows, recognition +sequences, event program and prepared parser machine. The measured boundary is +prepared LR recognition through complete source parsing and artifact publication. + +The model's counted reductions and sequence operations locate algorithmic work. +They do not establish its CPU share. Three preliminary candidates were withheld: +combined stack-prefix traversal and unary value reuse, prepared production +closures, and fused undecorated unit dispatch. Unit dispatch reduced allocation +by roughly 2% in a long request but did not improve the representative GQL CPU +median. No execution algorithm from those candidates is in this change. + +* Official implementation study + +The installed Gerbil compiler's =gerbil/compiler/optimize-call.ss= accessor +specializer distinguishes a final class from an extensible class. For a final +class, it emits =%#struct-direct-ref= when the input still needs a check; only an +already admitted unchecked calling convention selects =%#struct-unchecked-ref=. +The ordinary extensible case emits =%#struct-ref=. Mutator specialization has the +corresponding distinction. This is the compiler's existing contract. + +The study follows [[https://gerbil.scheme.org/reference/dev/profiler.html][Gerbil's statistical profiler]] and the installed +=tools/gxprof.ss= continuation sampler. Separate prepared and full-source runs +sampled 30,000 representative GQL requests at a one-millisecond interval. +Prepared recognition produced 2,027 continuation samples; full-source parsing +produced 2,995. Top-frame generic structure access accounted for 218 and 285 +samples respectively (about 10.8% and 9.5%). The LR checkpoint loop itself was +40.8% and 31.2%; sequence bounds, alias construction, action selection, table +lookup and source publication also appeared. Inclusive call graphs overlap and +must not be added as disjoint CPU percentages. + +These samples identify an implementation candidate, not a precise causal CPU +breakdown. The installed SDK did not package the profiler command and its +source referenced older standard-library imports. A local adapter retained its +sampling and analysis routines; the adapter, workload and raw profiles remain +ignored study evidence. Profiling and compiler-interface loading affect the +sampled heap. Performance admission therefore uses separate processes and the +unmodified request sampler, without the profiler or source-interface expansion. + +* Scheme representation contract + +#+begin_src scheme +;; Execution metadata has a fixed engine-owned layout. +(defstruct lr-action-row + (ascii-literals literals tokens eof layout-start layout-next casefold-literals) + transparent: #t final: #t) + +;; Native access remains checked for an unadmitted receiver. +(lr-action-row-tokens row) +#+end_src + +** Question: why does this exploit Scheme's strengths? + +Named native records expose ownership and field meaning to readers and Gerbil's +compiler. The final declaration states that execution metadata has no subclass +protocol. The compiler specializes ordinary accessors without manual unsafe +primitives, duplicated dispatch, positional vectors or foreign AST storage. +Public grammar composition and POO extension remain separate responsibilities. +Existing runtime initialization setters retain their mutation contract; final +means a closed type, not immutable fields. + +** Question: how is this scientifically checked? + +The compiled action-index module contains seven generic =##structure-ref= +operations in the retained reference and seven =##direct-structure-ref= +operations in the candidate. Default checked access rejects scalars, lists, +vectors and an unrelated token record. Complete semantic suites cover reduction +order, checkpoints, persistent sequences, field/alias events and publication. +The performance experiment checks independent public artifacts before timing. +Neither generated-code inspection nor the abstract Lean model alone establishes +runtime equivalence or improvement. + +* Independently loaded matched requests + +The reference production revision is =9a5d6d9956ca1fcc1e1b37e62c077917a69e734e=. +Its compiled library was retained before applying the final declarations. +Both images execute the same compiled workload under the same Gerbil SDK +(=2591dcd= with the configured patch set). Each sample group runs in a fresh +process. The loader selects retained reference products or candidate products; +the runtime class flag is checked as false or true respectively. Baseline +source identities, every retained product and the workload digest are saved +in a repository-relative manifest under ignored =.data/parser-lr-unit-study/=. +This host declaration is not a compiler attestation. + +Four groups run in reference, candidate, candidate, reference order. Each stage +has forty samples of 1,000 calls. Preparation, validation and artifact hashing +are outside the clock. The representative GQL query, a 64-value GQL RETURN, +a 64-term FHIRPath addition and Arithmetic =1= are tested. Prepared results +and an independently materialized GLR result publish the same canonical artifact +as full-source parsing. All four artifact hashes match across every group. + +The SDK's unchanged 4,096-character UTF-8 encoder supplies the environment +control. Arithmetic exercises the changed shared parser machine and is retained +as a small grammar workload. It cannot be an unchanged implementation control. +The encoder's candidate/reference aggregate CPU median ratio is =0.994453=; +its repeated median bands overlap and satisfy the existing 5% drift margin. + +| Workload | Prepared LR CPU ratio | Complete source CPU ratio | +|----------+-----------------------+---------------------------| +| Representative GQL | 0.920224 | 0.925755 | +| GQL RETURN, 64 values | 0.929880 | 0.943689 | +| FHIRPath addition, 64 terms | 0.899957 | 0.944110 | + +Every ratio divides the largest candidate CPU median by the smallest reference +CPU median. All six are below =0.95=. For each stage, the largest candidate CPU +P95 is at most =1.05= times the largest reference CPU P95. These are batch CPU +quantiles for 1,000 calls, not individual-request latency quantiles, and they do +not establish concurrent throughput. Arithmetic prepared medians are below one +millisecond per batch; its full-source tail varies substantially between runs. +No small-grammar P95 improvement is claimed. + +Large-request intervals all include collections. Their allocation sample count +is zero, so allocation bytes are unavailable. No allocation benefit is inferred +from counter deltas crossing a GC. This study admits a repeated CPU improvement +only; the existing backend allocation admission remains unchanged. + +* Native admission and evidence ownership + +=t/benchmarks/lr-request-stack/record-access.ss= owns the workload and complete +artifact comparisons. =record-evidence.ss= reads all four supervised outputs, +requires exact class flags, complete sample indices, summary/sample agreement, +artifact equality, successful exits and the fixed CPU/control/tail thresholds. +It recomputes percentiles using the existing Scheme sampler. Duplicate completion +and changed-artifact mutation witnesses are rejected. Python does not compute +performance statistics. + +The full engine collector retains model, source/FFI/test/benchmark preparation, +Scheme conformance and request observations. The optional record-study review +also binds all four raw receipts, baseline products, current products and the +baseline manifest. Native Ascent separately requires =record-cpu-improvement= in this scoped +review and reviews content identity and dependencies. A successful ordinary +measurement cannot substitute for a missing or rejected CPU admission. +Its evidence is not a Scheme-to-Lean refinement theorem, a native compiler proof +or permission to execute a parser change. + +* Reproduction and qualification boundary + +Build the reference revision in a separate checkout with the same SDK and retain +its compiled library. Build the current workload against both images, preserving +all numbered native load products. Execute the workload with the existing +=preload-runtime-module= owner; do not import source interfaces into timed +benchmark processes. Save the supervised outputs as =reference-a.log=, +=candidate-a.log=, =candidate-b.log= and =reference-b.log= beside =baseline.json= +and =baseline/lib/=. Select the upstream Ascent ABI artifact through +=ASCENT_ENGINE_LIBRARY=, then run: + +#+begin_src sh +uv run --project formal/evidence --locked python -m unittest discover -s formal/evidence -v +uv run --project formal/evidence --locked python formal/evidence/collect_engine.py --record-study .data/parser-lr-unit-study +#+end_src + +The standard =t/benchmarks/build.ss= owns both new benchmark modules. Build +supervision retains 240 seconds total and 30 seconds inactivity; runtime retains +120 seconds and 5 seconds. Cold-loader failures and rejected candidates remain +failed local observations. No CI job, wrapper compiler, deadline increase or +reduced semantic gate is introduced. + +The =2cd75d9= CI failure was a stale package-ownership test requiring the removed +Scheme Rust dependency and an old ASP pin. The precise test repair landed as +=9a5d6d9=. That revision's CI completed successfully after retrying a Darwin +preparation inactivity failure with unchanged thresholds. Local semantic results +and matched CPU admission are separate from the new revision's remote CI. + +* Local closure + +The final collector completed nine passed observations: Lean, Quint, four +standard build owners, seven Scheme modules, the composed request sampler and +strict record CPU admission. Native Ascent established all five scoped +requirements, returned no missing requirements and kept execution authorization +false. Eleven host/native controls passed; native Scheme also rejected duplicate +completion and changed-artifact receipts. Local semantics covered 77 Cases; +two belong to pre-existing uncommitted tests, whose edits are not included in +this engine commit. Remote exact-head CI remains a distinct clean-source gate. + +The existing LR backend sampler separately reported unstable control and no +admitted complete-request benefit. That negative result remains intact; it does +not establish or invalidate the separately loaded record experiment. The +record study's raw samples, products, manifests and review packet remain ignored +local artifacts, with content identities and repository-relative input names. diff --git a/docs/research/engine-reduction-generation.org b/docs/research/engine-reduction-generation.org new file mode 100644 index 00000000..67be682f --- /dev/null +++ b/docs/research/engine-reduction-generation.org @@ -0,0 +1,275 @@ +#+title: Shared engine reduction generation + +* Scope and ownership + +The optimization target is the engine implementation, across language packages. +Benchmarks and complete language products verify the implementation; they do +not define a language-specific optimization target. This module follows one +edge of the canonical grammar -> prepared machine -> semantic reduction -> +canonical artifact path. It complements the +[[file:scheme-deflanguage-design-synthesis.org][DSL architecture synthesis]] and +[[file:lr-request-cost-boundaries.org][complete-request cost study]]. +The [[file:request-owned-lr-stacks.org][request-owned stack experiment]] records a larger +representation change rejected after matched native measurements, with a +cross-language benchmark retained for subsequent engine work. +The subsequent [[file:semantic-sequence-representation.org][semantic sequence study]] follows the same +request controls into repeated representation validation at action/publication +boundaries. +The [[file:unary-reduction-corridors.org][decorated unary corridor study]] distinguishes retained semantic +actions from the lifetime of their intermediate immutable stack heads. +The [[file:scheme-reduction-abstractions.org][Scheme reduction abstraction study]] applies official Gerbil staging +patterns to the shared operand-action interpreter and its preparation contract. + +The shared owner is =src/compiler/fused-reduction.ss=. Its checked strategy +consumes production IDs, RHS operands and field/alias actions from canonical +Parser IR. It emits both recognition and event reductions. Language authors +keep declaring syntax and rules; no new language option or public grammar +composition operation is introduced. + +* Repeated stack expressions + +Previously the compiler constructed an independent =nth-tail= expression for +every operand. For a production of width n, the emitted stack-prefix bindings +contained 2n + n(n-1)/2 =cdr= expressions: n for the state remainder, n for +the value remainder, and the triangular sum for operand extraction. The +generator itself repeatedly constructed those nested expression trees. + +The new generator walks the semantic stack once in emitted =let*= bindings, +retaining each suffix needed by the next operand. The state remainder still +uses one linear expression. Width zero retains the original stack; width one +needs no intermediate suffix binding. Operand names remain in source order, +although the stack is consumed from the last RHS operand toward the first. + +#+begin_src scheme +;; Width-three prefix inside a generated reduction clause. +;; v0, v1, v2 are consumed by the original source-order action expression. +(let* ((remaining-states (cdr (cdr (cdr states)))) + (v2 (car semantic-values)) + (tail2 (cdr semantic-values)) + (v1 (car tail2)) + (tail1 (cdr tail2)) + (v0 (car tail1)) + (remaining-values (cdr tail1))) + ;; The existing offset, field/alias action and goto publication follow here. + (values v0 v1 v2 remaining-values remaining-states)) +#+end_src + +Why is this design preferable? The canonical compiler shares stack suffixes +once, instead of requiring grammar authors or individual backends to encode +their own extraction algorithm. Scheme lexical bindings express that sharing +directly, and the same generated action algebra serves both semantic backends. +Why is the claim scientifically testable? For every emitted rule, the test +counts stack-prefix =cdr= expressions independently of the generator helper and +requires exactly 2n. It also compares complete artifacts against the generic +executor, including source order, fields, aliases, precedence and rejection. + +* Evidence and limits + +The checked-in HCL and Arithmetic reduction modules were regenerated through +the shared strategy. HCL source changed from 126076 to 125628 bytes; Arithmetic +from 25310 to 25246. These are source sizes, not native image sizes or runtime +allocation measurements. The important structural result is removal of the +quadratic operand-expression construction, not a percentage inferred from +these two small grammar widths. + +Eighteen source cases passed for strategy contracts, both languages' independent +recognition/event backends and the all-language accepted/rejected ABI byte +fixture. The receipt is unarchived local execution. +The generic executor remains the semantic control; generated modules retain +the same grammar digest and backend admission contracts. The generated state +and value tails are private implementation bindings. + +This does not establish a native speedup. Gambit or the C compiler may already +share repeated reads during optimization. Native compilation and execution +qualification must be reported separately, and this study does not reinterpret +a source-expression bound as a P50/P95 result. + +The shared compiler and both emitted modules compiled with official +=gerbil compile -O=. Native GxTest then passed eight cases, including the +structural read bound and complete recognition/event artifacts for both language +families. The child completed in 15.578 seconds, exit zero, with =HARNESS-OK=, +final =OK=, no supervision reason and no missing receipt. The candidate load +path was a temporary native overlay; admission traces and +emitted-source controls are retained in the receipt +unarchived local execution. + +Two earlier native attempts were not admitted: one reached the five-second +idle gate during untraced test import, and one exposed a stale local +=parser-test~0.scm= without a usable native stage product. Recompiling the two +test-language dependencies and using the existing dependency/actual-interface +trace closed this qualification. No timeout was increased and no new test +runner was added. This native result is semantic/compilation evidence, not a +performance result or the PR's remote CI result. + +* Prepared recognition reductions borrow the semantic stack + +The next implementation owner is =src/runtime/lr-parser.ss=. Previously its +deterministic generic executor copied a width-n semantic-stack prefix into a +source-ordered list for every reduction wider than one. =reduce-value= consumed +that list immediately. Checkpoints and retained fragments require the original +immutable stack and semantic values, not this transient list. + +Preparation now stores reversed RHS operand references once per runtime. +Recognition reduction borrows the existing stack, bounds the consumed prefix, +applies each operand's closed field/alias chain in declaration order, and +prepends the completed operand to restore source order. No language-specific +branch, new DSL declaration or semantic callback is introduced. The prepared +references are shared when selecting the event runtime. They cost one vector +and reversed operand lists at preparation; preparation is excluded from the +steady-state measurements below and is not claimed to become cheaper. + +#+begin_src scheme +;; Conceptual width-three recognition reduction. The canonical action chains +;; remain attached to their operands; only the private stack traversal changes. +(let loop ((operands '(rhs2 rhs1 rhs0)) + (stack semantic-stack) + (children '())) + (if (pair? operands) + (loop (cdr operands) (cdr stack) + (recognition-sequence-append + (apply-closed-actions (car operands) (car stack) offset) + children)) + children)) +#+end_src + +Why is this design preferable? The engine owns the stack orientation and +normalizes operand references once. Authors keep declaring source-order syntax +and rules. Scheme's tail calls consume persistent stack suffixes without +copying them; the semantic algebra has no user callback whose operand +evaluation order could become observable. Action order within an operand, +source order in the product, byte offsets and retained stack identity remain +explicit obligations. This is a runtime algorithm change, not a request to +language authors to simplify their grammar manually. + +Why is the claim scientifically testable? =t/lr-stack-reduction-test.ss= +compares complete canonical artifacts with the independent source-order, +materialized GLR reducer for widths 0, 1, 2, 8 and 32, both semantic backends, +and chained fields/aliases. Existing nullable, layout and fragment-transfer +controls cover retained siblings and incremental use. The removal bound is +the sum of RHS widths for generic deterministic recognition reductions wider +than one. Unary reductions already borrowed their value; generated reductions +and GLR are outside this removal bound. + +The reusable workload is =t/benchmarks/semantic-stack/benchmark.ss=. It builds +the same prepared grammar and immutable input tokens once, admits the complete +product against GLR, warms both reduction-only and reduction-plus-publication +routes, and reports CPU/allocation statistics for both backends. Grammar +construction and scanning are deliberately outside these samples. Publication +is measured separately so that an allocation transfer cannot pass unnoticed. + +* Rejected universal reverse-order event construction + +The first candidate also prepended event programs while consuming the stack. +Complete artifacts agreed, but semantic equality did not establish equivalent +construction cost. =event-program-append= has a sequential append builder; +right-associated joins can force different chunk/tree work. In sequential +native baseline/candidate/candidate/baseline runs, width-32 identity operands +with complete publication had per-call median CPU values of 0.01295, 0.02117, +0.01493 and 0.01287 ms. This candidate is rejected for the event backend. + +The final implementation borrows recognition stack values while preserving +source-order event reduction and its append builder. These are private backend +algorithms for the same admitted action algebra, not a legacy DSL branch. +GLR retains its independent materialized reducer and constructor interning. + +The initial native experiment used 20 samples of 100 iterations per route. +For recognition width 8, complete-product allocation was 7426.08 -> 7042.08 +bytes/call for identity operands and 16514.08 -> 16130.08 for decorated +operands. Width 32 was 20146.08 -> 18610.08 and 56498.08 -> 54962.08, +respectively. The same differences held for reduction-only measurements and +both baseline/candidate pairs. Width one was unchanged. Allocation admission +excludes samples crossing GC; the fractional constant includes amortized +measurement overhead. These synthetic grammar numbers are not language +P50/P95 or Rust FFI performance claims. CPU variation prevents a general +latency improvement claim. + +Initial receipts are unarchived local execution. Keep the negative event result as +part of the algorithm decision, rather than promote its lower allocations. + +* Final implementation qualification + +The executor binds the two checked stack suffixes through closed +=reduction-stack-tail= calls. The macro emits its continuation once and the +recognition reducer reads the borrowed original values. Event source-order +materialization uses its own closed prefix procedure. These functions have no +captured parser request or grammar callback. This also keeps the stack-prefix +loops outside the large executor continuation during native compilation. + +The final module and new test compiled with official Gerbil =-O= and the SDK's +native C optimization in 16.682 seconds, exit zero, no supervision reason. +Actual interface and GCC function/pass output supplied progress; the five-second +idle gate remained unchanged. The receipt is +unarchived local execution. +Gambit's [[https://gambitscheme.org/4.9.7/manual/][documented stdio buffering setting]] was used for compiler +diagnostics so completed output lines are visible immediately. This is I/O +configuration, not a timer generating progress or a GC/heap workaround. + +Seventeen final source cases passed in 5.670 seconds with actual zero exit, +and two native GLR parity cases passed in 6.746 seconds. Receipts are +unarchived local execution. + +Final sequential native ABBA receipts are +unarchived local execution. Each completed 20 samples of 100 iterations +for every route, passed complete-product admission and exited zero. The final +recognition allocation differences remain 384 and 1536 bytes/call at widths +8 and 32, in both reduction-only and complete-product routes. Width one and +every event route retain baseline allocation. For width-32 identity event +publication, per-call median CPU is 0.01259, 0.01278, 0.01287 and 0.01284 ms; +the earlier reverse-join candidate's increase is absent here. Other routes +contain CPU variation, so no general speedup is inferred. + +Earlier qualification failures remain failures: source all-language loading +hit idle and total gates, interrupted compiler variants hit the idle gate, +and one source invocation attempted to interpret the compiled FFI exit helper. +The final source test path isolates only that existing native exit helper; +parser modules remain source-owned. This is not permission to count an =OK= +marker without process completion. The language test dependencies have a +separate official native compile receipt, +unarchived local execution (32.976 +seconds, exit zero), rather than relying on stale test =.scm= products. + +The native all-language ABI assertion and generated HCL/Arithmetic controls +passed nine cases in 38.634 seconds, actual exit zero, =HARNESS-OK= and final +=OK=. All ten language entries supplied accepted/rejected products: twenty +byte-for-byte controls against the existing Rust fixture. Candidate LR interface +admission is recorded in +unarchived local execution. + +These cost and initial native product controls used parent =dd07195=. Before +publication the checkout advanced to =0906fa3=, including the independent +result-profile and private word/projection refinements. Thirty-seven source +integration cases then passed, including the incoming result-profile rules, +in 26.183 seconds with zero exit and no supervision reason. The receipt is +unarchived local execution. +This distinction prevents an earlier compiled overlay from being reported as +proof of every incoming module. Exact-head clean CI remains separate evidence. + +The incoming native refresh initially found an unbound +=result-projection-parts-field=: compiling its consumer first admitted the old +local =result-profile.ssi=. Compiling the producer, generator and consumer in +dependency order succeeded in 22.646 seconds. The refreshed native overlay then +passed all nine controls and twenty language ABI products again in 14.162 +seconds, zero exit, no supervision reason and no missing final receipt: +unarchived local execution. +The package's standard dependency scheduler owns this order in normal builds; +these isolated compiler calls do not establish a second build system. + +* Remaining shared-engine implementation boundary + +The [[file:shared-parser-cost-sampling.org][shared sampling owner]] now closes the cross-language measurement dependency +on the GQL workload module. Selected-derivation counts and independent prepared/ +complete-product controls remain in place; subsequent recognition/reduction +experiments use one sampler and one allocation/GC attribution contract. + +The next implemented preparation boundary and its admission/cost controls are +documented in [[file:prepared-lr-action-plans.org][prepared LR action plans]]. + +Follow the same canonical rule/action algebra through generic LR execution, +generated reductions, retained recognition and canonical event publication. +Establish which representations must survive for checkpoints, GLR and replay, +and which can be eliminated for an admitted deterministic execution. Reuse +the existing semantic backend owner rather than add grammar-level controls. +Before changing that route, compare accepted and rejected complete products +across independent language families, and include retained/incremental products +where the ownership contract requires them. Core work is the compiler/runtime +change; the corpora and benchmarks are its admission evidence. diff --git a/docs/research/event-fold-prepared-state.org b/docs/research/event-fold-prepared-state.org new file mode 100644 index 00000000..39c0149a --- /dev/null +++ b/docs/research/event-fold-prepared-state.org @@ -0,0 +1,58 @@ +#+title: Request-isolated prepared EventFold state + +* Ownership and execution + +The engine now prepares a private EventFold program from closed strategy data. +Preparation snapshots pairs and strings, preserves DAG sharing, rejects cycles +and executable values, validates typed initial state, and indexes helper names. +No source, runtime handle, GC root, request cache or mutable working state is +retained in the program. Existing language declarations remain the author path; +this is not a new callback DSL. + +Each root/helper layout contains an identity-keyed name index and immutable +initial values (Boolean, unsigned integer or empty stack). A request copies the +initial-value vector. Stack updates, parameter overrides and temporary bindings +belong to that request. Temporary storage is allocated only when used. Removing +and rebinding a declared slot preserves the existing missing-update policy; +duplicate declarations retain first-binding semantics. + +The existing declaration macro captures one program at module initialization. +Org's private POO strategy similarly holds the engine program and invokes it +through a thin adapter. There is no downstream compilation cache or new process +launcher. The dynamic =run-event-fold= entry prepares explicit caller data and +uses the same executor; it is not a second parser. + +* Qualification and evidence limits + +The first native qualification used =8e98e830= plus this candidate, the existing +Gerbil SDK and the existing Just owners. All 61 EventFold cases passed, including +five new controls for request isolation, temporary bindings, first declarations, +source snapshots/UTF-8, helper parameters and malformed/cyclic input. The hard +idle watchdog, assertion propagation and final =OK= requirement remain intact. +Subsequent upstream alignment requires fresh qualification; these results are +not relabelled as a different source revision. + +The consumer's =021f97a7= diagnostic baseline completed 2,032 physical native +requests for 2,000 documents: 32 parser batches plus 2,000 document-semantic +crossings. Its first sample reported 754.850ms batch owner thread CPU and +810.139ms all-request owner thread CPU. Rust syntax/graph/record stage wall sums +were about 50ms. These counters have different scopes; overlapping worker wall +sums, VM GC intervals and operation waits cannot be added as exclusive costs. + +The observed baseline gate still failed. Wall times were unstable under other +host workloads: diagnostic best-of-five was 1.402s; a separate untraced run was +2.868s. Neither result is an optimization claim or a replacement for earlier +602ms/597ms historical samples. The original 100ms threshold is unchanged. + +* Remaining closure + +State preparation removes repeated layout construction and per-slot cells, +not all strategy interpretation. =fold-statements= and predicate evaluation +still inspect forms at runtime. Direct native strategy specialization remains +an engine responsibility. Static byte-set preparation and per-document semantic +crossings also require attribution and matched controls before further changes. + +Prepared-state functionality, complete consumer parity, both runtime choices, +1,000/10,000-document concurrency and the unchanged latency gate are separate +admissions. Native compilation or a passing unit suite alone closes none of the +remaining performance gates. diff --git a/docs/research/event-fold-recognition-architecture.org b/docs/research/event-fold-recognition-architecture.org new file mode 100644 index 00000000..a9550cde --- /dev/null +++ b/docs/research/event-fold-recognition-architecture.org @@ -0,0 +1,276 @@ +#+title: EventFold recognition architecture audit + +* Scope and decision + +Audit the recognizer, not only the event representation. Ordered lossless +events are an output contract; they are not an algorithm for selecting rules. +Retain that contract provisionally. The current recognizer is not qualified +as a high-performance architecture: syntactic AOT lowering and code interning +do not establish shared evaluation, bounded scan visits or active-state dispatch. + +* Research obligations and concrete implementation boundaries + +| Research principle | Current implementation | Missing admission | +|--------------------+------------------------+-------------------| +| Typed algebraic parsing: deterministic choice and progress | Closed typed state/helper IR; recursive helpers rejected | No grammar-wide nullable/first/overlap proof for contextual Org rules | +| Staging: remove static dispatch rather than interpret grammar | Generated Scheme expressions and transitions | Thousands of private calls still represent expression structure; code sharing is not value sharing | +| flap: fuse recognition stages and eliminate ephemeral tokens | Direct event tape avoids nested result lists | Output tape is not flap's discarded lexer token stream; do not delete required output events | +| Shared immutable observations | Expression interning; adjacent probe fusion | Frame-level scan facts and control/effect barriers need explicit admission | +| Ordered semantic actions | Request slots, ordered statements and request-owned tape | Any dispatch/fusion must retain writes, queries, exceptions and emission order | +| Input progress and bounded work | Byte loops progress; future queries have separate indexes | No whole-program scan-visit bound; helper acyclicity alone does not prevent repeated traversals | + +Primary sources: A Typed, Algebraic Approach to Parsing (PLDI 2019), +https://www.cl.cam.ac.uk/~jdy22/papers/a-typed-algebraic-approach-to-parsing.pdf; +flap: A Deterministic Parser with Fused Lexing (PLDI 2023), +https://arxiv.org/html/2304.05276. Their deterministic/LL(1) assumptions and +published timing ratios are not established for this Org recognizer. + +* Concrete findings + +The actual generated horizontal-prefix procedure expression-3471 scans from +start to the first non-horizontal byte. Offset procedure 3473 calls it; 3476 +calls 3473. The diagnostic product reports 4518 calls to 3471 over 24 complete +Org controls. This is direct evidence of dynamic repeated work, not exclusive +CPU attribution or an average claimed for each document. + +The condition lowering also normalized nested and/or results to Boolean values +when only branch truth was required. Branch contexts can retain native Scheme +short circuiting; published Boolean slots must still normalize to #t/#f. + +Helper-local state vectors, repeated source probes, source encoding, event +storage growth and final publication are separate allocation boundaries. +Removing one cannot establish removal of the others. The 71.459 ms historical +batch is not fully explained by GC; wall minus CPU does not name the cause. + +The reset/increment summary experiment conservatively removes unconditional +current-byte counter transitions from the exceptional alphabet. Lean proves +the affine/reset summary fold; Quint keeps lifecycle/publication responsibility. +Actual AOT controls pass but CPU remains near the control's 43-44 ms in the +serial reverse round. The implementation is withdrawn: static trigger removal +does not identify the executed dominant path. A changing-input native call +census is needed before claiming additional shared-scan/state work is material. + +Live paired realtime/monotonic probes reproduce non-GC request stalls with +involuntary OS descheduling. They separate complete-tape comparison and probes +from the parser interval. This is not retrospective classification of older +batches, an exclusive CPU profile, or permission to change kernel/scheduler +configuration. Both CPU work and off-CPU wall inflation remain relevant. + +The changing-input native census observes roughly 95000 calls to multiple +residual state-selection nodes for 1000 documents, while encoding/publication +each runs 1000 times. Call frequency is not exclusive CPU attribution. Whole +residual byte partitioning expands code/RSS and is withdrawn; separating idle +state from trigger-byte admission also fails to produce a stable major gain. +Both pass source-bound models and actual tapes but fail the 10 ms gate. Their +negative outcomes are retained, not compensated by selecting a fastest lane. +Next admission must establish dependency support per independent consumer and +executed coverage rather than repeatedly test one all-zero machine guard. No +counter is declared irrelevant from its name and no invalidation cache is added. + +* Execution restructuring + +Preparation must separate immutable source/frame observations from stateful +transitions. Lexical frame facts have no request lookup table or invalidation +cache. Only total, state-independent observations may be moved before a branch; +state-dependent ranges, future queries and unknown operations remain ordered. +Every source-bound change receives new facts. Helpers own independent frames. + +State execution must use admitted necessary-byte/active-state facts, not a +hard-coded list of Org markers. Unknown effects reject specialization. Run +coalescing is restricted to overwrite-only transitions with a retained trigger +residual. The current run planner is existing infrastructure, not proof that +every plain-text path is coalesced. Measure executed coverage before expanding +admission; do not add a mutable guard cache already rejected by matched tests. + +* Non-overlapping verification responsibilities + +The current experiment stages conservative necessary-prefix tries per +immutable line frame, across ordered consumers. A frame-local Boolean guards +each admitted original condition, retaining each transition only once. +A negative classification bypasses impossible conditions, including their +total state reads. Positive classification retains the original condition +and suffix tests; it does not infer a successful rule. +Queries, errors and unknown effects preceding a prefix prevent admission. +The anchors are the original line start and the admitted total horizontal +prefix scan. The latter is bound once and reused by the original consumers; +offsets derived from mutable state are not classified. Both anchor tries feed +one conservative Boolean: an unrelated positive trigger cannot admit a rule, +and an absent trigger cannot suppress an earlier query. Nested source frames +discard both facts. This covers expanded byte predicates as well as literal +prefix operations without introducing a line-index cache. +Source-bound changes, loops and helper handoffs remain separate owners. +The residual preserves else transitions, writes and event order. This adds +neither a request cache nor a new runtime. Native code growth, RSS and matched +timing must qualify it; fewer source predicates alone cannot admit the change. + +Lean: denotational refinement of scan reuse, branch truth/observable order, +overwrite-only runs and trigger residuals; arithmetic source-progress and +operation-count bounds. Preconditions must be explicit. Abstract theorems do +not prove the concrete Scheme compiler or physical milliseconds. + +Quint: finite histories of frame activation, stale-fact rejection, ownership +and publication. Keep these separate from Lean's algebra. Bounded seeded runs +are not exhaustive proofs of all runtime histories. + +Actual AOT: generated-unit identity, complete tapes for independently prepared +changing inputs, configuration overrides, frame boundaries, Unicode and error +controls. Interpreter/compiler agreement on one strategy is not independent +evidence that the strategy implements all Org syntax. + +Performance: unchanged 1000 changing-request <=10 ms and >=100000 requests/s +preflight first. Only then run 1000/10000/100000 requests for twenty batches +each, reporting per-request P95/P99 in microseconds, batch wall in milliseconds, +throughput in requests/s and whole-process peak RSS in bytes. Matched controls +must be sequential and host contention recorded. Slower candidates are removed; +original executables and all negative receipts are preserved. + +* Concrete restructuring candidates and evidence boundaries + +The frame-fact/condition candidate passed five frame controls, four condition +controls and 24 complete actual Org AOT controls. Its immutable scanner fact +is lexical, not a request memo table. A strict control/candidate/candidate/control +preflight reports 83.344/47.997/47.991/50.837 ms. The first control contains +a 13.505 ms request spike; do not infer a large causal speedup from that sample. +Candidate P95 is 67/65 us, P99 114/108 us, throughput approximately 20835/20837 +requests/s, RSS 28803072/28819456 bytes. All four fail the unchanged gate. +GC remains five and GC-spanning counter deltas are not admitted allocated bytes. + +An opt-in lexical-state owner layout replaces helper private vectors with +local scalar variables. Mutating statements cannot escape into by-value +interned procedures; read-only expressions can receive current scalar values. +Independent helper frames and explicit overrides retain their owners. Root +state crosses the per-line callback and finish phase and retains an explicit +shared vector. This +is a generated execution-region representation change, not a language DSL, +new runtime or ABI. It requires complete AOT and matched timing before admission. +Lean's frame_representation_refinement makes transition commutation an explicit +premise; it does not prove concrete Scheme lowering. Quint keeps the separate +activation/ownership/publication histories. Compilation size and time are costs, +not permission to inline the whole grammar or widen qualification timeouts. + +The all-frame lexical experiment is rejected: 41.072/104.958 ms against +46.292/44.342 ms controls; P95 57/154 us, P99 65/1168 us, RSS +28147712/28196864 bytes and six GCs versus five. The second request maximum is +8.471 ms and wall minus CPU is 47.950 ms. No causal attribution is inferred +from these counters. A lower median or RSS cannot admit the unstable candidate. +The source lowering of cross-callback root state was removed. Its immutable +generated product and both negative receipts are retained, not selected as a +qualified baseline. The helper-only candidate is also rejected: 145.675/44.563 ms +against 44.977/48.229 ms controls; P95 299/62 us, P99 2130/103 us, RSS +27426816/27361280 bytes and seven GCs versus five. The slow lane has 76.494 ms +wall-minus-CPU; its cause remains unproven. Both lexical owner lowerings and +their compiler flag were removed. Fewer calls and lower RSS cannot admit them. +The active source retains vectors plus frame-fact/conditional-truth lowering. + +* Rejected one-step state settling candidate + +An overwrite-only admission was too restrictive for first-byte capture and +escape-reset transitions. The candidate computes constant post-state facts +at compile time, retaining the first ordinary-byte transition in order. Only +facts common to both branches are usable. The residual must preserve those +facts and contain only total overwrites whose reads do not intersect residual +writes. Prefix bounds and entry guards must remain unwritten by the entire +entry transition. Events, queries, exceptions, helpers and source-frame changes +reject this admission. The scanner leaves the first trigger for the original +transition and never transfers the event owner or caches a request value. + +For the unchanged actual Org IR, static eligible loops increase from 33 to 47 +out of 285. This is preparation coverage, not executed coverage or a speed claim. +Lean's settled_run_refinement owns the conditional stable-region algebra; +Quint's peeled phase owns same-epoch entry-before-commit lifecycle ordering. +Neither substitutes for concrete admission tests or complete AOT tapes. +The <=10 ms preflight remains mandatory before the twenty-batch matrix. + +V1 failed matched physical qualification: 48.930/47.077 ms versus +46.374/47.903 ms controls. V2 preserves already admitted kernels and avoids +duplicate entry/residual execution on singleton runs. It passes the new-head +Lean, Quint, native ASCENT and 24 actual AOT controls, but also lacks a stable +throughput benefit: a calm ABBA gives control 44.537/46.412 ms and candidate +45.936/45.341 ms. Candidate P95 is 62/62 us, P99 86/85 us, throughput +21769.418/22055.094 requests/s and RSS 28803072 bytes. All lanes have five +GCs and fail the unchanged gate. An earlier control has 137.279 ms wall and +64.381 ms CPU, with a 27.865 ms request spike; that disturbed sample cannot +establish a candidate speedup. All raw receipts remain available. + +Both production candidates, their extra admission tests, compiler projection +and model additions were removed. The prototype and source snapshot remain +as rejected evidence, not active runtime infrastructure. Increased static +eligibility does not establish executed hot-path coverage; unchanged GC and +RSS also do not prove the allocation origin. Next architecture work must +measure and fuse repeated recognition across consumers, rather than infer +whole-engine benefit from a local transition's reduced operation count. + +The private checkout fast-forwards to PR #13 head b5e1895 without discarding +owned edits. The new request-owned LR changes do not modify Fold compiler, +tape or source-line inputs; the Fold artifact scope is distinct from complete +qualification of that LR executor or downstream FFI/Tokio/Orgize. + +* Dominating recognition facts across consumers + +The active prototype binds repeated start-anchored delimiter, word and key +observations once in the first dominating consumer's lexical region. Later +consumers reuse scalar offsets, while state writes and direct tape actions +retain their original order. It adds no request cache or intermediate token +list. Conditional first observations, state-dependent bounds, queries/errors, +loops, opaque instructions, helper handoffs and source-frame changes block +this transformation. Existing unconditional trim/content scan experiments +remain rejected and are not re-enabled. + +The actual generated Org product contains nine lexical consumer facts and +7423 static private call sites versus 7443 in the frame-fact control. These +are structural observations, not dynamic coverage, allocation or speed proof. +The initial cross-helper proposal found zero eligible scans in all 27 actual +helper descriptors and was removed. Sharing code cannot be substituted for +sharing an executed source observation. Lean models ordered pure consumers; +Quint models same-extent borrowing and invalidation histories. Actual full +tape comparison and matched wall/throughput gates still decide admission. + +The source-bound Lean/Quint/Scheme-to-actual-AOT join passes. Two matched ABBA +preflights give candidate wall44.855-45.843 ms/1000 changing requests, request +P9562-64 us/P9971-86 us, throughput21813.581-22294.059 requests/s and whole +process peak RSS27.390625-27.4375 MiB. All runs have five GCs and fail the +unchanged10 ms preflight. Controls include a disturbed52.707 ms lane and a +45.827 ms lane nearly tying the candidate. No large/stable speedup or allocation +improvement is admitted, no baseline is promoted, and no full matrix starts. +This is only a dominating delimiter-fact candidate; ordinary-text dispatch +and whole-engine consumer fusion remain open architecture obligations. + +* Independent AOT closure and anchored recognition + +The installed official compiler's =compile-obj= retries while an =.o.lock= +exists. The earlier failed overlapping links left part-11's lock unowned; +subsequent serial attempts waited there, not in a proven kernel deadlock. +After a process inventory and exact-path open-file check, the lock was moved +to retained evidence. Both independent single-body executables linked +successfully, sequentially, without SDK or kernel changes. Test entrypoints +now reject existing object locks without removing another build's evidence. + +The horizontal-anchor extension shares one admitted immutable indentation +observation with both classification and original consumers. State-dependent +offsets, queries, helpers and frame changes remain opaque. Static generated +private call sites are 7420 versus 7423 in the single-anchor predecessor; +static counts are not dynamic work or performance evidence. + +Both versions pass 24 complete actual Org AOT tapes. Observer-free, sequential +ABBA preflights use the same pinned SDK, 1000 changing requests, 2856770 input +bytes, immutable corpus and complete independent expected OEV1 tapes: + +| Control / candidate | Wall ms | Request P95 us | Request P99 us | Requests/s | Peak RSS bytes | +|---------------------+---------+----------------+----------------+------------+----------------| +| Delimiter control 1 | 47.230 | 63 | 100 | 21172.983 | 28770304 | +| Anchored candidate 1| 44.626 | 63 | 96 | 22408.461 | 28917760 | +| Anchored candidate 2| 45.312 | 63 | 124 | 22069.209 | 28901376 | +| Delimiter control 2 | 46.060 | 63 | 96 | 21710.812 | 28770304 | +| Single-body control 1| 51.264 | 68 | 280 | 19506.866 | 28835840 | +| Anchored candidate 3| 44.320 | 63 | 117 | 22563.177 | 28917760 | +| Anchored candidate 4| 44.919 | 64 | 97 | 22262.294 | 28917760 | +| Single-body control 2| 49.605 | 66 | 186 | 20159.258 | 28966912 | + +All lanes complete independent semantic comparison, have five GCs and exit70 +at the unchanged 10 ms gate. Earlier single-body lanes range43.006-45.728 ms, +so the slower later controls do not establish a large stable speedup. Candidate +memory is higher than the delimiter controls. Raw allocation counter deltas +across GC are not admitted allocated bytes. These are engine Fold measurements, +not FFI/Tokio, Python or public Org::parse acceptance. The remaining ordinary +byte-loop work is not eliminated by this frame-prefix prototype. No production +baseline is promoted and no twenty-batch workload matrix runs. diff --git a/docs/research/event-fold-source-accumulation.org b/docs/research/event-fold-source-accumulation.org new file mode 100644 index 00000000..d78cb9d6 --- /dev/null +++ b/docs/research/event-fold-source-accumulation.org @@ -0,0 +1,100 @@ +#+title: Native EventFold source accumulation + +* Ownership + +The engine owns =fold-source-lines=, the byte-bound source-line traversal used +by both the public line-event strategy and the prepared EventFold executor. +Org grammars do not implement another line scanner or event builder. + +* Algorithm + +The traversal threads a caller-owned accumulator through each complete line, +including CR, LF, CRLF and a final unterminated line. Byte boundaries use native +UTF-8 length; an empty source does not invoke the visitor. The general +=source-line-events= interface retains its original forward event sequence. + +EventFold already emits into a persistent reversed event chain. Its executor +now carries that chain across source lines and final transitions, then restores +forward order exactly once. It no longer reverses and recopies each line's +events, or reverses the full event list twice to remove the provisional root +finish before appending final transitions. The chain remains request-owned; +there is no shared mutable queue, kernel patch or Rust syntax implementation. + +* Admission + +The native Fold catalog includes controls for empty input, unterminated input, +consecutive endings, mixed UTF-8 CRLF/CR/LF byte bounds, multiple emissions per +line and final transitions after all line events. Existing helper, state, +outline and invalid-program cases remain mandatory. + +This change reduces event-spine construction and traversal, not statement +interpretation. Prepared predicates and statements still require further +engine specialization. Allocation reduction is not a measured throughput win; +consumer conformance and matched performance receipts remain separate gates. + +* Generated native publication + +The closed Scheme generator now consumes its private reversed outer spine +with =reverse!= at publication. Each request allocates its root cell; every +transition and admitted helper prepends fresh cells. Helpers receive and +return that spine synchronously, and the closed algebra cannot expose it to a +callback or retain it in a state slot. Quoted start/finish payloads and token +payloads are not mutated. The general callback-based line-event API and the +independent interpreted oracle retain their existing publication algorithms. + +The native executable catalog adds one retained-result/caller-mutation +control for each of its six products. It compares mixed UTF-8/CRLF/CR input +and empty input with the independent oracle, holds both results across +repeated calls, truncates their caller-owned outer cells, and requires fresh +subsequent calls to retain complete output. The catalog therefore contains +96 controls, not 90. Source admission also requires destructive publication +in generated code; no public event format, digest recipe or ABI changes. + +This removes exactly one newly allocated outer cell per published event +relative to copying =reverse=. It does not remove event payloads, parser +recognition, metadata evaluation or full graph publication. In particular, +existing =ResultProfile= constructs recognition results, while +=GraphProjection= generates a Rust syntax projection table; neither currently +provides a native EventFold selection plan that suppresses irrelevant output. +A sparse consumer view must be compiled by the engine from admitted +declarations and checked against full native parsing, not implemented as a +Rust scanner or inferred from a passing source-generation test. Strict +downstream latency gates remain separate from this allocation argument. + +* Explicit native frame bytes + +The generated product now traverses its already encoded source with +=fold-source-byte-lines=. Each line owns one byte slice and its decoded +string; offsets come directly from the source byte cursor. Shared generated +procedures receive =line-bytes= explicitly. Nested source bounds and helpers +decode their slice without encoding that string again. Constant prefix bytes +are generated at compile time, rather than allocated by each predicate call. +The interpreter keeps its independent string traversal and dynamic view. + +This follows the ownership discipline in Gerbil's =std/text/parser/ebnf/lexer= +cached stream state and =std/list/list-builder= private accumulation. The +official buffered =std/sync/channel= synchronizes cross-thread producers; it +is not an appropriate replacement for a synchronous request-local event +spine. A shared builder also has continuation re-entry semantics, so the +generator does not introduce a shared queue or mutable global byte cache. + +These are fewer source traversals and transient allocations, not a claimed +end-to-end win. Native oracle parity, retained-result isolation, and the +unchanged complete consumer latency gate must qualify this candidate. + +Local admission rebuilt the optimized SDK, compiled the native control +executable, and passed all 96 oracle/override/publication controls. Helper +controls now also execute prefix and byte predicates inside nested bounds. +The five source-admission cases passed under the existing five-second genuine +output watchdog. Two earlier startup attempts failed that watchdog during +overlapping compilation; neither is admitted as passing evidence. No consumer +latency result is established by these engine controls. + +The native byte cursor now caches its current byte and next position, using +fixnum operations after the u8vector entry contract. The guarded next cursor +never exceeds the vector length, including terminal CR and CRLF cases. The +[[file:../../t/benchmarks/event-fold-frames/README.org][native frame benchmark]] retains a benchmark-only generic cursor reference +and full-output parity at 1,000/2,000/10,000 documents. Five source controls +(including wrong source-type rejection) and all 96 independently compiled +Fold controls pass after this change. CPU observations and wall-time +regressions remain separately reported; no consumer latency gate is relaxed. diff --git a/docs/research/event-fold-whole-path-performance.org b/docs/research/event-fold-whole-path-performance.org new file mode 100644 index 00000000..4d79d874 --- /dev/null +++ b/docs/research/event-fold-whole-path-performance.org @@ -0,0 +1,851 @@ +#+title: EventFold whole-path performance design and admission +#+startup: overview + +* Decision + +** Overwrite-only ordinary-byte segments + +The =tape-run= candidate changes the execution unit from one byte to one +admitted contiguous ordinary-byte segment. Its static classifier depends on +the actual IR, not a fixed document scenario. Entry zero-mode facts are checked +on every segment; write targets must not change them. A stable skipped-prefix +bound may be admitted only after the cursor reaches it. Only total, pure, +state-independent overwrite families can execute once at the segment's last +byte. Events, queries, helpers, state read-modify-write and frame changes reject +coalescing. Exceptional bytes and active modes retain the original body. + +Exceptional-byte cofactoring removes unreachable effect barriers before the +entry facts are reapplied. Facts come from explicit guards, never prior request +values. The actual inline kernels qualify with 16/12 guard cells and seven +overwrite fields; 33 of 285 static loop forms admit runs. This is not a dynamic +frequency or physical performance proof. There is no runtime cache or cache +invalidation maintenance. Lean owns the overwrite-last algebra; Quint retains +separate lifecycle/publication histories. Concrete compiler admission still +requires source regressions and actual native complete-tape conformance. + +Five admission controls and all 24 real AOT tapes pass. Two full sequential +control/run/run/control preflight rounds show candidate walls +62.097/65.347/103.421/62.059 ms for 1000 changing requests. Controls are +132.238/76.472/87.628/363.383 ms; the large wall-minus-CPU pauses and host CPU +contention prevent a qualified speedup claim, and one corresponding pair +regresses. Keep every receipt; do not select only the fast candidates or slow +controls. The <=10 ms gate fails in every lane. Baseline promotion, PR push and +the three-load twenty-batch matrix remain prohibited until preflight passes. +No inferred 100000-request time substitutes for an actual measurement. + +** Effect-admitted elimination of inert observations + +The opt-in =tape-pruned= compiler pass removes nested =if= forms only when +both rewritten branches are empty and evaluating the predicate is total and +pure. Unknown predicates, future-line queries and unproved range checks remain +observable. In a validated byte loop, membership scans from its cursor through +clamped forward steps have ordered in-frame bounds; their discarded results +can be removed. Reversed ranges and =with-source-bounds= invalidate this +admission. No request value cache, global owner or invalidation maintenance. + +Four regression cases cover pure nested observations, query/error barriers, +ordered writes/events, validated forward ranges, reversed ranges and frame +changes. All 24 actual Org AOT complete tapes and source-owned controls pass. +Lean owns clamped-range arithmetic and pure discarded-condition equivalence; +the concrete Scheme effect checker remains a separate implementation boundary. +Quint's lifecycle/publication model is unchanged, not a second algebra proof. + +Strict sequential tail/pruned/pruned/tail preflight: tail 74.197/73.520 ms, +pruned 69.201/71.196 ms per 1000 changing inputs. Candidate P95 90/96 us, +P99 144/156 us, throughput 14450.658/14045.733 requests/s, CPU 69.001/69.988 ms, +five GCs and peak RSS 29474816/29491200 bytes. Control P95 94/97 us, +P99 218/186 us, throughput 13477.634/13601.741 requests/s, CPU 73.473/73.235 ms, +five GCs and RSS 29687808/29671424 bytes. Approximately five percent diagnostic +wall improvement, not the requested throughput breakthrough. All exit 70 at +the unchanged <=10 ms gate; no full matrix or baseline promotion. + +Separate named-helper owner elimination and token-local offset reuse passed +semantic controls but regressed matched wall time and were removed. Their +static reductions (zero owner assignments and fewer calls) are negative +evidence: operation-count changes cannot replace actual runtime acceptance. +Retained negative artifacts and detailed boundaries are recorded in Orgize's +=PERFORMANCE_WORK_RECEIPT.md=; no kernel/SDK or Automation changes. + +** Effect-only direct-tape call graph + +The request-owned tape already mutates in place, but interned statement +procedures returned it and their callers reassigned =events=. The opt-in +=tape-tail= product removes that internal result plumbing. The root still +retains and publishes the tape; named source helpers retain their current +contract. List-producing Fold and default generation remain unchanged. +There is no runtime value cache, invalidation maintenance or extra scan. +Installed Gerbil =compile.ss= enables =proper-tail-calls=; Gambit's headers +separate continuation-frame allocation from movable heap allocation. + +Static generated call-site analysis changes 3689 definitions / 7734 private +calls / 0 confirmed tail sites / 3281 owner assignments to +3648 / 7679 / 2127 / 70. Tail counts are conservative syntactic evidence, +not executed counts, optimizer guarantees or exclusive CPU attribution. +Two structural tests, 24 complete actual Org AOT tapes and the source-owned +canonical/wire controls passed. Lean adds ordered effect/private-owner +threading refinement; Quint separately checks publication and invalidation +histories. Native Ascent binds those observations to the actual copied AOT +artifacts; required matched-performance remains missing, not passed. + +Strict ordinary/tail/tail/ordinary 1000 changing-request preflight: +ordinary 96.537/90.398 ms, candidate 72.575/73.322 ms. Candidate P95 96/96 us, +P99 142/142 us, throughput 13778.849/13638.471 requests/s, CPU 72.440/73.006 ms, +GC 5, peak RSS 29638656/29671424 bytes. Ordinary GC 27 and RSS +30654464/30572544 bytes. The unchanged 10 ms gate still fails (exit 70); +there is no twenty-batch, FFI/Tokio/Python/Org graph acceptance or promotion. + +Matched request-local no-GC allocation means, including observation overhead: +plain-links 122976 -> 15904 bytes (327/332 samples), quote-heavy +231408 -> 24976 bytes (318/332), doc 122384 -> 31888 bytes (325/333). +GC-interrupted requests are excluded. These do not establish all-request +allocated bytes, peak heap or attribution of every byte to continuation frames. +Repeated horizontal scans remain separate work; this change removes the +call-graph continuation burden rather than claiming scan elimination. + +** Rejected active-state and guard-validity kernels + +Two further engine candidates reached real AOT controls, not only a model. +Both preserved the 24 complete native Org event tapes and failed the unchanged +1000-request 10 ms gate. Their source integrations were removed; compiled +negative artifacts and complete timing receipts are retained. + +Removing the all-zero restriction from the common-byte cofactor regressed +strict ordinary/active/active/ordinary walls from 91.652/90.694 ms to +137.734/137.686 ms. Active P95 was 341/344 us, P99 465/465 us, throughput +7260.371/7262.903 requests/s and 57 GCs versus 27 for ordinary. Peak RSS fell +from approximately 30.6 MB to 29.9--30.0 MB, insufficient to justify the loss. +Byte rejection alone leaves active-state work that idle specialization avoids. + +A loop-local invalidated guard cache also regressed: ordinary 118.218/104.248 ms, +cached 133.954/140.565 ms, cached P95 351/321 us, P99 526/567 us and +7465.249/7114.146 requests/s. GC was 29 versus 27; RSS 29.85--30.00 MB versus +30.62--30.67 MB. The second candidate has 27.094 ms wall-minus-CPU, without +request-level causal attribution. Guard invalidation maintenance did not +remove the dominant allocation path. GC-spanning raw allocation counters are +not admitted allocated-byte comparisons. Lean's arbitrary-state ordered +refinement obligations and Quint's lifecycle histories passed separately; +neither establishes a runtime gain or complete compiler proof. No twenty-batch +matrix, baseline promotion, FFI/Orgize acceptance or upstream push occurred. + +** Guarded idle-state specialization and request allocation + +The new opt-in engine plan derives uint guard cells from admitted IR rather +than Org-specific names. It retains the original body outside the all-zero +guard. Ordered writes invalidate individual facts; opaque, helper and frame +effects invalidate all facts. Known tape appends, cache queries and possible +range errors are observable but do not alias the private slot vector: retain +their evaluation and order without discarding unrelated zero-state facts. +Read-only configuration cells must not enter an idle guard. The initial plan +included nonzero inline policy, preventing that ordinary path from entering +its specialization; the corrected plan intersects guard candidates with +actual loop writes. Tests cover that regression and tape-effect preservation. +The corrected plan passed fresh actual AOT controls but still fails the +physical throughput preflight; it remains opt-in, not the accepted baseline. + +A whole-loop common-byte plan combines this zero-state specialization with +necessary-byte regions. Only the common fallback region is generated as a +fast kernel; exceptional bytes retain the complete original body. The validated +current-byte membership table is reused, not a new source scan. Current native +Ascent evidence binds model/source controls and 24 actual AOT tape controls; +matched-performance remains missing and executionAuthorized=false. + +Strict control/ordinary/idle/idle/ordinary/control preflight: common-byte +walls 88.284/90.422 ms per 1000 changing requests against 122.040/120.904 ms +current-source control. P95 112/113 microseconds, P99 312/356 microseconds, +throughput 11327.081/11059.255 requests/s, CPU 88.186/90.279 ms, GC 27 and RSS +30605312/30621696 bytes. Control GC is 56 and RSS 29442048/29491200 bytes. +No-GC observed allocation means are 122976/231408/122384 bytes per +plain-links/quote-heavy/doc request, including observer overhead (38-52 percent +lower than identical diagnostic control). RSS rises about 1.2 MB. Every +preflight still fails exit 70 at 10 ms; no full matrix or promotion follows. +These are pure Scheme Fold/direct-tape boundaries, not FFI, Rust/Tokio, Python +or public Org parsing acceptance. All processes completed; negative receipts +and immutable baseline executables remain. + +Request-local allocation observations exclude complete-tape comparison from +the measured parser interval. In no-GC samples, the current control records +74912192 bytes over 334 plain-links requests, 133141712 bytes over 277 +quote-heavy requests, and 65326608 bytes over 333 doc requests. These are +admitted SDK counter deltas including observation overhead; requests with GC +are not admitted allocation samples. They are not peak heap or RSS figures. +Compact helper arguments leave all three sums exactly unchanged. Its strictly +sequential same-source A/B/B/A walls are 122.866/133.232 ms against +140.371/129.364 ms control; one wall regresses and both fail the 10 ms gate. +That candidate's source integration is removed, with negative receipts kept. +It is not a solution for the observed request allocation pressure. + +Lean owns guarded transition refinement with an explicit specialization +equivalence premise, not a proof of the whole Scheme transformation. Quint +owns activation, invalidation and re-entry histories (four directed tests, +1000 seeded bounded histories), not duplicate Boolean/uint algebra. Native +ASCENT now binds copied real AOT binaries, generated units, program IR and +complete expected tapes along with the current model/source tests. A passing +model/conformance bundle still leaves matched-performance missing and cannot +authorize a baseline promotion. The compact bundle precedes the corrected +idle plan and must not be reused as its current-source proof. + +** Dynamic call attribution and bounded call-graph candidates + +A count-only product avoids per-leaf clocks, timing closures and frames. It +still performs aggregation-table updates and is diagnostic-only; zero timing +fields are placeholders, not measured zero cost. Its production binder rejects +performance admission. All 24 actual Org controls passed with the unchanged IR +SHA256 =d37b4b160c9d956a549e86486e0b590b0cbdbf2a273ccb16f683c100d3158c38=. +The native counter product recorded 84538 private-helper calls across 928 +executed helper identities. The leading row is a horizontal-prefix scanner +with 4518 calls; multiple subsequent offset helpers invoke it again. Simple +backslash and brace byte checks recorded 2124 and 2007 calls respectively. +Call frequency is not an exclusive CPU-cost measurement. + +An opt-in horizontal-prefix candidate binds the immutable root-line offset +once and removes the fact at source-bound transitions. Six source controls and +24 actual Org AOT controls passed. Sequential A/B/B/A 1000-request preflights: +control 126.312/144.855 ms, candidate 118.076/176.429 ms. Candidate P95 was +322/376 microseconds and P99 370/1341 microseconds; control P95 344/357 and P99 +423/478 microseconds. GC remained 56 in every batch. Candidate throughput was +8469.122/5668.002 requests/s, RSS 29327360/29261824 bytes; control throughput +7916.904/6903.455 requests/s, RSS 29507584/29523968 bytes. No gain or gate pass +is admitted; default remains disabled. The historical nested-guard control is +not an isolated control of only the prefix rewrite because earlier builder +and shared-probe edits also differ. No causal percentage improvement is claimed. + +The rejected experimental graph pass substituted only bounded nonrecursive scalar +leaves without private callees or loops. Parallel lexical argument bindings +preserve call-by-value evaluation without duplicating expressions. It retains +leaf definitions for first-class references and excludes event-producing +procedures. Full IR-to-AOT refinement and physical performance are still +required; this is not authorization to inline the entire Fold graph. Six source +controls and 24 actual Org AOT controls passed. A preliminary group was excluded +because one process yielded before the following invocation; all its receipts +are retained. A new strictly sequential A/B/B/A group (each process explicitly +awaited to completion) failed: control 131.747/126.461 ms, candidate +214.002/158.763 ms. Candidate P95 638/431 microseconds, P99 1666/789 microseconds, +throughput 4672.854/6298.697 requests/s, RSS 29687808/29622272 bytes, CPU +154.238/146.770 ms and 56 GCs. Control P95 344/335 microseconds, P99 466/463 +microseconds, throughput 7590.306/7907.576 requests/s, RSS 29540352/29491200 +bytes, CPU 130.865/125.615 ms and 56 GCs. The candidate failed both the unchanged +10 ms gate and this control comparison; the historical-control boundary above +still applies, so no isolated causal cost claim is made. + +Both failed candidate source entrypoints and graph rewrites have been removed. +Generated products, binaries and negative receipts remain for investigation. +The count-only diagnostic and its regression controls remain, not production +optimizations. Expression and statement source hashes match their pre-turn +values. Counts show real redundant work, but neither removing one scan nor +expanding small leaves establishes the needed order-of-magnitude improvement. + +** Whole-loop byte-region execution + +Status: experimental planner retained for conformance/model work, NOT enabled +in the default Fold lowering. The recursive candidate failed its unchanged gate +and its default integration was withdrawn after the measurements below. + +=fold-byte-regions= classifies a validated current byte once for an entire +loop body, rather than testing every independent state-machine gate for ordinary +text. The current candidate descends through decisions and lexical joins but +not source-bound changes or nested byte loops. A mismatching necessary-byte condition contributes its original else +branch; matched conditions retain their full predicate and query suffix. +Identical ordered bodies are grouped with the standard hash table and ordered +list builder. Source frames, helper handoffs and nested loops remain separate +owners; cofactoring does not descend through them. This is an execution-plan +change, not a request cache or a grammar-specific alphabet whitelist. + +Engine controls passed (100 canonical/90 wire/six source cases). Lean proves +the pure-prefix mismatch/query-trace obligation and composition through ordered +state transitions, assuming each individual compiler refinement is established. +Quint tests every byte with both state values and a same-byte state mutation +witness; its Fold suite now has eight checks alongside seven publication checks. +Neither establishes a physical clock bound or complete IR/compiler refinement. +Both the initial top-level and subsequent recursive candidates passed all 24 +actual Org controls. The top-level candidate failed its 1000-request preflight +at 131.683/136.540 ms; those singleton results establish no gain. Source inspection +showed nested state decisions left unpruned, leading to the recursive candidate. +It generated 5056 private definitions: code-size/working-set cost is an explicit +admission concern. Native preflights failed at 231.825/112.404 ms for 1000 +requests, P95 340/175 microseconds, P99 582/392 microseconds, throughput +4313.599/8896.481 requests/s, RSS 32030720/31981568 bytes. GC fell from 56 to +46, but the first request maximum reached 58237 microseconds. Its measured +batch wall minus process CPU was 106.641 ms versus 0.228 ms in the second run. +Whole-process involuntary context switches were 221/61 and page faults 599/599; +these do not identify the specific request stall as scheduling or GC. Allocated +bytes remain unavailable across GC. The condition control was 132.240 ms, +CPU 124.873 ms and RSS 29409280 bytes. No favorable singleton was admitted. + +Default lowering and predicate sources were restored exactly to their +pre-experiment SHA256 values: statements +=dd6b0b99175f4e694a3059e77b82c8a009587859f89ddf1f5bed7fb0e039a8ca=, +expressions =33dbe3c2ea47bf88bae0c1aba4bd42b6e170aec56eb1ccff61e9def4e4e86906=. +The planner/model controls remain research evidence, not a speed claim. The +remaining architectural obligation is the generated execution/call graph and +its live continuation/state traffic, not importing a library or reducing GC +in isolation. Neither diagnostic counts nor model step bounds establish time. + +After withdrawal, the fresh native evidence join =.fold-restored-final= passed +Lean, seven publication/eight Fold Quint checks, and the Scheme control owner +(100 canonical/90 wire plus six source cases). Native ASCENT leaves +=matched-performance= missing and =executionAuthorized= false. A preceding +link failure (=libcrypto= not found) is retained; the successful retry used the +already installed OpenSSL library directory, not an SDK rebuild or widened gate. + +** Actual Org integration and model coverage + +The shared-scan and standard-builder candidate was regenerated and compiled +into the actual Org Fold product in =target/fold-stdlib-real= in the private +consumer checkout. All 24 native Org controls passed. Its IR input SHA256 +=d37b4b160c9d956a549e86486e0b590b0cbdbf2a273ccb16f683c100d3158c38= +matches the preserved nested-guard control; the three fixtures are unchanged. +The candidate contains three shared-probe regions and 3577 private procedures +across 56 units. These are static counts, NOT measured dynamic call costs. + +| Obligation | Current evidence | Remaining boundary | +| Shared immutable scan | Lean exact classification and ordered consumers | Compiler summary/refinement over the complete IR | +| Guard eligibility | Lean frame/write/opaque rejection | Correct computation of every implementation summary | +| Guard evaluation order | Executable Lean Boolean subset with query traces and a harmful-reordering witness | All instructions, exceptions, aliases and source helpers | +| Scan lifecycle and consumer interleavings | Quint invalidation, rescan, stale-use mutation and ordered-consumption histories | Mapping actual compiler frames and effects to lifecycle actions | +| Native behavior | Engine 100 canonical/90 wire controls and actual Org 24 controls | Complete public syntax/recovery/query coverage | +| Physical performance | Unchanged 1000-request/10 ms preflight | A successful matched native run; models do not prove time | + +=formal/quint/FoldPlan.qnt= augments, rather than replaces, the seven publication +checks with eight Fold lifecycle checks and 1000 seeded histories of at most +20 transitions. These are bounded simulations, not exhaustive Scheme/VM +verification. =collect_models.py= binds both source models and their commands. + +Lean exclusively owns denotational equivalence, ordered query traces, byte +guard refinement and source-visit bounds. Quint exclusively owns the execution +history of scan validity, invalidation barriers, rescan and consumer admission; +it does not repeat Lean's Boolean/byte equivalence checks. Neither model proves +clock time or validates its own abstraction against generated AOT code. Actual +IR-to-generated-helper mapping, native semantic controls and matched runtime +receipts remain separate obligations. The existing AOT receipts predate this +model revision and must not be relabeled as qualification of the revised chain. + +The ordinary benchmark static link failed once on disk exhaustion; a subsequent +retry stopped without child compiler activity and was terminated narrowly. +A source-driver SDK attempt failed because the products were static-only; no +interpreted Fold result was admitted and that temporary entrypoint was removed. +The independent full-program optimization experiment was stopped after about +18 minutes of construction, with C compilation reaching roughly 3.5 GiB RSS. +It produced no admitted runtime measurement and is not the default build route. +A missing OpenSSL header on its first C compilation was also a build dependency +failure, not a runtime measurement. No kernel/SDK modification or performance +gate relaxation was used. + +The next lowering candidate distinguishes condition truth from strict Boolean +publication, following the installed compiler's =generate-runtime-if%= and its +=current-compile-boolean-context=. Short-circuit bodies are normalized once +when published as Boolean values, not at every nested branch condition. Engine +controls (100 canonical, 90 wire and six source cases) and all 24 actual Org +controls passed. Lean additionally proves truth/ordered-trace preservation and +a witness forbidding removal of normalization at value-publication boundaries. +These are bounded model properties, not complete implementation refinement. + +An independently prepared 1000-input corpus retains the original changing +indices 1000--1999 and complete canonical OEV1 outputs. Only untimed oracle +construction is removed from the native executable dependency closure; the +existing per-request clock, full tape comparison and 10 ms gate are unchanged. +The initial candidate preflight failed: 140.334 ms batch wall, 7125.857 requests/s, +request P95/P99 391/509 microseconds, process CPU 136.041 ms, 56 GCs and RSS +29376512 bytes. Allocation bytes are explicitly unavailable across these GCs. +This singleton is not a qualified gain or a full twenty-batch matrix. A matched +preserved-control build is required before assessing relative speed; interpreter +or static procedure counts must never be substituted for this measurement. + +The preserved nested control now builds with its original runtime closure. +Two final strictly sequential diagnostic preflights per lane used the identical +prepared corpus/protocol/SDK: control 131.021/126.030 ms, candidate +129.231/121.161 ms. Candidate request P95/P99 were 327/437 and 311/405 +microseconds; throughput 7738.081/8253.481 requests/s and RSS +29360128/29392896 bytes. Both lanes still collected 56 times and allocation +bytes remained unavailable. The small provisional difference is not an accepted +optimization or an architectural breakthrough: every 10 ms gate failed. An early +pair overlapped due to an unawaited running session; its receipts are retained +but excluded from matched claims. No full matrix or promotion was performed. +The intermediate standard/shared control's stalled link was stopped by exact +task PID; no task compiler or benchmark remains active. + +** Rebased evidence-chain integration + +The study checkout is now based on parser PR #13 commit +=2cd75d9f3a449a83dbb474cb618571e24ab31d61=. Previous Fold experiments +and immutable benchmark controls were preserved; their old results do not +qualify the new source head. The upstream Lean/Quint/Scheme/native ASCENT +chain is a verification owner, not a second parser or runtime scheduler. + +=formal/ParserEngine/Fold.lean= models shared immutable classification and +counts source visits. It proves agreement with two independent predicates, +preservation of ordered arbitrary state consumers, and necessary-first-byte +reordering across total Boolean state reads. The compiler must independently +establish source/bounds stability and effect barriers; the model does not +prove implementation refinement. Its full-scan visit bound is NOT a twofold +speed assertion: two early-exit probes may visit fewer bytes. Physical time, +allocation, GC and Rust capacity require matched implementation measurements. + +The model observation now binds the actual Fold analysis/planning/lowering +sources in addition to the Lean sources and dependency locks. The complete +engine collector's existing LR/GQL benchmark is not the real Org Fold +preflight. Its successful observation must not close the 10 ms Org gate. +Failed Fold receipts remain failures even when model and conformance pass. + +** Shared scanning and necessary-byte plan qualification in progress + +The engine now prepares adjacent admitted any-in probes over identical bounds +as one immutable two-bit classification table. It scans once, then executes +the original branches in order. State writes to bound dependencies, helper +handoffs, source-frame changes and opaque effects block fusion. The real Org +product contains three such regions and passes its 24 native wire controls. +Dominating content-end values are also reused within the matching probe and +byte-loop frame; nested source bounds discard that proof. This product has +passed engine differential controls but has not passed the 10 ms preflight. + +Literal patterns contain nested conjunctions. A further compiler candidate +exposes their necessary cursor byte before total state reads, without moving +queries, errors or opaque effects. Conjunction flattening must preserve order: +the installed std/list/list append-map did not preserve the returned inner +term order in this use, and the new order assertion rejected that candidate. +The subsequent implementation uses the official =std/list/list-builder= +=with-list-builder= macro: traverse conjunctions in source order and emit each +leaf once, without recursively copying child lists with =append=. This is a +compiler-planning allocation change, not a runtime throughput claim. Native +differential qualification and actual Fold timing remain separate gates. +No benchmark assertion was weakened. + +** Standard-library contracts in the model + +Standard-library selection is now an explicit dimension of the Fold admission +model. =LibraryAdmission= requires preservation of order, observable effects, +and ownership, plus a matched runtime performance gate. Its theorem rejects +a failed runtime gate regardless of the library-selection flag. The ordered +builder sequence contract is modeled with Lean's =List.flatten= and proved +equivalent to ordered =foldr List.append=. This proves a sequence property, +not refinement of the Gerbil macro or a GC/time guarantee. + +The installed SDK sources =src/std/list/list.ss= and +=src/std/list/list-builder.ss= were inspected before this change. Reuse the +official hash-backed deduplication for compiler byte sets and the official +builder for ordered compiler output; do not replace the request-owned byte +tape with a list. Compiler lists and runtime event buffers have different +ownership and allocation requirements. The builder's mutable tail remains +local to one compiler traversal: no continuation escape or shared publication +is introduced. The current =append-map= implementation must not be assumed +to satisfy ordered inner-list emission; differential order controls govern +that choice. A library import alone never establishes performance evidence. + +=just evidence-fold-controls = now links +the existing Python observation adapter to the actual Just-owned Fold +qualification and native Ascent review. The fresh run established Lean, +Quint and native differential Scheme conformance; =matched-performance= +remained Missing and =executionAuthorized= stayed false. The collector does +not claim actual Org conformance or start a performance matrix. Its packet is +=.data/parser-evidence/fold-controls.json=. The standalone native admission +controls also passed all eleven cases. No performance baseline was promoted. + +These refinements do not eliminate ordinary-text metadata transitions that +are required by later delimiters, nor qualify full FFI, Tokio or graph costs. +The validated-byte control and accepted baseline remain unchanged. No full +twenty-batch matrix starts until the real upstream preflight passes. + +Shared-probe plus content-end singleton preflight: 142.433 ms for 1000 changing +requests, 7020.845 requests/s, request P95/P99 164/194 us, CPU 142.236 ms, +six collections and peak process RSS 183042048 bytes. Its sequential preserved +validated-byte control: 143.669 ms, 6960.444 requests/s, P95/P99 165/233 us, +CPU 142.839 ms, six collections and RSS 179585024 bytes. The 0.86 percent wall +difference does not establish a meaningful improvement, especially with higher +process RSS and uncontrolled host background work. Both reject the unchanged +10 ms gate. RSS includes retained oracle/input/code, not exclusive parser +allocation or Rust capacity. These results are not full FFI/Org acceptance. + +The nested necessary-byte candidate subsequently passed 100 canonical and 90 +independent wire cases, plus all 24 actual Org native controls. Its singleton +preflight was 132.841 ms/1000 changing requests, 7527.796 requests/s, +P95/P99 151/173 us, CPU 132.837 ms, six collections and RSS 180174848 bytes. +This is approximately 7.54 percent lower wall than the sequential preserved +143.669 ms control, not twenty-batch qualification. It still fails the 10 ms +gate by 13.28 times, and RSS is slightly higher. The accepted baseline is not +promoted. Necessary-byte preparation reduces irrelevant guard evaluation; +it does not eliminate all state transitions or private procedure costs. + +** Direct-consumer experiment, not performance admission + +The current engine candidate emits OEV1 directly from the same admitted Fold. +The grammar fixes numeric kind IDs at generation. Request-owned byte storage +reserves once per event and seals before publication; no event descriptor list +or canonical-list materialization sits between production and the consumer. +The existing canonical product remains an independent control. Ordinary Org +request/batch source wiring selects the generated tape product and passes an +admitted byte slice, avoiding a string decode/encode round trip; this wiring +has passed focused native Rust/FFI integration controls, not performance +acceptance. Diagnostic operations 5/7 +still measure the older decomposed canonical path, not this ordinary path. + +The tape specialization prepares ordered byte decision regions before private +procedure expansion. Only a validated loop byte and a total guard prefix may +select a region; duplicate rules preserve order and residual state conditions. +Errors, query-cache effects and opaque prefixes prohibit bypass, while nested +source bounds discard the proof. The actual Org product has 59 such regions; +the immutable old product has none. This is structural evidence, not a claim +that all repeated source scans or generic state work have been removed. + +The current compiler also reuses the dominating validated byte for equality +and singleton byte-set predicates. This removes repeated offset arithmetic, +source reads and range checks without removing their admission: the enclosing +loop proves start <= cursor < limit <= end. Safe total state guards can move +behind a byte test to reject irrelevant branches before slot reads. Queries, +opaque operations and errors retain their order; a nested source frame drops +the proof. Ninety-two predicate references in the real generated Org product +use this refinement. Whole-range LaTeX/trigger prescan fusion remains open. + +The production program was separately generated under orgize/fold (document, +inline and tape); its staged closure contains all 58 tape units and the newly +compiled event-runtime/event-tape. The stage owner compiles both modules to +avoid selecting a stale interface. Five release Rust binaries then passed +public parser/Tokio controls (11 cases), index handoff (23), owned projection +(7), tracked Org parity (3) and host child-wait ownership. This admits those +actual FFI semantics, not parallel Scheme execution or an end-to-end speedup. + +Final controls passed 3 analysis, 2 decision-effect/order, 3 tape-owner/source, +6 source, 100 native canonical differential and 90 native independent wire +cases. Full actual Org generation, native link and 24 complete wire/override +controls passed separately. The matched throughput adapter compares the old +canonical product plus the existing two-pass OEV1 consumer against direct +OEV1, with grammar catalogs outside timing and a separate interpreter/wire +oracle. The 10 ms preflight is unchanged. No full twenty-batch acceptance, +baseline promotion, kernel change or Automation restart follows from controls. + +The sequential matched OEV1 preflight failed for both products. Direct: +171.141 ms, 5843.135 requests/s, request P95/P99 212/247 us, CPU +170.744 ms, six collections and process peak RSS 178126848 bytes. +List plus existing consumer: 198.975 ms, 5025.757 requests/s, P95/P99 +254/304 us, CPU 198.918 ms, eight collections and peak RSS 179077120 +bytes. These are single 1000-request batches, not a qualified matrix or +exclusive allocation/Rust-capacity measurements. The direct path is still +approximately 17 times above the 10 ms gate. No full matrix or production +baseline promotion was performed. This eliminates an intermediate consumer +representation but does not close residual source scanning/state execution. + +The subsequent loop-byte candidate versus the previous direct product uses +the same consumer, changing corpus/indexes and identity. Sequential singleton +preflights: 142.709 versus 181.078 ms, 7007.267 versus 5522.482 requests/s, +P95/P99 157/192 versus 221/284 us, CPU 142.525 versus 178.319 ms, both seven +collections, process peak RSS 180797440 versus 186269696 bytes. The candidate +still fails by 14.27 times; no full matrix follows. Twenty-batch acceptance, +exclusive allocation and Rust capacity remain unqualified. Complete oracle +generation takes most of process wall and is outside request timing; do not +interpret the 53.30/54.71 second process walls as Fold latency. Equal GC counts +and CPU near batch wall do not support a GC-frequency or non-CPU-wait cause. + +Optimize the upstream compiled Fold as a complete execution plan, rather than +accumulating isolated encoder, channel or capacity changes. The installed +official implementation is the implementation reference, not evidence that +our generated program has inherited its performance. Downstream FFI, graph, +Tokio and Python qualification follows an admitted upstream engine result. + +The baseline remains unchanged. No kernel/SDK modification, unsupported SMP +pass, acceptance relaxation or Automation restart is part of this design. +This document proposes work; it does not claim the architecture is implemented. + +** Architecture correction after the rejected local execution-plan experiment + +The subsequent combined frame/current-byte/dependency-ABI/owned-buffer candidate +also failed and has been removed from ordinary lowering. It passed 100 native +differential cases and 24 complete actual Org controls, but its matched actual +1000-request preflight took 245.616 ms versus the immutable old-algorithm +219.381 ms. Throughput was 4071.396 versus 4558.280 requests/s; request +P95/P99 were 193/6280 us versus 234/730 us. CPU was 245.615 versus +217.961 ms; GC counts were 15 versus 10. Whole-process peak RSS was +204324864 versus 190578688 bytes, including oracle/input retention and code. +This is a regression, not a qualified architecture or memory improvement. +Raw allocation-counter deltas under GC do not establish allocated bytes. +No full twenty-batch matrix was started; the unchanged 10 ms gate rejected +both products. Negative artifacts remain in =.fold-execution-final=, but +ordinary source no longer uses that experiment. A flat buffer followed by +canonical list materialization did not qualify elimination of consumer work. + +The expression-level scan hoists, prefix guards and event-run emitter are +removed from ordinary lowering. A 1.1 percent singleton difference with a +failed gate does not justify accumulating a new production execution path. +The conservative analysis tests and negative receipts remain; they are not +an admitted compiler optimization. Inherited byte-input, graph projection and +single-byte lowering changes are outside this rollback. + +The architectural seam is before procedural Fold expansion, not after each +expression has become a shared procedure. The current actual Org inline +strategy performs a LaTeX prescan, an inline-trigger scan, and a conditional +per-byte state pass. Its generated body updates boundary facts through +separate shared procedures. The diagnostic 84538 calls / 1767 events identify +an execution-amplification hypothesis, not an uninstrumented cost attribution. + +The next implementation must retain source-pattern identity, precedence, +state dependencies and effect barriers in one engine-owned execution plan: + +1. Source classification: partition byte/prefix tests and prescan facts by + source frame and cursor; assign shared immutable byte classes once in the + compiled product. Keep context-dependent and future queries explicit. +2. State execution: lower the admitted ordered decisions into bounded typed + control regions with lexical cursor/facts. Do not turn every small leaf + into a cross-unit call, inline the whole grammar, or move state/error + effects past a decision. Typed refinement and SSXI metadata follow the + installed compiler's optimization boundary. +3. Event consumption: integrate the producer with its actual event-tape and + graph consumer, retaining one owned output and numeric kind metadata. + Merely replacing token lists or adding a packed producer is insufficient: + RFCs 0013/0015 contain slower consumer results despite producer fusion. +4. Public integration: the same admitted plan owns Scheme AOT execution and + its FFI boundary. The Org pack supplies declarations, not a second generic + scanner/dispatcher; Rust/Tokio remains transport/scheduling, not evidence + of Scheme parsing parallelism. + +Acceptance requires complete independent events and existing recovery/override +controls, structural evidence that ordinary text avoids irrelevant rule work, +and the unchanged actual-Fold preflight. No new execution backend or result +representation is admitted until its actual consumer is measured. Encoding, +allocation and GC remain measured components; none is assumed to explain the +entire approximately twentyfold throughput gap. + +* Implemented experimental slice; admission remains failed + +The diagnostic compiler mode now emits nested call observations only in a +diagnostic product. Its binder rejects production performance admission; +ordinary products do not import or call the observer. The upstream native +actual-Fold case compares complete results against an independent engine +interpreter, without an Orgize runtime, FFI, Tokio or Python boundary. + +Twenty-four diagnostic controls emitted 1767 events and entered shared +procedures 84538 times; frame-start horizontal scanning was entered 4518 +times. Counts identify repeated work, not its uninstrumented wall share. +Observer intervals include measurement overhead. Publication already uses +destructive reversal of the request-owned event spine, so proposing ordinary +=reverse!= as a new allocation optimization would be redundant. + +The experimental source-fact cell reuses horizontal start, trimmed end and +content end only for identical request bytes and bounds. It follows admitted +state slots without changing their ordinals; state-dependent values and events +are not cached. Nested clamped successor expressions are collapsed at compile +time. Six source controls and 100 native differential controls passed. + +The observer-free actual-Fold preflight still FAILED: 1000 changing requests +in 0.201576 s, 4960.908 requests/s, request P95 219 us and P99 361 us; CPU +0.201187 s and nine collections. Allocation under GC remains unqualified, +and this receipt has no RSS measurement. This is not an accepted speedup: +a same-harness old-algorithm control is required before relative claims. +The baseline is untouched and twenty-batch admission has not started. + +A matched resource-controlled old-algorithm preflight measured 0.197398 s; +the candidate measured 0.195332 s. Request P95/P99 were 227/397 us and +219/280 us respectively. Process maximum RSS rose from 191987712 to +192413696 bytes, including retained oracle/input storage, and collections +rose from seven to eight. These singleton observations do not establish a +stable benefit. The source-fact cache and successor collapse were conservatively +rolled back; diagnostic tooling and negative receipts remain. Neither product +is admitted. The next transformation must address dependency/effect staging +and shared execution structure rather than promote this small difference. + +* Current evidence and boundaries + +At parser source =461a4bfe3a0101c42e71327dc372d8fd2ab640e7=, the privately +compiled actual Org product has 3746 private procedure definitions across its +generated part files, totalling 2319323 source bytes. This is a static count, +not a dynamic call count or proof that the procedure split is the bottleneck. +The source compiler interns code bodies into procedures; it does not generally +reuse the results of repeated computations at execution time. + +The actual changing-input Fold preflight uses three complete Org fixtures and +an independent interpreted oracle. A fresh control measured 1000 requests in +0.209919 s, 4763.742 requests/s, request P95 241 us and P99 614 us. Batch wall +includes sample recording and complete event comparisons; request latency is +the compiled parser call. Neither boundary includes FFI/Tokio/Python delivery. +One preflight is not a qualified twenty-batch baseline. + +The full pre-encoded-marker/owned-normalization candidate measured 0.216429 s, +P95 236 us and P99 431 us, and was rejected and rolled back. Its wall was +3.10% slower in this single comparison. It also included earlier experimental +byte lowering, so the difference cannot be causally assigned to encoding. +CPU was 0.215711 s and ten collections occurred. Raw allocation-counter deltas +under GC are not allocated bytes; no RSS or Rust-capacity benefit was measured. +The semantic controls passed, but correctness alone did not admit the candidate. + +* Official source principles and their application + +References are read directly from the installed SDK's =current/src= tree. + +| Official owner | Observed mechanism | Required engine application | +|----------------+--------------------+-----------------------------| +| gerbil/compiler/optimize.ss | type collection/refinement before optimized calls; SSXI metadata | Preserve admitted argument/result types and optimizer metadata across generated units; inspect generated code before claiming unboxing/inlining | +| gerbil/runtime/util.ss | constrained inline rules with explicit AST patterns | Specialize bounded pure leaves without indiscriminate whole-grammar expansion | +| gerbil/runtime/hash.ss | generic tables default to equal?; specialized key families exist | Stable grammar queries must not repeatedly process large structural descriptions; choose the representation before choosing a table | +| std/string/utf8.ss | typed bounds and official UTF-8 length/encoding primitives | Decode/validate input once at the appropriate owner; retain malformed scalar and boundary rejection | +| std/io/util.ss and std/io/bio/memory.ss | bulk subu8vector-move! into explicitly owned destination buffers | Fuse known producer/consumer copies where ownership permits; do not expose borrowed mutable buffers | + +These observations support design choices, not a speed guarantee. The rejected +copy candidate demonstrates why fewer allocations alone do not admit a change. + +* One request plan, four coordinated transformations + +** Experimental execution-plan implementation + +Historical experiment only: the lowering integration below has been rolled +back after the architecture correction; the measured binary is retained as +negative evidence, not the current ordinary compiler product. + +=event-fold-analysis.ss= now summarizes admitted state reads/writes and +effects, conservatively marking unknown operations, dynamic range errors and +future-query cache work. Shared scan lowering uses guaranteed evaluation +dominance inside a single expression, never a request cache or reuse across +state/bounds/helper changes. Pure necessary byte guards may reject impossible +branches only when the entire condition is effect-free and total. + +The implementation uses installed official =:std/list/list= operations +=delete-duplicates/hash=, =duplicates= and =append-map= rather than private +uniqueness/counting/flattening loops. Event runs use the official +=:std/list/list-builder= at compile time and emit linear =let*= chains, +preserving offset evaluation, allocation order, zero-token suppression and +the existing payload ABI. This fuses intermediary emitter calls; it does NOT +eliminate token payload cells or constitute compact event/sink closure. + +Three dependence/effect cases, six source cases, 100 native differential +controls and 24 full actual-Org/override controls passed. An initial emitter +duplicated conditional chains, expanding the test source to 6175306 bytes; +that obsolete compilation was stopped. Linear generation is 279011 bytes +(the original unoptimized fixture was 200134 bytes). Fixing an experimental +regression is not a performance improvement over the baseline. This execution +plan remains experimental pending observer-free actual-Fold admission. + +The matched observer-free preflight completed with nonzero exit 70 for BOTH +control and execution-plan candidate; neither passes admission: + +| Metric (1000 changing requests, one batch) | Control | Candidate | +|------------------------------------------+---------+-----------| +| Batch wall, ms | 218.790 | 216.313 | +| Throughput, requests/s | 4570.593 | 4622.931 | +| Per-request P95, us | 237 | 230 | +| Per-request P99, us | 812 | 507 | +| Process CPU during batch, ms | 217.207 | 216.207 | +| GC count during batch | 10 | 10 | +| Whole-process peak RSS, bytes | 203063296 | 192905216 | + +The approximately 1.1 percent singleton wall difference is not a qualified +speedup. The 10 ms / 100000 requests/s gate is still failed; no twenty-batch +matrix, baseline promotion or downstream acceptance follows. RSS includes +code, retained inputs and independent interpreted oracle preparation, not +exclusive request allocation or Rust reserved capacity. The raw allocation +counter deltas (399372032 / 397958912) are not admitted allocated bytes under +GC. Whole-process walls (50.33 / 50.02 s) include oracle preparation and must +not be reported as the measured batch wall. Receipts are +=.fold-value-control/plan-control-resources.log= and +=.fold-plan-final/real-resources.log=. + +** 1. Stage grammar knowledge; preserve execution semantics + +Build-time admission should produce dependency/read/write/effect information +for Fold expressions and transitions. The generated request should consume +that prepared plan, not repeatedly interpret or structurally hash grammar data. +Intern equivalent static future-query specifications once into compact product- +local identifiers; dynamic heading levels remain a separate key component. +The scope is the request, including nested parse restoration, not a global +content cache. Different query semantics must never share an identifier. + +The present prefix-family lowering already builds a trie for a restricted pure +alternative family. Extend shared classification only where precedence and +effect analysis permit it; reuse this facility rather than introduce a second +dispatcher. Ordered alternatives, state mutation, source-bound changes and +helper effects must retain their original order. + +** 2. Eliminate repeated work, not merely procedure calls + +Construct a typed execution DAG with source-only computations separated from +state-dependent predicates. Compute shared line bounds, whitespace ends, +prefix classification and other admitted scans once per applicable frame. +Use dominance and dependency information for reuse; changing start/end, +helper parameters or a written slot invalidates dependent values. A global +memo table or text snapshot is not a substitute for this analysis. + +Represent small state values in the owning lexical frame when they do not +escape or cross procedure joins. Keep explicit slots where aliasing is needed. +Bound specialization by generated size and compilation/retained-code cost; +making all 3746 procedures inline is not the design. Inspect emitted SSXI/SCM +and the actual call graph to qualify the intended compiler optimization. + +** 3. Fuse event production with its admitted consumer + +Current token transitions allocate token lists and an outer event spine; +later consumers traverse the materialized result. Establish event counts, +bytes and traversal costs before selecting a representation. Evaluate an +engine-owned compact event builder consumed directly by publication/projection, +with immutable kind metadata and request-owned output storage. + +Use the existing event/sequence architecture and its ownership contracts; RFCs +0013 and 0015 already record producer/consumer fusion and slower consumers. +Do not build a competing event framework or assume a packed representation is +faster. A changing-input complete-result oracle must compare every event, kind, +offset, ordering and recovery decision. The old representation may remain a +test oracle during evaluation, not a permanent second production parser. + +Offsets select physical width only after admitted bounds; overflow must reject +or use an already-supported wider representation, never truncate. Publication +requires a frozen owner/lease; cancellation, errors and retained results must +not permit reuse of live mutable storage. No per-token FFI callback is allowed. + +** 4. Keep one admitted source representation and bounded storage + +Map the full lifecycle: Rust input -> byte ownership/validation -> Fold scans +-> events -> metadata/projection -> output ownership. Remove only demonstrated +decode/encode/copy duplicates. Encoding a required result is not redundant. +Borrowing bytes does not authorize bypassing UTF-8 validation or retaining a +Rust pointer after its lease. Buffers and pools must have explicit reset, +retention and high-water policies rather than unconditional worst-case reserve. + +Measure Scheme natural GC and Rust live length/reserved capacity together. +Prefer a supported bulk primitive over hand-written per-byte FFI. Any retained +memory reduction must be evaluated with CPU, wall and tail latency; lower GC +without better total cost is not a throughput improvement. + +* Attribution before transformation + +Use one upstream diagnostic case containing the actual admitted Org Fold +product; no Rust queues or Python runtime participate in this gate. Count +dynamic predicate/helper calls, repeated source ranges, future-key lookups, +index-build visits, emitted events and output bytes. Separate encoding, +Fold evaluation, future-index construction and event publication intervals. +Parent/child intervals overlap; never sum them as exclusive costs. + +Optional observers are diagnostic-only. Preserve observer-free immutable +control/candidate executables for admission; do not qualify performance with +instrumentation enabled or compare incompatible harnesses. Sampling overhead, +clock resolution, CPU totals and GC counter limitations must be reported. +Never modify installed SDK/kernel code to add attribution. + +* Acceptance order and stop rules + +1. Preserve the exact baseline executable, grammar/compiler identities, corpus, + SDK, configuration and resource conditions. Rebuild rejected diagnostic + libraries before reuse; revision pins alone do not identify loaded code. +2. Validate complete semantic equality, Unicode/CR/LF/CRLF, malformed UTF-8, + empty slices, helper reentrancy, overrides, ownership and overflow paths. +3. Run the upstream actual-Fold 1000-request preflight. Require at least + 100000 requests/s and at most 0.010 s batch wall. Reject grossly inadequate + candidates before repeating long matrices. Tail values are reported only. +4. Only after preflight passes, run changing input for twenty batches EACH of + 1000, 10000 and 100000 requests. Hard walls are 0.010/0.100/1.000 s. + Report request P95/P99 in us or ms, batch wall in s, throughput in requests/s, + process RSS in MiB, capacity in bytes and explicitly scoped GC/allocation + counters. Do not present percentiles of twenty batch totals as request tails. +5. Reject slower candidates conservatively; retain negative receipts and revert + only owned changes. Promote nothing based on a small synthetic grammar, + fewer allocations, one faster sample or a compiler flag change. +6. Admit accepted upstream changes to the existing parser PR, then align + immutable GitHub revisions and actual compiled consumers. Qualify Orgize + publication/projection, FFI and asynchronous delivery separately. Rust queue + concurrency is not Scheme parsing parallelism. Unsupported SMP remains an + explicit skip, not a successful acceptance result. + +The 100000 requests/s objective gives an inclusive average budget of 10 us per +request, not a P95/P99 standard. No phase budget or memory limit is silently +invented. Output size and fixture complexity remain explicit; constant-size +bridge calls and tiny line grammars cannot satisfy the real workload gate. + +* Implementation status + +The encoding/copy candidate is rolled back. The whole-plan dependency analysis, +value-reuse lowering and compact producer/consumer path are proposals, not +implemented or qualified. The next implementation starts with the upstream +request plan and attribution, and must justify which transformation addresses +the measured dominant cost before another downstream acceptance matrix. diff --git a/docs/research/history-commit-model-implementation.org b/docs/research/history-commit-model-implementation.org new file mode 100644 index 00000000..2bb0b00f --- /dev/null +++ b/docs/research/history-commit-model-implementation.org @@ -0,0 +1,118 @@ +#+title: History commit: model, lifecycle and implementation + +* Concrete question + +Can an older Source parse completion replace a newer committed scanner history? +Same-thread execution alone does not exclude this: the Scheme publication callback +can call =language-handle-parse= recursively before returning to its outer caller. + +The existing ABI keeps history per handle and commits after publication succeeds. +Before this change, that success check did not recheck request identity or handle +liveness. An inner parse could commit its current history, then the outer callback +would overwrite it with the older request's history. + +* Three responsibilities, one obligation + +| Layer | Executable owner | Obligation | +|-------+------------------+------------| +| Lean | [[file:../../formal/ParserEngine/Reuse.lean][Reuse.commitHistory]] | Preserve the current history when generation differs, publication fails or ownership is released; a current successful commit selects its pending value | +| Quint | [[file:../../formal/quint/HistoryCommit.qnt][HistoryCommit]] | Explore nested callbacks, inner/outer failure and release; reject the mutation that overwrites a newer history | +| Scheme | [[file:../../src/ffi/language-handles.ss][language-handle-parse/publish]] and [[file:../../scripts/tests/history-commit-test.ss][real callback controls]] | Bind request generations and handle liveness around the actual parser and callback; verify fresh payload parity and retained history | + +These are complementary obligations. Lean proves a value-selection function for +arbitrary history values and generations. Quint checks abstract lifecycle +histories; it does not parse Bash or prove OS scheduling. Scheme executes the +actual Source engine, incremental scanner, payload publication and history store. +There is no verified compiler extraction or universal Scheme refinement theorem. + +* Actual admission + +The private handle entry now contains an exact-integer generation. An admitted +parse advances it before recognition; malformed UTF-8 or a rejected publication +therefore still supersedes an older admitted attempt. Invalid arguments rejected +before this admission boundary do not advance it. + +After its callback returns successfully, the outer call commits only when: + +1. The callback accepted publication. +2. The captured generation still equals the handle entry's current generation. +3. The handle table still contains that exact entry. + +Generation increments are exact Scheme integer arithmetic, without wrapping. +Handle identifiers are already non-reused and bounded by the ABI's integer range. +A released entry cannot become current again through a replacement registration. +All parsing remains confined to the existing runtime owner; this is re-entry +admission, not a concurrent Scheme VM or a new cache. + +A superseded request may still return its complete payload to its own caller. +The fix protects reusable history; it does not retract an already copied C result +or impose an asynchronous latest-result policy on the synchronous ABI. Descriptor +queries do not advance generation. Existing failed allocation/copy behavior +continues to retain the currently committed history. + +* Directed implementation controls + +The real controls exercise: + +- Inner successful commit followed by successful outer completion. +- Inner successful commit followed by an outer callback exception. +- A newer malformed UTF-8 attempt, followed by successful outer completion. +- A newer rejected publication, followed by successful outer completion. +- Handle release from the publication callback. +- Metadata queries and parsing on another handle do not supersede this request. +- Native payload equality against fresh handles, and no leaked handles. + +The independent script owns one runtime thread and uses the repository's existing +disposable native-child exit contract after its assertions. The complete evidence +collector executes it in a separate child before Suite admission, preserving the +Suite workers' own runtime affinity. It refuses to continue when these controls +fail, and binds the later semantics observation to their receipt. + +The new Quint finite preservation check evaluates 1,944 candidate states over its +explicit domain, with seven directed tests and a separate 1,000-history sampled +run. Its =latest= field is a specification observation of the newest committed +snapshot, not an additional implementation cache. The generation-omission mutation +is detected after an inner commit. This does not establish temporal fairness. + +* Reproduction + +#+begin_src sh +just formal +just formal-protocol +just test-history-commit +#+end_src + +Use the normal native package build before the Scheme command. CI runs the real +history controls in both Scheme semantic platform lanes. Model observation input +sets bind both the implementation owner and witness source, alongside Lean/Quint +sources and pinned tools; changing the witness invalidates a stored observation. + +* Observations and performance boundary + +Before the implementation fix, the first real callback control failed with +=outer completion overwrote newer history=. The supervised process reported +=error-output= and no final receipt; this is a retained negative control. + +After the fix, Lean built successfully with the new nested-commit and failed +publication theorems depending only on =propext=. The first actual implementation +control run exited zero in 7.759 seconds with =HISTORY-COMMIT-CONTROLS-OK=. + +This is a correctness repair found by comparing model and implementation duties. +A generation increment and a commit guard add runtime work. No CPU improvement is +claimed. Existing complete-request benchmarks retain responsibility for admission +of future performance changes; abstract cost equations do not turn this fix into +a speedup. + +The expanded same-owner controls, including rejected nested publication, exited +zero in 5.352 seconds. The existing 17-case language ABI Suite also passed with +successful process exit in 8.287 seconds, preserving UTF-8, per-handle isolation, +failed publication and ordinary history reuse behavior. The complete formal +package passed 61 directed Quint tests across eight modules, seven sampled +invariant runs and the locked Lean build. Source-binding and native-product +invalidation host controls passed. These observations do not claim that the +entire native ASCENT evidence review or every CI job has completed. + +The final controls additionally verified that metadata queries and another +handle's parse do not supersede the current request. All six control markers +and final =HISTORY-COMMIT-CONTROLS-OK= were admitted with process exit zero in +5.463 seconds. Total and silence limits remained 90 and 5 seconds. diff --git a/docs/research/lr-conflict-source-origins.org b/docs/research/lr-conflict-source-origins.org new file mode 100644 index 00000000..f227c4f1 --- /dev/null +++ b/docs/research/lr-conflict-source-origins.org @@ -0,0 +1,137 @@ +#+title: LR conflict diagnostics from effective Scheme declarations +#+startup: overview + +* Research question and implementation boundary + +How can a compact Scheme grammar remain understandable after the engine lowers +its expressions into synthetic productions? This module closes the rejected LR +action diagnostic part of the [[file:dsl-engine-contracts.org][integrated DSL study]]. The implementation +baseline is PR #13 =95de0d9= plus this source slice. It does not change CFG choice, +precedence, conflict policy, production numbering or the canonical LRSpec schema. + +Zimmerman's [[https://arxiv.org/pdf/2209.08383][Practical LR Parser Generation]], Section 2.3, studies intermediate +grammar transformations that defer reductions. Our inference is that generated +helpers must remain attributable to author expressions before investigating a +similar transformation. The paper does not prove this repository's lowering or +source attribution. We implement attribution and test it independently; this +slice does not implement CPS lowering or claim an XLR execution bound. + +* One occurrence contract + +The shared lowering owner accepts an optional diagnostic observer. Each emitted +production reports the selected rule, expression child path and original +GrammarExpr. Child indices count semantic expression children: field, alias, +precedence and unary operators use index zero; sequence and choice use their +ordered child positions. Wrapper names and precedence ranks are not children. +Equal expression values at different paths remain different occurrences. + +The observer reconstructs an indexed sidecar after an LR action pair is rejected. +Normal compilation does not replay lowering or construct source paths. The +sidecar is neither a language entry slot nor part of a published runtime table. +Existing normalization remains the authority for selected POO source rows. + +The appended =lrConflict= diagnostic contains: + +- =state= and =terminal= at the rejected action pair; +- =actions=, with =expressionOrigin= for each reduce production; +- =shiftItems=, each matching state item with its dot and =expressionOrigin=. + +Each origin contains =rule=, =expressionPath=, =loweringOperation= and =source=. +The source is the selected normalized row, including its composition history +when available. A shared shift has several candidate items; the diagnostic lists +them instead of assigning it to an arbitrary production. These are candidates +in that state, not a shortest derivation or a proof that every item wins a +precedence comparison. Unavailable reader coordinates remain unavailable. +Raw rule lists receive empty source rows. Existing error messages and irritants +are preserved, followed by the new structured evidence. + +* Scheme witness + +#+begin_src scheme +;; grammar.ss: the author states syntax, without naming helper productions. +(rules + (start (seq (optional "a") (optional "a")))) + +;; Engine-side diagnostic occurrence shape, shown without state numbering. +;; The two equal optional expressions have distinct identities. +((rule . start) + (expressionPath 0) + (loweringOperation . optional) + (source)) +((rule . start) + (expressionPath 1) + (loweringOperation . optional) + (source)) +#+end_src + +** Question: why is this DSL design better for authors? + +The engine owns helper expansion and diagnostics. Authors can write nested +syntax directly and receive failures in those same terms. POO overrides select +the effective rule and source together; an inherited rule's replaced declaration +is not reported as the active owner. No diagnostic configuration or generated +helper naming convention is added to =grammar.ss= or =parser.ss=. + +** Question: why is the design scientifically justified? + +The identity is structural and falsifiable: selected rule plus child path, not +expression equality or a generated symbol name. An observer replay must produce +exactly the same ordered productions as ordinary lowering. All four construction +strategies consume the same origin contract while retaining their own state +numbers and admission behavior. These controls establish the tested attribution +invariant; the paper supplies a transformation motivation, not a correctness +certificate or a performance result. + +* Code owners and qualification + +| Responsibility | Owner | Executable control | +|----------------+-------+--------------------| +| Preserve occurrence during lowering | =src/compiler/lr.ss= | =t/lr-conflict-origins-test.ss=: equal optional helpers under an alias have separate paths; replayed productions equal ordinary productions | +| Attach rejection evidence | =src/compiler/lr-compiler.ss= | Shift/reduce and reduce/reduce failures retain their messages and both action witnesses | +| Resolve effective source ownership | Existing =src/compiler/normalize.ss= | An ambiguous POO override is reported as selected source across four constructions | +| Retain construction semantics | =t/resolved-grammar-test.ss= and =t/lr-conflict-origins-test.ss= | Existing nullable, productivity, precedence and LR construction controls run with the new compiler; focused origin controls belong to the native conformance grammar suite | + +The focused native qualification uses the existing Scheme test harness and +process watchdog, with a five-second real-output idle gate. No project script, +standalone qualification binary or author-facing compatibility path is added. + +** Local receipt (2026-10-10) + +The final LR producer compiled natively with the existing optimize/static +options: =26.667s=, propagated exit zero and final =OK=. The focused contract +test module also compiled natively: =34.346s=, exit zero and =OK=. The existing +semantic module did not have a reusable native object, so both test sources were +executed through the repository's GxTest source path against the final native +producer overlay. All =41= cases passed (=37= existing semantic cases and =4= +new origin cases), =27.035s=, exit zero, both =MODULE-OK= markers and final =OK=. + +The source suite selection is reproducible with the normal prepared project: + +#+begin_src sh +gerbil test -g -v 5 t/lr-conflict-origins-test.ss t/resolved-grammar-test.ss +#+end_src + +Local raw receipts are unarchived local execution. Temporary +overlays are local evidence, not committed bootstrap sources or CI artifacts. + +Unsuccessful attempts are retained as negative evidence: the first added test +form had an unmatched parenthesis; placing the new tests in the large existing +semantic module hit the five-second GCC idle gate (=82.157s= total); the nested +diagnostic exception closure also hit that gate (=51.701s= total). The tests were +separated into their own module and the exception boundary became a named +function. Native loading of the existing semantic module failed because its +object was unavailable; its unchanged source passed through GxTest. None of +these attempts is admitted as a success, and the five-second idle gate was not +relaxed. + +The native conformance module catalog and composition suite both include the +new module. Exact-head Linux, macOS and Rust AOT CI remain separate gates. + +* Remaining boundaries + +This diagnostic is produced when table admission rejects an action pair. It is +not a serialized origin catalog for admitted GLR forks, runtime rejection, +scanner programs or generated backend entries. It does not reconstruct exact +reader coordinates from normalized values or infer an ambiguous input string. +Those mechanisms require their own source identity and witness contracts. +No performance benefit is claimed for this diagnostic change. diff --git a/docs/research/lr-owned-stack.org b/docs/research/lr-owned-stack.org new file mode 100644 index 00000000..3e85a699 --- /dev/null +++ b/docs/research/lr-owned-stack.org @@ -0,0 +1,375 @@ +#+title: Request-owned LR stacks and immutable parser frontiers + +* One engine implementation unit + +This study follows [[file:engine-record-access.org][closed engine record access]] and +[[file:parser-evidence-chain.org][the model and implementation evidence chain]]. It concerns the actual LR +executor and semantic operand access, rather than a language-specific adapter. +Grammar declarations, canonical IR, field/alias meanings and public artifacts +retain their existing contracts. + +The optimization hypothesis is to remove the driver's two list-spine allocations +per admitted shift/reduction and its two reduction-prefix traversals. Complete +unobserved requests may own state/value vectors. Checkpoints, arbitrary stack +observers, generated reductions and the event backend retain immutable list +frontiers. These are execution domains with different ownership obligations, +not language interfaces or legacy grammar versions. + +* Measured cost and rejected approaches + +A fresh statistical continuation profile at =e6cebe9= sampled 30,000 prepared +representative GQL requests at a one-millisecond interval. Of 3,982 top-frame +samples, =lr-run-checkpoint= contributed 1,901 (47.7%), =##table-access= 453 +(11.4%), the prepared action selector 287 (7.2%), alias construction 187 (4.7%), +and sequence-for-action 90 (2.3%). These are sampled frames, not additive causal +CPU shares. Among 631 selected table/hash frames, 488 also contained the action +selector. Attribution identifies a hypothesis; it cannot establish its benefit. + +A finite literal-ID/request-classification candidate passed selector identity +and Unicode controls but did not meet the long complete-request CPU margin. +Its first implementation retained about 9 MB more live heap; compact/selective +variants still lacked the required long-request benefit. Gambit string-key +specialization, Gerbil's string-table owner, and directly expanded AST kernels +also lacked the declared complete-request benefit when evaluated separately. +None of those isolated candidates has an admitted published performance claim. +The subsequent combined candidate +stages the shared AST constructor bodies together with owned stack execution; +its independent earlier failure is retained. +Their failed, timed-out and completed exploratory receipts remain ignored study +data. A source-load timeout and a test expecting the wrong resume return shape +are retained as failures, separately from later passing checks. + +This redirects work to the executor itself. Its old recognition path borrows +semantic operands but still walks both list prefixes to pop them and allocates +two new stack pairs for each successful transition. The vector policy changes +those operations without removing semantic children or postponing publication. + +A complete-source continuation profile of the owned-stack candidate sampled +30,000 representative GQL requests. Among 2,608 top-frame samples, the owned LR +driver contributed 919 (35.2%), the streaming source driver 304 (11.7%), +recognition-event publication 280 (10.7%), table access 211 (8.1%), the lexical +scanner closure 175 (6.7%), and the action selector 130 (5.0%). Inclusive stacks +overlap and are not additive CPU shares. In particular, the lexical closure's +=prepare-generated-lexer= enclosing name does not mean preparation runs per +request: source inspection shows retained scanners prepared at machine creation. + +The source candidate also carries lexical winner selection through a direct +ordered loop, and publishes ordinary child lists through a direct visit loop. +These preserve candidate order, length/precedence/declaration ties, token +identity and event order. Emitted Scheme confirms the lexical request loop +rather than a newly captured fold callback. Neither this emitted-code observation +nor sampled frame counts establish CPU benefit. + +* Official Scheme implementation study + +[[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][Sperber and Thiemann, The Essence of LR Parsing]] separates grammar-static work +from dynamic execution through specialization. This study uses that principle; +it does not reproduce their generator or inherit their performance results. + +Gerbil =2591dcd= =std/io/bio/input.ss= stages a finite operation protocol with +=defrules __bio-read=. Its =std/struct/pqueue.ss= grows owned vector storage by +copying a bounded range. The parser uses checked public =subvector-move!= and +=subvector-fill!=. No unchecked primitive, compiler-metadata override or global +request buffer is introduced. Inactive semantic cells are cleared to immediate +zero; only the active prefix is meaningful, and discarded heap references are +not retained until the next growth or request. + +* Shared transition body and storage ownership + +=lr-stack-storage.ss= owns the static storage operations and the explicit +token-only observer capability. =lr-parser.ss= expands one transition body into +the two policies. =reduction-plan.ss= expands one right-to-left operand algorithm +with either a list cursor or a vector index. Generic backend/action selection +remains in preparation; no new generic dispatch occurs per semantic child. + +#+begin_src scheme +;; One semantic traversal, specialized by its cursor expressions. +(def (reduce-lr-stack-plan actions stack offset) + (reduce-lr-cursor-plan actions offset + (cursor stack (car cursor) (cdr cursor)))) + +(def (reduce-lr-vector-plan actions storage top offset) + (reduce-lr-cursor-plan actions offset + (cursor top (vector-ref storage (fx- cursor 1)) (fx- cursor 1)))) + +;; One transition definition; the boolean is a compile-time storage choice. +(define-lr-driver #f lr-run-persistent-checkpoint) +(define-lr-driver #t lr-run-owned-checkpoint) +#+end_src + +** Question: why does this exploit Scheme and Gerbil's strengths? + +Hygienic rules stage representation operations into a shared semantic algorithm. +There is no second hand-maintained interpreter and no per-transition storage +object dispatch. Private buffers are lexical request state; field/alias factories +continue receiving ordinary child values and offsets, never buffer views. +Named records express the observer's capability. Direct checked calls let the +compiler retain argument/type knowledge instead of passing an optional-arity +driver through an opaque procedure variable. + +** Question: how is this scientifically checked? + +The claim is bounded by independent artifact equality, model invariants, native +semantic controls and matched complete-request measurements. A sampled hotspot, +a Lean theorem, or fewer source-level allocations cannot substitute for the +matched result. The first vector trial also exposed small-grammar dispatch +regression; it was corrected before confirmation, rather than widening the +control margin. + +* Frontier and proof obligations + +Owned state/value buffers are never stored in a checkpoint or returned to a +caller. Fork/failure handoff copies only the active prefixes into newest-first +lists. Selective GLR receives the existing exact frontier and cumulative counts, +without replaying deterministic work. Generated steps and arbitrary stack +observers keep their list protocol. Token-only observers cannot access buffers. + +=ParserEngine/OwnedStack.lean= proves active-prefix length, roundtrip projection, +state/value depth alignment, prefix-preserving growth/clearing extensionality, +and the push/frontier relation. It reuses the existing reduction/LeanPoo +preparation model. The native bulk operations, mutation correspondence, GC, +compiler correctness and concurrent foreign-call lifetime are implementation +obligations, not silently proved by the abstract Array model. Quint's existing +finite publication protocol remains a separate source-generation/cancellation +obligation. + +Scheme controls cover empty/unary/wide/deep decorated productions, nullable +anchors, position views, independent declaration trees and GLR, generated/event +execution, frontier resumption after an intervening request, token-only versus +arbitrary stack observers, and source ABI publication. An independent raw-rule +scan checks literal action identity, first duplicates and fresh Unicode strings. + +* Reproducible admission + +=t/benchmarks/lr-request-stack/record-access.ss= retains the existing real +workload: representative GQL, 64-value GQL RETURN, 64-term FHIRPath addition, +arithmetic =1=, and an identical SDK UTF-8 encoder. Each child independently +checks its full artifact against GLR, then measures 40 batches of 1,000 calls +for prepared LR and complete source parsing. A batch CPU quantile is not a +single-request latency quantile. Natural GC remains inside measurement. + +Before confirmation, =scope.json= declares GQL as the two benefit targets and +FHIRPath/arithmetic as controls. The native =record-evidence.ss= checker reads +all four supervised outputs, recomputes quantiles from raw samples, checks +40 x 1,000 completeness, image markers, all four artifact hashes and exits. +It requires worst candidate CPU median / best reference CPU median below +=0.95= for every target stage, at most =1.05= for every control stage, and the +existing =1.05= CPU P95 bound. The SDK control retains its fixed drift and +repeated-band-overlap checks. Role declarations must partition all four +families exactly; duplicate, missing and unknown roles are rejected. + +The reference is the independently retained =e6cebe9= product closure. Both +images use byte-identical compiled workload products. Product/source hashes, +raw receipts, failures and manifests live under ignored +=.data/parser-goto-stack-study/=; earlier rejected confirmations remain under +=.data/parser-owned-stack-study/=. Repository documentation does not contain +host paths or copied execution logs. The full collector binds the declared +scope, both product closures and native admission into Ascent. Python, managed +by =uv=, transports/hashes observations; Scheme owns statistical admission and +native Ascent owns relational inference. CI retains its existing owners and +budgets; no workflow or glue-script layer is added. + +#+begin_src sh +uv run --project formal/evidence --locked python -m unittest discover -s formal/evidence -v +uv run --project formal/evidence --locked python formal/evidence/collect_engine.py --record-study .data/parser-goto-stack-study +#+end_src + +* Qualification result + +The original owned-stack four-block confirmation failed. Its representative +GQL prepared-LR median bound was approximately =0.726=, but its complete-source +bound was approximately =1.086=; the latter does not satisfy =0.95=. This remains +a failed four-stage CPU improvement, despite passing semantic/model controls. + +A subsequent traversal candidate completed all four supervised blocks and +matched full artifacts, but native admission rejected the SDK control's +nonoverlapping repeated median bands. Its raw representative complete-source +quantiles also do not establish the required five-percent improvement. These +receipts are retained separately from the original failure; they cannot be used +to admit the later combined AST-staging candidate. + +The combined AST-staging confirmation also failed the unchanged native CPU +check: its conservative representative prepared-LR ratio was approximately +=0.754=, while complete source was approximately =1.068=. The SDK control passed +in this confirmation. The failed complete-source result remains explicit. + +A subsequent streaming-stack ablation kept AST and lexical code unchanged and +used persistent stacks only for source-fed calls. Its single representative +screen did not establish the required complete-source benefit. This screen is +not a repeated causal conclusion or four-stage admission; the temporary dispatch +condition was reverted. The restored source was subsequently rebuilt through +the unchanged native source build owner before the boundary study below. + +The combined source passed Lean, Quint, four native build owners, 80 Scheme +Cases across seven Suites, the existing real-request measurement owner, and its +four-claim native Ascent review before the ablation. That review did not request +=record-cpu-improvement=, so its empty Missing relation does not admit the CPU +optimization. The uv-managed evidence controls passed 11 tests separately. + +At that point the combined implementation remained unadmitted. The required claim +still covers BOTH prepared LR and complete source for BOTH GQL targets. Native +semantic/model observations, evidence-transport controls and previously green +remote CI are separate from that CPU requirement. No failed receipt is converted +to a passing stage-limited claim, and no unadmitted candidate is pushed. + +* Whole-request boundary diagnosis + +The native =request-cost.ss= workload supplies the exact significant tokens from +the representative request through both prepared and streaming LR, then measures +publication alone, LR plus publication, and the actual source entry. It requires +complete artifact equality and an independent GLR product before measurement. +All five intervals retain natural GC and raw samples. Separate medians cannot be +subtracted as an additive decomposition or substituted for the four-stage gate. + +| Interval (40 x 1,000 requests) | Reference CPU median ms | Staged/owned CPU median ms | +|-------------------------------+-------------------------+-----------------------------| +| Prepared LR | 80.402 | 55.544 | +| Callback-fed LR | 81.425 | 56.309 | +| Publication | 8.254 | 8.188 | +| LR plus publication | 90.402 | 89.224 | +| Complete source | 105.777 | 103.644 | + +The prepared intervals had collections in 22/40 reference batches and 13/40 +candidate batches. Mean CPU excluding the observed GC CPU was 58.630 versus +55.423 ms. Complete-source intervals had collections in 40/40 versus 37/40 +batches, with corresponding execution means 81.743 versus 80.326 ms. These are +diagnostic observations, including the profiler's limitations; the original +inclusive CPU quantiles remain the admission authority. Allocation reduction +changed the collection distribution much more than execution CPU. Therefore +the prepared median improvement alone did not predict complete-request benefit. +This does not establish a collector defect. + +A singleton sequence operand now prepends to a borrowed ordinary suffix with +one immutable cons. It avoids a branch record followed by list materialization; +views retain their explicit relocation structure. Native tests cover suffix +sharing, independent prefixes, moved bounds, field/alias products and deep +composition. Lean states the ordered-list equivalence, without claiming native +allocation cost. + +The combined singleton/private-array confirmation remained a failure: its +conservative representative prepared ratio was approximately =0.688= and full +source approximately =0.976=. The rejection is retained. + +* Admitted symbolic lookup and private storage obligations + +Canonical production admission requires a Symbol nonterminal. The transition +can therefore use the native =assq= protocol for short goto rows, preserving +the first original pair. General-purpose association indexes retain their +existing equality contract. A hygienic rule expands this finite protocol into +the transition without an intermediate generic association procedure call. + +#+begin_src scheme +(defrule (goto-target gotos-expr state-expr lhs-expr) + (let* ((gotos gotos-expr) (state state-expr) (lhs lhs-expr) + (index (vector-ref gotos state)) + (entry (if (or (pair? index) (null? index)) + (assq lhs index) (hash-get index lhs)))) + (and entry (cdr entry)))) +#+end_src + +This exploits the DSL's admitted symbolic domain and Scheme's native identity +lookup. The grammar author supplies no dispatch IDs, caches or extra metadata. +The independent GLR and declaration-derived semantic oracles remain controls. + +The installed SDK's =std/struct/pqueue.ss= combines typed boundaries with a local +array invariant. The private LR storage follows the same ownership principle: +initialization admits aligned frontiers, reductions check width against active +depth, and pushes grow before writing. Only accesses to request-owned state/value +buffers lower their repeated native checks. Borrowed frontiers, checkpoint export +and bulk growth/clearing remain checked. No module-wide unchecked declaration or +compiler metadata replacement is introduced. Lean discharges natural-number +index inequalities conditional on admitted capacity; it does not certify native +allocation, fixnum limits or machine code. Scheme controls exercise empty, unary, +wide/growing frames, nullable aliases, failure snapshots and fork handoffs. + +The goto candidate's boundary preflight measured prepared LR at 49.240 ms and +complete source at 93.661 ms per 1,000 calls. Admission uses the separate repeated +request study below. The boundary workload remains in the existing evidence +request owner; CI has no additional lane or budget. + +* Repeated request admission + +The reference retains production revision =e6cebe9=; the candidate applies these +changes to =a7eb67a=. Four serial blocks use reference/candidate/candidate/reference +order, each with 40 batches of 1,000 calls per stage. The native Scheme checker +retains the original strict ratio: maximum candidate median divided by minimum +reference median must be below 0.95 for all four GQL targets. + +| Request | Prepared LR ratio | Complete source ratio | +|---------------------------------+-------------------+-----------------------| +| Representative GQL | 0.6129 | 0.9410 | +| GQL RETURN with 64 values | 0.9304 | 0.8488 | +| FHIRPath addition with 64 values | 0.7802 | 0.8738 | +| Single arithmetic control | 1.0227 | 1.0158 | + +All four required GQL targets pass. The existing control median and P95 bounds, +SDK drift and overlapping repeat bands also pass. These are observed matched +admission results, not a statistical significance or universal speed claim. +An earlier cold reference attempt exhausted the unchanged 120-second owner budget; +its failed record remains separate. Repeating through the original frozen library +completed within that budget. The retained reference copy is byte-identical to the +execution library; this does not prove the cause of the cold timeout. + +The CPU receipt joins the existing Lean, Quint, Scheme conformance, build and +matched-performance receipts through native Ascent inference, with no missing +required evidence. The measured native runtime images and engine products remain +byte-identical after qualification. Static conformance preparation regenerates +auxiliary Scheme/C/object products separately. The local conformance run contains +85 cases, including one pre-existing uncommitted GQL profile case; that unrelated +edit is excluded from this change's publication. Twelve uv-managed evidence control +tests pass. Formal models retain their conditional scope and do not authorize +publication. Darwin preparation passes in 125 seconds under the unchanged +240-second total and five-second idle budgets with CI-equivalent host tools. +The existing closed native semantic/research/all-language benchmark profile also +passes. Earlier local link failures remain retained as environment failures: +the installed SDK referenced an obsolete OpenSSL directory, followed by system +symbol resolution failure with the unrelated host toolchain on PATH. The exact +generated GQL test unit cold-compiles with pinned Gambit/GCC in 2.73 seconds; +this does not resolve or explain the earlier remote idle timeout. Exact-head +remote CI remains a separate gate. + +* Native conformance lowering + +The unchanged-head Darwin retry again failed the five-second output deadline, +now while compiling the publication-test module host. It did not establish a +parser semantic failure. An isolated comparison uses the SDK's public +=compile-module= API, identical module basename, =O1=, multiple-host policy and +trusted tail-call flags. Enabling the existing Gerbil optimizer reduced that +unit's GCC CPU from 2.67 to 0.96 seconds and GCC reported allocation from 180 MB +to 77 MB. Both local compiler runs completed under the original watchdog; these +observations do not prove every cause of the remote stall. + +The existing conformance build owner now optimizes test construction before +native lowering. It retains all assertions, the compiler's native tail-call +contract and all qualification budgets. Complete local preparation passes in +114.85 seconds. No compiler-pass suppression, new CI lane or synthetic progress +producer is introduced. Exact-head remote CI must independently admit this change. + +The existing source conformance discovery also passes all 108 modules. Initial +local attempts omitted the CI source load path, then encountered partial cached +native test interfaces and a stale partial ABI-generator wrapper. The existing +CI =prepare-gxtest!= owner and targeted =std/make= cleanup restored source +ownership. Their failed observations remain separate from the passing suite; +no assertion was removed or skipped to obtain it. + +The first final evidence run after static preparation was rejected during first +Suite loading with an idle timeout. The collector now reuses the existing +=prepare-gxtest!= source owner, rather than assuming static test interfaces are +valid source inputs. The subsequent full chain passes five preparation/build +receipts, all eight targeted Suites (85 local cases), both real-request owners +and the original four-stage CPU check. Native Ascent has no missing required +evidence. All 591 retained production/benchmark native images are byte-identical +to the measured candidate. The rejected chain remains failed. The twelve +uv-managed evidence rejection/admission controls also pass after this change. + +The subsequent [[file:lr-unary-chain.org][unary-chain study]] starts from this admitted production +revision and retains its own negative results and complete-request gate. + + +The next complete engine unit is [[file:compact-recognition-children.org][compact recognition children and constructor continuations]], +including action-selection negative results, logical representation proofs and +unchanged complete-request admission obligations. + +The failed compact representation and subsequent real-request native sampling +lead to [[file:lr-request-dispatch-cost.org][official inline contracts at the token boundary]]. +Its semantic, compiler-lowering and CPU-admission claims are kept separate. diff --git a/docs/research/lr-request-cost-boundaries.org b/docs/research/lr-request-cost-boundaries.org new file mode 100644 index 00000000..fe161629 --- /dev/null +++ b/docs/research/lr-request-cost-boundaries.org @@ -0,0 +1,482 @@ +#+title: LR request cost boundaries and Scheme-native instrumentation + +* Research question and responsibility + +Why can the public GQL parser report request P50/P95 without explaining where +the request spends its time? At =8e0f92c=, =measure-parser-stages= recorded only +=observed-complete-request= for the representative GQL query. The shared Source +engines had stage hooks, but the ordinary streaming LR path did not. + +This is an instrumentation coverage gap, not evidence of a Gambit GC defect or +a grammar declaration defect. The engine owns the boundaries. Language authors +continue to declare grammar, parser metadata and tests through their existing +three interfaces; they do not add timing callbacks to grammar rules. + +* Official implementation study + +The installed Gerbil SDK source used for this investigation is +=0.19.2591dcd.patchf5cedd8168cb=, upstream source directory =2591dcd=. +=src/std/misc/ports.ss= defines =with-input= and =with-output= using hygienic +=defrules= that lower delayed bodies to ordinary procedures. +=src/gerbil/runtime/util.ss=, =DBG-helper=, preserves arbitrary result values +using =call-with-values= and =apply values=. These are the relevant patterns: +use the compiler's syntax facilities and preserve Scheme result semantics. +They do not imply that every debugging facility is allocation-free. + +The existing =with-parser-cost-stage= macro follows that separation: when the +request-local observer is absent, its expression executes directly; only the +observed branch calls the measurement procedure with a thunk. The measurement +procedure preserves multiple values and reports completion on dynamic exit. +No kernel modification, second scanner or language-specific timing DSL is +needed. Disabled calls still read the observer parameter; this study does not +claim zero overhead. + +* Implemented boundaries + +The fast streaming path retains the same scanner, generated driver selection, +deterministic checkpoint driver, fallback and publication behavior. + +| Stage | Measured operation | +|-------+--------------------| +| source-recognition | Selected streaming LR drive, including its scanner callbacks | +| artifact-materialization | Successful artifact construction or failure artifact construction | +| fallback-lexical-analysis | Remaining source tokenization after an admitted GLR fork | +| fallback-lr-execution | LR/GLR suffix resumption after the fork | +| generated-source-product | Optional generated complete-source backend attempt | + +Scanning and LR execution are interleaved inside the streaming drive. Measuring +each token callback would insert an observer and process-statistics snapshots +into the hot loop. The coarse fused boundary is deliberate and accurately +named. Further separation requires a matched experiment that accounts for +instrumentation cost, not subtraction of independent medians. + +Stage receipts describe invocations. A driver retry can produce several +receipts with the same stage name in one request. The benchmark retains every +raw receipt and reports its stage receipt count; that count must not be read +as an independent request count. Missing stages are not zero-cost stages. + +* Two design questions + +** Why is this a better use of Scheme's DSL and macro facilities? + +The common engine can expose diagnostics through hygienic syntax without +requiring duplicated timing plumbing in =grammar.ss= or a new POO slot in each +language. POO remains responsible for loader inheritance, metadata and parser +policy. Ordinary Scheme values and procedures express the measurement itself. +The following is the actual engine pattern and public benchmark usage: + +#+begin_src scheme +;; Engine-owned expression boundary; the macro's disabled branch is direct. +(with-parser-cost-stage 'artifact-materialization + (make-success-parse-artifact grammar-digest source tokens root trivia?)) + +;; A consumer supplies the existing public parser, not another implementation. +(measure-parser-stages parse-gql +gql-representative-query+ 20) +#+end_src + +** Why is the result scientifically testable? + +Each sample must equal an unobserved complete reference artifact and pass +source-bound admission. Accepted, rejected and empty GQL inputs additionally +exercise observation through the public parser. Allocation samples are admitted +only when the process-statistics interval has no GC and a nonnegative counter +delta. P95 uses the existing nearest-rank statistics index. CPU percentiles, +allocation admission counts and complete product identities remain separate +fields. A future optimization can fail these controls or fail to improve a +matched native measurement; the design does not assume that it must win. + +* Initial receipt and limits + +The representative query has 138 characters and publishes 315 events. With +observation enabled, 20 source-execution samples reported: + +| Stage | CPU P50 ms | CPU P95 ms | Admitted allocation P50 bytes | +|-------+------------+------------+------------------------------| +| source-recognition | 1.7015 | 1.8880 | 3304672 | +| artifact-materialization | 0.2550 | 0.2880 | 644000 | +| observed-complete-request | 1.9710 | 2.1850 | 3975904 | + +All 20 allocation samples were admitted in each row. Full product equality and +source admission passed outside the measured request. The temporary receipt is +unarchived local execution; it is execution evidence, not a +tracked benchmark fixture. + +This establishes that recognition owns most observed CPU and allocation for +this query. It does not isolate scanning from LR, identify a particular rule, +qualify all grammar sizes, or establish native end-to-end latency. Loading and +grammar compilation are excluded after the reference parse. Percentiles from +20 samples are diagnostic, not a stable service latency distribution. Stage +medians must not be summed to estimate total latency. + +The next optimization target should be selected inside the shared recognition +implementation using matched complete-product controls and native execution. +The present change closes observable coverage without changing DSL semantics. + +* Qualification boundary + +The focused source run passed 52 cases, including public GQL accepted/rejected +products, HCL controls, multiple values, original conditions, observer dynamic +scope and byte-for-byte ABI fixture parity across all language packs. The +official SDK also compiled =parse-cost.ss= and =parser.ss= into native modules. + +A partial native execution experiment was not admitted: the local dependency +cache contained an artifact interface without the current source-bound +validation export. Reordering source and compiled dependencies subsequently +terminated with exit 139. This establishes neither a parser regression nor a +successful native run. Qualification requires one current native dependency +graph; mixing an isolated rebuilt parser with stale interfaces is insufficient. +At that point, complete native execution remained an outstanding gate. + +* Unified native qualification and allocation accounting + +The follow-up used the official package =clean= and =build.ss compile= entry, +four std/make workers, and the existing Scheme process supervisor. The budget +remained 240 seconds with a five-second no-output gate. Two initial attempts +were rejected by that gate during cold dependency admission, before an adequate +build receipt. Compiler verbosity alone did not expose the cold import work. +The existing expander import tracer then reported actual interface admission +alongside the SDK compiler logs; the unified build completed in 154.258 seconds, +exit 0, with no timeout or error reason. No heartbeat or longer deadline was +introduced. This is optional native qualification, not a source-test binary +prerequisite. + +The subsequent execution selected the rebuilt +=.gerbil/lib/gerbil-parser/src/runtime/parser.ssi= and completed 20 GQL samples, +each with full reference-product equality and source-bound admission: + +| Stage | Native CPU P50 ms | Native CPU P95 ms | Admitted allocation P50 bytes | +|-------+-------------------+-------------------+------------------------------| +| source-recognition | 0.1090 | 0.1230 | 125888 | +| artifact-materialization | 0.0170 | 0.0190 | 45392 | +| observed-complete-request | 0.1315 | 0.1540 | 180432 | + +All 20 allocation intervals qualified in each row. The source and grammar +digests match the earlier source-execution receipt. Evidence is in +unarchived local execution. The benchmark entry itself +is source-loaded while the production parser dependency graph is compiled. +Observer overhead remains included. This establishes successful execution of +the unified native graph; it does not diagnose the exact instruction that +failed in the earlier mixed graph, admit cold-start latency, or establish a +before/after parser optimization speedup. + +The source/native difference also prevents treating interpreted diagnostic +allocation as native parser allocation. Native recognition remains the largest +observed stage, but it requires about 126 KB per observed sample in this receipt, +not the 3.30 MB measured during source execution. + +The older GQL component benchmark now uses the same allocation admission as +the request benchmark. It retains =allocation-counter-delta= for every sample; +only a no-GC, nonnegative interval contributes to allocation percentiles. +=allocationSampleCount= records the admitted population. An empty population +publishes =#f= for P50/P95, while a genuine zero-allocation sample remains zero. +Actor batches retain raw process counters but do not attribute them to one +caller. Twelve source controls cover these cases and the existing complete +GQL component products. + +Fourteen focused GQL/request controls subsequently passed against the rebuilt +native production graph. The all-language ABI fixture test passed separately +through its ordinary source-owned entry, as configured in CI. An initial attempt +to force compiled test imports selected an old test interface whose generated +Scheme body was not a valid source module; that combined harness was rejected. +The successful receipts are +unarchived local execution. Production native +qualification does not make stale compiled test products authoritative. + +This conservative policy is consistent with the official +[[https://raw.githubusercontent.com/gambit/gambit/master/lib/_kernel.scm][process-statistics implementation]] and +[[https://raw.githubusercontent.com/gambit/gambit/master/lib/mem.c][bytes_allocated implementation]] inspected on 2026-10-09: the statistics vector +includes allocation and collection counters, while the allocation expression +contains an occupied-still-object term. These upstream master sources are +structural references, not the identity of the installed SDK. They support +keeping raw counters, heap state and admitted interval estimates distinct; +they do not prove a collector defect or authorize changing the allocator. + +* Matched native component receipt + +The current =execution-counts=, =reduction-counts= and =matched-stages= modules +were compiled against the unified production graph. The existing workload +remained 40 samples of 100 calls per component, with complete artifact +comparisons and untimed execution/reduction controls. The supervised run +finished in 43.882 seconds including traced cold admission, exit 0, with +=GQL-STAGES-OK= and no timeout reason. Most of this elapsed interval is outside +the timed batches; it is not parser latency. Receipt: +unarchived local execution. + +| Component | CPU P50 ms per 100 calls | CPU P95 ms per 100 calls | Allocation P50 bytes per call | Admitted allocation batches | +|-----------+-------------------------+-------------------------+-------------------------------+----------------------------| +| global-lexing | 1.169 | 1.223 | 50788 | 40 / 40 | +| prepared-lr | 8.568 | 9.009 | 82484 | 39 / 40 | +| artifact-publication | 1.317 | 1.360 | 44324 | 40 / 40 | +| full-source | 12.095 | 22.589 | 173196.32 | 38 / 40 | + +Full-source wall P50/P95 per call were 0.1220/0.6866 ms. Its actual wall-P95 +batch was sample 12: 68.662 ms wall, 14.043 ms CPU, zero GC, and 17319632 +admitted allocation bytes. Thus GC does not explain this particular P95 batch. +The signed wall-minus-CPU observation was 54.619 ms; it does not identify a +scheduler, OS or library cause. Independent wall/CPU P95 values must not be +subtracted to diagnose this batch. + +Prepared LR remains the largest CPU component among these controls. Global +lexing is not the streaming scanner's selected lexical-mode path, and prepared +LR does not include source scanning, so component medians are not an additive +decomposition. These results support investigating the shared recognition +path with its 37 shifts and 288 reductions before proposing grammar-specific +runtime code. They do not establish a speedup or a GC problem. + +* Reduction-width specialization experiment + +** Question and Scheme design + +Does explicitly separating epsilon, unary and wider stack pops improve the +shared native LR path enough to justify more expanded code? The existing +=with-pop-reduction= macro already borrows the top semantic stack for a unary +production and allocates a source-order value list only for wider productions. +The candidate separated widths zero and one from the generic countdown loop, +retained the same underflow condition and evaluated all inputs once. Its three +branches duplicated the hygienically bound continuation body. + +The relevant Scheme abstraction stays inside the engine: + +#+begin_src scheme +(with-pop-reduction states semantic-values count + (source-values remaining-values remaining-states) + ;; Existing semantic reduction and goto publication consume these bindings. + (reduce-value production source-values offset make-recognition-fragment)) +#+end_src + +Why does this serve the DSL design? The macro binds the engine's private +representation protocol without asking grammar authors to declare stack +operations or choose another backend. POO loader/metadata composition remains +separate. Why is the choice scientific? Equivalent syntax is not sufficient: +the compiler sees ordinary Scheme in either version, so the candidate must +preserve complete products and justify its expansion cost in native controls. + +** Controls and execution boundary + +Nineteen source cases passed for empty/unary operand actions, retained siblings, +fragment transfer and GQL component products. The baseline was the unified +native production graph at =f750b96=. The candidate rebuilt only +=src/runtime/lr-parser.ss= with the same SDK and optimization settings, retaining +the other compiled production modules. Each child asserted its selected LR +=.ssi= path before running the unchanged 40 x 100 component workload. Timing +processes ran sequentially in ABBA order. + +An initial ABBA attempt stopped in its third child after =GQL-STAGES-OK= because +the child did not exit within the no-output gate. A repeat using ordinary +=exit= also stopped after a completed baseline receipt. Neither incomplete +sequence was admitted. The final experiment used the repository's existing +disposable native child exit contract, which flushes receipts and exits only +after the benchmark returns. This is the same ownership boundary already used +by =t/support/qualification/plans.ss=; no new project runner or timeout change +was required. All four final children exited zero, with no reason or missing +completion receipt. This does not diagnose the earlier teardown stall. + +** Native ABBA results and decision + +| Order | Variant | Prepared LR CPU P50 ms / 100 | Full-source CPU P50 ms / 100 | Full-source CPU P95 ms / 100 | +|-------+---------+-----------------------------+-----------------------------+-----------------------------| +| A1 | baseline | 8.653 | 11.876 | 15.690 | +| B1 | candidate | 8.517 | 11.713 | 16.790 | +| B2 | candidate | 8.445 | 11.759 | 17.420 | +| A2 | baseline | 8.734 | 11.860 | 12.477 | + +All four runs reproduced the admitted complete artifact and untimed reduction +controls. Admitted prepared-LR allocation remained 82484 bytes per call in +39/40 batches; full-source allocation remained 173196.32 bytes per call in +38/40 batches. The arithmetic mean of the two run medians decreased about +2.4% for prepared LR and 1.1% for full-source. This is a descriptive summary, +not a confidence interval or a tail-latency guarantee. Full-source P95 did not +improve consistently. The generated LR Scheme body grew from 141221 to 145903 +bytes, about 3.3%; this is generated-source size, not native binary size. + +The candidate was reverted. The small full-product CPU change, unchanged +allocation and inconsistent tails do not justify adopting body duplication on +this evidence. The result does not prove that every width specialization is +useless, nor that the compiler emits identical instructions. It rejects this +candidate as the next production optimization. Further work should investigate +representation or repeated action work that can remove measured allocation or +substantial complete-request cost, rather than manually replacing a countdown +loop on the assumption that native lowering is inadequate. + +Temporary final receipts are +unarchived local execution +and their =-supervision.log= peers. The superseded failed attempts remain under +unarchived local execution for +audit. After restoring the original macro, the checkout was advanced to +=9c4b763=, preserving the separately integrated shared region-opener index. +The experiment's ABBA timings remain evidence for =f750b96=, not a performance +claim about that later revision. + +After the fast-forward, 32 controls passed for region scanning, epsilon/unary +reductions, fragment transfer, GQL diagnostics and all-language ABI fixture +parity. The first local native rebuild stopped after 72.954 seconds at the +five-second no-output gate; it is a failed receipt. The same official incremental +build then completed in 10.628 seconds, exit zero, with no reason. Both retained +the original budgets. These receipts are +unarchived local execution. + +The rebuilt =9c4b763= production graph subsequently passed the original native +GQL 40 x 100 workload with complete product controls and =GQL-STAGES-OK=. +The owned child completed in 39.983 seconds including cold admission, exit zero, +with no reason or missing receipt; evidence is +unarchived local execution. This qualifies +the selected native execution after the fast-forward without promoting the +rejected width candidate or reusing older-head CI as current evidence. + +* Literal lookup eligibility and repeated conversion + +The next candidate used the prepared action row to bypass literal lookup and +=string-upcase= when the row contained no literal entries. Its predicate +checked the existing ASCII vector and the empty-list/table representation; +it added neither a user declaration nor a new row field. Layout dispatch, +exact-literal priority, token-kind fallback and original action-pair identity +were preserved. Twenty-two source controls passed, including empty, ASCII, +Unicode and table-backed literal rows. + +The representative GQL request has zero literal-free states among its 279 +non-EOF action observations. Therefore the candidate's bypass is never selected +on this workload. Four sequential native children used the same =9c4b763= +production graph, with only the candidate action-index and LR modules replaced. +Each asserted the selected LR interface path, executed the original 40 x 100 +complete-product controls and exited zero without a supervision reason or +missing receipt. The timeout and five-second no-output gate were unchanged. + +| Order | Variant | Prepared LR CPU P50 ms / 100 | Full-source CPU P50 ms / 100 | Full-source CPU P95 ms / 100 | +|-------+---------+-----------------------------+-----------------------------+-----------------------------| +| A1 | baseline | 9.795 | 12.936 | 20.775 | +| B1 | candidate | 10.604 | 14.887 | 23.100 | +| B2 | candidate | 10.896 | 15.929 | 22.907 | +| A2 | baseline | 11.117 | 18.196 | 25.796 | + +Admitted prepared allocation was 82484 bytes per call in 39/40 batches and +full-source allocation was 173196.32 bytes in 38/40 batches in every run. +Baseline medians also drifted substantially across the sequence. These timings +do not support a stable CPU improvement, and the unchanged allocation agrees +with the zero bypass eligibility. The candidate was reverted; its native +receipts remain in unarchived local execution. This rejects this +candidate for the measured GQL request, not every possible grammar workload. + +The permanent untimed execution diagnostic now distinguishes three quantities: +=caseFoldObservations= counts the current executor's conversion condition after +an exact-literal miss, =caseFoldTokens= counts distinct input positions requiring +conversion, and =repeatedCaseFoldObservations= counts additional conversions on +those positions. It consults canonical action rows without performing another +conversion, retains the complete directed-artifact comparison and excludes EOF. +This is a GQL prepared-execution diagnostic, not an instrumented layout or GLR +trace and not a timing decomposition of streaming recognition. + +For the representative request, the counts are 21, 14 and 7 respectively. +Although there are 288 reductions, most action observations hit an exact +literal or do not reach the conversion condition. A per-token conversion cache +could eliminate at most those seven repeated conversions on this request; +the action counts alone do not justify it as a major optimization. Uppercase +and lowercase keyword controls check that exact hits short-circuit conversion +while admitted shifts remain identical. Further representation work should +be prioritized against the measured complete-request allocation, rather than +assuming every reduction repeats case folding. + +After this experiment, the checkout was fast-forwarded to =cb6ff10=, preserving +the separate region-trie rejection study. These ABBA measurements qualify +=9c4b763= only. The diagnostic changes and source controls run on the later +checkout; they do not establish a native performance gain on that revision. +The final source diagnostic/index qualification passed eleven cases with +=HARNESS-OK= and final =OK=, exit zero, with no failed cases. Its receipt is +unarchived local execution; the representative count +receipt is unarchived local execution. + +* Bounded recognition-sequence materialization + +After rejecting repeated case folding as the major next target, this study +followed semantic representation through operand actions and publication. +=recognition-sequence-append= retains immutable branch operands. The previous +=recognition-sequence->list= delegated every non-list sequence to the generic +visitor, which allocated a pending-list cell and a three-slot vector for each +deferred right branch. Those frames preserve relocation state even for ordinary +unrelocated branches. This is engine representation work, not a language-pack +configuration problem or evidence of a Gambit GC defect. + +The candidate keeps Gerbil's official =:std/list/list-builder= macro as the +output-list owner. Its installed SDK implementation expands collection to a +single locally owned chain. A private traversal visits ordinary branches +directly while depth is below 64, appends list leaves with the standard +=for-each=, and delegates deep, relocated or event-program subtrees to the +existing iterative visitor. The direct traversal does not traverse child node +bodies and has a fixed recursion bound. It changes no field/alias action, +sequence ownership, token identity, parser setting or grammar declaration. +The cutoff is a storage safety bound, not a language precedence or indentation +parameter. Its value was not tuned against the benchmark. + +#+begin_src scheme +;; The same sequence contract covers persistent concatenation and publication. +;; A language author does not select traversal or temporary-frame storage. +(let* ((left (list (make-recognition-child #f (make-token 'number "1" 0 1)))) + (right (list (make-recognition-child 'item (make-token 'number "2" 2 3)))) + (sequence (recognition-sequence-append left right))) + (equal? (recognition-sequence->list sequence) (append left right))) +#+end_src + +Why is this interface preferable? Persistent semantic composition remains one +engine contract while temporary storage stays private; language packages need +no additional declaration to obtain the improvement. Why is the claim +scientifically testable? The old and candidate native modules execute the same +tokens and complete artifact controls, and admitted no-GC allocation measures +the actual request interval independently of the temporary-frame hypothesis. +This example alone is not performance evidence. + +** Matched native evidence + +The sequential ABBA runs kept the original 40 x 100 sample/batch sizes and the +same production dependency graph. The isolated candidate replaced only +=src/runtime/funcs.ss=, compiled with official =gerbil compile -O=. Each child +asserted the selected =funcs.ssi= path before dependency admission, reproduced +the complete artifact and derivation controls, and exited zero with no reason +or missing completion receipt. The original 60-second budget and five-second +no-output gate remained in force. + +| Order | Variant | Prepared LR CPU P50 ms / 100 | Full-source CPU P50 ms / 100 | Full-source CPU P95 ms / 100 | +|-------+---------+-----------------------------+-----------------------------+-----------------------------| +| A1 | baseline | 8.665 | 12.144 | 16.915 | +| B1 | candidate | 8.509 | 11.621 | 17.484 | +| B2 | candidate | 8.620 | 11.800 | 16.742 | +| A2 | baseline | 8.641 | 12.332 | 16.954 | + +Every baseline admitted 82484 prepared-LR bytes per call and 173196.32 +full-source bytes per call; every candidate admitted 77252 and 167964.32. +The decrease is 5232 bytes per call, approximately 6.34% of prepared LR and +3.02% of the full-source interval. Each run admitted 39/40 prepared batches and +38/40 full-source batches. Global lexing and artifact publication allocations +were unchanged. This supports reduced allocation on the complete measured +request, not a workload-independent percentage or an allocation-free parser. +The mean of the two full-source CPU medians decreased approximately 4.31%; +that descriptive value is not a confidence interval. P95 did not improve +consistently, so this study makes no tail-latency claim. + +The runtime sources are identical between the native baseline's =9c4b763= and +the candidate checkout's =cb6ff10=; the intervening commit changed a region +study and controls only. The earlier local case-fold diagnostics do not occur +inside any timed interval. Receipts are +unarchived local execution +and their =-supervision.log= peers. Thirty-four source controls passed before +native timing, including a 10000-child sequence, composed relocation, persistent +fragments, GLR completion and event-program publication. The mixed-view control +also checks that relocation does not spill into a sibling and untouched child +identity is retained. Production-build and final language-alignment qualification +are recorded separately from this matched experiment. + +** Production qualification + +The final source suite passed 35 cases, including all-language ABI fixtures, +with final =HARNESS-OK= and =OK=, exit zero and no failed cases. The official +=std/make= production build completed in 125.764 seconds with exit zero and no +supervision reason; real compiler/module output satisfied the unchanged +five-second progress gate. These receipts are +unarchived local execution. + +The first unified-production GQL child failed at the original 60-second total +budget during cold module admission, before =GQL-STAGES-OK=. It is not a passing +receipt. The identical command and budgets then completed in 10.620 seconds, +exit zero, no reason or missing receipt, reproducing the candidate's 77252 +prepared-LR and 167964.32 full-source admitted bytes per call. This confirms +that the change is present in the official production graph. It does not +establish why cold admission differed or remove the first failure. Receipts are +unarchived local execution. diff --git a/docs/research/lr-request-dispatch-cost.org b/docs/research/lr-request-dispatch-cost.org new file mode 100644 index 00000000..86036735 --- /dev/null +++ b/docs/research/lr-request-dispatch-cost.org @@ -0,0 +1,341 @@ +#+title: LR Request Dispatch and Official Inline Contracts + +[[file:lr-owned-stack.org][Request execution research index]] + +* Evidence and decision + +The compact-child study reduced allocation but failed complete CPU admission; +its production changes were restored. An independent macOS native sample then +observed ten seconds of the retained baseline executing real representative +GQL complete source requests. The compiled workload performs 30000 requests +per invocation and checks the complete final product; ten invocations completed +with the original 120-second watchdog. Sampling is an attribution experiment, +never a performance-admission run. + +The sample contained 8437 main-thread snapshots. Top-of-stack observations +included 1005 in the LR driver module, 529 in publication, 406 in tokens, +393 in reduction plans and 303 in the action index. These are module-level +observations, not exclusive Scheme-function timings. Gambit's generated module +handlers combine multiple procedures and tail calls obscure their Scheme +callers. GC, runtime primitives and unresolved frames remain in the report; +they must not be assigned to a parser subphase without further evidence. + +This evidence supports studying hot checked access boundaries alongside action +selection. It does not prove that token access alone accounts for a particular +CPU percentage or that inlining will improve a complete request. + +* Official implementation mechanism + +Gerbil's =core/contract.ss= defines =definline= in terms of a lifted dispatch +rule and a normal contracted procedure. =std/time/precise.ss= uses it for +accessors; =std/text/parser/char-set.ss= uses it for small predicates. The +compiler exports inline rules through SSXI. The experiment follows that public +mechanism rather than introducing a parser-specific accessor macro framework. + +#+begin_src scheme +(definline (token-kind (token : :t)) => :t + (vector-ref token 0)) + +;; Both forms retain checked vector reads; neither caches a token's contents. +(token-kind source-token) +(map token-kind source-tokens) +#+end_src + +The deliberately broad argument and return type preserves the original vector +protocol. The constructor still creates the same four-element vector; =token?= +still validates its complete original shape. This experiment changes direct +call lowering, not token admission, metadata, lexical modes or action priority. + +** Question: why use this Scheme/Gerbil design? + +A single procedure body supplies both first-class calls and compiler-staged +calls. The official expander and optimizer own lifting, hygienic argument +bindings and inline metadata. Parser consumers keep their existing names and +contracts; no extra runtime object or public DSL configuration is required. + +** Question: why is the design scientifically justified? + +The baseline sample supplies a concrete investigation boundary. Scheme tests +must compare direct and first-class reads, mutation visibility, malformed +vectors, constructor behavior and complete LR/GLR products. Compiler output +must show actual call lowering. These semantic and lowering observations remain +separate from the unchanged four target CPU margins and control gates. + +* Observed lowering + +The candidate's standard source build completed with exit zero. Its expanded +LR module contains no direct references to the four public token getter +procedures, versus 45 source occurrences in the retained baseline expansion. +The candidate exports compiler inline metadata and retains ordinary procedure +bindings. This establishes lowering at those source sites, not a runtime call +count, a byte-size improvement or complete-request CPU benefit. + +* Qualification + +The standalone inline candidate was rejected. Keep all four target CPU ratios below 0.95, +all original controls and tails, and the SDK repeat-band overlap. Bind measured +source and actually selected native images. Python uses the locked =uv= project; +Scheme computes statistical admission and native ASCENT reviews receipts. +CI retains its existing owners and budgets. No allocation-only or stage-only +claim can authorize a push. + +** Standalone candidate result + +Ten Scheme modules passed 107 Cases. The standard source and benchmark builds +completed after retaining initial signature and startup failures. Interrupted +measurement sequences remain censored, with all partial samples retained. +The final four-role sequence completed. The aggregate SDK repeat control +passed, but native Scheme admission rejected representative GQL prepared LR +with conservative ratio 0.98181 against the original 0.95 bound. ASCENT retained +=record-cpu-improvement= as missing. No push +was authorized by that evidence. A lower complete-source median does not +replace the prepared-LR margin or the control gates. + +A subsequent candidate combines the same checked inline boundary with scalar +endpoint handoff in the existing AST constructors. It retains the original +child representation. Logical endpoint equivalence, runtime identities and +matched full-request CPU are separate obligations; their effects must not be +added from independent microbenchmarks. This combined candidate is pending. + +** Combined candidate result + +The combined =parser-request-bounds-study= binds the three changed engine +owners and both retained native closures. Symmetric import-only preparation +completed before the four-role sequence; its logs are separate from request +samples. All four 40-by-1000 roles completed with the unchanged budget. + +| Mandatory CPU target | Conservative candidate/reference ratio | +|----------------------+----------------------------------------| +| Representative GQL prepared LR | 0.941906 | +| Representative GQL complete source | 0.926462 | +| Long RETURN prepared LR | 0.907502 | +| Long RETURN complete source | 0.883317 | + +Each target ratio meets the original 0.95 bound. These are descriptive target +results, not an admitted overall benefit. The identical SDK encoder reference +medians were 7.061 and 7.079 milliseconds per 1000 calls; candidate medians +were 7.120 and 7.177. Their aggregate ratio 1.011103 meets the five-percent +ratio bound, but the two repeat intervals do not overlap. The original SDK +control therefore rejects the complete experiment. No source or threshold was +changed to turn this trajectory into a pass. + +The full existing evidence command then passed Lean, Quint, all five build +owners, ten Scheme modules with 108 Cases and the real-request benchmark checks. +The locked =uv= evidence tests passed 13 Cases, including actual native ASCENT +review and source/native-product drift rejection. Native ASCENT established +those receipts while retaining =record-cpu-improvement= as missing and denying +execution authorization. Benchmark execution and semantic matching do not +establish CPU admission. + +#+begin_src sh +uv run --project formal/evidence --locked python formal/evidence/collect_engine.py \ + --record-study .data/parser-request-bounds-study +#+end_src + +The candidate remains uncommitted and unpushed. CI definitions are unchanged. +This result supports further qualification of the combined call boundary; +it does not qualify a GitHub revision or waive the SDK overlap requirement. + +* SDK repeatability: controlled measurement ownership + +The rejected request-bounds trajectory remains rejected. Its raw SDK samples +allocate the same 4144336 bytes per 1000-call batch in both roles. Collections +occur only in sample 38 or 39; the median separation also appears in batches +without GC. The observed median separation therefore cannot be explained as +one role's P50 including a timed collection. Baseline heap sizes differ by +about one MiB, which is an observation rather than a causal explanation. + +A fixed four-role diagnostic placed an identical source-level SDK probe before +and after the existing request workload. Probe medians changed within a role, +and substantial wall/CPU differences were observed. The probe's source closure +is not the original compiled control closure. These diagnostic samples cannot +replace admission samples or prove that heap state causes the original gap. +All four roles and their negative observations are retained. + +Native identity inspection found another uncontrolled boundary: the original +roles selected different compiled images for both the SDK caller in +=record-access= and the shared sampler in =parser-stage-cost/benchmark=. +Identical source text does not establish an identical native caller. Build +metadata and code placement can differ even when the encoder primitive itself +is unchanged; binary differences alone do not establish a CPU cause. + +The follow-up =parser-shared-control-study= freezes the baseline caller and +sampler images into both roles, including their phase-zero images. Each engine +keeps its own frozen runtime closure. The collector rejects differing selected +measurement-image digests and reference-image drift, in addition to candidate +source/product drift. Existing evidence tests exercise these rejection paths. +This changes experimental ownership, not parser thresholds or CI lanes. + +The experiment retains the original reference/candidate/candidate/reference +order, 40 samples of 1000 calls, complete-product comparisons, four mandatory +CPU margins, control tail limits and SDK repeat-interval overlap. Import-only +preparation is symmetric and outside timing. A new result cannot convert the +previous failed trajectory into a pass. Measurement-image equality is a +necessary control condition, not a performance result or a remedy proven to +make SDK timing reproducible. + +** Shared-image experiment result + +The first shared-image four-role qualification also failed. Reference SDK +medians were 7.257 and 7.762 milliseconds per 1000 calls; candidate medians +were 7.063 and 7.115. Their aggregate ratio was 0.944004, outside the original +five-percent control bound, and the repeat intervals were disjoint. The +representative GQL prepared-LR conservative ratio was 0.967533, independently +below the required benefit. Native Scheme rejected admission and ASCENT +retained the missing CPU claim. Image equality therefore did not restore +repeatability or qualify the candidate. + +An attempted native-call interception failed because changing an exported +binding did not redirect the compiled caller. Its complete failed receipt is +retained; no SDK closure was captured, so this is not a before/after result. +The replacement benchmark explicitly owns a compiled encoder closure: + +#+begin_src scheme +(call-with-record-sdk-control + (lambda (encode expected) + (measure-parser-component '(sdk-native-before utf8) 40 1000 encode expected) + (record-request-main) + (measure-parser-component '(sdk-native-after utf8) 40 1000 encode expected))) +#+end_src + +** Question: why use this Scheme ownership boundary? + +A continuation receives one native encoder closure and its complete expected +result. Its payload remains live across both phases. The official compiler +keeps direct imports; the diagnostic does not depend on mutable-binding +interception or source-interpreted encoder bodies. The ordinary admission +entry still uses the same 4096-character input, 40 samples and 1000 calls. + +** Question: what can this experiment establish? + +The =record-repeatability= entry compares the same compiled closure before +and after a complete request workload in one VM. Each batch validates its +complete output. Shared caller/sampler digests remove native-image variation +between roles. Heap snapshots and no-GC samples remain observations; a time +change does not identify heap state, scheduling or hardware frequency as its +cause. Extra diagnostic phases cannot replace ordinary admission receipts. + +[[https://kar.kent.ac.uk/33611/][Kalibera and Jones (2013)]] distinguish uncertainty between builds, +executions and iterations and motivate repetition at the level that varies. +[[https://users.elis.ugent.be/~leeckhou/papers/oopsla07-stat.pdf][Georges, Buytaert and Eeckhout (2007)]] separate startup and steady-state +measurement and discuss scheduling, locality and system effects. These support +separate diagnostics for native-image, VM and sample variation. Their Java/JIT +results are not evidence about Gerbil AOT, nor do they authorize changing the +fixed parser admission rules. Our two-repeat range is an admission control, +not a statistically calibrated confidence interval. + +** Encoder-entry identity and output scheduling + +The complete retained/fresh/retained contrast did not support payload age as a +uniform explanation. Most fresh medians were near the surrounding retained +medians, while the ordinary entry sometimes remained slower. Standalone Gambit +C lowering of the retained factory source showed two encoder closure entries: +one inside the exported factory and another copied into its same-module +=main= caller. Their instruction sequences perform the same primitive call; +this establishes duplicated entry locations, not a proven timing cause in the +previously measured binary. + +The =sdk-control= module now owns the encoder factory separately from request +measurement. The request module's lowered Scheme contains no encoder primitive +body. The ordinary entry returns the measured encoder to the diagnostic, which +checks =##closure?= before comparing =##closure-code=. Four complete roles +confirmed actual code equality between ordinary, retained and fresh encoders. +The internal primitives are used only for this checked diagnostic, not parser +contracts or unchecked runtime access. Shared-image admission also binds the +factory's selected images in both roles. + +Despite entry equality, one role observed substantially higher CPU medians +than the others. Code duplication therefore cannot explain all repeatability +failures. These observations do not establish a specific hardware-frequency, +processor-placement or background-thread cause. + +The next fixed contrast examines output scheduling on one retained native +encoder: immediate/deferred/immediate. Immediate mode writes and flushes every +row between timed batches; deferred mode retains the already collected rows +and emits them after all batches. Encoding, semantic validation, 40 samples, +1000 calls per sample, initial GC and natural timed collections are unchanged. +The default remains immediate for request progress. No timeout or CPU threshold +changes. This is a measurement-interference diagnostic; it cannot be admitted +as an ordinary nine-stage receipt. + +** Canonical CPU clock intervals + +The output-scheduling contrast completed all four roles. Deferred SDK medians +were 6.735000000004043, 6.734000000005014, 6.735000000002489 and +6.760000000004651 milliseconds in reference/candidate/candidate/reference +order. The intended shared endpoint was 6.735 milliseconds, but binary +subtraction of cumulative floating CPU counters separated it numerically. +This is a concrete false separation at a timer tie, independently of the +larger earlier fluctuations. The diagnostic remains a diagnostic. + +Gambit's =os_time.c= getrusage path converts integral timeval seconds and +microseconds to floating seconds. The installed Gerbil precise-time library +keeps seconds and subsecond fields integral. The sampler now reconstructs +bounded cumulative CPU timestamps as integral nanoseconds before subtracting +user and system intervals. It publishes both =cpu-ns= and display milliseconds; +summary nanoseconds are independently reconstructed by native admission. + +Admission uses these exact intervals and the original exact fractions +=19/20= and =21/20=. Target comparisons remain strict, control comparisons +inclusive, and SDK repeat ranges must still overlap. No tolerance is added. +A Scheme regression checks equal clock intervals with different origins, +separate user/system contributions, and a genuine one-microsecond gap that +must remain separated. Ten Scheme modules pass 109 Cases with the standard +120-second total and five-second idle watchdog. + +The ordinary SDK control now defers row output until its unchanged 40 batches +complete. Parser stages retain immediate progress output. Every ordinary +receipt must also contain one checked encoder-entry identity witness; missing +or duplicate witnesses are rejected. All these conditions flow through native +Scheme admission and the existing ASCENT evidence collector, with unchanged +CI lanes. The =parser-sdk-canonical-study= binds engine and shared measurement +sources before fresh sampling. Earlier failed or censored trajectories are +not rounded, pooled or retroactively accepted. + + +** Fresh ordinary-request admission + +The source-bound =parser-sdk-canonical-study= completed the fixed +reference/candidate/candidate/reference sequence. Every ordinary role retained +40 samples of 1000 calls for each of the eight parser stages and the SDK +control, complete artifact equality, a checked shared encoder-entry witness, +and a successful supervised exit. The two reference SDK medians were 7.002 +and 7.013 milliseconds; candidate medians were 6.942 and 7.101 milliseconds. +Their intervals overlap and the aggregate candidate/reference ratio is +=14043/14015=, approximately 1.002. This admits the original calibration rule; +it does not establish uniform SDK timing or explain every earlier fluctuation. + +| Target | Stage | Conservative candidate/reference CPU P50 ratio | Original strict limit | +|--------+-------+------------------------------------------------+-----------------------| +| Representative GQL | prepared LR | 569/609 = 0.934319 | < 0.95 | +| Representative GQL | full source | 21649/23675 = 0.914424 | < 0.95 | +| GQL RETURN | prepared LR | 122713/137575 = 0.891972 | < 0.95 | +| GQL RETURN | full source | 16958/19099 = 0.887900 | < 0.95 | + +Native Scheme admission passed all four target comparisons, both FHIRPath +and arithmetic control stages, and every parser-stage CPU P95 non-regression +check. Ratios use the worse candidate repeat and better reference repeat; +no execution is excluded and no earlier failed study is pooled into this one. +The combined engine change is official checked token inlining plus staged +recognition bounds. These results do not isolate each component's marginal +contribution, claim a new lexer algorithm, or qualify unrelated languages. + +The first complete evidence collection exposed duplicate paths where shared +measurement images intersected the retained baseline closure. Source and +native snapshots reject duplicate inputs before observation. The collector +now forms a sorted distinct union at this admission boundary, retaining every +artifact identity once. The failed collection is retained separately. All +13 UV-managed evidence tests pass, including source drift, candidate image +mutation, mismatched measurement images and baseline image mutation rejection. +The complete evidence chain is rerun after this collector correction; local +admission and exact-head remote CI remain separate qualification boundaries. + + +The corrected complete collector exited zero. Native ASCENT established Lean +model checking, Quint protocol checking, all five Scheme build receipts, +Scheme conformance, matched request performance and repeated CPU improvement; +its =Missing= relation is empty. This is evidence admission, not an execution +permission: =executionAuthorized= remains false by design. The local working +checkout ran ten Suite modules and 109 Cases, including existing unstaged +controls; the committed revision's clean CI is a separate check. No workflow, +timeout, sample count or admission limit was changed for this qualification. diff --git a/docs/research/lr-request-model-foundation.org b/docs/research/lr-request-model-foundation.org new file mode 100644 index 00000000..08da030e --- /dev/null +++ b/docs/research/lr-request-model-foundation.org @@ -0,0 +1,268 @@ +#+title: LR request modeling before optimization selection + +* Order of work + +Research and current implementation establish the model vocabulary and its +hypotheses. Model qualification then identifies feasible optimization questions. +Only a candidate whose obligations can be established proceeds to an engine +change and complete-request Benchmark. This change extends the research and +model layers; it adds no runtime optimization or new performance result. + +* Paper findings and transfer limits + +Sperber and Thiemann's [[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]], sections 3.1 and 4.1, +separates grammar-static choices from dynamic input. Its binding-time work and +list-loop specialization motivate auditing repeated operations at their actual +binding scope. Constructor elimination must preserve computation order and +multiplicity. Their functional parser is not our explicit-stack executor; +the paper does not establish that our token metadata can be shared. + +Wagner and Graham's [[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]], sections 7 +and 8, ties reuse to context/content obligations and discusses when matching +conditions cost more than the reconstruction they avoid. It motivates charging +admission and maintenance work rather than treating a correct reuse guard as a +speedup. Its subtree reuse theorem does not authorize metadata reuse here. + +The following scope and cost definitions are derived from those principles and +our code. They are our model hypotheses, not claims made by either paper. + +* Current code audit + +| Owner | Current behavior | Model consequence | +|-------+------------------+-------------------| +| [[file:../../src/runtime/lr-parser/checkpoint.ss][LR driver]] binding | Runtime, widths, modes, action selector, reduction plans and goto index are bound at driver entry | Grammar-static binding is already present; repeated preparation is not inferred from a reduction count | +| LR input admission | Empty =rest= invokes =next-input= using lexical mode and optionally LR state | An extent starts at actual input admission, not at an offset or mode name alone | +| LR successful reduce | Both executor paths retain =rest= while updating states and values | A stable input can span several reductions, but action selection still depends on the new LR state | +| LR shift | Consumes =cdr rest= | The previous input extent ends; a cache cannot cross this boundary by pointer coincidence | +| [[file:../../src/runtime/lr-parser/recognition.ss][Capture reduction]] | Capture-enabled reductions project token kind/lexeme and construct a pair for nonempty input | Repeated metadata work is a question to measure, distinct from stack prefix work | +| [[file:../../src/runtime/token.ss][Token representation]] | A four-field vector with a string lexeme; no immutable token contract or mutation revision is enforced | Pointer equality and source identity do not prove projection stability | +| Checkpoint/failure/fork return | May pause execution or transfer into selective GLR | The proposed local scope is conservatively withdrawn; resumed and fallback execution need fresh admission | +| EOF | Empty input projects =#f= and may invoke the scanner again on a later iteration | No pair allocation can be saved at EOF; an empty-list sentinel is not an admission generation | + +This audit identifies a missing implementation premise: the current token vector +and lexeme can be mutated, but they carry no revision that establishes the +model's stability key. No metadata-sharing implementation is admitted by this +model-only change. It also does not change scanner calls, action selection, +semantic callbacks or fragment pair identity. + +* Lean responsibilities + +[[file:../../formal/ParserEngine/Lookahead.lean][Lookahead.lean]] uses a logical key with =owner=, =source=, =contract=, +=extent= and =revision=. The contract must bind grammar, prepared plan and +lexical policy. Extent identity denotes actual admitted input. Revision denotes +metadata mutation; neither is an implemented hash or pointer address. + +=admitted_projection= requires both key equality and an explicit producer +premise: the stored value equals the pure projection of its basis. +=extent_observations_equal= preserves the ordered sequence of those projected +values within one stable extent. Supersession and mutation reject the old key. +An included counterexample shows why equal keys alone cannot prove correctness. +Effects, mutable object identity, atomic snapshot construction, Scheme compiler +correctness and the producer premise remain implementation obligations. + +For an extent with =r= capture reductions, projected metadata cost =M=, +per-reduction check cost =C= and scope setup cost =S=, define: + +#+begin_example +Repeated metadata: r * M +Scoped metadata: 0 when r = 0 + M + r * C + S when r > 0 +Break-even: r * C + S <= (r - 1) * M when r > 0 +#+end_example + +These are weighted abstract operation costs in one chosen unit. Weights are +hypotheses to validate, not CPU samples. The scoped expression describes a +hypothetical demand-driven strategy and does not select a runtime design. +A singleton extent strictly regresses when =C + S > 0=. An EOF extent with +=M = 0= cannot pay positive admission overhead from saved materialization. +The model retains these negative regimes alongside a beneficial example. + +=SharedWork= names preparation, scan, LR, other capture work, validation, +publication, FFI and scheduling. Each =ExtentWork= carries its own reduction count and materialization, check +and setup weights, so EOF and lexeme-dependent snapshot checks need not share +ordinary-token costs. Complete-request baseline/candidate costs add all extent +costs to the shared terms, including =failedAdmission= work. =complete_request_break_even= includes every +extent, so singleton losses cannot be discarded. Common work may cancel only +under the explicit equal-work assumption. =complete_request_with_extra_work= +charges additional work instead of silently canceling it. The equations are +conditional arithmetic statements, not complete parser semantics or CPU proofs. +Scheme-only measurements set consumer FFI/scheduling scope explicitly; they +cannot qualify an unmeasured Tokio request. + +* Quint responsibilities + +[[file:../../formal/quint/LookaheadScope.qnt][LookaheadScope.qnt]] models scope histories, not LR language acceptance. +=Bind= remains the atomic specification of a successfully established snapshot +and its producer premise. The additional =Observe=, =CommitObserved= and +=DiscardObserved= actions expose the interval between coherent observation and +admission. Commit requires the observed key to remain current; mutation, EOF +admission and supersession invalidate an in-flight observation. Retry requires +an explicit discard and a new observation. =Read= requires active scope and +exact basis/current key equality. +Reduction retains the admitted extent; shift, EOF admission and mutation change +its identity. Source/contract change, cancellation, failure, supersession, +completion, pause and fallback withdraw observations. A new scope starts cold. + +Directed witnesses show that ignoring mutation revision or treating EOF as a +permanent sentinel permits stale observations. The finite coordinates are +=0/1= abstractions without wraparound. Each individual action is atomic; +=Observe= still assumes a coherent initial key/value observation. Torn field +reads within that observation, real reentrant callbacks, unbounded token +streams, queue fairness and liveness are not established by sampled histories. +A checked pre/post generation read or an exclusive immutable owner must be +established by a future implementation; merely adding a post-check is not a +proof of that producer premise. +The Lean producer premise and Quint =Bind= abstraction are a named assumption +bridge, not a checked translation between the tools. + +* Model-derived decisions and remaining evidence + +| Question | Model result | Next evidence before implementation | +|----------+--------------+-------------------------------------| +| Can repeated capture metadata be scoped? | Conditional on stable producer/key and request-wide break-even | Establish immutable admission or mutation detection, including callbacks and string mutation; count its overhead | +| Does the 64-record request benefit? | Not established; total production count/width histogram is insufficient | Record reductions per actual input extent, including singleton, zero-capture, EOF and fallback paths | +| Can EOF be treated like an allocated lookahead pair? | No; the current EOF materialization has no pair allocation | Separate EOF weights and scanner admission behavior | +| Can LR action selection be cached with token metadata? | No such theorem; action depends on changing LR state | A separate state/action equivalence model would be required | +| Can a local capture gain prove end-to-end CPU improvement? | No; every extent and changed common cost must be charged | Existing matched complete-request Scheme Benchmark, then a distinct Rust/Tokio consumer qualification | + +The next step is an observation-only trace over existing complete requests to +populate the extent distribution and audit stable-value admission. If the +hypotheses or break-even condition fail, reject this candidate and use the same +cost decomposition to investigate another engine cost. Do not implement a +cache merely because some extents contain repeated reductions. + +* Reproduction + +#+begin_src sh +just formal +just formal-protocol +PYTHONPATH=formal/evidence python3 -m unittest formal/evidence/test_model_inputs.py +#+end_src + +The evidence collector binds both model checks to the actual LR driver, capture +owner, token representation and existing controls. New Lean files are included +by the existing source inventory. Model success still leaves native conformance +and matched performance obligations open. + +* Local qualification + +Lean 4.34.0 built successfully (43 jobs). Projection admission and ordered +extent observations report no axiom dependencies; arithmetic cost theorems +report =propext=, =Classical.choice= and =Quot.sound=. No =sorry= or new axioms +were introduced. + +The locked Quint 0.33.0 package passed all ten modules: 81 directed tests and +nine sampled invariant runs. The new scope module passed 15 directed tests and +1,000 sampled histories of at most 20 steps with seed 1. These runs establish +only the declared bounded checks, not exhaustive or unbounded safety. + +The host input-binding control passed, including drift of the new protocol, +actual LR driver and token representation. There is no runtime change, new +native conformance qualification, Rust consumer result or CPU claim in this +model foundation. The previous CI repair at =46b7938= completed successfully; +it is a separate revision and does not qualify this new source. + +* Staged admission and failed-work accounting (2026-10-15) + +Revisiting section 3.1.3 of Sperber and Thiemann and the actual token accessors +exposed a gap in the initial atomic =Bind= assumption. Construction cannot be +moved across arbitrary effects just because a later key check succeeds. +The current code reads a kind and a mutable lexeme reference; retaining that +reference is not the same as taking a stable content snapshot. The extended +Quint model makes changes between observation and admission explicit. Its +unchecked-commit mutation witness demonstrates stale publication when that +interval is ignored. It does not claim the Scheme runtime supplies revision +numbers or coherent immutable snapshots today. + +Lean now records failed snapshot work for each extent, even when its reduction +count is zero. =retryWork= charges each failed attempt's materialization, +validation and discard. For a used extent, the refined condition is: + +#+begin_example +r * C + S + failedAdmission <= (r - 1) * M +#+end_example + +The earlier demand-driven =r = 0= formula applies only before failed or eager +admission work. =unused_extent_can_pay_failed_admission= formalizes that limit. +A retained counterexample has eight reductions and a favorable no-retry cost, +yet loses after five failed attempts. These are symbolic examples, not observed +request frequencies. Retry counts and lexeme-copy/hash work must be measured +before selecting an implementation. Unbounded retries have no proven progress +or latency guarantee; model safety does not justify a spinning retry loop. + +Local validation passed Lean (43 jobs), the complete locked Quint package +(89 directed tests across ten modules and nine sampled invariant runs), and +source-input binding. The scope module contributes 23 directed tests and 1,000 +sampled histories of at most 20 steps, seed 1. No runtime optimization or new +CPU claim accompanies this model extension. + +* Follow-up implementation qualification + +[[file:sparse-capture-model-first.org][Sparse capture]] applies the paper/code-first sequence to retained LR frontiers, +with separate Lean, Quint, semantic and complete-request CPU evidence. The +[[file:capture-metadata-model-first.org][metadata classification experiment]] retains its rejected CPU observation. + +* Pending input transport and the next selector boundary + +The current [[file:../../src/runtime/lr-parser/checkpoint.ss][driver]] creates a singleton input list at admission, retains +it across reductions, and consumes it on shift. The [[file:../../src/runtime/lr-parser/prepare.ss][action selector]] takes +that list on every query, checks EOF, and projects its first token. Lexical mode +belongs to scanner admission; a later reduction can change LR state before the +same pending token is consumed. The source owner already retains that admitted +mode for post-shift publication. + +The functional representations in Sperber and Thiemann motivate studying this +transport boundary; they do not justify replacing a list with a scalar while +silently recreating the list at every action query. Lean Lookahead now proves +that arbitrary ordered reductions preserve the pending token/mode pair and its +shifted publication. The representation must also retain unconsumed lookahead +on fork, failure and pause. No token-field caching or mutation epoch is admitted. + +Count transport list cells with =A= actual admissions, =Q= converted nonempty +action queries and =E= escapes needing conversion: + +| Representation | Transport cells | +|----------------+-----------------| +| Existing list input | A | +| Scalar with list conversion at each query/escape | Q + E | +| Scalar with native selector input | E, plus independent selector/guard costs | + +=scalar_input_break_even= charges all conversions. =request_transport_regresses= +lifts the comparison to complete logical request work only under equality of +every shared term; this does not establish a physical CPU cost. The stronger +=per_query_conversion_regresses= rejects the adapter when queries exceed +admissions. For the logical 64-record action inventory, 192 shifts and 322 +reductions already imply 514 nonempty queries when those reductions precede +nonempty lookahead; EOF queries themselves allocate no singleton and must not +be counted as nonempty conversions. The actual EOF/nonempty distribution needs +an execution trace before a numeric cell saving is admitted. The Lean example +with 514 conversions assumes those reductions have nonempty lookahead and is +explicitly a counterexample, not a measured trace. + +The next candidate therefore needs an engine-level selector accepting token/EOF +directly, with list materialization only at existing external frontiers. It must +cover literal precedence, case folding, layout, generated direct steps, failure +and selective GLR before implementation. Current list-based selectors and +extension factories remain the semantic authority during this study. + +Quint SparseCapture now independently tracks admitted modes and emitted modes, +requires the emitted sequence to be the exact consumed admission prefix, and +rejects another scan or trivia admission while lookahead is pending. Ten directed +histories include reductions changing lexical mode before shift and a fork with +unconsumed lookahead. The first control used =enabled=, which the pinned +TypeScript backend cannot execute; the executable shared admission guard now +owns that check. This is bounded protocol evidence, not an LR grammar oracle. + +* Implemented selector input and captured ownership + +[[file:selector-input-model-first.org][The implementation study]] records the upstream LeanPoo update, exact input +transport, physical executor admission, bounded goto preparation and additional +complete-request CPU qualification. The earlier per-query conversion warning +remains an implementation constraint for generated steps. + +The [[file:singleton-scanner-model-first.org][prepared scanner-start follow-up]] records rejected representation experiments, +product-start producer premises and the bounded CI supervision repair before +claiming any additional complete-request CPU benefit. + +The [[file:capture-fold-cost-model.org][capture allocation study]] models terminal-index demand, shared transition +contexts and full-request cost obligations, retaining rejected CPU results. diff --git a/docs/research/lr-semantic-kernel-staging.org b/docs/research/lr-semantic-kernel-staging.org new file mode 100644 index 00000000..a2931a33 --- /dev/null +++ b/docs/research/lr-semantic-kernel-staging.org @@ -0,0 +1,405 @@ +#+title: LR semantic kernel staging + +* Cost boundary and implementation owner + +This study continues [[file:prepared-lr-action-plans.org][prepared action admission]] and +[[file:lr-request-cost-boundaries.org][complete request cost attribution]]. The latter identifies recognition as the +largest measured CPU stage. Compiled language publication is a separate cold +path. This change targets the shared LR engine, without language declarations, +request caches or additional author configuration. + +[[file:../../src/runtime/reduction-plan.ss][Reduction preparation]] lowers admitted canonical operand actions to private +procedures. Its finite backend/action generic selects the recognition and +event kernel factories once when the private backend catalog initializes. +Grammar preparation reads that closed catalog. Names are lexical constants; +semantic values, offsets and fragment constructors remain invocation inputs. +The LR driver owns admission, stack bounds, layout transitions, precedence and +branch selection. Zero-width reductions return the backend's empty value; +unary reductions apply their prepared step directly, without an operand loop +or an identity join. Generated reducers share the engine's expression emitter; +canonical projection retains an independent interpreter. Canonical IR remains +inspectable and serializable data. + +Deterministic recognition traverses the borrowed stack backwards and prepends +completed operands. GLR traverses source-order values forwards with its own +request-local fragment interner. Event preparation occurs when the eligible +event runtime is selected; its forward append association is preserved. +Installing generated steps invalidates future selection without changing +retained checkpoints or their prepared kernels. + +* Research transfer and official Scheme patterns + +Jones, Gomard and Sestoft's [[https://studwww.itu.dk/people/sestoft/pebook/][Partial Evaluation and Automatic Program Generation]] +(1993) separates known inputs from later inputs and constructs a residual +program. Here the known inputs are the admitted backend, action kind and name; +the later inputs are request values and positions. This is a small explicit +specialization of the semantic interpreter, rather than a general partial +evaluator or a parser-generation theorem. Preparation cost and residual calls +must be measured; specialization alone does not establish a speedup. + +The installed Gerbil compiler's =compiler/method.ss= separates =compile-e= +method selection from the hygienic =with-context= rule. The SDK's +=std/vector/vector.ss= owns =vector-map/index=. We use the same responsibility +separation: a finite preparation generic, bounded vector construction, named +execution algorithms and one hygienic rule for the two forward traversals. +Higher-order composition binds an admitted action chain once, preserving its +declaration order; the hot traversal does not use generic dispatch. + +* Scheme witness and the two design questions + +#+begin_src scheme +;; Grammar authors declare meaning, not a backend traversal or callback. +(field outer (alias Renamed (field inner (token word)))) + +;; Illustration of the private preparation/execution boundary: +(let ((field (car +recognition-action-factories+)) + (alias (cdr +recognition-action-factories+))) + (let ((apply-inner (field 'inner)) + (apply-name (alias 'Renamed)) + (apply-outer (field 'outer))) + (lambda (value offset constructor) + (apply-outer (apply-name (apply-inner value offset constructor) + offset constructor) + offset constructor)))) +#+end_src + +** Question 1: Why is this better for the DSL and its engine? + +The declaration remains a structured semantic specification. Macro expansion +and POO composition produce canonical data; engine admission then selects a +closed executable operation. Lexical closures exploit Scheme's binding rules +to stage fixed names without capturing a request. Hygienic syntax shares the +ordered traversal while preserving its two distinct join operations. The +benefit comes from these cooperating mechanisms beneath =grammar.ss=, +=parser.ss= and =parser-test.ss=, rather than S-expression notation alone. + +** Question 2: What makes this scientifically testable? + +For every admitted chain, the prepared operation equals the canonical action +interpreter applied in declaration order. The empty chain is identity. An +induction on chain length preserves values and supplied offsets; a second +induction on operand count preserves the executor's existing association and +source order. Neither argument permits moving GLR interners into preparation +or reversing event joins. This equivalence statement must also specify the +origin supplied to an empty operand; agreement between two executors alone +does not establish the source-origin contract. + +* Nullable origins across execution owners + +An independent expected tree exposed an error in both the original and staged +LR reducers. An empty alias preceding a token received the enclosing +production's end offset. The parent then inherited that incorrect start. +The smallest witness is: + +#+begin_src scheme +(source-file + (node SourceFile (seq (node Before (empty)) word (node After (empty))))) +;; For word spanning [0, 1): Before is [0, 0), After is [1, 1), +;; and SourceFile is [0, 1). Reusing a prepared runtime must not retain +;; either boundary from an earlier request. +#+end_src + +For raw operand values v_i and production lookahead L, define b_n = L and +b_i = start(v_i, b_(i+1)). An empty marked operand i receives b_i; a +nonempty action computes its own span from its children. This suffix +recurrence preserves source order without introducing a second empty-value +representation. The deterministic stack already traverses backwards. Forward +GLR and event executors build a linear suffix index lazily, only when a +marked empty input requires it. The independent canonical projection applies +the same origin contract through its literal action interpreter. + +The generated Scheme owner uses static nullability to emit shared suffix +bindings only for nullable aliased operands. Each needed bound is computed +once, and nonnullable productions emit no new boundary work. Arithmetic's +generator calls this engine-owned expression emitter instead of maintaining +another action interpreter. Rust native reduction traverses operands backwards +to establish boundaries, then restores source-order groups before flattening. + +[[file:../../t/fused-reduction-test.ss][Generated recognition and event controls]] compare a fresh grammar's complete +artifacts with a manually constructed tree. [[file:../../rust/gerbil-parser-runtime/tests/unit/nullable_boundary.rs][The Rust unit control]] supplies +the same empty/token/empty production at offsets 0, 17 and 3. These controls +test source semantics independently of the original implementation; the +original's incorrect result is a falsifier, not a compatibility requirement. + +[[file:../../t/lr-stack-reduction-test.ss][Stack controls]] construct the expected semantic tree independently of both +LR executors at widths 0, 1, 2, 8 and 32, including nested field/alias chains. +The GLR, deterministic recognition and event products must equal that oracle. +Preparation admission, retained backend selection, fragment transfer, layout, +contextual parsing and language ABI suites cover the surrounding contracts. + +[[file:../../t/benchmarks/lr-request-stack/benchmark.ss][The native benchmark]] accepts original runtime procedures as explicit +experiment inputs. It verifies every complete artifact against the public +parser, measures preparation plus one complete request separately, and compares +prepared execution plus publication in alternating CPU pairs. Its language +matrix is GQL RETURN, FHIRPath addition and Arithmetic addition at 64 and 512 +units. Natural collection and the existing CPU resolution/allocation admission +rules apply. Prepared-artifact CPU is not whole-source parser CPU, retained +heap, or a claim about generated Arithmetic execution. + +* Qualification boundary + +The initial candidate passed 82 cases in ten modules with final =OK= and actual +exit zero. The LR module's bounded native build exits +zero in 71.353 seconds; the benchmark module build exits zero in 50.556 seconds. +These are local observations, separate from exact-revision platform CI. + +Earlier compiler attempts failed the existing five-second idle or 120-second +total gate. The original interpreter control also failed during native macro +compilation. A static C object was rejected as a dynamic macro module; the +control must be a loadable SDK module before comparison. Actual compiler GC +and register-allocation diagnostics expose work; no heartbeat, optimization +reduction or timeout extension is admitted. Generated diagnostics remain +unarchived local execution, outside Git documentation. + +The first implementation selected a generic method for every operand action. +Its GQL preparation-plus-request CPU P50 increased from 241.676 to 421.816 ms +at 64 units. It is rejected. The corrected owner selects two factories once, +then traverses admitted actions with closed construction procedures. The +later catalog correction below moves these fixed choices out of grammar +preparation entirely. This negative result explains why a preparation boundary alone is insufficient: +the frequency and lifetime of selection also matter. + +The first combined six-case process exceeded its fixed total budget. Cases +can be qualified separately through =compare-lr-staged-case=; its default +200-call alternating batches retain natural GC and complete product admission. +The whole matrix remains available through =compare-lr-staged-actions=. + +The corrected implementation passes 93 source-owned Scheme cases in eleven +modules with actual exit zero, including independent nullable-origin controls, +both generated semantic backends, layout restoration and language ABI alignment. +The optimized Rust unit executable passes 146 cases with actual exit zero. +Fresh platform builds remain a separate CI gate. A private generated-HCL native +build failed its idle gate; incomplete products were isolated before testing. +The new grammar's generated procedures are evaluated directly by the source +suite, so this receipt does not assert fresh native compilation of every +generated language module. + +The 64-unit comparisons use twenty samples and twenty alternating pairs of +100 complete requests. GQL cold CPU P50 is 262.936 ms for the original and +266.985 ms for the corrected owner; the former 75% regression is absent in +this observation. Paired request CPU P50 ratios are 0.932 for GQL, 1.031 for +FHIRPath and 0.988 for Arithmetic. Their P10/P90 intervals span both sides of +one. FHIRPath cold P50 is 4.033 versus 5.009 ms, an unfavorable observation. +These mixed results do not admit a stable latency improvement or a +language-wide performance claim. Allocation medians and non-GC batches cannot +replace complete-request CPU evidence. + +All six language/scale controls subsequently pass complete artifact equality +and their unchanged process budgets, with 25 requests per batch at 512 units. +The 100-call GQL 512 process exceeded its total budget; that failed observation +is retained and is not admitted as a complete comparison. Clippy with warnings +denied and the two owned Rust files' format check also pass. + +Across the six controls, preparation adds about 700 KB even for Arithmetic's +small grammar. Its 64-unit cold CPU P50 is 0.606 versus 1.150 ms. This rejects +the per-preparation factory selection as the final lifetime: the four generic +choices belong to the closed backend catalog's module initialization. +Grammar preparation subsequently binds only admitted names. Initialization +cost still exists and is distinct from repeated preparation and request cost; +this is not a global grammar or request cache. The catalog is private and +closed, so it has no caller-visible method-registration invalidation contract. + +The catalog revision compiles natively with the SDK's existing optimization +settings and passes the same 93 Scheme cases. Its 64-unit cold preparation +plus request observations are: + +| Family | Original CPU P50 ms | Catalog CPU P50 ms | Original allocation bytes | Catalog allocation bytes | +|--------+---------------------+--------------------+---------------------------+--------------------------| +| GQL RETURN | 247.499 | 239.119 | 107079088 | 107107328 | +| FHIRPath addition | 4.373 | 4.186 | 3135536 | 3140000 | +| Arithmetic addition | 0.633 | 0.474 | 1047504 | 1048656 | + +The grammar-independent 700 KB preparation penalty is absent; remaining +increments depend on the admitted action plans. This establishes the corrected +lifetime boundary, not a retained-heap or initialization-cost reduction. +Twenty pairs of 25 requests produce CPU P50 ratios 0.941, 0.920 and 1.064, +respectively; all P10/P90 intervals still cross one. Stable request CPU benefit +remains unproved. The combined six-case process stops at its unchanged total +gate after these three cases; it is not a passed whole-matrix receipt. +Large cases use the per-case entry, and CI separately validates the complete +public request matrix against a freshly built integrated graph. + +* Integrated CI control calibration + +At revision =842767b=, the fresh Ubuntu build passes the 48-clause complete +public parser CPU gate: production P50/P90 ratios are 0.7784/0.9265. The +12-clause, 384-call gate is rejected: production P50/P90 is 0.8606/0.8813, +but the identical before-control P90 is 1.10996, outside the unchanged 1.10 +stability limit. Its after-control P90 is 1.02133. The production benefit +alone cannot admit the failed control or the whole CI job. + +The rejected control group's paired totals are 251.356/278.994 ms, with +70.630/95.819 ms GC CPU and three/four collections. The extra collection +accounts for most of the paired CPU difference. The integrated calibration +therefore uses 768 calls per short-request batch and 256 per long-request +batch, keeping twenty eight-leg groups, every measured sample, +natural GC, identical before/after controls, the 0.90--1.10 stability limits, +production P90 below 0.95, and the existing 120-second/five-second gates. +This doubles measured work to reduce the relative size of a collection; +it does not change parser output, remove failed observations or admit the +new CI run in advance. The original long-request receipt above remains +attributed to its 128-call batch. + +* Native test procedure ownership + +The macOS job at =842767b= stops while compiling the resolved-grammar test +object: actual compiler output becomes quiet for more than five seconds. +This is a native compilation failure, separate from parser execution and the +Ubuntu CPU-control rejection. Additional register-allocation and optimization +diagnostics also fail the local silence gate and are rejected; diagnostic +volume does not resolve compilation cost. + +The SDK's =std/test/unit.ss= expands each =test-case= into an anonymous +procedure nested in the suite procedure. Gambit's native host grouping can +therefore collect a large suite into one compilation unit even with multiple +hosts enabled. The resolved-grammar suite now owns named top-level case +procedures and registers them through the SDK's =TestCase= and +=test-case-add!= operations. Assertions remain in Scheme, and suite discovery +and its exported identity remain unchanged. No additional test framework, +compiler optimization exception or process wrapper is introduced. + +An independent reader compares all 37 original titles and complete assertion +bodies with the registered named procedures: every body is structurally +equal. The source-owned resolved-grammar and test-style suites pass all 43 +cases with actual exit zero. Fresh static Scheme generation also exits zero. +Fresh static native object compilation completes in 69.64 seconds with actual +exit zero and no silence-gate violation, retaining the SDK's optimizer and +existing multiple-host policy. Full platform CI remains a separate admission +requirement; this single local object cannot qualify the complete build. + +* Shared diagnostic module publication + +At =ba81c0e=, Ubuntu passes the calibrated CPU gates but the GQL component +report terminates with signal 139. Its first loader warning identifies the +undefined =measure-parser-component__0= entry referenced by the previously +compiled =matched-stages= consumer. This is a generated-interface mismatch, +not a parser timeout or a failed artifact-equality assertion. + +The conformance build already owns the complete shared stage sampler and its +GQL consumers. A later CI support command unnecessarily recompiles the sampler +with the bare =gxc= CLI. The SDK's =gxc-main.ss= defaults to =gerbil/base= and +sets that language before compilation. This differs from the compiler context +that generated the consumers: the former wraps optional arguments in a local +procedure, while the latter publishes the fixed-arity =__0= and =__1= entries. +Overwriting the native producer cannot preserve those earlier references. + +Fresh Scheme generation reproduces both shapes. The conformance-context +sampler defines =__%=, =__0=, =__1= and the public wrapper; the exact bare-CLI +command defines only the public wrapper with a local optional-argument +procedure. Both generation processes exit zero, confirming that successful +compilation alone cannot establish producer/consumer interface agreement. + +CI now retains the sampler published by its conformance owner. The additional +LR benchmark and actor consumers compile against that retained interface. +There is no ABI adapter or legacy entry. The closed executable, native module +loads and complete GQL reporting remain distinct exact-head CI requirements. + +* Disposable matrix process completion + +At =ebb254f=, the complete Ubuntu and Rust CI jobs pass. macOS passes native +conformance compilation, Scheme tests and both GQL reports. Its shared LR +matrix emits all six family/scale results and =LR-REQUEST-STACK-OK=, then +remains alive after the benchmark returns. The supervisor stops it with +exit 143, =idle-timeout= and no missing marker. This is rejected: a completed +receipt cannot substitute for successful process completion. + +The matrix invocation now uses the same checked =test-child-process-exit!= +fixture as the existing prepared GQL and native ABI children. It flushes +receipts and exits only after the matrix returns normally. Assertions and +exceptions still precede that call, and the parent still requires actual +exit zero and the complete marker under the original 90-second/five-second +gates. This belongs to the disposable test invocation; the reusable benchmark +procedure and parser runtime do not terminate their caller. Independent C and +Rust lifecycle tests continue to qualify production ownership separately. + +A local full repeat using the available staged products exceeds the original +90-second budget after 233 of 240 measurement rows; it is rejected, and does +not qualify the fresh integrated graph. A terminal-protocol smoke control +uses one sample and one request for each of the twelve matrix stages: all six +complete-artifact controls return, the marker is flushed, and actual exit is +zero in 57.89 seconds. This admits that process-completion path only. Fresh CI +retains twenty samples and twenty requests, and must admit the full matrix. + +* Complementary CPU observations + +At =7f93f63=, Ubuntu rejects the 48-clause control despite production P50/P90 +0.84905/0.88740. Before-control P10 is 0.89421; after-control P10 is 0.89166. +In before group 14, identical left/right code consumes 394.131/352.436 ms, +including 142.419/101.369 ms GC CPU and eight/six collections. In after group +9, the corresponding totals are 437.726/376.318 ms, including +170.288/103.445 ms GC CPU and eight/six collections. The completed run is +rejected; preceding admitted revisions cannot qualify this one. + +Increasing the nominal batch further would approach the original 120-second +budget on the slower recorded runner. The shared sampler instead subdivides +each nominal quantum and places both complementary eight-leg blocks inside +each observation. Each variant still executes exactly four nominal quanta; +odd call counts split into floor and remainder, and an indivisible one-call +quantum retains eight legs. Twenty observations remain disjoint: no +overlapping windows, repeated admission attempts or sample removal are used. + +The pure request planner is tested through work conservation, positive exact +sub-batches and equal weighted leg-position moments through degree two, +including odd call counts and both observation orientations. Each timed +sub-batch still validates its complete expected result and resolves at least +one millisecond. Receipts record actual sub-batch calls, leg count and requests +per variant. All GC CPU remains charged; no collection is forced between +variant switches. Thresholds and total/idle budgets remain unchanged. + +This balances temporal exposure and changes natural-collection phase sampling +without increasing total parser work. It does not prove immunity to arbitrary +GC or host noise; the complete fresh CI controls must still qualify it. + +The final sampler builds with the conformance compiler policy in 36.27 seconds +and exits zero without a silence violation. Nine source-owned benchmark and +style cases pass in 36.78 seconds. The execution test performs real complete +arithmetic requests and independently counts 20,481 calls per variant: one +warmup plus the declared twenty groups. These receipts establish the planner +and its consumer's work budget; fresh public TLA+ CPU admission remains separate. + +At =e1acd02=, fresh Ubuntu admits both complete public parser cases. Their +production P50/P90 ratios are 0.83850/0.84736 for 48 clauses and +0.85803/0.88438 for 12 clauses; both identical before/after controls pass. +Keyword and identifier lexical controls also pass. The smaller literal-prefix +control rejects: before P10/P90 is 0.87358/1.14591 and after is +0.81683/1.14505, despite production P90 0.68653. The whole job is rejected. + +Before group 9 executes one million requests per variant. Its CPU totals are +130.901/104.157 ms; GC CPU is 36.914/10.479 ms with seven/two collections. +The non-GC totals are nearly equal. This subcase needs more work relative to +that collection-count imbalance. Its nominal quantum becomes 750,000 calls, +three times the prior value; keyword and identifier retain 250,000. The +observed preceding elapsed times, 14.30 and 30.28 seconds, leave headroom +under the unchanged 120-second limit for this bounded calibration. Twenty +groups, the complete outputs, every GC interval and both 0.90--1.10 controls +remain required. The admitted public-parser observations above remain tied +to their revision and unchanged work budgets. + +At =b793f0f=, the 48-clause control passes again, with production P50/P90 +0.85467/0.86281. The 12-clause control rejects: before P10 is 0.89505 and +after P10/P90 is 0.89117/1.12283; production P90 is 0.93351. Before group +10 compares 301.930/270.241 ms including 110.742/79.472 ms GC CPU and +seven/five collections. Its non-GC totals are 191.188/190.769 ms. +The roughly 32 ms collection-count difference dominates the failed ratio. + +The short public case's fixed nominal quantum therefore becomes 1,536 calls. +The preceding complementary-plan elapsed times are 35.35 and 50.26 seconds; +doubling work fits the original 120-second budget on those recorded runners. +This improves the collection-count resolution while retaining twenty groups, +every natural collection, full artifact validation and the original percentile +limits. The 48-clause case retains 256 calls, and the literal-prefix calibration +above remains independently required. No rejected run becomes admitted. + +At =6733f32=, the 1,536-call short case exceeds the unchanged 120-second +budget after both twenty-group before/production series and eight after-control +groups. All completed identical-control observations remain within 0.90--1.10, +but the incomplete series cannot qualify either performance or completion. +The same runner's long case takes 81.35 seconds, compared with the preceding +41.26-second receipt: elapsed capacity must be checked on the actual runner. + +The short case's fixed nominal quantum becomes 1,024 calls, between the +768-call control-resolution failure and the 1,536-call budget failure. +Scaling the completed work leaves headroom for all sixty groups under the +original budget. Fresh CI must independently admit this intermediate quantum; +neither the partial controls nor the elapsed projection constitute admission. +No group count, CPU resolution, percentile limit or timeout changes. diff --git a/docs/research/lr-table-admission.org b/docs/research/lr-table-admission.org new file mode 100644 index 00000000..7803540c --- /dev/null +++ b/docs/research/lr-table-admission.org @@ -0,0 +1,764 @@ +#+title: Canonical LR table admission and staged execution +#+startup: overview + +* Prepared capability and request-local layout execution + +The table contract derives layout capability from canonical productions. The +execution boundary must preserve that decision: ambient dynamic parameters are +request data, not evidence that a prepared runtime admits layout semantics. +At parent =27f5471=, a plain runtime accepts one =word= token normally, but the +same runtime rejects it inside an outer column table and reference frame. The +old action lookup consults =current-layout-columns= without its own capability, +so an unrelated frame can remove the ordinary shift. + +The LR executor now captures its prepared layout capability once alongside +case policy and indexes. Its deterministic loop and both GLR lookups pass that +value to the private action selector. Only a layout-capable runtime consults +dynamic source columns. Existing GLR memo keys and acceptance already use +runtime capability; branch-local layout shifts and closing reductions retain +their existing owners. This fixes capability leakage without changing grammar +declarations or introducing a request-time generic dispatch protocol. + +The tokenized source boundary independently binds its own columns and empty +initial frames for every request. Plain requests bind no columns; layout +requests construct columns from their own source. Parameter restoration protects +an enclosing request after acceptance or rejection. Layout ordinary literals +also use the same prepared casefold lookup as plain LR, followed by the existing +guard/column decision; a second whole-string conversion path is removed. + +#+begin_src scheme +(def runtime + (lr-prepare (compile-lr-spec '((root (alias Root (token word)))) 'root))) +(def tokens (list (make-token 'word "x" 0 1))) +;; These values belong to an enclosing source request, not runtime's grammar. +(parameterize ((current-layout-columns (make-layout-columns "x")) + (current-layout-frames '((100 . "outer")))) + (let-values (((root rest) (lr-parse/prepared runtime tokens))) + (null? rest))) ; => #t, matching execution outside the dynamic bindings + +;; Source ownership is bound once at the existing request boundary. +(parameterize ((current-layout-columns + (and (lr-runtime-layout? (parser-machine-runtime machine)) + (make-layout-columns source))) + (current-layout-frames '())) + (parse-tokenized/with machine digest source tokens parse-significant policy)) +#+end_src + +** Question: why is this design preferable? + +The symbolic grammar determines the immutable prepared capability. Dynamic +parameters carry only request-owned coordinates and branch frames. Keeping +these roles distinct lets the same prepared parser run from ordinary code, +callbacks or nested services. The existing Scheme request scope handles binding +and restoration, while named LR algorithms retain explicit prepared arguments. +POO/macro composition continues to select the declaration before lowering; +authors add no layout switch, column builder or cleanup callback to their three +interfaces. The existing casefold index is reused rather than implementing a +second lexical algorithm for layout requests. + +** Question: how is the claim scientifically tested? + +The matched counterexample changes only ambient parameters around the same +runtime and tokens. Permanent controls compare complete artifacts outside and +inside those bindings, under both requested semantic preferences, with a plain +grammar and a dynamic-precedence fork grammar. The independent materialized GLR +entry exercises its fast and memoized lookups; a streaming driver checks token +callbacks. Separate source controls execute success/rejection/success and require +the original outer parameter identities after each request. ASCII and Unicode +casefold controls check row identity, case-sensitive token fallback, primary +shift and blocked-shift reduction fallback. They do not prove every possible +layout grammar balanced or every compiler-generated fallback equivalent. + +The [[https://small.r7rs.org/attachment/r7rs.pdf][R7RS-small report]], Section 4.2.6, specifies temporary dynamic parameter +values and restoration. The installed Gerbil =2591dcd= runtime +=src/gerbil/runtime/control.ss= delegates =call-with-parameters= to Gambit's +=##parameterize1=. This is the existing language mechanism, not a replacement +scheduler or private dynamic-binding implementation. The report motivates the +scope boundary; actual Gerbil controls qualify the implemented request behavior. +No CPU, allocation or concurrent actor performance benefit is inferred. + +** Qualification + +The native parent counterexample reports =input does not match LR parser= only +inside the ambient layout context. The baseline child exits zero in 6.368 +seconds with final =OK=. This records +the reproduced bug, not success of the repaired implementation. + +The repaired layout, LR and source request owners and the focused layout module +compile natively with optimization/static products in 70.397 seconds, exit zero +and final =OK=. Fourteen consumer suites pass 131 cases in 26.984 seconds. The +matrix includes the four new isolation/streaming/casefold/restoration cases, +both semantic preferences, deterministic and dynamic fork grammars, generated +Scheme/Rust controls, resolved source origins, complete parser products and +ten-language ABI byte equality. Both supervisors retain the five-second real +progress gate and report no rejection reason or missing completion output. + +Dependency products +are reused in the native overlay; this is not a clean-checkout package build. +No Rust runtime source or language declarations change. Exact-head Linux, +macOS and Rust platform CI remain a separate delivery gate. This qualification +closes the reproduced request-capability leakage and tested restoration paths; +it does not establish arbitrary layout equivalence or actor concurrency safety. + +** Checkpoint and contextual execution admission + +Request isolation alone does not establish resumability. At parent =36c9f84=, +=lr-initial-checkpoint= accepts a layout runtime, although =lr-checkpoint= and +its prefix snapshot carry neither source columns nor branch reference frames. +The contextual executor uses this checkpoint record and likewise admits a +self-consistent layout product. The native baseline explicitly constructs both +objects; it does not claim that a particular continuation corruption was +observed. Its terminal catalog contains the fixture's actual =word= token, so +the counterexample is independent of a mismatched scanner declaration. + +The LR owner now defines =require-lr-checkpoint-runtime!= and applies it before +semantic backend selection or checkpoint construction. Contextual preparation +uses that same requirement before building positions or a scanner plan. The raw +contextual entry delegates to prepared admission and execution, eliminating its +separate validation/indexing/scanner path. Both entries consequently own their +published digest strings; mutation of the caller's product cannot rewrite a +previously published raw artifact. These are execution requirements derived +from the existing continuation representation, not new language author slots. + +#+begin_src scheme +;; A continuation must own every value needed to resume execution. +(def (require-lr-checkpoint-runtime! runtime) + (when (lr-runtime-layout? runtime) + (error "LR checkpoints require a runtime without layout"))) + +(def (lr-initial-checkpoint runtime tokens) + (require-lr-checkpoint-runtime! runtime) + (lr-initial-checkpoint/selected + (lr-runtime-for-current-semantic-backend runtime) tokens)) + +;; Raw and prepared calls share product admission and private plan ownership. +(parse-source/contextual/prepared + (prepare-contextual-parser machine product) source) + +;; Full source layout parsing retains its GLR owner. The high-level incremental +;; entry returns a complete artifact and no reusable checkpoint for this runtime. +(let-values (((artifact tokens modes snapshots) + (parse-source/checkpoints layout-machine source 1))) + (and (parse-artifact-valid? artifact) + (not tokens) (not modes) (equal? snapshots '#()))) +#+end_src + +*** Question: why is this design preferable? + +The symbolic production capability is checked by the owner of continuation +state. Plain checkpoint, contextual preparation and full layout execution +therefore have explicit, consistent requirements. Scheme procedures reuse one +admission algorithm and one owned contextual plan; POO/macro declarations still +lower through the same machine without duplicated author configuration. +Supporting layout continuations later requires an owned coordinate/frame +representation with branch semantics, rather than an ambient parameter switch. + +*** Question: how is the claim scientifically tested? + +The parent control admits both incomplete representations. The repaired control +requires the same error for initial checkpoints, prepared contextual admission +and raw contextual admission under both semantic preferences. Independent full +source controls compare complete artifacts, exact round trips and absence of +snapshots for single-line success, multiline success and rejected layout input. +The contextual identity control mutates the original product without freezing +the reference artifact; ordinary HCL, Unicode and rejection paths must still +agree with prepared execution. This establishes the tested capability and +ownership boundary. It does not implement layout continuation capture, +contextual GLR branch state or prove arbitrary layout grammars balanced. + +*** Qualification + +The native parent baseline admits both objects and finishes with exit zero and +=OK= in 19.871 seconds. Two preliminary +fixture attempts fail on a missing native helper and an undeclared HCL token; +neither is used as capability evidence. The final fixture uses the actual +layout grammar's =word= terminal. + +The LR producer emits its optimized/static products. Initial consumer admission +fails because the reused overlay precedes the new interface; the first retry +still has the same ordering error. With the new overlay placed last in the +environment setup (and therefore first in the resolved load path), a fresh +consumer process compiles the source owner and both focused test modules in +57.451 seconds with exit zero and =OK=. The contextual fixture helper compiles +in 16.309 seconds. The initial test admission reports its missing native helper +and is retained as a failed receipt; it is not a passing semantic run. + +The final seventeen-suite native matrix passes 167 cases in 23.001 seconds, +including contextual ownership, layout admission/full-source parity, fragment +transfer, deferred scanner continuations, both semantic preferences, generated +products and ten-language ABI bytes. All final +supervisors retain the five-second real-progress gate, report no rejection +reason or missing completion output and propagate the child exit code. +These are reused dependency overlays, not a clean-checkout package build. +The integrated Org qualifier preserves the historical body and checks 104 +local links and 17 heading targets. No performance benefit is claimed; +fresh exact-head platform CI is a separate gate. + +** Compact request-owned source coordinates + +The request capability boundary also identifies a source-size cost in the shared +engine: layout preparation encoded the source with Gambit's native UTF-8 encoder +and then stored one boxed-vector slot per byte, including EOF. This allocation +is incurred even when the significant token stream is short. No language +declaration or POO metadata can eliminate that engine-owned representation cost. + +The retained design uses a =u32vector= where the column bound fits. Every +character advances a column by at most eight (a tab); newline/return reset it. +Consequently =8 * string-length(source)= bounds every column, independent of +UTF-8 byte width. Above the conservative u32 admission bound the same builder +uses exact Scheme integer slots. Neither path truncates columns or narrows the +language domain. Queries remain constant-time and continuation-byte offsets +retain the leading character's column. + +A hygienic =defrule= stages the storage operation once into the builder. It +expands directly to the selected vector setter, avoiding a callback or storage +dispatch for each byte. Read dispatch is private to the coordinate owner; the +LR engine and the three author interfaces still consume the same source-column +semantics. The official native UTF-8 encoder retains character-conversion +ownership. On a 64-bit VM the column payload falls from eight to four bytes per +offset; the temporary encoded source remains part of preparation cost. + +#+begin_src scheme +;; A tab advances at most eight columns per source character. +(def (make-layout-columns source) + (let* ((bytes (string->utf8 source)) (size (+ (u8vector-length bytes) 1))) + (if (<= (string-length source) (quotient #xffffffff 8)) + (build-layout-columns bytes (make-u32vector size 0) ##u32vector-set!) + (build-layout-columns bytes (make-vector size 0) vector-set!)))) + +;; Storage is an engine-owned value, not a language-author configuration. +(parameterize ((current-layout-columns (make-layout-columns source))) + (layout-token-column token)) +#+end_src + +*** Question: why is this design preferable? + +The DSL declares layout semantics; the engine selects a bounded physical +representation and stages its primitive operation. Scheme's hygienic syntax +therefore supplies specialization without adding author slots, generic registry +lookups or per-byte procedure calls. POO remains responsible for extensible +declarations, while this closed storage choice belongs to the named coordinate +algorithm. Exact integer storage preserves the mathematical contract outside +the packed domain without introducing an alternative grammar architecture. + +*** Question: how is the claim scientifically tested? + +An independent boxed-vector oracle checks every byte, UTF-8 interior and EOF for +empty input, ASCII, tabs, CR/LF/CRLF and Unicode scalar-width boundaries. Explicit +controls check 80000-column tab input, Unicode after CRLF, missing request context +and wide exact-integer reads. Existing layout action guards, complete artifacts, +GLR branches, nested request restoration and all-language ABI controls qualify +the consumers of the new value. The native conformance catalog owns the new +fixture and suite. + +The committed [[file:../../t/benchmarks/layout-columns/benchmark.ss][coordinate benchmark]] uses the existing shared batch sampler +for preparation and complete prepared-GLR products, including lexing, significant +token selection, recognition and canonical artifact construction. Both storage +variants must produce equal complete TLA+ artifacts. The workload exposes source +size with 32768 whitespace characters; it is not a representative language-wide +request distribution or a claim about GQL latency. Natural collections and +allocation admission retain the shared owner's policy. + +A rejected first candidate traversed source characters with the official +=:std/string/utf8= length helpers and eliminated the encoded byte vector. Its +admitted preparation allocation fell from 295897.6 to 131737.6 bytes/call, but +batch CPU P50 rose from 2.181 to 5.408 ms for ten calls; complete-product batch +CPU P50 rose from 6.461 to 8.304 ms. Those block observations falsify adoption of +that traversal on this workload. The retained builder restores native encoding +and byte traversal; these rejected measurements do not qualify its performance. + +Reproduce the current semantic and measurement contracts from the repository +root with declared dependencies installed: + +#+begin_src sh +gerbil compile -O +gerbil env gerbil test -v 5 t/layout-column-storage-test.ss t/layout-boundary-test.ss +gerbil compile -O t/fixtures/layout-columns.ss t/benchmarks/layout-columns/benchmark.ss +gerbil env gxi -e '(import :gerbil-parser/t/benchmarks/layout-columns/benchmark) (benchmark-layout-columns)' +#+end_src + +Historical local observations are not archived replay receipts. The committed +workload and assertions are the reproduction entry; exact-head platform CI is +an additional delivery gate. + +*** Native storage qualification and rejected alternatives + +The checked u32 writer also regressed preparation CPU (3.450 ms versus 2.202 ms +for ten calls). It is not retained. The final private writer uses +=##u32vector-set!=, following the installed Gerbil standard library's +=std/serde/unmarshal.ss= =defparse-hvector= specialization. The encoder produces +valid widths, the allocated buffer contains every byte plus EOF, and the proven +column bound covers every stored value. Only private writes use this primitive; +public token reads remain checked. Compiler optimization settings are unchanged. + +Four sequential native runs change only the layout producer: boxed, packed, +packed, boxed. All complete-product controls pass. Each component has twenty +samples of ten calls; all allocation intervals are admitted. Both boxed runs +allocate 295897.6 bytes per coordinate construction and 512984 bytes per complete +product. Both packed runs allocate 164609.6 and 381696 bytes respectively: the +same 131288-byte reduction occurs in preparation and the complete product. +This is approximately 44.4% and 25.6% less allocation for this padded workload, +not retained-heap measurement or a language-wide allocation claim. + +Packed preparation batch CPU P50 is 1.969/2.019 ms; boxed observations are +2.145/2.136 ms. Complete-product CPU P50 is 6.000/5.454 ms for packed and +5.789/5.558 ms for boxed. These observations do not establish a complete-request +CPU improvement. Adoption is limited to the stable allocation reduction with +preserved byte coordinates and complete products. The first baseline attempt +failed the unchanged idle gate because its copied overlay omitted phase-zero +native artifacts; it is excluded from the completed four-run sequence. + +The final native matrix passes 172 cases across eighteen suites in 44.874 +seconds, with propagated exit zero, final =OK= and the unchanged five-second +real-progress gate. It includes every byte oracle, layout boundaries, full +ten-language ABI bytes, contextual ownership and fragment/deferred continuations. +An earlier merged run failed language ABI admission after only the concurrent +scanner producer was rebuilt. Its exported trie macro had changed representation; +rebuilding the direct =lr-action-index= consumer restored consistent expansion +and the entire matrix passed. That failed mixed graph is retained as a build +qualification failure, not evidence of a packed coordinate mismatch. The native +coordinate build, final bounded writer and macro consumer compile normally; +no optimizer setting or watchdog threshold is relaxed. Research and audit links +are checked by the Scheme qualifier. Clean exact-head CI remains separate. + +* One boundary from productions to executable tables + +This completes the structural table boundary identified in +[[file:prepared-lr-action-plans.org::*Complete production admission before execution products][the production admission study]]. Production semantics and LR table structure +are related obligations with distinct owners. =src/compiler/lr.ss= admits +production declarations. =src/compiler/lr-table.ss= admits the resulting action +and goto vectors before indexes or executable products are published. + +The source audit at parent =5a6a70b= found that preparation indexed rows without +checking target ranges or uniqueness. A valid terminal key could conceal an +out-of-range shift/reduce reference, a singleton fork or duplicate row entries. +The old association/index helpers selected the first duplicate; that utility +behavior did not establish canonical table legality. + +The shared gate establishes: + +- Nonempty action vectors and equally sized goto vectors, with proper rows. +- Canonical terminal keys, symbolic goto keys and no repeated key within a row. +- Shift and goto targets within the state domain; reduce targets within the + admitted production domain. Numeric references are nonnegative Fixnums. +- Three-element shifts with the existing precedence contract, two-element + reductions, EOF-only acceptance and matching nonassociative terminal metadata. +- Flat forks with at least two atomic alternatives. Layout guards contain a + shift and a reduction or flat reduction fork. Request-generated layout-shift + actions are not a published table declaration. +- Layout guards and layout terminal keys require layout capability derived from + admitted productions. Preparation uses the same predicate to enable source + columns and request-local frames; table shape alone cannot enable that protocol. +- Action keys belong to the canonical production terminal catalog (including + EOF); goto keys name a declared production LHS. Marked operands retain their + base-symbol membership and unused declared rules remain admissible. +- The graph from state zero has a reachable EOF acceptance. Shift edges in + forks/guards and goto edges all participate. Conflict resolution may leave + disconnected states; admission preserves their existing identities. + +Standalone LR compilation, prepared execution, fused Scheme strategy admission +and raw-IR Rust publication consume this gate. The source build catalog discovers +the new production module through the existing PackageSpec. The action validator +captures state and production domains once per admission; queries retain their +existing indexes and captured executors. Row and entry objects are returned +unchanged. As with prepared productions, callers must not mutate admitted IR. + +* Scheme witness + +#+begin_src scheme +;; One state and one production: every referenced offset is zero. +(let ((productions '((0 root ((terminal token word)) concat #f))) + (actions '#((((terminal token word) shift 0 #f) ((terminal eof) accept)))) + (gotos '#(((root . 0))))) + (validate-lr-tables productions actions gotos)) + +;; Each is rejected before preparation or Rust publication. +(let (productions '((0 root ((terminal token word)) concat #f))) + (validate-lr-tables productions '#((((terminal token word) shift 1 #f))) '#(())) + (validate-lr-tables productions '#((((terminal token word) reduce 1))) '#(())) + (validate-lr-tables productions '#((((terminal token word) fork (reduce 0)))) '#(()))) +;; A separate negative control: ordinary productions cannot publish a guard. +(validate-lr-tables '((0 root ((terminal token word)) concat #f)) + '#((((terminal token word) layout-guard (shift 0 #f) (reduce 0)))) + '#(())) +#+end_src + +** Question: why is this design preferable? + +The Scheme DSL states grammar meaning; users do not configure table validation, +index storage or backend dispatch. Structural matching expresses the closed +action algebra, while a shared row traversal owns domain and uniqueness checks. +POO strategies inherit the same canonical admission rather than supplying their +own definition of a legal table. These mechanisms implement the staging boundary +of the [[file:engine-reduction-generation.org][integrated reduction study]]; they add no author slots or compatibility path. + +Layout capability is declared by canonical production meaning: a layout-end +action or a layout-start/layout-next operand, including marked operands. One +structural matching function now serves table admission and runtime preparation. +This uses Scheme's symbolic action algebra to move a protocol obligation into +the engine, rather than asking grammar authors to duplicate an enable flag. + +The table gate preserves original entries, including the identity required by +GLR receipts and checkpoint execution. Generic association helpers still support +first-match duplicate lookup as utilities; a canonical parser table cannot use +that behavior to bypass admission. Each responsibility has its own control. + +** Question: how is the claim scientifically tested? + +For s states and p productions, every admitted shift/goto target is in [0,s) +and every admitted reduction is in [0,p). Matching each atomic branch extends +that invariant to flat forks and guarded reduction alternatives. Uniqueness +is checked with a preparation-owned equality table, independently for each row. +These are structural invariants, not a proof of LR conflict resolution or that +the table recognizes its declared grammar. + +Falsifiers include an admitted out-of-range reference or malformed wrapper, +changed original-entry identity, modified valid artifacts, or a Rust output file +published after admission failure. Focused index controls test boundaries and +malformed rows; existing utility controls retain first-match behavior. Full +generated/GLR artifacts, layout closure and ten-language ABI fixture bytes provide +independent valid-product controls. Raw Rust publication has separate failure +controls for target bounds and duplicate keys. + +Layout falsifiers additionally include a guard or layout key admitted with plain +productions, or loss of capability through an operand's field/alias wrapper. +Positive controls cover end actions, bare start markers and marked next markers; +the source layout suite exercises actual request columns and frame transitions. + +Validation adds a row traversal and per-row equality hash tables; lookup cost +and hashing literal keys contribute to preparation work. No worst-case total +cost bound or measured preparation/request benefit is claimed. Reachable +acceptance is checked below; it does not certify goto completeness, conflict equivalence, lexical +realizability. Layout correspondence here is a necessary capability condition, +not proof that markers are balanced or that every guard has the correct fallback. +Backend-specific capability admission retains its existing owners. + +* Production-relative vocabulary and reachable acceptance + +This follows the same admission boundary through all four consumers, rather +than adding a language-specific validator. The =0c46a44= baseline gate admits +an undeclared token key, an undeclared goto symbol and an isolated cycle with no +acceptance reachable from state zero. Their numeric offsets and row shapes are +valid; structural shape alone cannot establish the production/table relationship. + +The implementation reuses =production-terminal-catalog= from the LR owner. +Its index includes EOF at offset zero; Scheme's truth semantics correctly treat +that zero as present. A separate equality set admits production LHS symbols. +No second terminal-normalization policy is introduced. Operand actions are +projected by the existing canonical catalog, so field and alias annotations do +not change terminal membership. Dead declared rules remain legal composition +inputs; absent table symbols do not become implicit declarations. + +#+begin_src scheme +(def productions '((0 root ((terminal token word)) concat #f))) +(def actions '#((((terminal token word) shift 1 #f)) + (((terminal eof) reduce 0)) + (((terminal eof) accept)))) +(def gotos '#(((root . 2)) () ())) +(validate-lr-tables productions actions gotos) ; same vectors and entries + +;; An accepting row does not legalize an undeclared grammar symbol. +(validate-lr-tables productions + '#((((terminal token unknown) reduce 0) + ((terminal eof) accept))) '#(())) ; rejected +;; An isolated accepting cycle cannot be entered from the initial state. +(validate-lr-tables productions + '#(() (((terminal token word) shift 1 #f) + ((terminal eof) accept))) '#(() ())) ; rejected +#+end_src + +** Question: why does this exploit Scheme and keep complexity in the engine? + +The symbolic IR supplies the vocabulary directly. Structural matching traverses +the closed shift/fork/layout algebra without another opcode registry or author +configuration. POO-selected strategies consume one canonical gate before +execution or publication. The graph worklist uses the installed Gerbil SDK's +=:std/struct/queue= implementation: typed final queue storage, enqueue/dequeue +operations and a state-sized visited vector. Each discovered state is enqueued +at most once. Graph traversal is bounded by states plus action/goto alternatives; +symbol hashing and terminal-catalog construction remain separate admission costs. +These mechanisms execute during preparation/publication, not token lookup. + +** Question: what does the research prove, and what falsifies this contract? + +Knuth's [[https://doi.org/10.1016/S0019-9958(65)90426-2][On the Translation of Languages from Left to Right]] (1965), definitions +and Section II, separates grammar symbols, derivations and recognizer +construction. The [[https://harrymoreno.com/assets/greatPapersInCompSci/2.5_-_On_the_translation_of_languages_from_left_to_right-Donald_E._Knuth.pdf][primary paper scan]] is the research source. Our vocabulary check +and reachable-acceptance check are necessary artifact conditions derived from +that separation; they do not decide the LR(k) property or prove table/grammar +equivalence. A reachable goto is an abstract graph edge, not proof of a viable +runtime reduction stack. No accepted input string is inferred from this graph. + +An initially stronger proposal rejected every disconnected state. Actual GQL +qualification falsifies it at state 1001: static conflict resolution removes +shift edges while retaining the original state identities. That implementation +is withdrawn. The final contract checks reachable acceptance and preserves +disconnected state rows; it neither trims nor renumbers the automaton. + +Independent falsifiers include an undeclared key admitted despite an otherwise +reachable accept, an isolated accepting cycle admitted from state zero, missing +second-fork or layout-primary edges, and lost marked-operand membership. Positive +controls retain unused declarations, cycles, goto-only paths and disconnected +rows beside a reachable acceptance. Runtime rejection and Rust publication use +the same malformed inputs; publication must neither create a missing file nor +overwrite an existing sentinel. Existing compiler, layout, fused execution and +ten-language ABI controls qualify the complete consumer boundary. + +* Rust target representability after canonical admission + +The next audit follows the same staged contract to the actual Rust model. +=engine/model.rs= represents syntax aliases as =u16=, state/production references +as =u32= and each production's dynamic precedence as =i32=. Canonical Scheme +precedence accepts Integer. The emitter previously printed that integer directly, +so canonical legality could produce Rust source with an out-of-range literal. + +=validate-rust-lr-capability= now runs after shared production/table admission +and before the output-string buffer. It checks kind/reference domain capacities, +rejects production-derived layout capability unavailable in the Rust model, +and checks dynamic precedence against the inclusive signed 32-bit bounds. +Layout uses the shared production predicate, including marked operands and unused +rules; it is not a second DSL interpretation or an author-configured feature flag. + +#+begin_src scheme +;; Canonical Scheme legality does not inherit a consumer's fixed-width limit. +(validate-production-semantics + '((0 root () concat (dynamic 1267650600228229401496703205376)))) + +;; The target owner applies this additional obligation before source emission. +(let (precedence (production-precedence production)) + (when (and precedence (eq? (car precedence) 'dynamic) + (not (<= -2147483648 (cadr precedence) 2147483647))) + (error "Rust LR dynamic precedence exceeds i32 capacity" production))) +#+end_src + +** Question: why does this preserve the Scheme design advantage? + +Symbolic productions carry meaning independently of a target's machine widths. +The engine admits this meaning once; each publication target owns its additional +representation obligations. Scheme matching and first-class traversal keep that +boundary explicit without expanding grammar.ss/parser.ss author configuration. +Both raw IR publication and descriptor publication use the same Rust entry, so +both paths enforce the target capacity gate. Descriptor policy admission keeps +its separate owner. + +** Question: what evidence establishes the target claim? + +The falsifier is Rust source emitted with an out-of-range production precedence. +Controls cover both exact endpoints, zero, each adjacent overflow and a bignum. +The same rejected target IR must still prepare under Scheme. An existing output +file must retain its contents after target rejection; an oversized kind catalog +must create no file. Layout controls include end actions, start operands and +marked next operands. Existing ten-language ABI bytes and deterministic source +controls establish that currently supported products retain their output. + +The u32 domain bound is inspected directly against the Rust model; no test +allocates billions of states or productions. Admission of an individual i32 +precedence alone did not close accumulated branch scores. The runtime obligation +identified here is addressed by the following checked accumulation contract. +This is target publication correctness, not a runtime performance claim or +proof of complete Rust/Scheme semantic equivalence. + +** Checked cumulative precedence and global failure + +The Rust configuration, completed candidate and public receipt now carry i64 +scores. Individual generated productions retain their admitted i32 representation. +Reduction computes checked i64 addition before draining values or truncating +states. Capacity failure returns =dynamic-precedence-overflow= at the current +source boundary and leaves the configuration unchanged. The same reduction owner +serves deterministic, contextual and selective GLR execution. + +Selective GLR treats this diagnostic as a global failure, like its existing +artifact and branch-budget failures. Discarding an overflowing branch as a normal +parse rejection could admit an incorrect lower-scoring winner. Both branch orders +must fail even when another branch has already completed. + +#+begin_src scheme +;; Integer policy has the same meaning before target representation is chosen. +(+ 2147483647 2147483647) ; => 4294967294 +(+ -2147483648 -2147483648) ; => -4294967296 +(+ 2147483647 -2147483648) ; => -1 +;; Rust accumulation preserves these values; target capacity failure is explicit, +;; never a wrap, clamp, panic or silently discarded GLR alternative. +#+end_src + +*** Question: why is this an engine responsibility? + +Grammar declarations state precedence meaning. Configuration arithmetic and +candidate comparison implement that meaning; grammar authors should not configure +integer widths, exception handling or branch-discard policy. One shared reduction +operation checks the target's capacity before mutation, and one GLR owner classifies +the resulting failure. The receipt reports the same wider score that selected the +winner, with no legacy narrow-score alias. + +*** Question: what closes the numerical claim? + +Full parser controls cross both i32 limits, preserve branch ordering and retain +equal-score ambiguity after positive/negative cancellation. Seeded i64 boundary +controls verify failure before state/value/cursor/score mutation. Internal GLR +controls put a completed branch before and after an overflowing branch, for both +positive and negative overflow. Existing complete parse/event/contextual controls +qualify the shared owner. This closes checked accumulation and propagation within +the explicit Rust target domain; it does not claim arbitrary-precision Rust scores +or successful execution of every canonical Scheme Integer grammar. + +* Native qualification and evidence boundary + +** Production-relative admission closure + +The final shared gate and both changed index/publication test modules compile +natively with optimize/static enabled: 39.392 seconds, exit zero and final =OK=. +The complete consumer matrix passes 72 cases in 51.872 seconds, exit zero, +=reason=#f=, no missing receipt and final =OK=. Its eleven modules cover LR +indexes, stack reduction, semantic preparation, fused reduction, Rust generation, +layout boundaries, ten-language ABI parity, event programs, unary corridors, +ranked literals and scanner profiles. The three new table cases include several +independent graph and symbol witnesses; the publication case checks absent and +existing files for every malformed input. + +The production-shape fixtures now share an already accepting graph. This keeps +their positive controls independent of the new table gate and prevents malformed +production/operand tests from succeeding merely because their graph is invalid. +Existing complete parser controls still use real compiled grammars. + +Qualification keeps the five-second idle fence and a 120-second total budget. +The withdrawn all-states-reachable implementation fails GQL at state 1001: +27.763 seconds, exit one and =error-output=. A source test run reaches GQL Rust +source generation but fails the idle fence after 37.865 seconds; it is not a +success. The changed test modules are then compiled normally, without changing +production optimization or timeouts. Two subsequent runs expose old empty-graph +fixtures (24.509 and 18.406 seconds, =error-output=); those are consolidated before +the successful complete matrix. No parser speedup is inferred from these elapsed +times. Fresh native module qualification is not clean-checkout exact-head CI. + +** Checked accumulation closure + +The =ad8c6a3= remote Rust job rejected the inline =score_tests= module under +=RUST-AGENT-PROJECT-003=. The preceding runtime and Clippy receipts therefore +did not establish workspace policy admission. The branch-order control now +lives in [[file:../../rust/gerbil-parser-runtime/tests/unit/precedence_capacity.rs][the dedicated unit-test source]], mounted from the parser with =#[path]=. +This retains access to private execution state without placing test bodies in +the production source. No policy severity or ignored-directory setting changes. +Runtime semantics, workspace source policy and exact-head platform CI are +independent obligations; all three must be checked before delivery closure. + +After relocation, all 145 runtime library cases pass again: 7.720 seconds, +exit zero, no rejection reason and no missing receipt. The existing workspace +policy test product, built against the locked =9b95fd6= policy and supplied with +Cargo's manifest-directory environment, scans the current sources successfully: +one case, 1.403 seconds, exit zero. This is current-source policy execution, +not a new clean compilation of the policy tool. Workspace formatting and +=git diff --check= also pass. The remote failing job is not relabeled a success. + +Cold policy-tool compilation attempts hit the unchanged 180-second total fence. +The first completes log processing after 231.857 seconds with +=reason=total-timeout=; no success is admitted. Verbose multiline diagnostics +and an overly broad =Running= filter caused substantial log-processing cost. +The temporary consumer now samples actual diagnostic lines and preserves test +results and Cargo completion output. A malformed temporary consumer and an +initial direct policy execution without =CARGO_MANIFEST_DIR= are also rejected; +the latter is rerun with Cargo's required environment before the successful +current-source receipt. These qualification failures are not parser timings. + +The final Rust runtime suite passes all 145 cases in 64.892 seconds, exit zero, +with no rejection reason or missing required output. The warnings-denied Clippy +library/test gate passes in 52.108 seconds. Workspace formatting is checked +without rewriting unrelated files. These receipts use the native 1.95 compiler, +the existing optimized Cargo profile and the unchanged five-second idle fence. + +Earlier silent and LLVM-pass-only builds are rejected for idle progress +(6.245 and 71.450 seconds); an unsupported LLVM diagnostic option is rejected +by the compiler before compilation. The first complete test execution passes +144 cases and fails the new ranking control because its declared root kind +does not match the winning branch. The fixture is corrected before final +qualification. None of these attempts is a successful receipt. + +Official [[https://rustc-dev-guide.rust-lang.org/tracing.html][rustc tracing]] supplies real frontend activity; verified LLVM pass-number +diagnostics supply backend activity. A temporary Scheme stream consumer samples +verbose compiler events while preserving errors and test results. It emits no +timer heartbeat and changes no optimizer setting. Full unsampled tracing has +substantial output-handling overhead; these diagnostic timings are not engine +performance evidence. + +** Rust target capacity follow-up + +The parent native emitter printed all three overflowing precedence controls in +8.162 seconds, exit zero. The first temporary probe failed on an unbound +re-exported language binding (9.669 seconds); resolving exports through official +module import corrected the probe. That first failure is not target evidence. + +The final native emitter built in 32.608 seconds. Seven source GxTest suites +passed all 42 cases in 22.200 seconds, including boundary/prepublication controls +and twenty accepted/rejected ABI products across ten language entries. Successful +receipts report exit zero, no rejection reason, no missing required output and +final =OK= under the unchanged five-second real-progress gate. Exact-head platform +CI remains separate from these local controls. + +After concurrent PR commits through =f75aa18=, the target change is rebased and +its owner rebuilt (41.740 seconds). Initial cross-checkout qualification fails +on incomplete compiled test modules; two partial native dependency combinations +also exit 139 at the arithmetic ABI control. These failures do not qualify the +rebased graph and are retained in the =rebased-*-test= receipts. + +The scanner, action index, LR runtime and machine are then rebuilt sequentially +in this checkout's dependency environment. Official Scheme expansion emits each +phase, and SDK Gambit compiles it with real compiler progress options. The final +native batch receipt is 210.095 seconds, exit zero; the temporary all-pass GCC +diagnostics produce 6.5 million lines and include substantial supervisor output +handling. That receipt is neither a package-build benchmark nor parser timing. +The final +eleven-suite source run passes 69 cases in 61.918 seconds, exit zero, final =OK=, +no missing required output and twenty exact ABI products. It includes the merged +casefold/scanner controls, event execution and budget-one unary continuation +controls. It does not repeat or relabel the other study's performance results. + +** Production-derived layout capability follow-up + +At parent =af52bc3=, the native prepared runtime accepted three inconsistent +controls: plain concat productions paired with a layout guard, a start key or +a next key. The retained baseline completed in 12.677 seconds with exit zero. +All four publication owners now pass admitted productions to the table gate; +the runtime consumes its shared layout predicate rather than a second classifier. + +The table/layout/compiler native batch passed in 62.396 seconds. The combined +runtime/generator attempt failed the unchanged five-second idle gate (exit 137, +159.927 seconds); an isolated runtime retry also failed (50.044 seconds). +An overlapping generation attempt failed on an incomplete macro product, so +consumers were subsequently built in dependency order. These attempts are not +successful qualification receipts. + +Official Gerbil =compiler/driver.ss= makes =gsc-cc-options= ignore ordinary +CC options for the macro phase. The newly emitted runtime macro source was +therefore compiled directly with official Gambit and explicit real compiler +progress options; it passed in 24.709 seconds under the same watchdog. This +qualification adjustment changes neither engine optimization nor idle limits. +The subsequent fused Scheme/Rust native batch passed in 44.575 seconds. +Seven source GxTest suites then passed all 39 cases in 27.894 seconds, with +complete artifact controls, actual layout requests, rejection before Rust file +creation and twenty accepted/rejected ABI products across ten language entries. +Every successful process receipt has exit zero, no rejection reason, no missing +required output and final =OK=. These are local native receipts, not exact-head +platform CI or performance measurements. + +** Original physical table controls + +The retained parent runtime admitted all five malformed table controls in +16.695 seconds, exit zero. The first temporary baseline probe instead failed +with a pair error because its quoted vector omitted one row-list level; that +failure remains recorded and is not target-range evidence. The corrected probe +uses explicit vector -> row -> entry -> terminal nesting, as do the Rust controls +and Scheme witness above. + +The canonical owner, new table module and LR compiler completed native +compilation in 112.424 seconds. The runtime, fused Scheme and Rust consumers +completed in 174.309 seconds. Both receipts report exit zero, no rejection +reason, no missing required output and final =OK= under the unchanged five-second +real-progress idle gate and official SDK optimization settings. Build elapsed +time is not preparation timing or parser throughput evidence. + +Seven source GxTest suites passed all 38 cases against the final native owners +in 78.591 seconds. They cover valid original-entry identity, malformed physical +tables, utility duplicate behavior, production admission, generated/GLR complete +artifacts, source layout boundaries, raw Rust file rejection and twenty +accepted/rejected products across ten language entries matching ABI fixture +bytes exactly. The complete harness and process receipts both pass; local native +overlays remain distinct from clean-checkout exact-head platform CI. diff --git a/docs/research/lr-unary-chain.org b/docs/research/lr-unary-chain.org new file mode 100644 index 00000000..93acac00 --- /dev/null +++ b/docs/research/lr-unary-chain.org @@ -0,0 +1,193 @@ +#+title: LR Unary Chains: Complete-Request Cost and Rejected Optimization + +* Question and baseline + +This study continues [[file:lr-owned-stack.org][the request-owned LR study]] from production revision +=b5e1895=. The question is whether carrying consecutive unary reductions in +registers removes enough execution work to pass the unchanged four-stage CPU +gate. Earlier improvements against =e6cebe9= are not incremental evidence here. + +* Actual complete-request attribution + +The representative GQL source was parsed 30,000 times with the pinned Gerbil +=2591dcd= SDK. The workload checks its final result against the initial complete +product. Statistical continuation sampling produced 4,242 samples: + +| Top continuation | Samples | Share | +|------------------+---------+-------| +| Owned LR driver | 1409 | 33.2% | +| Streaming source driver | 518 | 12.2% | +| Recognition event publication | 383 | 9.0% | +| Gambit table access | 371 | 8.7% | +| Generated lexer closures | 281 | 6.6% | +| Prepared action selector closures | 209 | 4.9% | +| Raw table lookup | 134 | 3.2% | +| Alias kernels | 131 | 3.1% | +| Sequence normalization | 97 | 2.3% | + +These are sampled top frames, not additive component CPU shares. GC and the +sampler remain in the observation. A closure may carry its enclosing factory's +name: =prepare-generated-lexer= does not establish repeated lexer preparation. +The source prepares its mode scanners before requests and calls those closures +during token scanning. No GC defect or universal workload ranking is inferred. + +The tracked =t/benchmarks/lr-request-stack/request-profile.ss= entry uses the +existing benchmark build owner. Its =source= and =prepared= modes retain the +same 30,000 real calls and result check. It is a workload, not another parser +or statistical admission implementation. + +The pinned SDK advertises =gerbil prof= but does not contain its dynamic +=:gerbil/tools/gxprof= module. Its retained source frontend also imports the +unavailable =:std/sort= module. Therefore an official CLI profiling command is +not qualified on this installation. This observation uses an ignored +source-derived adaptation of the upstream heartbeat sampler with the compiled +workload; it does not add a production sampler or CI lane. Sampling remains +independent of the original matched CPU gate. The missing SDK tool is a +reproducibility limitation, not evidence of parser execution overhead. + +* Scheme design and scientific hypothesis + +Gerbil's =std/list/list.ss= names the traversal, carries intermediate results +through recursive bindings, and separates collection from final publication. +The existing LR storage policy already owns mutable vectors for unbounded, +unobserved requests. A unary reduction changes their active heads while +retaining depth and the predecessor state. This suggests carrying those heads +through a selected chain and committing once: + +#+begin_src scheme +;; Conceptual operation over admitted unary transitions, not a public DSL. +(let chain ((state state) (children children) (row row) (actions actions)) + (if (admitted-unary-transition? row) + (let-values (((next-state next-children) (reduce-selected-unary row children))) + (chain next-state next-children (select next-state input) (fx+ actions 1))) + (values state children actions row))) +#+end_src + +Question: why is this a useful Scheme design? Lexical ownership and proper tail +calls express the live state directly; the caller commits storage at one +boundary. Existing canonical operand factories retain field/alias semantics. +No user grammar configuration, new dispatch mechanism or AST omission is needed. + +Question: what makes the optimization hypothesis scientific? The unchanged +artifact, selector-call count, nullable boundaries and failure/GLR handoff are +independently checked. The complete request is measured against a separately +retained current baseline. Failure of the fixed CPU gate rejects the candidate +even when the representation argument and conditional Lean theorem are valid. + +This follows the engine's existing prepared-action and stack-equivalence design; +it does not introduce a new parsing theorem or change the selected grammar. + +* Contract hazards and negative results + +The first implementation selected the chain's exit action again in the outer +driver. It measured representative prepared LR at 53.641 versus 50.139 ms per +1,000 parses and complete requests at 100.673 versus 95.306 ms. It failed the +performance gate and also added selector invocations. Its source and receipts +remain separate from the corrected experiment. + +The corrected implementation carries the selected exit row to the outer driver, +reads the actual request boundary for each reduction, and leaves streamed EOF +processing in the original driver so arbitrary input callbacks retain their +invocation contract. It retains all AST aliases and fields. A conditional Lean +fold theorem states that repeated head replacement equals one final commit; +it does not certify native memory, termination, selection or performance. + +The corrected candidate's first complete-request block still fails the target +margin: representative GQL prepared/full CPU medians are 51.503/98.015 ms versus +50.139/95.306 ms. Long RETURN medians improve only from 144.105/259.605 to +142.062/252.689 ms. A second candidate block is retained; all four roles are +submitted to the original Scheme admission before any positive claim. + +* Execution identity and qualification boundaries + +Gerbil =src/gerbil/runtime/loader.ss= resolves the last contiguous =.oN= file for +each module, beginning with =.o1=. Checking only =.o1= is insufficient when later +native versions exist. The independent baseline retains the complete library, +and the experiment binds the selected native versions and workload closure. + +The first cold baseline attempt reaches the unchanged 120-second total timeout. +It remains a failed receipt. A repeat with the same library and budgets exits +zero in 46.532 seconds; no causal explanation of the cold-load difference is +established. Measurement order is reference A, candidate A, candidate B, +reference B. Each block includes 40 samples of 1,000 calls for both stages of +all four admitted families and the identical SDK encoder control. + +The four target stages still require =max(candidate median) / min(reference +median) < 0.95=. FHIRPath and arithmetic remain non-regression controls; P95 and +SDK repeat-band checks remain unchanged. Native Scheme admits the statistics; +uv-managed Python transports and hashes evidence; native Ascent determines +missing claims. There is no CI timeout change or additional CI lane. + +* Final disposition + +The corrected four-block experiment is rejected. Conservative CPU median ratios +are approximately 1.027/1.028 for representative GQL and 1.001/0.994 for long +RETURN (prepared/full). None passes its fixed target margin. The arithmetic +prepared control also exceeds its non-regression margin (approximately 1.074). +These ratios summarize the retained blocks; the authoritative native admission +rejects at its first target failure. Its SDK repeat-band check passes with ratio +0.985643. No later target or control is relabeled as admitted. + +The candidate's source and 1,403 selected native modules remain bound in ignored +=.data/parser-unary-chain-study/=, alongside the original timeout and rejected +first implementation. The baseline executes =lr-parser~0.o58= and the corrected +candidate executes =lr-parser~0.o61=; they are distinct native products. + +The candidate passes all 18 stack-reduction Cases, including the added deep +decorated chains, request offsets, exact selector counts and failure handoff. +The Lean build passes its conditional equivalence theorem. Native Ascent records +the failed CPU receipt and reports =record-cpu-improvement= missing with +=executionAuthorized: false=. This is a rejected performance candidate, not a +completed positive engine qualification. + +Production =src/runtime/lr-parser.ss= is restored to =b5e1895= and rebuilt. The +regression contract, official-profiler workload and research result remain local +for the next engine study. CI configuration and thresholds are unchanged; no +GitHub commit is published for this candidate. + +* Controlled continuation-materialization ablation + +The corrected function-based candidate allocates 45,824.336 bytes per +representative prepared request versus 41,104.336 in both reference blocks. +Full requests allocate 127,648.720 versus 123,888.720 bytes. Emitted SCM returns +a four-value object and checks/extracts that object at each chain boundary. +This motivates a controlled compilation experiment, rather than another +source-level claim that fewer vector writes must be faster. + +A hygienic =with-owned-unary-chain= rule directly binds the chain exit state +to its caller's continuation. It retains selector calls, request-local +boundaries and the same semantic kernels; it removes the four-value return +and its caller extraction. The emitted SCM shows the direct nested recurrence. +It follows the compiler's syntax-owned binding/continuation pattern without +introducing an extensible object or public grammar operation. + +One complete native preflight reproduces baseline allocation exactly: +41,104.336 prepared and 123,888.720 full bytes per representative GQL request. +CPU medians are still 56.163/100.050 ms per 1,000 calls; long RETURN measures +142.237/252.656 ms. Its SDK control is 7.114 ms. The 18 semantic Cases pass, +including exact selector counts. This is evidence of eliminating the added +materialization cost, not a passing four-block CPU result. The preflight is +rejected and no additional repeated-benefit claim is made. + +The exact macro candidate and selected native images remain in ignored +=.data/parser-unary-continuation-study/=. Production is restored again. The +experiment demonstrates why a better Scheme expression layer and lower +allocation must still face the unchanged whole-request CPU gate. It does not +establish that vector writes, aliases or GC dominate CPU, and it does not +justify retaining a slower executor. + +* Restored-source closure + +After the final restoration, the unchanged production LR source is rebuilt by +the existing complete collector. All five build/source-preparation receipts, +all eight semantic Suites (86 Cases), the real-request workload owner, Lean and +Quint pass. The separate uv-managed native Ascent controls pass 12 tests. +The fixed repeated CPU admission remains failed and native Ascent reports only +=record-cpu-improvement= missing. Its negative result is preserved rather than +turning semantic validity or allocation evidence into a CPU claim. + +PR #13 remains at =b5e1895= with its executed checks successful and configured +Scheme CPU checks skipped. No source change to production LR or CI is retained +from the rejected candidates, and no commit is pushed. The local additions are +the deep-chain/selector regression contract, reusable request workload, +conditional model theorem and this integrated negative study. diff --git a/docs/research/module-loading-reuse.org b/docs/research/module-loading-reuse.org new file mode 100644 index 00000000..94aae8fe --- /dev/null +++ b/docs/research/module-loading-reuse.org @@ -0,0 +1,348 @@ +#+title: SDK module loading reuse and source admission + +* Ownership + +Gerbil's =runtime/loader.ss= owns module dependency initialization, its loaded +module table, concurrent waiters and cached initialization failures. +=expander/module.ss= owns expansion contexts and delayed module evaluation. +=import-module= takes a =reload?= flag followed by an =eval?= flag; unconditional +=#t #t= reconstructs the imported context rather than reusing it. + +The parser delegates these responsibilities to the SDK. This work changes build +and test preparation. It does not change scanning, LR parsing, GPA1 publication +or the steady-state Tokio/FFI parse algorithm. + +* Source admission + +[[file:../../src/language/module-input.ss][language-module-descriptor]] records only SHA-256 source identities, not +module contexts or evaluation results. For file inputs, it reads the complete +source and normalizes the path. First admission, changed content, or an explicit +=reload?: #t= request uses official expander reload. Unchanged content reuses the +SDK context. Export selection and descriptor admission still execute on every +request. The identity is recorded only after successful evaluation and selection. + +This does not depend on file size or timestamp. Missing files, invalid syntax, +failed initializers and missing descriptor exports remain errors. Failed source +admission does not establish a successful identity. Before any actual file +reload, the previous identity is withdrawn because official expansion installs +a replacement context before its delayed initializer is evaluated. A corrected +file, including restoration of the previously successful bytes, is therefore +re-admitted normally after a failed replacement. + +Library inputs beginning with =:= reuse the SDK's library snapshot. A caller +editing such an input explicitly requests reload. Reloading a module does not +recursively invalidate imported dependencies: changing imported helpers requires +an appropriately refreshed build host or explicit SDK admission of those changed +modules. Inputs must remain stable during each generation request. + +[[file:../../src/ffi/rust-runtime-aot.ss][rust-runtime-source]] reuses its installed grammar-support module. File +content admission applies to the supplied grammar path, including calls through +the build C ABI. No per-parse content hashing is added to runtime language handles. + +* Test preparation + +[[file:../../t/fixtures/tla-sany-differential/preload.ss][The test preloader]] delegates initialization to official =load-module=. +The official =list-modules= state decides whether dependency inspection is +needed. Known entries, including cached failures, go straight to the SDK; +warm admission does not read compiled wrappers or traverse their graph. +There is no second persistent table claiming runtime modules loaded successfully. + +Cold entries retain wrapper inspection solely to report actual dependency +admission under the existing silence gate. A request-local traversal guard bounds +cycles and disappears on return or exception; it is not a module success cache. +Interface admission uses =import-module= without forced reload, retaining the +real nested expander trace. Logging reports actual starts and completions, with +no synthetic heartbeat. Total and idle-timeout gates remain unchanged. + +The source import scanner remains because FFI tests need compiled dependencies +before source evaluation. Its in-progress marker prevents cyclic source walks; +exceptions clear that marker, so malformed source can be fixed and retried. +Official runtime initializer failures remain cached until =reload-module!=; +the preloader propagates those failures on every attempt. + +* Controls and reproduction + +#+begin_src sh +just test-module-loading +just benchmark-module-loading generation +just benchmark-module-loading preload +#+end_src + +[[file:../../t/benchmarks/module-loading/controls.ss][Scheme controls]] exercise unchanged descriptor identity, content edits, +real DSL edits with changed generated Rust, explicit/automatic generation parity, +missing files/exports, failing initializers, repeated failure propagation, +official runtime reload recovery and malformed-source retry. CI executes them +in both Scheme platform semantic lanes. + +[[file:../../t/benchmarks/module-loading/benchmark.ss][The Scheme CPU benchmark]] uses process user plus system CPU from +=##process-statistics=. Complete generation compares production admission against +forced support/grammar reload, retaining identical Rust output checks. The +preload experiment compares official runtime admission against the +[[file:../../t/benchmarks/module-loading/reference-preload.ss][frozen former preloader]], with fresh local preparation state and an already +loaded SDK closure. It measures repeated diagnostic preparation, not a cold +startup or a complete parse request. + +Both variants receive the same output handling. Logging completed batches is +outside measured CPU intervals. Warmup, before-reference/reference, candidate +production/reference and after-reference/reference are explicit. Every leg and +group contributes; no outliers are discarded. Control P10 must be at least 0.90, +control P90 at most 1.10, and candidate P90 below 0.95. Execution remains bounded +by 90 seconds and 5 seconds without real progress. + +* Exploratory observations + +The original four-leg, 20-group generation run with four calls per leg failed: +before P10/P90 =0.750332/1.506497=, candidate P90 =0.024901=, after +=0.652532/1.302397=. The 20-call preload run also failed its controls: +before =0.790618/1.179499=, candidate P90 =0.000241=, after +=0.833490/1.105312=. Neither establishes an admitted performance claim. + +Increasing generation to 16 calls per leg still failed: before +=0.775688/1.138560= and after =0.924308/1.138511=, despite candidate +P90 =0.039524=. The initial unreported warmup also triggered the unchanged +5-second idle gate; completed-batch reporting subsequently made the actual work +visible. These negative observations are retained rather than treated as passes. + +The eight-leg generation group also failed: before P10/P90 +=0.922577/1.116147=, candidate P90 =0.058664=, after +=0.936217/1.115300=. The final sampler distributes the same number of generation +calls across more alternating legs to balance natural collection ownership. + +The 32-leg, ten-group generation run remained rejected: before P10/P90 +=0.871094/0.992500=, candidate P90 =0.058020= and after +=0.813078/1.280293=. Final groups aggregate twice as much measured +work while retaining the same total calls and order balancing. With five +groups, nearest-rank P10/P90 are the minimum and maximum: every control group +must satisfy the unchanged stability interval, and every candidate group must +satisfy the CPU threshold. No observation is removed. + +* Admitted complete-generation observation (2026-10-13) + +The final Scheme run on macOS arm64, using the installed Gerbil SDK and GCC, +completed 3,840 timed generation calls in 80.920 seconds. Each output matched the +complete expected Rust module. Five groups per phase contain 64 alternating +legs, four calls per leg; all groups passed the unchanged controls and CPU gate. + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before reference/reference | 0.944280 | 0.990447 | 1.033192 | +| Production/reference | 0.030124 | 0.030866 | 0.046970 | +| After reference/reference | 0.947154 | 0.990073 | 1.068137 | + +The worst production group used 95.30% less CPU than the matched forced-reload +control. This is complete repeated Rust source generation from an unchanged +records grammar, including source hashing, descriptor selection, emission and +output comparison. It does not establish cold-start acceleration, changes in +Rust execution, or steady-state parsing gains. The driver exited zero with +=MODULE-LOADING-CPU-PROOF-OK= and no missing receipt. + +The pure direct-load preloader triggered the 5-second idle gate during cold +initialization. That design was rejected. Cold dependency diagnostics were +retained while warm traversal was removed using official module state. +A subsequent 16-call preload batch measured approximately 0.19 CPU seconds; +its total planned calls exceeded the 90-second budget, so the owned child was +stopped early. The final preload experiment uses four calls per leg; neither +termination is an admitted CPU result. + +* Admitted warm-preparation observation (2026-10-13) + +The final cold-diagnostic/warm-reuse implementation completed 3,840 timed +preparation calls in 62.634 seconds. Five groups per phase contain 64 alternating +legs and four calls per leg. Natural collection and output handling remain in +the protocol; all groups passed. + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before reference/reference | 0.922730 | 0.968456 | 1.023916 | +| Production/reference | 0.001916 | 0.002081 | 0.026373 | +| After reference/reference | 0.938500 | 1.010287 | 1.014187 | + +The worst production group used 97.36% less CPU for first diagnostic preparation +of an already-loaded runtime closure, compared with a fresh frozen preloader. +This does not claim a speedup for a second call to the former preloader's own +local cache, cold initialization, a whole test run, or complete parsing. The +SDK owns all runtime initialization and cached failure behavior. The driver +exited zero with =MODULE-LOADING-CPU-PROOF-OK= and no missing receipt. + +* Final correctness and cold-host qualification + +Final content/failure controls exited zero in 5.166 seconds. The real compiled +ABI probe used the final cold diagnostic path, parsed 1,000 requests, verified +three complete generated Rust results and exercised invalid handle/input/build +requests. It exited zero in 7.349 seconds with =NATIVE-ABI-OK= and +=MODULE-COLD-HOST-OK=. The probe uses the existing disposable native-child exit +contract after all assertions; it does not qualify library teardown. + +The ordinary source FFI Suite printed all six successful cases and final =OK= +but stalled during exit. The frozen original preloader reproduced the same +post-Suite stall. Both runs failed the unchanged idle gate and are not admitted +process-success receipts. The repository's existing native-child exit contract +is used separately to qualify the returned Suite result without claiming a +teardown repair. + +The two CPU workloads are independent Ubuntu jobs in the existing manually +triggered matrix. They reuse exact-head prepared Scheme products and the +existing processor-affinity policy. Ordinary PR CI runs source/failure controls +on both Scheme platforms; existing FFI jobs own the real host assertions. + +To reproduce the separate cold-host check after normal library/fixture builds: + +#+begin_src sh +gerbil env gxi build-qualification.ss compile +gerbil env gxi build-ffi-tests.ss compile +gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .data/module-cold-host.log --require MODULE-COLD-HOST-OK \ + -- gxi t/benchmarks/module-loading/cold-host.ss +#+end_src + +The final six-case source Suite run, using that existing child-exit contract, +exited zero in 15.221 seconds with =TEST-SUITE-OK=, =HARNESS-OK= and final =OK=. +Any exception from the Suite prevents the explicit successful child exit. + +* Completed child lifecycle + +The macOS semantic lane at =857e6e6= reached every module-loading control +receipt but stalled after completion. Local reproduction returned +=exit 143, reason idle-timeout= after 22.397 seconds. The standalone control +now uses the existing disposable-child exit interface only after all checks +and cleanup complete. Exceptions still prevent that exit, and the qualification +supervisor still requires the success receipt and exit zero under the same +90-second total and 5-second idle budgets. The repaired local run completed +in 4.983 seconds with exit zero. This is a test lifecycle repair, not a parser +CPU optimization. + +* Failure restoration model and regression (2026-10-15) + +Auditing the official =expander/module.ss= showed that =core-import-module= +installs a new context in its registry before =eval-module= forces its delayed +initializer. A previous successful digest therefore cannot attest that registry +entry after a failed replacement. This differs from committed parse history: +the parser's publication owner can retain an old result, whereas the SDK has +already replaced its current module context. + +The new [[file:../../formal/quint/ModuleAdmission.qnt][ModuleAdmission]] protocol separates source bytes, current SDK context, +accepted identity, pending admission and failure. Its mutation witness retains +a historical marker after failure and demonstrates that it incorrectly skips +reload when the original source is restored. Eight directed histories cover +unchanged reuse, changed content, failed restoration/recovery, explicit reload +failure, rejected selection and repeated failure. Its sampled invariant check +runs 1,000 histories of at most 20 steps, seed 1. This does not model the SDK's +internal mutex, concurrent source edits or recursive dependency invalidation. + +[[file:../../formal/ParserEngine/Reuse.lean][Lean Reuse]] proves that withdrawal forces a new attempt even for the original +source, failed selection establishes no accepted identity, successful admission +can reuse, and explicit reload cannot be skipped. Source must remain stable +within each admission attempt; identities remain logical collision-free values. +The actual module owner, preloader, controls and CPU experiment are included in +the existing model source binding. Whole-request cost and CPU obligations remain +separate from those lifecycle statements. + +The new Scheme control first reproduced the defect: load =second= successfully, +fail a replacement twice, restore the exact =second= content, and observe the +cached =initializer failed= exception. It terminated with =reason error-output= +and no final success receipt. The fixed implementation withdraws identity only +before actual file reload and commits it after successful evaluation/selection. +Warm unchanged calls retain official context reuse. The full controls then +passed in 7.223 seconds, including =MODULE-RESTORED-SOURCE-OK=, source-edit Rust +output changes, repeated runtime failures and official explicit recovery. + +* Current measurement protocol + +Both module CPU experiments now share the existing +[[file:../../t/support/qualification/benchmark-qos.ss][benchmark host QoS admission]] with the capture experiment. On macOS this +sets and verifies the same host thread QoS before either variant is timed; +Linux keeps the existing processor-affinity policy. This is test support and +has no production runtime scheduling effect. That normalization run retained the original counts, order, natural +collection, all control groups and 90/5-second budgets. Successful standalone +experiments invoke the existing disposable-child exit only after all gates; +CI requires an anchored workload-specific marker and exit zero. + +The first current generation rerun without QoS admission was rejected. Before +P10/P50/P90 was =0.994409/1.041373/1.058727=, candidate +=0.027589/0.041004/0.057543=, and after +=0.983333/1.013864/1.113498=. The after maximum exceeded =1.10=; the apparent +candidate gain is not an admitted CPU claim. The run ended at 58.882 seconds +with an error and no success receipt. This negative result is retained. + +* macOS GCC link prerequisite + +The latest preparation failure at =edf9f8a= was =ld: library 'crypto' not found= +while linking =gparse=. The macOS SDK setup already exported OpenSSL pkg-config +metadata, but GCC's linker did not inherit a library search path for keg-only +OpenSSL. Setup now exports the discovered OpenSSL and SQLite library directories +through =LIBRARY_PATH=, preserving any existing value. A real GCC program linked +against =libcrypto= and ran successfully locally. This does not establish remote +CI success or change the selected Scheme compiler. + +The same 5-group/64-leg protocol with admitted QoS also failed: before +P10/P50/P90 =0.957872/0.992398/1.028283=, candidate +=0.040042/0.041113/0.052788= and after +=0.982706/1.063867/1.139041=. The run ended at 82.282 seconds with error-output. +QoS admission alone did not establish stable controls. + +The next bounded sampler allocates more work to reference stability rather +than candidate precision: four groups of 80 alternating legs, five calls per +control leg and one call per candidate leg. Each comparison still uses equal +calls on both sides. That is 3,200 control calls plus 320 candidate calls, +3,520 timed calls in total, with eight untimed warmup calls. Each control group +aggregates 400 calls instead of 256; all observations remain admitted or rejected +together. The same =0.90..1.10= reference bounds and candidate P90 below =0.95= +remain, with natural GC and the original 90/5-second watchdog. Four-group +P10/P90 are minimum/maximum, not population confidence bounds. This protocol +change and the two failed experiments are retained explicitly. + +* Current admitted complete-generation result + +After the failure-restoration repair and explicit host admission, the bounded +sampler completed 3,520 timed generation calls in 66.255 seconds. Every generated +module matched the complete expected Rust source. No groups were dropped. + +| Phase | Calls per leg | P10 | P50 | P90 | +|-------+---------------+-----+-----+-----| +| Before reference/reference | 5 | 0.975825 | 0.993883 | 1.031791 | +| Production/reference | 1 | 0.028644 | 0.028754 | 0.083046 | +| After reference/reference | 5 | 0.969325 | 0.970802 | 0.994358 | + +The worst candidate group used 91.70% less process user-plus-system CPU than +the forced support/grammar-reload control. This requalifies the existing SDK +reuse design with the repaired failure path; it is not a 91.70% improvement +over the preceding PR revision. Source hashing, export selection, generation, +output comparison and natural GC are inside timed calls. Startup, test-host +orchestration and diagnostics outside completed calls are not timed. It does +not qualify cold module loading, complete test-suite duration, Rust/Tokio +runtime execution or full parsing. The final workload-specific receipt appeared +and the supervisor returned =exit 0, reason #f, missing ()=. + +* Current admitted warm-preparation result + +The independent preload experiment completed 3,520 timed calls in 71.287 seconds +under the same revised sampler. Every group passed, with natural GC and output +handling included in the measured preparation calls. + +| Phase | Calls per leg | P10 | P50 | P90 | +|-------+---------------+-----+-----+-----| +| Before reference/reference | 5 | 0.955111 | 1.002432 | 1.073114 | +| Production/reference | 1 | 0.002682 | 0.002712 | 0.002990 | +| After reference/reference | 5 | 0.904189 | 0.957122 | 1.012461 | + +The worst candidate group used 99.70% less process CPU than first diagnostic +preparation with a fresh frozen preloader for an already-loaded SDK closure. +This does not compare two warm calls to the frozen preloader's local cache, +cold SDK initialization or full Suite execution. It requalifies the existing +SDK-state reuse design, not an incremental speedup over the previous PR head. +The workload-specific success receipt and exit zero were both observed. + +Final Lean qualification completed 43 jobs. The locked Quint package passed +97 directed tests across 11 modules and ten sampled invariant runs. Source +binding controls passed. Model qualification does not establish the SDK's +implementation or CPU result; the actual source/failure controls and matched +measurements above are the separate implementation and performance evidence. + +The optional full =gparse= rebuild was also attempted locally under a stricter +90/5-second exploratory supervisor. Direct cold macro loading produced no output +and hit the idle gate at 5.686 seconds. Repeating with real SDK dependency +tracing reached executable C compilation, then hit the same idle gate at +35.997 seconds during a compiler job. Neither is a completed-build receipt; +these attempts do not qualify native command linking or remote CI. The separate +GCC/libcrypto link control passed. The repository CI build owner retains its +existing build budget, and no timeout was widened by this repair. diff --git a/docs/research/parser-engine-formal-model.org b/docs/research/parser-engine-formal-model.org new file mode 100644 index 00000000..107dcedc --- /dev/null +++ b/docs/research/parser-engine-formal-model.org @@ -0,0 +1,271 @@ +#+title: Parser engine semantics and operation model + +* Objective and evidence boundary + +This model connects one complete deterministic reduction path: canonical +field/alias actions, retained factories, reverse LR operands, logical AST +children and sequence materialization. Production remains Scheme. Lean checks +universal properties of the reference algorithms; Scheme controls check the +implementation; matched benchmarks measure CPU and allocation. These are +three separate obligations, not interchangeable receipts. + +The executable package is [[file:../../formal/ParserEngine.lean][formal/ParserEngine.lean]]. It pins Lean 4.34.0 +and LeanPoo revision =0a111f6c5de43b261de5bba9287b48540d9f7c2e= in both +the Lake declaration and manifest. LeanPoo is a verification dependency only. +Its =Functional.Requirements.Factories= and =Selected= retain typed factories; +=prepare_ok_iff= establishes that successful preparation selected exactly +those factories. Our executor threads transformed children through that tuple: +LeanPoo's independent-result =build= is deliberately not used for sequential +actions. No new Scheme dispatch facility or author interface is introduced. + +* Papers, obligations and implementation owners + +Sperber and Thiemann's [[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]] separates grammar-static +information from request-dependent execution through partial evaluation. +Our immediate obligation is narrower: retaining selected action functions must +preserve their declaration order and semantics. This is not a reproduction of +the paper's full LR generator or its speed measurements. + +Wagner and Graham's [[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] motivates +explicit treatment of retained parsing structure. Here persistent child joins +are modeled separately from complete publication. Incremental invalidation, +reuse certificates and grammar recognition soundness remain later obligations; +existing [[file:../architecture/0010-persistent-recognition-snapshots.org][snapshot architecture]] owns those implementation requirements. + +| Obligation / Lean theorem | Scheme owner | Implementation controls | +|---------------------------+--------------+-------------------------| +| Prepared action chain equals interpretation: =prepared_plan_equivalent= | =src/runtime/reduction-plan.ss= | =t/fused-reduction-test.ss= | +| Prepared reverse reduction preserves source order and operand anchors: =prepared_reverse_reduction_equivalent= | =src/runtime/reduction-plan.ss= | =t/lr-stack-reduction-test.ss= | +| Field labels survive nesting; empty aliases retain supplied anchors | =src/runtime/reduce.ss= | =t/fused-reduction-test.ss= | +| Materialization preserves children and suffix: =materialize_into_exact= | =src/runtime/funcs.ss= | =t/recognition-sequence-test.ss= | +| Shift/reduction preserve state/value depth relation | =src/runtime/lr-parser.ss= | =t/lr-stack-reduction-test.ss= | +| Full publication has byte and event obligations | =src/runtime/artifact.ss= | =t/artifact-publication-cost-test.ss= | + +The table identifies conformance owners, not a proved refinement from Scheme +into Lean. Existing controls must be inspected and extended when either side +changes. =Examples.lean= also checks both action orders, nested fields, distinct +empty-operand anchors, empty leaves, suffix preservation and a prefix-copy +count. They complement the general theorems; finite examples do not prove them. + +* Cost model: count work before estimating time + +For a logical sequence with R leaf/branch records and N emitted children, +=materializeInto= is an executable suffix builder. =copyLeaf= counts one +emission per recursive cell; branches count one structural visit and combine +child counts. The theorem proves both its output and exactly R + N counted +operations, for arbitrary sequence shape and an arbitrary retained suffix. +The append-based =flatten= is only the reference denotation: its own runtime +is not claimed linear. This recursive Lean algorithm does not prove the +bounded control-stack behavior of Scheme's builder/iterative visitor, nor +cover position views or event-program sequences. + +=stackSequence= allocates branches rather than copying accumulated prefixes. +A branch has constant logical construction work; =join_allocations_linear= +counts n branch constructions. Repeated singleton append in the reference +list implementation copies 0 + 1 + ... + (n - 1) prefix cells; +=copied_prefix_triangular= proves 2 C(n) + n = n squared. This comparison +formalizes the engine's existing persistent representation; it is not a new +optimization or a measured speedup. Empty-join normalization is abstracted +away, so logical branch counts need not equal Scheme allocations. + +Prepared factories make no provider selections in the modeled request +executor. Interpreted action selections equal the action-list length. +Factory-body costs, field/alias boundary walks, allocations and GC are outside +that selection count. They must still be measured on real requests. + +=publicationWork= is an explicit contract charging every source byte and flat +event. Its lower-bound theorem is arithmetic under that contract, not a proof +of the SHA256 implementation or event publisher. Full digest and flat output +prevent a sublinear complete-publication claim even when recognition is local. + +* Question: why does this design exploit Scheme and its DSL? + +The advantage is staging and higher-order composition, not parentheses alone. +Canonical syntax describes actions once. Engine-owned generic selection turns +those declarations into retained functions; each request supplies its own +children and source anchors. Grammar authors need not select storage, dispatch +or traversal strategies. Scheme closures preserve that separation naturally. + +The existing engine pattern is illustrated below; this is an implementation +witness, not a new public DSL form: + +#+begin_src scheme +;; src/runtime/reduction-plan.ss: select before repeated requests. +;; Each retained action receives request-local children and offsets. +(prepare-operand-actions field-factory alias-factory actions) +#+end_src + +The LeanPoo tuple models selection provenance, while the parser model adds the +ordered context-threading law. A generic method may choose the implementation +at preparation time without forcing generic selection into every child visit. +The three language interfaces remain grammar.ss, parser.ss and parser-test.ss. + +* Question: how is the design scientifically checked? + +Each claim names its counted operation and assumptions, a kernel-checked +semantic law, a Scheme conformance owner and a measurement owner. The +integration theorem connects preparation to reverse-stack reduction instead +of treating local equalities as a complete parser proof. The materializer +counts the operations of a defined algorithm instead of assigning a desired +linear formula to an unspecified traversal. + +The remaining proof boundary is explicit: input/span admission, derivation of +nullable offsets, LR table/CFG soundness, GLR ambiguity, arbitrary external +actions, relocation views, Scheme refinement, digest correctness and reuse +validation are not proved. Operands carry already supplied individual anchors. +Stack-depth preservation requires admitted width; it does not validate goto +entries. No theorem establishes P50/P95, heap bytes or hardware performance. + +* Reproduction and qualification + +#+begin_src sh +cd formal +lake build +#+end_src + +Or run =just formal= from the repository root. Entering =formal= selects its +pinned toolchain; a root-level Lake directory option alone does not select it. +Every module is imported by the aggregate target. Warnings are errors; the +aggregate prints axiom dependencies of the main theorems. Proofs introduce +no custom axioms, sorry placeholders or native_decide shortcuts. Standard +Lean logical axioms reported by the build remain visible. + +CI adds this build to the existing contracts lane through a pinned official +Lean action. It does not add a qualification script or relax Scheme gates. +The existing complete-request and sequence measurement owners remain: + +- =t/benchmarks/semantic-sequence/benchmark.ss= for sequence representation. +- =t/gql/benchmark-profile-test.ss= for request phases and complete entry. +- =t/artifact-publication-cost-test.ss= for publication controls. + +This change alters no production execution path and makes no CPU speed claim. +Benchmark reruns become necessary when the corresponding Scheme implementation +changes. A successful formal build qualifies the model only; remote CI must +qualify the exact pushed revision separately. + +* Next coherent proof boundary + +Extend the model with checked LR transitions and table/grammar correspondence, +then prove an implementation refinement for the same reduction path. Use the +same action and source contracts in Scheme controls. The [[file:parser-evidence-chain.org][evidence-chain study]] now adds a finite Quint publication/cancellation +model and native ASCENT dependency review. Worker lifecycle and implementation +conformance remain open; the protocol model is not a second grammar authority. + +* Fold run specialization: distinct proof responsibilities + +* Whole-engine execution boundary: not another Fold-only speed model + +=Execution.lean= models ordered, dependent consumers over one immutable source +extent. The shared recognition fact is source-only; consumers retain their +original order and may observe earlier state writes. State includes the event +owner and rejection outcome, rather than only a Boolean mode. The general +=transition_list_preserves= theorem permits writes that preserve the admission +invariant; requiring every relevant cell to remain untouched is unnecessarily +strong. The source-visit count theorem is conditional logical work, NOT elapsed +time, an allocation estimate or proof that the Scheme optimizer is sound. + +The whole-span trial reuses =Fold.count_summary_identity=, rather than adding +a duplicate counter model. =Execution.count_summary_extent= proves that the +summary carry is true exactly when its suffix length equals consumed length; +=count_summary_length_encoding= justifies that ABI representation without +truncating the initial Scheme integer. It does not prove a C implementation or +physical speed. =RunPlan.qnt= separately models the synchronous borrowed phase: +no source invalidation, state commit or residual publication during the borrow, +and ordered commit/consumer execution after release. Directed Scheme tests and +complete actual AOT tapes bind implementation observations. Both span-leaf +implementations were slower and withdrawn despite passing those controls. +The model is an obligation for a future execution-region candidate, not a +claim that the retained production code now uses this foreign span executor. + +The rejected implementation derived local completion support after byte-region +classification and reapplies explicit entry facts after unreachable trigger +effects disappear. Literal zero resets preserve unsigned zero support. A +nonzero intermediate write, unknown handoff or observable publication rejects +this admission. The original triggering transition and event order remain +unchanged. Matched timing did not establish a consistent improvement, and the +admission/lowering policy was restored to the prior baseline. The invariant +checker and directed controls remain verification tooling, not an enabled +runtime optimization. This is one part of the engine execution plan, not whole-engine +performance closure and not a maintained state/text cache. + +The architecture audit must account for the complete request, not add up +instrumented leaf durations or mistake fewer guards for fewer executed visits: + +| Execution boundary | Implementation owner | Required observation | +|--------------------+----------------------+----------------------| +| Input admission and byte ownership | =src/runtime/source-lines.ss= | Encoding versus UTF-8 validation; retained source; no per-line decoding in the byte-span path | +| Source recognition and consumer execution | =event-fold-analysis.ss=, =event-fold-byte-dispatch.ss=, =event-fold-scheme-statements.ss= | Dynamic scans/bytes, state dependencies and helper/frame barriers; one admitted extent, ordered consumers | +| AOT closure and owner return | =event-fold-scheme.ss=, =event-fold-scheme-modules.ss= | Generated call graph versus actual call counts; lexical/slot ownership and retained outputs | +| Event allocation and publication | =src/runtime/fold-tape.ss= | Record count, growth/copies, final sealing and GC; no intermediate event-list authority | +| Downstream graph consumption | =graph-projection-runtime.ss= and the consumer adapter | Separate full-chain measurement; cannot inherit direct Fold timing | + +Existing =FoldPlan.qnt= owns source/bounds epochs, effect barriers and ordered +consumer histories. Its local-preserving-transition history requires the same +extent and scan basis; unknown transitions still invalidate. =RunPlan.qnt= +owns interval commit/residual ordering. No second grammar or duplicated Lean +semantic proof is introduced in Quint. Neither model proves processor +activation, garbage-collector behavior or Tokio latency. + +Real AOT conformance and observer-free complete-request timing remain mandatory. +The latest historical count-only census reports many repeated generated calls, +but its instrumentation and GC prevent treating those counts as exclusive CPU +or allocated bytes. The new architectural model is deliberately not a claim +that the 1000-request 10 ms gate has passed. The compiler-to-model dependency +analysis and complete pipeline cost attribution remain explicit open work. + +* Generated call-region fusion + +The call planner operates on the complete generated program before module +partitioning. Private expression references are counted across bodies and +value/export references. Only a unique call edge with scalar-reference +arguments and bounded expanded size is spliced. Shared consumers and helper +frame entries retain their boundary; effectful arguments are not moved. +Consumer effects and events retain order. Fresh lexical names preserve let, +let*, single-binding Gerbil let and named-loop scope. Quotes are opaque. +Final reachability prevents a rejected parent from deleting needed callees. + +The budget bounds generated code, not request concurrency or runtime dispatch. +No request cache, event representation or runtime graph is added. Production +direct tapes use this pass; list consumers and diagnostic products do not. +Six directed controls cover scope, shared calls, argument effects, skipped +errors, size rejection and mutable owners. Lean splice_call_preserves_order +proves the ordered transition-region contract, not Scheme alpha conversion. +Existing Quint FoldPlan checks a fused-call/frame invalidation history rather +than duplicating the semantic proof. The collector binds the planner, tests, +models and real AOT. Static call reductions are not performance admission. + +* Existing interval contracts + +=Fold.lean= owns pure transition refinement: =overwrite_run_last= and +=overwrite_run_residual= retain the final overwrite and the original trigger +transition; =run_partition= preserves contiguous source extents. These theorems +do not establish the compiler's eligibility analysis or physical scan cost. + +=RunPlan.qnt= owns lifecycle histories, not the same denotational proof: +frame invalidation prevents stale commits, a committed ordinary run must +execute its residual before the next scan, and a mutation witness rejects +stale admission. The seeded 1000-trace/20-step check is bounded testing, not +an exhaustive proof over all executions or a throughput guarantee. + +The model collector binds the ordinary-run admission and lowering, generated +module owner, borrowed byte scanner, source-line visitor, event-tape writer, +and implementation controls. Complete real AOT tapes and matched timing remain +separate evidence obligations. Model success never promotes a candidate that +fails the 1000-request <=10 ms preflight. + +The foreign ordinary-run leaf experiment passed boundary and complete-tape +controls but did not improve matched whole-call timing and was reverted. +The retained models describe the existing run contract, not a deployed foreign +scanner. Diagnostic =t/benchmarks/parser-throughput/stalls.ss= reads the +installed Gambit header's GC counters and OS resource counters; its extra +probes and cold first call make it diagnostic only, not acceptance timing. + +* Explicit Lean and Quint responsibilities + +[[file:../architecture/0019-model-responsibilities.org][Model responsibility boundary]] maps paper arguments to separate algorithm, +lifecycle, implementation and performance obligations. Lean's new +=ParserEngine.Reuse= proves whole-result identity admission and counted +preparation amortization including per-request validation. Quint's +=ReuseAdmission= checks source/grammar/plan invalidation, cancelled/failed +completion and superseded ownership with independent mutation witnesses. +Neither qualifies cross-edit subtree reuse or a runtime speedup. diff --git a/docs/research/parser-evidence-chain.org b/docs/research/parser-evidence-chain.org new file mode 100644 index 00000000..ec9f5512 --- /dev/null +++ b/docs/research/parser-evidence-chain.org @@ -0,0 +1,302 @@ +#+title: Parser model and implementation evidence chain + +* One optimization chain + +The optimization unit is an engine change with four distinct obligations: + +| Evidence layer | Owner | Qualified claim | +|----------------+-------+-----------------| +| Algorithm semantics and counted operations | Lean + LeanPoo | =lean-model= | +| Finite publication protocol | Quint | =quint-protocol= | +| Implementation/product conformance | Existing Scheme suites | =scheme-conformance= | +| Complete-request matched measurements | Existing Scheme benchmark admission | =matched-performance= | + +Python transports observations and content identities into native Scheme +ASCENT. ASCENT derives usable receipts, missing requirements and transitive +invalidity. It neither implements the parser nor proves Lean theorems or +computes new performance statistics. The evidence adapter is verification-only. +The production grammar.ss / parser.ss / parser-test.ss interfaces are unchanged. + +This extends [[file:parser-engine-formal-model.org][the reduction and sequence model]]. The existing +[[file:parser-execution-evidence.org][measurement admission]] remains authoritative for actual requests. +Passing a model does not establish implementation conformance or improvement. +The [[file:lr-owned-stack.org][request-owned LR stack study]] applies the chain to execution storage and +immutable-frontier publication. + +* Source study and dependency ownership + +The designated Gambit study file =evidence/suite-review-rules.ss= was absent +in the inspected checkout. Its adjacent =review-rules.ss= and +=research-rules.ss= supplied the study pattern: finite current-input snapshots, +receipt/claim dependencies, cycle rejection and explicit missing premises. +Those local study files are not imported or copied into the parser runtime. + +The reusable implementation is +[[https://github.com/tao3k/gerbil-ascent/blob/64742af763486f37e4318ed26f83b8bec46e4906/python/src/ascent_engine/review.py][gerbil-ascent's pinned EvidenceReview]]. Its Python binding delegates relational +execution to the native Scheme engine. =formal/evidence/pyproject.toml= and =uv.lock= +pin that source revision and runtime Python dependencies. The ABI library +remains an upstream build product: this repository checks its sidecar ABI and +library SHA256 before loading it, then records the artifact, sidecar and Python +binding-source identities. A valid digest establishes artifact identity, not +the authenticity or correctness of the upstream build declaration. + +Quint 0.33.0 and its dependency integrity records are fixed by the repository's +=package.json= / =package-lock.json=. Lean retains its separate toolchain and +LeanPoo lock. No native library, environment-specific receipt or raw output is +committed. Generated observations live under ignored =.data/parser-evidence/=. + +* Protocol model and its limits + +=Publication.qnt= models a prepared request's source generation, cancellation +and publication. A result can become public only while its basis matches the +current generation and it has not been cancelled. Editing withdraws an old +published result. Rejected results remain private. + +Seven deterministic tests cover successful publication, stale requests, +cancellation, edit withdrawal, and mutation witnesses that omit admission. +The closure test evaluates every state in the explicit 360-state domain: +source/basis generations 0--2, five phases, two cancellation values and four +output-generation values. It checks one-step preservation from every valid +state for each admitted transition. This is finite evaluation, not Apalache +verification, an unbounded temporal proof, a liveness theorem, or conformance +of the current actor/session implementation. The exact test command and domain +are included in the observation's bound inputs. + +* Receipt admission and traceability + +=parser_evidence.py= records command arguments, working directory, deadline, +exit code, expected markers, output digest, executable digest and declared +input-file digests. Commands execute without a shell. Nonzero exits, missing +markers, timeouts and input changes do not produce passed facts. Source/input +paths are repository-relative and cannot escape through parent components or +symlinks. Tool versions, dependency locks and relevant source closures must be +included by the caller; a launcher fingerprint alone does not identify all +interpreter or imported dependency contents. + +Each immutable observation has a content ID derived from its full record. +Its raw output and record are stored under that ID. Review rechecks current +source/executable/output/metadata contents. Deleted or changed inputs invalidate reuse. +Receipt dependencies identify those same content IDs. Upstream Scheme ASCENT +rejects absent, failed, stale and circular dependencies, including transitive +invalidity. The review packet contains the exact input facts, observations, +native result and reviewer identities for subsequent inspection. + +Host observations are trusted local observations, not signed attestations. +The host still owns input-set completeness, checker selection and claim scope. +The optional =required= argument supports scoped reviews; the model collector +uses the complete four-layer requirement set. Even a complete set produces +=REVIEW_REQUIRED= rather than execution authorization. No result packet is a +substitute for inspecting its assumptions and the actual benchmark admission. + +Python's deadline covers its direct checker child. It is not a replacement for +the existing Scheme descendant-process supervisor. Scheme qualification calls +must retain their own established process-control owner. + +* Executable Scheme witness and design questions + +This is the upstream review's relational pattern, not a new parser DSL: + +#+begin_src scheme +;; CurrentInput and ReceiptInput are supplied observations. +;; Recursive ReceiptPath keeps transitive dependency identity explicit. +((ReceiptPath R D) <-- (ReceiptDepends R D)) +((ReceiptPath R D) <-- (ReceiptPath R X) (ReceiptDepends X D)) +((Unusable R (lit 'circular-justification)) <-- (ReceiptPath R R)) +((Unusable R (lit 'stale-or-missing-input)) + <-- (ReceiptInput R K V) (not (CurrentInput K V))) +#+end_src + +** Question: why does this exploit Scheme's strengths? + +The parser retains its higher-order prepared functions and engine-owned +representations. A second Scheme engine supplies declarative relational review +and retained derivation components through its C ABI. Python supplies bytes and +observations; it does not duplicate Scheme inference with imperative rule loops. +This preserves producer ownership and keeps language authors free of verification +or storage machinery. Component reuse may reduce repeated review work, but no +review-time speedup is claimed without measurement. + +** Question: how is this scientifically checked? + +Lean proves properties of its algorithms; Quint evaluates a named finite +transition domain; Scheme tests compare semantic products; benchmarks preserve +matched measurement controls. Each observation binds the relevant source, +tool/lock context and raw output. Mutation and negative controls test whether +invalid evidence is rejected. The combination exposes missing proof and +measurement obligations instead of converting one successful layer into a +whole-engine claim. + +* Reproduction and CI boundary + +From the repository root: + +#+begin_src sh +just formal +npm ci --ignore-scripts --prefix formal/quint +just formal-protocol +uv sync --project formal/evidence --locked +#+end_src + +Select an upstream-built native ABI artifact and its adjacent JSON receipt +through =ASCENT_ENGINE_LIBRARY=. With the prepared environment: + +#+begin_src sh +uv run --project formal/evidence --locked python -m unittest discover -s formal/evidence -v +uv run --project formal/evidence --locked python formal/evidence/collect_models.py +#+end_src + +The collector executes real Lean and Quint commands and sends their saved +observations to native ASCENT. Its expected result establishes the two model +claims and explicitly reports =scheme-conformance= and =matched-performance= +as missing. Success qualifies that model-layer experiment only. + +Eleven host/native controls cover nonzero exits despite success markers, +missing markers, source mutation during execution, retained timeout, changed +source/log/metadata rejection, dependency cycles, failed transitive dependencies, +partial bundles, measurement without CPU admission, and absent execution authority. + +CI adds locked Quint tests and Python host controls to the existing contracts +lane. The native ASCENT integration is locally exercised against an upstream +artifact; CI does not rebuild an unrelated Scheme dependency or pretend that +host tests qualify native inference. A future native CI gate needs an immutable +upstream artifact provisioned by that project's build owner. + +* Engine implementation study + +[[file:engine-record-access.org][Closed engine records]] applies this chain to measured LR recognition and AST +construction. It includes the official compiler contract, independently loaded +reference products, complete artifact equality, repeated native CPU admission +and rejected preliminary candidates. The infrastructure alone makes no parser +speed claim and implements no new statistics or parser semantics in Python. + +* Complete engine observation entry + +=just evidence-engine= executes the four evidence layers in one review. It +uses the same locked uv project as =evidence-controls= and =evidence-models=. +The model-only entry remains useful for diagnosing missing runtime evidence; +it does not substitute for this entry. + +The sequence is: + +1. Execute Lean and Quint against their declared source and dependency inputs. +2. Build production Scheme, the standard test driver, FFI test owners and the + composed benchmark owner. These are =scheme-build= prerequisite receipts, + not semantic evidence. Native =std/make= and =gxc= own compilation. +3. Execute seven Scheme suites covering prepared reductions, fused actions, + persistent sequences, publication costs, the GQL stage controls and native + language admission. Require process success, the exact suite count and + the test driver's final =OK=. +4. Execute the existing GQL stage sampler and the matched GQL, FHIRPath and + arithmetic request backend sampler. Both retain their existing semantic + artifact checks and sampling/admission rules. +5. Submit source-bound observations and receipt dependency edges to native + Ascent =EvidenceReview=. Conformance depends on every build receipt; + measurement depends on those receipts and conformance. Failed preparation + stops dependent execution and leaves both runtime requirements missing. + +The production declarations, Scheme source closure, tests and workloads are +hashed before and after each command. Runtime observations additionally bind +prepared library products. Logs and metadata have content identities and +are checked again by the review host. This supplies finite provenance for +explicitly chosen inputs; it is not a proof that the supplied closure includes +all external compiler, operating-system or hardware influences. + +Build supervision uses the existing 240-second / 30-second inactivity +budgets. Runtime tests and measurements use the existing 120-second / +5-second inactivity budgets. The Python direct-child deadline exceeds the +Scheme supervisor's budget; Python adds no descendant-process supervisor. +The previous five-second exploratory build failures remain failures. + +=matched-performance= means that the matched measurement executed and its +output is available for review. It does not mean that an optimization won. +The Scheme sampler's =LR-BACKEND-PERFORMANCE-ADMISSION= output remains the +owner of control stability and complete-request benefit admission. Ascent +establishes receipt dependency validity, not CPU improvement or a +Scheme-to-Lean refinement theorem. No new CI job or compiler is introduced. + +* Scheme dependency boundary + +The parser package no longer imports =gerbil-scheme-rust= from Scheme. Native +language admission uses SDK UTF-8 decoding, a Unicode scalar predicate and +canonical SDK re-encoding. This rejects surrogate encodings as well as +noncanonical byte encodings without foreign admission allocation. Existing +ASCII admission, Unicode boundary and malformed-input controls remain in +=language-abi-test.ss=. No Unicode throughput benefit is claimed from the +dependency removal. Rust code generation still has its separate Cargo IR +consumer; deleting a Scheme package dependency does not remove that Rust +contract. + +* Input-set transport and local admission results + +Each observation retains its complete sorted =(relative-path, SHA256)= input +manifest. For native review, the host transports one SHA256 commitment of the +canonical manifest as =ReceiptInput= and recomputes the same commitment as +=CurrentInput=. Missing files omit the current commitment; changed file +contents change it. Ascent owns the equality and dependency inference. This +reduces repeated relational input rows without weakening per-file mutation +checks or introducing Python inference. + +The local upstream artifact still rejects requests beyond its ABI transport +limit. The earlier expanded-input request returned status =-4=; it is retained +as a failure, not hidden by increasing an engine limit. Compact commitments +allow the existing source-bound review to execute under that same artifact. +The ten native/host controls include a complete runtime bundle and rejection +of both semantic and measurement receipts through a failed build prerequisite. + +The CI at =77d13f7= completed successfully. Its configured CPU/diagnostic skips +are not benchmark qualification. New local engine observations apply to their +recorded source and prepared-product manifests, including any local study +changes; they do not retroactively qualify that CI revision. + +* Executed complete-chain qualification + +The complete local entry passed both model checks, all four Scheme build +prerequisites, six semantic suite owners and the composed request workload. +Native Ascent returned =Missing = []= and =executionAuthorized = false=. +The six semantic owners executed 15, 10, 8, 10, 11 and 17 Cases respectively. +Their source manifests include local study edits; these counts qualify that +recorded local manifest rather than an unchanged Git head. + +Preparation uncovered two separate admission failures: stale helper +interfaces, and a generated Scheme phase body without its compiled test-helper +object. The standard benchmark build owner now owns the required arithmetic +and HCL test helpers as well as the sampling workload closure. Compilation +uses =std/build-script= / =std/make=; Python no longer lists compiler targets. +The completed source harness uses the existing disposable-child exit owner +only after its =main= returns successfully. Earlier =OK= output followed by +an inactivity failure remains a failed observation. + +The matched workload also exposed an asymmetric diagnostic catalog: the +recognition semantic replay has no event-program counterpart for eligible +FHIRPath and arithmetic machines. Such diagnostic pairs are explicitly +reported as =LR-BACKEND-PAIR-UNSUPPORTED=. A missing =full-source= pair still +raises an error. Sampling counts, ABBA/BAAB order, negative-control admission +and the five-percent decision margin are unchanged. + +The latest complete workload reported =control-stable = false= and +=complete-request-benefit-admitted = false=. Those negative decisions remain +in its receipt. Completion of all evidence layers is not an optimization win. +The representative GQL stage experiment identifies prepared LR as the next +cost investigation: one completed local run reported CPU P50 of 6.712 ms per +100 prepared parses, versus 9.058 ms per 100 complete requests. Lexing and +publication controls reported 0.763 and 0.836 ms per 100 calls. These are +independent prepared controls, not an additive source-execution decomposition +or a portable performance target. No parser backend default changes. + +* Independently admitted engine CPU change + +The optional =--record-study= entry additionally requires +=record-cpu-improvement=. A =matched-performance= receipt cannot establish it. +The new claim is produced only by the Scheme =record-evidence.ss= checker, +whose strict repeated CPU, tail and unchanged-SDK control checks consume four +independently loaded complete-request receipts. Native Ascent rejects the +missing or failed admission even when the ordinary measurement layer passed. +A native negative control checks this distinction explicitly. + +CPU admission depends on all build receipts, semantic conformance and the +ordinary request measurement. Its input closure additionally includes the +reference products, reference source/product manifest and all four raw logs. +The host's detailed manifests and the upstream ABI identity remain separate +from a proof of the compiler or the completeness of the supplied inputs. +See [[file:engine-record-access.org][the engine implementation study]] for the workload scope and conservative +results. CI retains the existing contracts and native semantic lanes. diff --git a/docs/research/parser-execution-evidence.org b/docs/research/parser-execution-evidence.org new file mode 100644 index 00000000..a9a831f3 --- /dev/null +++ b/docs/research/parser-execution-evidence.org @@ -0,0 +1,243 @@ +#+title: Parser execution evidence and receipt admission +#+startup: overview + +Part of [[file:shared-ranked-scanner-execution.org][the integrated execution study]]. +Historical revisions, failed experiments and receipt limits retain their original scope. + +* Executed interleaved backend qualification + +** Protocol and admission fixed before observation + +The earlier recognition-then-event diagnostic did not qualify performance. The +same benchmark now prepares both variants once and alternates timed batches +within each workload. Even-numbered groups execute recognition/event/event/ +recognition (ABBA); odd groups execute event/recognition/recognition/event (BAAB). +Each variant has two equally sized batches in every group. Both variants warm +up before one initial GC per workload. Switching variants does not force GC; +naturally occurring collections stay in request CPU and wall observations. + +The existing Scheme sampler exposes =measure-parser-batch= for this scheduling. +It returns the original CPU, wall, GC, allocation-admission and product controls +without introducing a new statistics implementation. Ordinary component callers +retain their previous warmup and initial-GC behavior. Batch counts must be exact +positive integers divisible by calls per thunk; no truncated invocation count +can be reported as a complete batch. Changed semantic products reject the +observation. The extended profile suite passes ten cases, including these +measurement obligations. + +For every paired group, the CPU ratio is summed event CPU divided by summed +recognition CPU. The P10/P50/P90 statistics describe these twenty paired ratios; +they are not a confidence interval, individual request percentiles, or a ratio +of independently selected CPU medians. Before execution, the practical admission +margin was fixed at five percent: + +- P90 below 0.95: =observed-benefit=. +- P10 above 1.05: =observed-regression=. +- Otherwise: =not-admitted=. + +GQL is the negative control because both preferences retain recognition. Its +paired median must remain within 0.95--1.05 and its empirical band must contain +one at both scales and measured boundaries. Failure blocks whole-engine benefit +admission. Passing this center check alone does not establish narrow dispersion. +The global benefit marker additionally requires both non-GQL complete-request +routes to meet the benefit criterion; it is a conservative universal-benefit +check, not a proposal to change the default backend. + +** Actual native receipt + +The caller runs at source baseline =a36c4d1= with the paired benchmark changes. +It explicitly admits the fresh =lr-parser~0.o6= and restored +=lexical-lexer~0.o6=. The shared sampler and benchmark caller are compiled with +the fixed =2591dcd= SDK and GCC. Heap cap remains 3 GiB, total bound 120 seconds, +and idle bound five seconds. Benchmark compilation first rejected an incomplete +form and then a malformed binding; both failures are retained as build failures. +The corrected benchmark compiles successfully in 27.137 seconds. + +The source profile suite passes ten cases, then all twelve paired workloads +complete: three languages, two sizes, prepared LR and complete public source. +Twenty groups contain four batches of twenty calls each: 19,200 timed calls in +total, excluding warmup, source-route witnesses and semantic reference admission. +Both prepared variants publish the independent materialized GLR reference; +public variants and untimed route witnesses publish the same source-bound +canonical artifact. Products are checked after every batch. The complete child +exits zero in 53.291 seconds with =TEST-SUITE-OK=, +=LR-REQUEST-BACKENDS-OK=, =LR-BACKEND-PERFORMANCE-ADMISSION= and +=BACKEND-CONTROL-OK=. + +| Family, units | Boundary | CPU ratio P10 | P50 | P90 | Observation | +|---------------+----------+---------------+-----+-----+-------------| +| GQL 64 | prepared LR | 0.761 | 0.995 | 1.115 | not admitted | +| GQL 64 | full source | 0.197 | 0.999 | 1.100 | not admitted | +| FHIRPath 64 | prepared LR | 1.275 | 1.336 | 1.358 | observed regression | +| FHIRPath 64 | full source | 0.327 | 1.208 | 1.647 | not admitted | +| Arithmetic 64 | prepared LR | 1.383 | 1.416 | 1.443 | observed regression | +| Arithmetic 64 | full source | 0.635 | 0.989 | 1.020 | not admitted | +| GQL 512 | prepared LR | 0.557 | 1.016 | 2.000 | not admitted | +| GQL 512 | full source | 0.592 | 1.447 | 2.121 | negative control fails | +| FHIRPath 512 | prepared LR | 0.814 | 1.404 | 2.353 | not admitted | +| FHIRPath 512 | full source | 0.638 | 1.106 | 1.982 | not admitted | +| Arithmetic 512 | prepared LR | 0.547 | 1.421 | 3.307 | not admitted | +| Arithmetic 512 | full source | 0.455 | 0.530 | 2.281 | not admitted | + +** What the test establishes + +The small prepared event routes have repeatable regression observations under +the declared empirical criterion. FHIRPath's admitted allocation ratio is 0.959 +in 19 of 20 groups; Arithmetic's is 0.790 in all twenty, with no timed GC in +that prepared Arithmetic workload. Lower allocation therefore does not imply +lower CPU even in an interval without collection. Those component regressions +are observations from this caller, not a portable causal percentage. + +Large complete-source workloads fail allocation comparison: none of the twenty +groups has four no-GC intervals. GQL, FHIRPath and Arithmetic encounter 25, 32 +and 21 collections respectively across their eighty timed full-source batches. +These are whole-VM collections during the sole caller, not an allocation delta +that can be attributed across collections. Every batch retains its raw counters +and collection snapshot in Scheme output. + +The identical-executor GQL full-source control fails its median margin at 512 +units. The superficially favorable Arithmetic full-source median, 0.530, also +has a band spanning 0.455--2.281 and is not a speedup. Collection placement in +short interleaved batches is a concrete confounder; repeated identical source +allocation and unchanged generated execution identify it as a measurement +boundary that needs resolution before a complete-request CPU claim. The receipt +alone does not prove it is the only source of variation. + +The executable gate reports =gql-fallback-stable #f= and +=complete-request-benefit-admitted #f=. It passes semantic qualification and +rejects performance admission. Default execution remains recognition. These +results justify studying event representation and GC-sensitive request sampling; +they do not justify enabling event execution, filtering out costly GC samples, +increasing timeout budgets, or declaring a whole-engine speedup. +* Qualification output completion + +A real Rust compiler trace exposed a separate test-cost defect. The child had +completed its test, but the qualification parent continued processing a roughly +200 MB output receipt for several minutes. Inspection found a redundant final +UTF-8 conversion, line-list construction and per-line regexp check after the +monitor had already stopped. Incoming error prefixes were already checked by +the streaming reader. That final pass duplicated its work outside the active +watchdog and retained additional whole-log representations. + +The process owner now reuses the reader's prefix state. Bare =ERROR= at EOF is +the remaining delimiter-free case; =ERROR!= and =*** ERROR= are already rejected +while streaming. Identifier prefixes such as =ERROR_COUNT= remain ordinary +output. The required-receipt path decodes captured bytes only when a caller +actually declares receipts. Byte capture, log storage, status propagation, +owned-session cancellation and configured deadlines retain their original +contracts. The new tests explicitly provide a byte sink while invoking this +raw-output protocol inside a Suite that otherwise captures text. + +** Executable receipts + +=qualification-output-test.ss= exercises ordinary prefixes, three error forms, +bare error across the 65,536-byte read boundary and complete eight MiB output. +All four cases pass. The large child returns exactly 8,388,608 captured bytes, +status zero and no reason in 0.668 seconds, including result construction. The +whole Suite qualifies in 12.108 seconds with exit zero and final =OK=. Bootstrap +and completed harness import are actual phase markers; no heartbeat or deadline +extension is used. + +The current Rust library's 145 tests pass and its qualification parent exits +normally in 8.017 seconds while handling a 200,000,405-byte diagnostic console +receipt. This is a functional and bounded-output receipt, not a matched parser +CPU comparison. LLVM pass diagnostics retain the compiler and optimization +settings; they are confined to local troubleshooting. + +The existing native ownership controls also pass with GCC 16, including six +concurrent cold initializations, caller-mailbox preservation, UTF-8 argv, +nonzero/signal exits, silent/continuous/closed-output timeouts, escaped-session +cancellation, unrelated-process survival, Unicode backpressure and immediate +error cancellation. A first run rejects =closed-output=; a narrow reproducer +returns the required =idle-timeout= in 1.245 seconds, and the complete recheck +passes without changing any budget. Control failures now report the expected +and actual reason, status, elapsed time and phase. This preserves the initial +negative result and improves future attribution. + +The standalone C fixture initially fails to link because the shell resolves +=ld= outside the SDK environment. The fixed SDK environment resolves the system +linker and GCC 16 builds the same fixture successfully. The compiler is not +replaced. The restored shared-engine semantic gate passes all 44 cases in +19.376 seconds. The allocation verifier recheck admits the previous 916 exact +intervals in 17.894 seconds, preserving its complete-request savings and CPU +rejection boundary. + +** Rust policy delivery + +PR head =ad8c6a3= fails =RUST-AGENT-PROJECT-003= because its precedence-capacity +unit test is inline in =engine/parser.rs=. The test is moved unchanged to +=tests/unit/precedence_capacity.rs= and mounted with =#[path]=. +The upstream =8bfc1fa= independently provides the same migration; rebase keeps +that owner and drops the duplicate local commit. The existing ASP policy +executable uses the current workspace manifest as its input; its policy source +matches this checkout. That source-policy gate passes in 1.568 seconds. The +145-test receipt above includes the moved assertion. Remote CI for the next +published head remains a separate gate. + +* Compiled ranked-scanner request controls + +The earlier rejected result-publication lowering left a specific evidence gap: +a compiled caller checking the entire observable scanner result. The new +=benchmark-ranked-literal-requests= entry retains the original component entry +and adds twelve workloads: depths 1, 4, 16 and 64, each accepting all prefixes, +admitting only the shortest prefix, or missing the first edge. Preparation is +untimed. Each stage uses twenty samples of 5,000 complete scanner calls; the +sampler checks the final batch result's name, endpoint, precedence and declaration +index. These are scanner calls, not whole-parser latency measurements. + +The experiment stores the selected prepared terminal and endpoint, allocating +the four-element result only at traversal exit. The trie, rank selection and +mode masks are unchanged. The same compiled caller runs reference, candidate, +candidate, reference in fresh qualified children. Each child executes 1,200,000 +scanner calls; the four children complete in 4.969, 6.620, 4.248 and 5.666 seconds, +respectively, with =RANKED-LITERAL-REQUESTS-OK= and exit zero. All commands retain +GCC, the 3 GiB heap, the 120-second total limit and five-second silence limit. +An initial incomplete Scheme form fails reader admission before measurement; +its corrected build completes in 9.148 seconds. + +| Accepting depth | Reference bytes/call | Candidate bytes/call | Saved bytes/call | Ratio of summed child CPU medians | +|-----------------+----------------------+----------------------+------------------+----------------------------------| +| 1 | 336.0672 | 336.0672 | 0 | 1.0032 | +| 4 | 912.0672 | 336.0672 | 576 | 0.8583 | +| 16 | 3216.0672 | 336.0672 | 2880 | 0.7675 | +| 64 | 12432.0672 | 336.0672 | 12096 | 0.6326 | + +The shared receipt verifier admits 936 collection-free allocation intervals. +The exact integer batch-counter savings agree with 192 bytes per eliminated +result spine on this native build. Shortest-only and miss controls retain exact +allocation. Depths 16 and 64 have both candidate CPU medians below 95 percent +of both reference medians. However, unchanged controls drift: the depth-4 miss +ratio is 1.1831 and depth-16 miss ratio is 1.2059. The global CPU control gate +therefore rejects every CPU benefit claim. These small batches do not resolve +portable CPU non-regression, and no complete-parser benefit is established. +The production scanner source and complete native/static artifact snapshot +are restored. This experiment is not shipped as a runtime optimization. + +** Shared evidence admission and qualification + +=t/benchmarks/parser-stage-cost/receipt.ss= owns complete sampler receipt +admission for both ranked scanner and complete casefold requests. Callers supply +the completion marker and expected stage, batch and sample counts. The reader +rejects repeated sample identities, duplicate summaries, missing completion or +observations, inconsistent raw allocation counters, nonuniform admitted +allocation and stale CPU summaries. Each comparison retains its own workload +policy and conservative CPU control gate; no language-specific reader is copied. + +Four permanent source test cases cover valid receipts and forged or incomplete +observations. They run with the seven existing LR/scanner suites: 48 cases, +eight =MODULE-OK= receipts, =TEST-SUITE-OK modules=8=, final =OK= and exit zero +in 24.126 seconds. The first invocation produces no bootstrap progress and is +rejected in 6.071 seconds by the unchanged silence gate. A subsequent invocation +records the actual bootstrap and harness-loaded boundaries; no periodic progress +output or deadline extension is used. The native shared reader and both +comparison consumers compile in 13.513 seconds; the final counter validation +rebuild completes in 11.277 seconds. + +The ranked-scanner comparison confirms allocation while reporting +=controls-stable? #f=. The complete casefold comparison is rechecked with the +same reader against its existing four native receipts. This recheck preserves +the earlier complete-request allocation evidence, and is not a new runtime +measurement or CPU claim. Raw local receipts are identified by the stems +=parser-ranked-reference-a=, =parser-ranked-candidate-a=, +=parser-ranked-candidate-b=, =parser-ranked-reference-b=, +=parser-ranked-shared-evidence= and =parser-casefold-shared-receipt-recheck=. diff --git a/docs/research/prepared-lr-action-plans.org b/docs/research/prepared-lr-action-plans.org new file mode 100644 index 00000000..d5acba4c --- /dev/null +++ b/docs/research/prepared-lr-action-plans.org @@ -0,0 +1,400 @@ +#+title: Prepared LR action plans and engine ownership + +This document records the initial indexed action-chain implementation and its +qualification. [[file:lr-semantic-kernel-staging.org][Semantic kernel staging]] continues the same admission boundary: +private prepared slots now contain executable steps or identity, while canonical +operand chains remain grammar data. GLR and deterministic execution share the +lowered operations; current stack tests use a manually constructed semantic +oracle to preserve independent evidence. + +* Design question and implementation + +The grammar -> canonical IR -> prepared machine -> reduction -> artifact path +must have one owner for semantic admission. This continues the +[[file:engine-reduction-generation.org][shared reduction study]]: generated reductions already inspect canonical +operands during compilation, while the shared deterministic executor previously +decoded marked operands and checked the production action on every reduction. +The new =lr-prepare= boundary checks the closed =pass=, =concat= and layout-end +production algebra and ordered, two-element =field=/=alias= actions with symbolic +names. The shared canonical predicate also admits generated Scheme and Rust +publication. Invalid +actions fail before a request starts, including productions not executed by +that request. Canonical grammar identity, GLR, replay and generated reductions +retain their existing owners. + +Each production receives one private source-order vector of action chains. +Recognition consumes this vector backwards alongside the newest-first semantic +stack; events consume it forwards alongside their source-order values. Neither +backend decodes marked grammar operands during deterministic reduction. The +vector holds existing action chains rather than callbacks or request values. +Like the existing prepared tables, it assumes the admitted canonical grammar +is not mutated after preparation. No public grammar model is duplicated. + +* Why is this preferable for the Scheme DSL? + +#+begin_src scheme +;; Author-facing syntax remains grammatical and compositional. +(source-file + (node SourceFile + (seq (field left (node Item word)) + (field right (node Item word))))) + +;; Illustrative private prepared representation, in source order: +#(((alias Item) (field left)) + ((alias Item) (field right))) + +;; Recognition reads slot 1 for the newest semantic value, then slot 0. +;; Events read slot 0, then slot 1, preserving sequential builder appends. +(vector-ref action-plan index) +#+end_src + +The benefit comes from the macro/canonical-IR/preparation boundary, rather than +parentheses alone. The engine resolves and validates the action algebra once; +authors declare nodes and fields without queue, traversal, storage or backend +controls. Gerbil's official =std/vector/vector= uses a length-bounded vector +mapping implementation. This change likewise stages fixed-size execution data; +it does not add another collection algorithm or replace Gerbil's compiler. +The private execution functions remain closed named procedures, avoiding new +capturing callbacks for each request and keeping plan access outside the LR +continuation's generated body. + +* Why is the claim scientifically testable? + +For width n, plan slot i is exactly the action chain of canonical operand i. +Recognition starts at n-1, applies that operand's actions in declaration order +and prepends its result. After k steps, the accumulated children are the +source-order results for operands n-k through n-1. Induction gives the same +result as the independent forward materialized GLR reducer. Events append in +increasing index order, preserving the existing event builder's association. +Width zero returns the respective backend's empty value. + +Executable admission covers widths 0, 1, 2, 8 and 32, nested field/alias chains, +distinct operand fields, retained siblings, nullable edits, layout boundaries +and fragment transfer. Malformed production and operand actions must fail +at preparation. Complete canonical products are compared with independent GLR +and generated reductions; allocation and CPU observations are separate claims. + +The semantic-stack benchmark now measures preparation independently from +reduction-only requests and complete artifact publication. This makes moving +cost across phases observable. An initial two-list action plan passed semantic +controls but added 1824 bytes per width-32 preparation and gave no stable +request CPU improvement. It was rejected in favor of a single indexed vector. +Its native ABBA receipts remain unarchived local execution; all exited zero. + +An earlier plan-access expansion also failed the five-second idle compiler +gate in 37.323 seconds. The closed-procedure revision compiled with the official +SDK options in 50.150 seconds, exited zero and retained the same idle gate. +These are qualification observations, not proof that generated source size +alone caused the compiler failure. No timeout or compiler optimization level +was relaxed. + +* Final vector qualification and cost boundary + +The final source control group passed 34 cases in 22.755 seconds, actual exit +zero: unarchived local execution. It includes +the incoming scanner dispatch controls at parent =f2eb2b8=. The first vector +compile failed the idle gate during Gambit's macro-interface generation, +despite that Scheme file being slightly smaller than the baseline. The retry +with official =gxc -gsc-flag -verbose= completed the LR module, latest scanner, +test and benchmark in 23.455 seconds, actual exit zero: +unarchived local execution. +The [[https://gambitscheme.org/4.9.2/manual/][Gambit manual]] documents =-verbose= as compiler activity tracing. +It supplies real phase output; it does not establish a causal performance fix. + +Final native ABBA receipts are unarchived local execution. All four runs completed +20 samples of 100 iterations for six preparation and 24 request routes, +admitted their independent complete-product controls and exited zero. +The baseline is the preceding =d69f015= LR implementation in the retained native +overlay; both sides use the same expanded benchmark instrumentation. This +isolates the LR change rather than calling an old full-language build current. + +| RHS width | Baseline prepare bytes/call | Vector prepare bytes/call | Difference | +|-----------+-----------------------------+---------------------------+------------| +| 1 | 4290.08 | 4242.08 | -48 | +| 8 | 9106.08 | 8834.08 | -272 | +| 32 | 25618.08 | 24578.08 | -1040 | + +The table records recurring allocation levels; some admitted samples add +2.56 bytes/call of measurement variation. The difference applies to decorated +and undecorated families. All 24 request routes retain baseline allocation. +The benchmark measures allocated bytes, not retained heap size. + +Preparation now performs additional semantic admission. Width-32 identity +preparation CPU medians are 0.006475, 0.007150, 0.007235 and 0.006460 ms/call +in ABBA order. Request CPU does not show a stable improvement: width-32 identity +recognition publication measures 0.008380, 0.008430, 0.008220 and 0.008380 ms; +decorated event publication measures 0.029335, 0.041980, 0.029675 and 0.029815 ms. +The second run's event outlier is retained. This establishes a compact staged +representation and earlier admission, not an end-to-end latency win. Runtime +reuse remains necessary to amortize preparation; no numeric reuse threshold +can be inferred from these observations. + +The final native integration passed twelve cases and twenty accepted/rejected +ABI products across all ten language entries in 28.068 seconds, exit zero, +=HARNESS-OK= and =OK=: +unarchived local execution. +It covers the candidate LR and latest =f2eb2b8= scanner interfaces, independent +GLR, generated HCL/Arithmetic recognition and event reductions, and exact Rust +fixture bytes. Local native qualification is distinct from clean exact-head CI. + +* Canonical action admission across product owners + +The subsequent source audit found a concrete mismatch: fused Scheme generation +required symbolic field and alias names, while prepared LR checked only the +two-element shape. Rust's raw-IR emitter had its own operand decoder and could +emit string field names. A well-shaped list was therefore insufficient to make +the same semantic declaration legal across execution products. + +=src/compiler/lr.ss= now owns =operand-actions-valid?=. Prepared LR, fused Scheme +strategy admission and Rust table publication call this predicate. Rust also +uses the canonical operand accessor instead of a duplicated decoder. Rust +validates all production chains before constructing output; file publication +already follows successful source construction. Unused productions cannot +bypass either preparation or raw-IR publication admission. + +#+begin_src scheme +;; Canonical semantic declarations, not an additional author interface. +(operand-actions-valid? '((alias Item) (field left))) ; => #t +(operand-actions-valid? '((field "left"))) ; => #f +(operand-actions-valid? '((alias #f))) ; => #f +(operand-actions-valid? '((field left) . invalid)) ; => #f +#+end_src + +** Question: why is this design preferable? + +Scheme's structural matching expresses the closed declaration algebra directly: +each action has exactly one operation and one Symbol name. The invariant belongs +to canonical IR, so a POO strategy override cannot change semantic name legality +and backend selection cannot choose a different contract. Authors retain the +existing node/field DSL. Validation occurs at preparation or publication rather +than inside repeated reductions; no request callback or generic dispatch is added. +This completes the staging boundary studied above without adding a new DSL. + +** Question: how is the claim scientifically tested? + +The falsifier is a malformed chain accepted by any product owner, a published +file following rejection, or changed complete output for a valid declaration. +The preceding native LR accepted four malformed names under each of two backend +preferences: eight =BASELINE-ACCEPTED= receipts, exit zero in 10.026 seconds. +The first temporary baseline probe failed because an interpreted script used an +unqualified binding after an evaluated import; its failure is retained and is +not semantic evidence. The corrected probe used qualified native bindings. + +The executable controls check proper and dotted chains, unknown actions, +non-symbol payloads, both backend preferences and unused productions. Rust +controls append an unused malformed production to real Arithmetic IR and require +failure before a temporary output file exists. Existing wide/nullable/ordered +reductions compare complete artifacts with independent GLR; HCL and Arithmetic +generated backends and deterministic Rust/GQL output retain their own controls. +This is contract closure, with no latency or allocation improvement claim. + +Qualification at parent =989e117= used the official SDK with the native candidate +overlay ahead of previously qualified dependencies. The canonical LR owner, +runtime and fused strategy compiled in 95.047 seconds; the Rust emitter compiled +in 34.007 seconds. Both exited zero with final =OK= and the unchanged five-second +real-output idle gate. Four source GxTest suites passed all 22 cases against these +native owners in 27.542 seconds. The separate all-language ABI control passed in +15.793 seconds: twenty accepted/rejected products across ten entries matched the +checked-in Rust fixture bytes exactly. These local receipts do not establish +clean-checkout platform CI or a measured speedup. + +Receipts: unarchived local execution. The initial baseline-script failure remains +unarchived local execution. + +* Complete production admission before execution products + +The same audit at parent =93d179c= extended beyond semantic names. Prepared LR +trusted production numbering and precedence shape before selecting an executor; +layout detection accepted any pair beginning with =layout-end=. A retained +native baseline demonstrably admitted an incorrect production ID, zero-width +=pass=, empty or symbolic closing boundaries, and a string precedence rank. +Five accepted malformed productions are recorded in the baseline receipt. This +is an admission gap, not evidence of a valid grammar parsing incorrectly. + +=validate-production-semantics= in the canonical LR owner now establishes: + +- A proper production list, with each five-element production ID equal to its + ordered table offset and a symbolic nonterminal owner. +- Proper operand lists containing canonical token/literal/layout terminals or + nonterminal references; marked operands have exactly one base and one admitted + ordered action chain. +- =pass= has one operand, =concat= has any proper operand list, and =layout-end= + has no operands. Bare =layout-end= means column/EOF closure; a boundary-bearing + action has one or more nonempty strings. +- Precedence is absent or a two-element =none/left/right/dynamic= declaration + with an integer rank, matching the existing grammar header contract. + +Standalone LR lowering, runtime preparation, fused Scheme admission and raw-IR +Rust publication all consume this validator before indexing or publication. +Compiler and runtime layout classification use the same canonical action +tag predicate; boundary payload validation belongs to the canonical validator, +so repeated layout reductions do not rescan declarations. The runtime's duplicate +classifier is removed. Backend restrictions +remain explicit: admitting a layout production does not make it executable by +Rust or by the fused Scheme provider. Repeated reductions consume already +admitted plans and do not add validation callbacks. + +#+begin_src scheme +;; Engine-owned canonical products lowered from ordinary author declarations. +(validate-production-semantics + '((0 root ((terminal token word)) pass #f) + (1 items () (layout-end "END") #f))) + +;; Each fails before indexing, backend selection or Rust file publication. +(validate-production-semantics '((1 root () concat #f))) +(validate-production-semantics '((0 root () pass #f))) +(validate-production-semantics '((0 root () (layout-end END) #f))) +#+end_src + +** Question: why is this design preferable? + +The structural algebra has one owner, so a language author or POO provider does +not need to know which backend is stricter. Scheme matching states arity and +scalar constraints directly; an ordered traversal establishes the table-offset +invariant. This uses the grammar/lowering/preparation staging boundary studied +above. It adds no author slot, versioned interface or compatibility branch. +Existing direct IR fixtures are corrected to canonical terminal forms instead +of retaining a shorthand that the producer never emits. + +** Question: how is the claim scientifically tested? + +The falsifiers are an accepted malformed production, changed valid output, +validation bypass in an unused rule, or an output file modified after rejection. +Controls enumerate incorrect/duplicate IDs, malformed wrappers and terminals, +wrong reduction widths, dotted layout actions, invalid closing strings and +malformed precedence. Valid controls include all three reduction forms, static +and dynamic ranks, tokens, literals, layout markers and nonterminal operands. +Source lowering has its own rejection controls. Rust appends dead malformed +rules to real Arithmetic IR and must preserve an existing output file; the +earlier absent-file control remains. Full GLR/generated artifact parity and +ten-language ABI bytes provide independent valid-product checks. + +The structural check is linear in the production and operand declarations, +including their action chains. This is additional construction/preparation work; +no preparation timing or end-to-end speedup is inferred from staging it outside +requests. It does not validate LR transition targets, conflict equivalence, +reference resolution or lexer realizability; those have separate owners. +The subsequent closed-reference section below extends this canonical owner +with reference admission; the historical qualification remains unchanged. +The subsequent [[file:lr-table-admission.org][shared table study]] implements target bounds, row uniqueness +and closed action/guard shape admission through the same publication sequence. + +Final qualification used the official SDK and native overlay at parent +=93d179c=. The canonical owner, layout runtime and LR runtime compiled in +91.073 seconds; final LR, fused Scheme and Rust publication consumers compiled +in 48.244 seconds. Both supervisor receipts report exit zero, no rejection +reason, no missing required output and final =OK=. Six source GxTest suites +then passed all 32 cases in 29.442 seconds against these native owners. This +includes 23 malformed production families under both backend preferences, +standalone rejection through five LR construction routes, existing complete +GLR/generated products and twenty accepted/rejected ABI products across ten +language entries matching the Rust fixture bytes exactly. + +Rejected attempts remain explicit: the initial build failed the unchanged +five-second idle gate at GCC compilation of the LR compiler, exit 137 after +121.143 seconds. A standalone retry completed in 35.727 seconds; final consumers +were subsequently rebuilt against the final canonical owner. The first source +test entry failed before executing cases because a newly added test form lacked +one closing parenthesis, exit 1 after 10.948 seconds. The final test receipt +follows that source correction; no assertion or timeout was weakened. + +Receipts are unarchived local execution. Failed +attempts remain =parser-production-contract-build-supervision.log= and +=parser-production-contract-test-supervision.log= in the same directory; the +intermediate retry is =parser-production-contract-compiler-retry-supervision.log=. +Local native controls do not qualify a clean-checkout exact-head CI run or +establish any performance benefit. + +* Closed rule references across canonical IR consumers + +The complete production contract now resolves every nonterminal operand against +the complete production-owner index, after admitting production and operand +shapes. This includes marked operands and unused productions. It reuses +=production-index-by-lhs= in the canonical owner, rather than introducing a +language registry or a second reference policy in each backend. Forward +references, self recursion, mutual recursion and declared unused rules retain +their production order and identities. A missing owner raises +=invalid canonical LR unresolved reference= with the production ID and original +operand before nullable/FIRST construction or executable publication. + +This boundary applies to standalone LR compilation, recognition/event +preparation, POO fused strategy admission and raw Rust source publication. +The table gate continues to check vocabulary, physical targets and reachable +acceptance. Source declarations may be composed before lowering; their final +canonical product cannot defer a missing rule to a request. + +#+begin_src scheme +;; A complete product: later is a forward reference; unused is declared. +(def productions + '((0 root ((marked (nonterminal later) ((field value)))) pass #f) + (1 later ((terminal token word)) pass #f) + (2 unused ((nonterminal unused)) pass #f))) +(eq? productions (validate-production-semantics productions)) ; => #t + +;; A symbolic operand alone does not make the product complete. +(validate-production-semantics + '((0 root ((nonterminal missing)) pass #f))) ; raises before preparation + +;; Being unused does not excuse an unresolved owner. +(validate-production-semantics + '((0 root ((terminal token word)) pass #f) + (1 unused ((marked (nonterminal missing) ((alias Item)))) pass #f))) +#+end_src + +** Question: why is this design preferable? + +Scheme symbols retain identity across the S-expression IR and the existing +=eq?= owner index. Structural matching extracts the base of marked operands, +preserving the field/alias chain as diagnostic evidence. The shape pass admits +the entire declaration set before reference resolution, so source order does +not become a restriction on grammar composition. Authors write grammatical +references in =grammar.ss=; they do not supply dependency lists, resolution +callbacks or backend switches. POO strategy overrides are checked against the +same canonical owner before generated forms are emitted. This uses the DSL, +composition and preparation stages together, rather than relying on parentheses +or macro syntax alone. + +** Question: how is the claim scientifically tested? + +The independent falsifiers are a missing used owner, a missing owner in a dead +marked operand, changed admission under the event backend, or any output +created/overwritten before rejection. Positive controls retain forward and +recursive declarations and exact list identity. Five LR construction routes +exercise rejection before analysis. The fused override control requires an +empty output port; Rust controls require both an absent file and a preserved +existing sentinel. Existing language ABI, generated reductions and complete +artifact controls establish that valid products remain consumable. + +Knuth's [[https://harrymoreno.com/assets/greatPapersInCompSci/2.5_-_On_the_translation_of_languages_from_left_to_right-Donald_E._Knuth.pdf][1965 primary paper]], Section I, defines production operands over the +grammar's intermediate and terminal alphabets. It does not require every +intermediate to have a production. Requiring an owner here is an additional +closed-product policy of this engine, not a theorem derived from that paper. +Likewise, an existing owner does not establish a finite derivation, LR(k), +lexical realizability or grammar/table equivalence. Finite entry admission +remains the compiler's productive-analysis obligation; unused declared cycles +remain valid composition data. The reference check is linear in declarations +and operands with an indexed owner lookup; its preparation cost is additional +work and has no admitted parser-speedup claim. + +** Qualification + +At parent =1cfb23a=, the native canonical owner accepted the missing-owner +witness, exit zero in 1.066 seconds: +unarchived local execution. This counterexample +establishes the admission gap; it is not evidence of a production request crash. + +The final native build compiles the canonical owner and the stack, fused and +Rust generation test modules with optimization and static products in 74.116 +seconds. Thirteen suites then pass 117 cases in 26.889 seconds, including all +five construction routes, both semantic preferences, empty fused emission, +absent/existing Rust output, resolved grammar/source-origin controls and +ten-language ABI byte parity. The existing complete generated-reduction and +artifact controls remain in that matrix. Both process receipts report exit +zero, no rejection reason, no missing output and final =OK= under the unchanged +five-second real-progress idle gate. + +Receipts are unarchived local execution. This is focused native +qualification over reused dependencies, not a clean-checkout full-language +rebuild or exact-head platform CI. At parent =1cfb23a=, Rust CI passed while +Linux Scheme was running and macOS Scheme was queued. The new source slice +requires its own exact-head CI. No request-performance benefit is claimed. diff --git a/docs/research/prepared-parser-representation.org b/docs/research/prepared-parser-representation.org new file mode 100644 index 00000000..6bbe414b --- /dev/null +++ b/docs/research/prepared-parser-representation.org @@ -0,0 +1,242 @@ +#+title: Prepared parser representation and reduction ownership +#+startup: overview + +Part of [[file:shared-ranked-scanner-execution.org][the integrated execution study]]. +Historical revisions, failed experiments and receipt limits retain their original scope. + +* Engine-owned semantic representation of installed entries + +The complete-route audit above identifies two independent axes: the prepared +LR runtime's captured semantic executor and the installed generated entry's +product. =src/compiler/machine.ss= now makes those obligations queryable through +=parser-machine-backend-representation=. This is an engine capability contract, +not another language declaration or a backend-selection policy. + +| Entry query | Contract | +|-------------+----------| +| prepared | Captured recognition or event-program executor | +| drive | Installed whole-source driver returns recognition values | +| source | Installed source specialization returns ParseArtifact or declines with #f | +| step | Installed active reduction step uses the captured runtime representation | +| event-step | Separately installed optional event-program reduction step | + +Missing optional entries return =#f=. Unknown kinds and non-machine values reject. +The =step= slot deserves an explicit distinction: on the base recognition runtime +it holds a recognition step; the selected event runtime captures the admitted +event step into its active step slot. The separate =event-step= slot describes +an optional installation on the base runtime, not a claim that every selected +runtime retains that installation slot. Backend installation admits the same +closed kind domain before any mutation; digest and duplicate-slot checks retain +their existing atomic admission behavior. + +The representation review found an installation mismatch, not just a missing +label: recognition =step= declarations previously admitted an event machine. +The direct machine installer and the full-batch precheck now reject that pairing. +The batch precheck runs before all mutation, so a valid earlier drive remains +uninstalled when a later step has the wrong semantic runtime. Existing raw LR +step operations remain low-level active-step operations; the machine declaration +owner supplies this generated recognition-entry obligation. + +#+begin_src scheme +;; Preference selects the prepared executor; it does not rewrite generated code. +(let* ((selected + (parameterize ((current-lr-event-program-enabled? #t)) + (parser-machine-for-current-semantic-backend machine))) + (prepared (parser-machine-backend-representation selected 'prepared)) + (drive (parser-machine-backend-representation selected 'drive))) + ;; For an eligible machine with an installed recognition drive: + ;; prepared => event-program, drive => recognition. + ;; A later parameter change does not change either captured obligation. + (values prepared drive)) +#+end_src + +** Question: why is this abstraction preferable? + +The generated entry kind already expresses an engine responsibility. Deriving +its representation obligation there avoids another metadata annotation in +=parser.ss= and another POO wrapper around an ordinary procedure. Captured runtime +identity supplies the prepared representation. Named closed-domain algorithms +are appropriate here; the existing CLOS preparation protocol remains responsible +for backend/capability selection. No generic query, grammar scan or backend +selection occurs when describing a captured machine. + +This slice makes an existing semantic obligation explicit. It does not enable +event generation for a recognition drive, change the default backend, clear a +generated entry, alter canonical IR or add another compatibility route. + +** Question: what makes the claim testable? + +=t/language-surface-test.ss= adds a control that selects an event runtime, installs +a recognition drive on that selected machine and verifies both distinct products +are described correctly. The base machine remains unchanged. Switching the +preference back to recognition must not change the retained event machine's +reported representation. A source specialization that declines remains a +ParseArtifact-producing capability with ordinary LR fallback. + +The existing =benchmark-lr-request-backends= caller now reports these representation +obligations alongside requested preference, prepared admission, generated entry +availability and actual untimed source stages. Its twenty-four routes still +publish the canonical recognition reference before timing is admitted. This +combination separates capability from the route actually taken: a source entry +can decline, and an observed generated-drive stage has its own recognition +contract. The new fields are emitted outside timed intervals. + +The previous =source-recognition= stage is an inclusive interval and cannot by +itself distinguish a generated drive from ordinary checkpoints. The shared +=src/runtime/parser.ss= now reports =generated-drive-execution= or +=prepared-checkpoint-execution= at the actual branch. The benchmark asserts the +expected completed branch before admitting cost samples. These child intervals +are nested inside =source-recognition= and must not be added to it as independent +costs. Disabled observation retains the macro's direct expression path; it does +not invoke an observer thunk. No token-level tracing is introduced. + +The CI-owned =t/language-source-contract-test.ss= checks Arithmetic's actual +generated-drive branch under both preferences and compares its canonical artifact +with the unobserved product. GQL accepted, rejected and empty requests retain the +ordinary checkpoint witness. Existing =t/parse-cost-test.ss= controls preserve +multiple values, thrown conditions and observer scope. These controls connect +the declared representation, actual branch and published artifact. + +The query describes the engine's admitted obligation; it cannot inspect an +arbitrary procedure body and prove its semantic correctness. Known generated +implementations need artifact/field/span equality controls. No new speedup claim +is made by this change, and a query result alone does not qualify a full request. + +** Qualification of the representation contract + +The native machine/interface constraint rebuild completes in 32.273 s, and the +actual route owner/caller rebuild in 31.856 s. The final native interface suite +passes nine cases (10.707 s); source cost and source-contract suites pass nine +cases over the native producer (13.255 s). The final native cross-language caller +validates all twenty-four prepared/public routes in 15.774 s, with two samples +of two calls per route. Arithmetic reports actual generated-drive execution; +GQL and FHIRPath report checkpoint execution. All publish the canonical reference +under both requested preferences. Each gate retains exit 0, final =OK= and the +five-second idle fence; the caller also emits =LR-REQUEST-BACKENDS-OK=. + +These are local product/route controls over the reused dependency closure, +not a fresh full-package build, a paired CPU comparison or exact-head CI. The +next measured optimization must preserve the contracts and qualify complete +requests; this slice supplies their executable admission boundary. + +The final rebase incorporates =0a1b326='s paired sampler and negative controls. +Native sampler/caller rebuilding completes in 51.951 s; the merged caller +completes twenty ABBA/BAAB groups with two calls per batch in 24.615 s. Twelve +paired workload summaries cover the twenty-four representation routes, with +completed actual-branch witnesses and canonical products. Exit 0 and both final +markers retain the five-second gate. The GQL identical-executor control is stable; +complete-request benefit admission is =#f=. This smaller-batch run closes merged +execution qualification, not a new portable performance claim. Historical +two-sample receipts above are execution controls from the earlier caller. + +* Prepared identity reduction elimination + +** Scope and rejected alternatives + +The first four native comparisons use the production graph at =0a1b326= and +requested event preference. Each comparison executes candidate/reference/ +reference/candidate children, with twenty batches of twenty calls for three +languages, sizes 64 and 512, and prepared/full-source boundaries: 19,200 timed +calls per comparison. Independent materialized GLR and complete canonical +artifacts admit the products before measurement; every batch checks its result. + +Three earlier candidates were withdrawn: + +- Borrowing the event stack and applying actions while recursive traversal + unwinds removes the source-order operand list. Prepared allocation falls + about 9.4% for FHIRPath and 8.6% for Arithmetic at 512 units, but CPU medians + and unchanged GQL controls do not establish an acceleration. +- Empty/unary semantic dispatch avoids the generic reducer and join. Large + prepared medians sometimes improve, while small routes and the unchanged + generated Arithmetic driver remain variable. It is not retained as a CPU win. +- A single checked traversal pops both stacks and collects event operands. + Allocation is unchanged. Complete-request and prepared CPU comparisons do + not justify replacing the existing pop protocol. It is also withdrawn. + +These completed comparisons are negative optimization evidence, despite passing +21 semantic cases each. No backend default or admission margin is changed. +The first identity candidate then removes its redundant semantic call and stack +publication, with classification still inside execution. Four native children +repeat a 48-byte reduction per identity occurrence in this SDK's admitted +allocation intervals. The later prototypes move that classification to +preparation; the earlier event-preference receipts are not its default-path CPU +qualification. + +The first prepared marker used =#f= for identity and vectors for other plans. +Although allocation fell, its default-path comparison repeated a 17.5% CPU +increase in small prepared FHIRPath (mean of two child medians). That prototype +is withdrawn. The homogeneous candidate instead interns one valid unary vector. +Compiler diagnostics distinguish the result: the false-marker helper returns +=(nullable vector::t)=; the homogeneous helper returns =vector::t=. This supports +the representation distinction, not a causal explanation for the CPU result. + +** Withdrawn preparation proof and execution + +=prepare-reduction-plan= first validates the production action and every ordered +operand-action chain. Exactly one operand with an empty chain uses a shared +valid unary plan, =(vector '())=. All plans remain source-order vectors. Identity +is a private pointer-equality proof against that shared prepared plan. No grammar hint, +loader schema change, public option or language-specific rule is introduced. +This stages a closed semantic fact, following the engine's existing prepared +executor and staged operand-fold design. + +Execution retains the original semantic value and immutable semantic-stack cell +for that certified identity. State popping, goto lookup, state publication, +action counting, budget consumption and recognition capture still occur. There +is no corridor jump or mutable checkpoint. Decorated and wider productions keep +the original reducers. The independent GLR implementation remains unchanged. +Both captured semantic backends share the same proof, even when a generated +entry independently owns a different representation. + +The tests expand identity/decorated RHS widths through 128 and run decorated and +identity unary chains through depth 16 under both semantic backends. Every +budget-one pause increments the action count by exactly one; complete artifacts +match independent GLR and unpaused execution. Existing event tests retain +Unicode, generated entries, relocation, history edits and retained roots. + +** Native artifact and latest-graph admission + +The initial HCL test load exposed a stale compiled machine consumer: an older +inlined event-step getter read the event-reducer slot after the runtime record +had changed. Rebuilding the machine consumer restores the correct slot before +any comparison. This is an artifact dependency failure, not a language-parser +failure or a reason to bypass the generated-entry contract. + +Three unary-build attempts stop at the five-second idle fence, with exit 137: +63.485, 55.941 and 69.529 seconds. GCC diagnostics alone do not establish progress +through every compiler stage. The existing Scheme native artifact observer and +GCC saved intermediate products expose actual writes; no synthetic heartbeat or +larger deadline is introduced. A simplified unary candidate compiles in 30.935 +seconds but is later withdrawn on cost evidence. The prepared identity +false-marker prototype compiles in 35.073 seconds; the homogeneous candidate +compiles in 22.007 seconds with the fixed =2591dcd= SDK and GCC. + +Before final admission, the checkout incorporates =95de0d9= and =11b2cf1=. +The changed LR compiler and its consumer rebuild in 25.506 seconds. Machine, +parser and benchmark consumers rebuild in 56.895 seconds. The final four-module +source test child passes 23 cases in 14.458 seconds, with exit zero, final =OK= +and =TEST-SUITE-OK=. An earlier new-test syntax failure is corrected before this +admission. These are qualified changed-module consumers, not a fresh build of +all package products or a Rust AOT receipt. + +The identity comparison uses the latest caller and default recognition preference. +Only the LR module prefix changes between children; the fresh compiler, machine, +parser, sampler and lexical artifacts remain identical. Each child verifies its +loaded native LR suffix: reference =o6=, homogeneous candidate =o13=; lexical =o6= +remains fixed. The reference LR source is unchanged between =0a1b326= and +=11b2cf1=. Heap cap is 3 GiB, total budget 120 seconds and idle budget five seconds. +Warmup and initial GC follow the unchanged component sampler; naturally occurring +GC remains in CPU and wall observations. Allocation uses only admitted no-GC +counter deltas and is reported separately from CPU. + + +The homogeneous identity candidate is also withdrawn from production. Across +four default-path children, 848 admitted allocation intervals repeat the same +48-byte saving per identity reduction. An exact Scheme checker verifies the raw +integer batch deltas, identity counts and zero delta in generated Arithmetic +source execution. This establishes an allocation result for the prototype only. +CPU observations remain unstable, including a 16.35% increase in the unchanged +generated Arithmetic control when averaging child medians. No CPU improvement +or retained runtime acceleration follows from this result. The wider RHS and +budget-one identity-chain semantic tests remain useful regression coverage. + diff --git a/docs/research/ranked-scanner-specialization.org b/docs/research/ranked-scanner-specialization.org new file mode 100644 index 00000000..f765fcc8 --- /dev/null +++ b/docs/research/ranked-scanner-specialization.org @@ -0,0 +1,590 @@ +#+title: Ranked lexical specialization and rejected candidates +#+startup: overview + +Part of [[file:shared-ranked-scanner-execution.org][the integrated execution study]]. +Historical revisions, failed experiments and receipt limits retain their original scope. + +* Ranked literal publication experiment: rejected lowering + +The current Coding Style research requires known private calls, checked +boundaries, and admitted ownership. Reading the pinned Gerbil SDK =2591dcd=, +=src/std/text/parser/ll1.ss=, shows =ll1-uint= accumulating its scalar result +while consuming digits. This motivates retaining a decision during traversal +and constructing an observable result only at the output boundary. It does +not establish that an additional accumulator is free in a compiled lexer. +The earlier rejected dense region-trie experiment remains rejected. + +The shared ranked literal trie originally constructs a four-element result +for each admitted accepting prefix along the matching path. A prefix chain +can therefore allocate O(A) result spines for A accepted terminals even though +only one escapes. Three candidates reduced publication to O(1) per successful +call while retaining the same trie edges and terminal selection: an explicit +endpoint plus a named publication helper; a prepared terminal width with +publication after traversal; and an explicit endpoint with direct publication +at each exit. Traversal and terminal admission complexity were unchanged. +No language-specific rule, unchecked annotation, compatibility mode or schema +change was added. None of these runtime candidates is retained. + +** Native qualification and negative result + +The GCC SDK, 3 GiB heap, 120-second total guard and five-second silence guard +were fixed. Reference artifact =scan~0.o3=, helper candidate =scan~0.o4=, +width candidate =scan~0.o5= and direct-exit candidate =scan~0.o6= were verified +from the runtime loader in fresh processes. Complete artifact snapshots +retained contiguous object suffixes. No build or test from this task overlapped +timed controls. Identical interpreted orchestration invokes native scanner +closures; these are component measurements, not a compiled-parser latency claim. + +Each block takes eleven samples of 1,000 scanner calls. Preparation occurs +outside timing. The catalog has 32 accepting prefixes of one through 32 ASCII +characters. Controls admit only the shortest terminal, use a singleton catalog, +and reject every terminal through the mode mask. Endpoint sums are checked +for every sample. Natural GC is retained; allocation intervals crossing GC +remain unadmitted. + +The prepared-width candidate used reference/candidate/candidate/reference +blocks with the following CPU medians in milliseconds per 1,000-call sample: + +| Workload | Reference A | Width candidate A | Width candidate B | Reference B | +|----------+-------------+-------------------+-------------------+-------------| +| Prefix chain | 1.054 | 1.048 | 1.045 | 1.270 | +| Short admitted | 1.216 | 1.077 | 1.076 | 1.000 | +| Singleton | 0.215 | 0.287 | 0.282 | 0.211 | +| All filtered | 1.342 | 1.383 | 1.062 | 0.974 | + +The named-helper candidate also measured singleton medians of 0.286 and +0.288 ms against the initial reference's 0.214 ms. Direct exit publication +was then bracketed by the previous final reference and a new trailing reference: + +| Workload | Reference before | Direct A | Direct B | Reference after | +|----------+------------------+----------+----------+-----------------| +| Prefix chain | 1.270 | 1.139 | 1.072 | 1.218 | +| Short admitted | 1.000 | 1.198 | 1.048 | 1.002 | +| Singleton | 0.211 | 0.245 | 0.286 | 0.220 | +| All filtered | 0.974 | 0.957 | 1.033 | 0.982 | + +All three candidates reduce prefix-chain median allocation from 6,464,576 +to 512,576 bytes. The reference admits 10/11 allocation samples in every +reported block; candidates admit 11/11. Short-admitted and singleton allocation +remain 512,576 bytes, and all-filtered allocation remains 320,576 bytes, with +11/11 admission. These figures include unchanged interpreted orchestration. +Every successful control emits =RANKED-CONTROL-OK= and exits 0. + +Publication allocation decreases substantially, but singleton CPU is worse +across the tested representations, and shortest-admitted CPU also worsens +in some surrounding comparisons. The final direct-exit candidate therefore +fails the admission gate. The original runtime source and complete native +artifact snapshot are restored. No complete-request or Tree-sitter improvement +is claimed. These controls do not identify whether argument passing, generated +control flow or call-boundary behavior explains the CPU cost. That diagnosis +requires a compiled caller and complete lexer/request controls before another +production representation change. + +** Permanent contracts and reproducibility + +The independent linear catalog oracle compares every source position across +all 64 mode masks for six overlapping, tied-rank and Unicode declarations. +It validates longest match before rank and declaration order, rejected longer +terminals retaining shorter accepted candidates, and end-of-source behavior. +Additional cases mutate caller-owned declaration strings and returned results +to prove preparation and future-call isolation. These tests are retained +against the restored production scanner. + +The repository-owned =t/benchmarks/ranked-literal/benchmark.ss= exposes the +four workloads with positive exact sample/iteration counts, checked endpoint +sums, per-call CPU/allocation output and the existing GC-aware allocation +admission rule. It can be compiled and invoked as a native caller; no alternate +engine, JSON result snapshot or process-per-case wrapper is committed. +Two candidate edits initially failed reader admission with an incomplete form. +Both were fixed before measurement; those failed builds are excluded from +successful receipts. + +The permanent benchmark compiled natively in 4.643 seconds. On the restored +=b61522e= engine, eight source modules passed 70 cases and the compiled +benchmark completed two 1,000-call samples for each workload in 23.403 seconds +total. Receipts include =TEST-SUITE-OK modules=8=, =OK=, +=RANKED-LITERAL-OK= and exit 0. After rebasing onto the concurrent Fold revision +=26f96a5=, two directly owned scanner modules passed six cases and the same +compiled benchmark completed again in 23.223 seconds, with all success +receipts and exit 0. The concurrent Fold change is outside this experiment's +performance scope; neither these tests nor the micro table qualify its new +byte-frame implementation. The shared scanner source has no retained diff. + +* Closed-start specialization of the shared regular DFA + +The current Scheme Coding Style research favors binding-preserving syntax, +prepared decisions and small named loops. Reading the pinned Gerbil SDK +=2591dcd=, =src/gerbil/core/contract.ss= (=definline=) shows signature-aware +dispatch expansion rather than a promise that every abstract call is free. +The SDK's =src/std/text/parser/ll1.ss= likewise retains compact loop state. +The transfer here is to remove state that cannot occur, using a private +hygienic lowering whose expansion is inspected before measurement. + +** State argument and implementation + +A regular scanner's initial mask contains only the six declared run bits, +so it lies in 0..63. Number starts use bit 32. If an admitted initial mask +lies below 32, no transition can introduce a number state: the generic +number transition only advances already active number states. The other +five run families intersect their active mask with the current character +mask; they can lose bits but never gain them. Every surviving subset is +accepting. This path therefore needs neither pending fraction/exponent state +nor an accepted-subset/accepted-end rollback pair. + +=make-ranked-regular-scanner= retains its prepared winner table and numeric +state machine. At matching entry, a nonnumeric initial subset selects the +existing monotone run algorithm, even when the grammar also declares a +number rule. Numeric starts keep their original rollback behavior. Modes +without number rules also use the same monotone lowering. The private +=scan-regular-run= syntax expands that algorithm at both call sites using +caller bindings and evaluates the initial offset once. No callback, table, +public scanner API, language metadata, schema version or compatibility route +is added. The final implementation adds no unsafe operation or type assertion. + +** Rejected contract lowering and generated-code inspection + +The first =definline= attempt with an empty argument contract failed macro +admission with =invalid match target=. Supplying a checked string contract +compiled correctly, and emitted Scheme showed the loop expanded at both +call sites, with a remaining =string?= check and parameter binding group. +A subsequent private string assertion relied on the caller's preceding +=string-length= and removed that check. It still failed the complete-lexer +control: batched HCL lexer medians were 60.20/61.50 microseconds for the +surrounding references and 64.80/65.95 for the candidate. Neither contract +version is retained. These measurements do not prove that the bindings +caused the difference. + +The final implementation uses Gerbil's ordinary hygienic =defrules= mechanism +for the private loop. Generated Scheme contains the expanded cursor/active +loop at each call site, with no helper call, contract check or contract +parameter binding group. Entry selection uses the proven initial range +instead of computing a bit intersection and then testing zero. + +** Native caller reference/candidate/candidate/reference control + +The shared engine benchmark is compiled with the same GCC-backed SDK before +all measured blocks. It invokes native scanner closures inside native loops; +GXI only starts the entry. The 3 GiB heap, 120-second total guard and five-second +silence guard remain fixed. Loaded artifacts are asserted: reference +=scan~0.o3= and final candidate =scan~0.o7=. Complete owned snapshots preserve +contiguous loader suffixes; other clients are unchanged. No build or test +from this task overlaps timing. The final scanner build passed in 10.568 +seconds; the preserved legacy benchmark and new request-cost entry compiled +in 15.193 seconds. + +Each block uses eleven samples. The six scanner controls run 100 calls per +sample. Long inputs contain 4,096 ASCII identifier characters, Unicode +identifier characters, spaces or decimal digits. Singleton and unmatched +inputs are one character. Complete HCL lexer samples run 20 calls on a +64-line assignment source. Independent =direct-lex-hcl= equality checks +complete token products before measurement and the last product of each +sample after measurement. Preparation and equality are outside timing. + +CPU medians below are microseconds per call: + +| Workload | Reference A | Candidate A | Candidate B | Reference B | +|----------+-------------+-------------+-------------+-------------| +| ASCII identifier | 62.84 | 26.48 | 26.48 | 56.95 | +| Unicode identifier | 194.85 | 173.38 | 146.16 | 203.53 | +| Whitespace | 58.11 | 29.76 | 26.44 | 66.38 | +| Numeric | 56.56 | 60.84 | 51.84 | 51.77 | +| Singleton | 0.07 | 0.03 | 0.03 | 0.03 | +| Unmatched | 0.02 | 0.05 | 0.02 | 0.02 | +| Complete HCL lexer | 60.60 | 63.50 | 56.00 | 61.85 | + +Successful scanner allocation remains 194.08 bytes per call, and unmatched +allocation remains 2.08, including measurement overhead; every scanner block +admits 11/11 allocation samples. Complete lexer median allocation remains +336,906.4 bytes per call with 10/11 admitted samples in every block. Natural +GC is retained, with the existing allocation admission rule unchanged. All +four blocks emit =REGULAR-MODE-OK= and =REGULAR-CONTROL-OK= and exit 0. + +Long ASCII and whitespace scan costs improve by roughly a factor of two +in both candidate blocks. Unicode scans also improve. Numeric and very short +controls vary; the short timings are too small for a portable speed claim. +Complete lexer CPU moves in opposite directions across candidate blocks, +so no stable complete-lexer or complete-parser speedup is established. This +is a shared engine state specialization, not closure of the Tree-sitter gap. +An earlier one-call complete-lexer protocol was visibly noisy and was +replaced by the same twenty-call protocol on both sides; its timings are +excluded from the final table. + +The existing =t/benchmarks/regular-mode/benchmark.ss= CLI and its independent +old-mode oracle are preserved. The adjacent =request-cost.ss= owns these +native caller controls and complete-token checks. No language implementation +file is modified. Permanent tests compare the subset DFA against independent +individual run scanners at every input offset across all 64 rule subsets, +including Unicode numeric characters and truncated fraction/exponent forms. + +** Qualification after the latest PR rebase + +After rebasing onto revision =6dbc112=, eight source test modules passed +71 cases: ranked regular and literal scanning, lexical plans and first-character +contracts, structured lexical behavior, parser runtime, region scanning, and +Bash. The regular suite includes the independent comparison across all 64 +rule subsets. The compiled =request-cost.ss= entry also completed its scanner +result and complete HCL token checks. The combined qualification exited 0 in +29.465 seconds, under the unchanged 120-second total and five-second idle +limits with a 3 GiB heap. This smoke run establishes semantic qualification; +the performance conclusions remain bounded by the native ABBA controls above. + +A subsequent remote LR update (=31c0667=) arrived during publication and was +rebased before delivery. Its runtime module rebuilt with GCC in 18.117 seconds. +The regular scanner, parser runtime, and Bash suites passed 42 cases together. +The direct GXI entry emitted both success markers but stalled during process +shutdown; the unchanged idle guard rejected that run after 18.046 seconds. +Repeating through the existing test-only process exit interface completed in +11.954 seconds with exit 0 and both required markers. That receipt qualifies +the merged runtime; it does not establish that the direct GXI shutdown issue +is resolved. + +* First-character competition staging and callback ownership + +** Source study and phase ownership + +The pinned Gerbil SDK revision =2591dcd= provides a useful distinction in +=src/std/text/parser/ll1.ss=. The =ll1-bind= and =ll1-begin= combinators compose +procedures before reader execution, whereas =ll1-or/list= retains a named loop +where input-dependent failure determines the next alternative. The =peeker= +constructor resolves the specification's outer representation before the +resulting predicate sees input. These are concrete implementation patterns, +not evidence that every closure composition improves performance. + +The existing [[file:scheme-dsl-research-basis.org][DSL research basis]] and +[[file:scheme-macro-binding-experiments.org][binding experiments]] retain +language declaration context and place derived execution mechanisms in the +shared engine. These experiments need no author macro, POO slot, grammar field, +unsafe assertion, or compatibility path. They investigate prepared dispatch +and callback ownership inside the existing ranked lexical algebra. + +Previously, an ASCII table selected the nonregular candidate list, but each +token folded that list with a source/offset callback and checked whether regular +and literal scanners participated. The participation facts are derivable during +preparation. The experiments below show why derivability alone is insufficient +to admit a new execution representation. + +** Historical execution candidate, withdrawn after the cross-language audit + +The initially retained candidate kept the existing ASCII table, mode admission, +conservative non-ASCII path, regular/literal dispatch and mode-miss fallback. +The private =scan-ranked-candidates= hygienic macro preserves the stable +source, offset and candidate bindings at its sole call site. It applies the +existing =prefer-generated-match= in a named loop without a new procedure +argument boundary. It preserves +all candidate calls and their order, including no-match results. The caller no +longer constructs a source-dependent fold callback for each token. There is no +new prepared procedure catalog, nested composition graph or transient interning +cost. This isolates callback ownership from the rejected staging designs. + +The original nonregular order, regular scanner and literal scanner order remain +unchanged. Token materialization and byte extents retain their existing owner. +No claim about eliminating all scanner allocation follows from removing this +one callback. The fixed GCC compiler owns ordinary procedure and sequence checks. + +** Independent semantic controls + +The initial staged prototype passes six modules with 55 cases in 21.287 seconds +and exit 0: first-character admission, lexical plans, ranked regular and literal +scanners, parser runtime, and Bash. The new first-character case uses independent +instrumented scanners to check evaluation order, longest match before priority, +a missing candidate, Unicode byte offsets and an empty ASCII projection. +Existing controls retain unfiltered reference dispatch, declaration-order ties, +directed mode fallback, all 64 regular subsets and complete incremental products. + +Native callers are held constant within each matched control series. Reference +and candidate lexical module objects are selected explicitly before process +admission. The 3 GiB heap, 120-second total bound and five-second idle guard +remain unchanged. Compilation and semantic admission are outside measured +request batches. + +** Rejected nested-composition prototype + +The first prototype composed a captured previous procedure and next scanner for +each ranked fold edge. The complete HCL lexer allocated 312,330.4 bytes instead +of 336,906.4 in both candidate blocks. Its CPU medians were 49.05 and 47.40 +microseconds per call, against reference values 98.75 and 58.50; the reference +itself varied considerably. Each block used eleven samples of twenty lexer +calls and checked complete tokens against the independent direct HCL lexer. + +That component result did not justify retention. Native complete requests use +twenty samples of twenty calls at sizes 64 and 512, with complete source-bound +artifacts admitted against independent materialized GLR output. The 512-item +GQL route regressed in both candidate blocks: 2.55185 and 2.50270 ms per call, +against references 2.29435 and 2.33710. Prepared LR allocation remained unchanged +while complete requests allocated fewer bytes. CPU improvements on some other +routes could not override this repeated regression. + +The prototype also increased HCL lexer preparation allocation from 353,360 to +1,023,200 bytes per call; all eleven samples in each block were admitted. +Preparation CPU medians were 0.185 and 0.196 ms for the references, versus +0.407 and 0.406 for the candidates. Those calls execute the factory once per +sample and check its returned lexer against independent complete tokens after +timing. The two cost boundaries therefore exposed both request and setup risks. + +A second staging prototype replaces nested composition with a named loop +and interns tuple keys before building procedure lists, so equivalent ASCII +classes do not repeatedly allocate those lists. It retains staged participation +and removes the per-token fold callback. Nested frames remain an investigation +hypothesis, not a proven isolated cause of the first prototype's CPU change. +The second prototype was also rejected, as recorded below. + +** Rejected staged-loop prototype + +The second prototype's complete-request ABBA still leaves the 512-item GQL +candidate above both references. Prepared LR CPU also varies, so these data do +not isolate the cause or establish a speed equivalence margin. The prototype +is not retained as a performance optimization. CPU values below are batch CPU +P50 divided by twenty, in milliseconds per call; allocation is admitted +bytes per call. Complete products remain admitted against independent GLR. + +| Route | Reference A | Candidate A | Candidate B | Reference B | Reference bytes | Candidate bytes | +|-------+-------------+-------------+-------------+-------------+-----------------+-----------------| +| gql-return 64 prepared-lr | 0.26665 | 0.22170 | 0.28530 | 0.21620 | 230932. | 230932. | +| gql-return 64 full-source | 0.36290 | 0.31155 | 0.35580 | 0.32865 | 542941.6 | 530589.6 | +| fhirpath-addition 64 prepared-lr | 0.22660 | 0.21915 | 0.22460 | 0.23470 | 255540. | 255540. | +| fhirpath-addition 64 full-source | 0.39165 | 0.35525 | 0.36910 | 0.35280 | 695181.6 | 678989.6 | +| arithmetic-addition 64 prepared-lr | 0.09380 | 0.12410 | 0.09520 | 0.09490 | 134804. | 134804. | +| arithmetic-addition 64 full-source | 0.15455 | 0.15155 | 0.15885 | 0.15105 | 462141.6 | 445949.6 | +| gql-return 512 prepared-lr | 1.77545 | 2.03600 | 1.98900 | 1.84755 | 1894004. | 1894004. | +| gql-return 512 full-source | 2.77905 | 2.96355 | 3.15610 | 2.79175 | 4330101.6 | 4231733.6 | +| fhirpath-addition 512 prepared-lr | 1.76920 | 1.76395 | 1.99520 | 1.79975 | 2104980. | 2104980. | +| fhirpath-addition 512 full-source | 3.59885 | 3.29520 | 3.55690 | 3.40980 | 5585421.6 | 5454541.6 | +| arithmetic-addition 512 prepared-lr | 0.79980 | 0.89065 | 0.75345 | 0.76800 | 1080980. | 1080980. | +| arithmetic-addition 512 full-source | 1.44400 | 1.17690 | 1.19105 | 1.18840 | 3696477.6 | 3565597.6 | + +HCL preparation allocation falls from the first prototype's 1,023,200 bytes to +656,624, but still exceeds the reference's 353,360. Eleven of eleven samples +are admitted in every block. Preparation CPU medians are 0.172, 0.353, 0.435 +and 0.175 ms in ABBA order. Thus reducing repeated prepared-key construction +improves that prototype's setup cost without qualifying it for retention. + +The complete HCL lexer still allocates 312,330.4 candidate bytes per call, +against 336,906.4 reference bytes. Its CPU medians are 75.35, 63.35, 50.10 +and 53.60 microseconds per call in ABBA order, with eleven samples of twenty +calls. Allocation admission is 11/11, 10/11, 10/11 and 11/11 respectively. +Those component results cannot override the complete GQL request risk. + + +** Procedure-boundary loop experiment and system contention + +A third candidate preserved the original preparation and dispatch but moved the +candidate fold into an ordinary named procedure. Preparation allocation stayed +at 353,360 bytes, and the complete HCL lexer allocated 312,330.4 bytes. However, +its HCL CPU medians were 97.35 and 116.75 microseconds per call, against references +54.25 and 55.50. Those timings do not qualify the procedure form as a CPU +optimization. The focused 512-item GQL ABBA also varies, including unchanged +prepared-LR controls: complete-request CPU medians are 3.38985, 3.53775, +3.36565 and 3.67730 ms in order. No stable complete-request speedup follows. + +A live process inventory during the last series confirmed concurrent GXI, GSC +and GCC compilation processes. The logs also show substantial wall-minus-CPU +intervals. This is evidence that the measurement environment was not exclusive; +it does not isolate the cause of any earlier prototype's CPU result or excuse a +regression. The procedure-boundary version is not retained. The next lowering +uses a private hygienic macro, following the qualified regular-run lowering in +the preceding section, to preserve call-site bindings directly. + +Two preliminary focused-control entries failed before sampling because they +used a parser-module namespace for the grammar-owned GQL machine binding. +The generated interface identified the correct grammar binding; the corrected +four blocks completed with every required marker and exit 0. These entry errors +are excluded from the performance table. + +** Rebase compilation boundary + +The work rebases onto =aff9375=. Byte-span helpers, source-line helpers and the +new completion-policy module compile before the affected LR runtime. The GCC +invocation for =lr-parser~0= then triggers the unchanged five-second silence +guard; the overall run exits 124 after 72.626 seconds, with an idle-timeout +reason and no completed new LR object. This failure is retained. No compiler +switch or deadline increase is used to admit it. The shared lexical module +with the hygienic loop independently compiles with GCC in 39.100 seconds and +exits 0. Its component receipts do not by themselves qualify that failed LR +rebuild or establish whole-head native closure. + +** Hygienic loop component control + +The final component series runs candidate/reference/reference/candidate in +that order. Each block uses the same compiled =request-cost.ss= caller, +independently checks complete HCL tokens, and asserts the loaded lexical phase. +The native lowering has no separate =scan-ranked-candidates= procedure: the +named loop is in the prepared scanner's lexical body. Ordinary checked engine +operations and the original prepared tables remain intact. + +| Boundary | Candidate A | Reference A | Reference B | Candidate B | +|----------+-------------+-------------+-------------+-------------| +| Complete HCL lexer CPU, us/call | 50.300 | 60.750 | 60.200 | 83.800 | +| complete-hcl-lexer bytes/call | 312330.4 | 336906.4 | 336906.4 | 312330.4 | +| complete-hcl-lexer allocation admission | 10/11 | 10/11 | 10/11 | 10/11 | +| HCL preparation CPU, ms/call | 0.175 | 0.221 | 0.203 | 0.202 | +| hcl-lexer-preparation bytes/call | 353360.0 | 353360.0 | 353360.0 | 353360.0 | +| hcl-lexer-preparation allocation admission | 11/11 | 11/11 | 11/11 | 11/11 | + +Preparation allocation is identical in all blocks. The complete lexer saves +24,576 admitted bytes per call (about 7.3 percent) in both candidates. Existing +regular scanner microcontrols retain their products and allocation. CPU is +reported with the known contention boundary; no stable complete-parser CPU +improvement or portable latency multiplier is claimed. These receipts qualify +the shared lexical component, not the previously failed fresh LR rebuild. + +A bounded retry compiles only the LR runtime and its current native benchmark +caller. It completes in 64.801 seconds with exit 0 under the same GCC toolchain, +heap, total limit and idle guard. The earlier failed run remains part of the +receipt. The successful retry resolves the fresh native object gate for these +related modules; it does not prove that shared-host contention is absent or that +all unrelated runtime owners have been requalified locally. + +The rebased test phase at =aff9375= completes all eight modules and 63 cases, +including completion-policy and generic dispatch contracts, and emits both +=TEST-SUITE-OK modules=8= and =OK=. The combined entry then fails while loading +the current benchmark's unbuilt selective-GLR fixture; its overall exit is 1, +so that combined command is not reported as a successful gate. Building only +that missing fixture completes in 15.494 seconds with exit 0. A separate native +control then checks explicit current LR and lexical phase identities, complete +GQL/FHIRPath/Arithmetic artifacts against independent GLR, HCL token equality, +and regular scanner products. It emits all four required markers and exits 0 +in 11.097 seconds. Source phase results and the corrected native gate remain +separate receipts; no repeated source test run is used to conceal the fixture +admission failure. + +* Cross-language audit of callback removal + +** Admission correction + +The preceding HCL scanner receipt is insufficient to retain a shared runtime +optimization. The macro was therefore audited on GQL, FHIRPath and Arithmetic +at both 64 and 512 units using the existing +[[file:../../t/benchmarks/lr-request-stack/benchmark.ss][complete-request benchmark]]. +The series order was candidate A, reference A, reference B, candidate B. Each +route used twelve samples of ten calls, one initial GC and natural timed GC. +CPU values below are batch medians divided by ten, not individual request +percentiles. Import, compilation, product validation and derivation diagnostics +are outside timed intervals. Each complete artifact agrees with the independent +materialized GLR interpreter and owns the original source. + +The LR runtime was freshly compiled from the semantic preparation refactor +included in =f3dff04=. All four children explicitly admitted =lr-parser~0.o6=; +only the lexical module varied between the original fold =.o1= and the private +macro =.o5=. Both use the same language modules and native benchmark caller. +The owned checkout was aligned with =b1b13e0= before qualification. The unchanged +GCC SDK, 3 GiB heap, 120-second total bound and five-second idle guard applied. +All four children exited zero with both required markers in 25.807, 29.299, +16.522 and 20.123 seconds respectively. + +| Family and units | Boundary | Candidate A ms | Reference A ms | Reference B ms | Candidate B ms | +|------------------+----------+----------------+----------------+----------------+----------------| +| GQL 64 | prepared LR | 0.2310 | 0.2638 | 0.2314 | 0.2437 | +| GQL 64 | full source | 0.5827 | 0.4148 | 0.3278 | 0.3369 | +| FHIRPath 64 | prepared LR | 0.2660 | 0.2180 | 0.2645 | 0.2184 | +| FHIRPath 64 | full source | 0.4186 | 0.3917 | 0.3599 | 0.3594 | +| Arithmetic 64 | prepared LR | 0.1068 | 0.1021 | 0.1034 | 0.0982 | +| Arithmetic 64 | full source | 0.1864 | 0.1514 | 0.1539 | 0.1490 | +| GQL 512 | prepared LR | 2.0286 | 1.9738 | 2.1479 | 1.9309 | +| GQL 512 | full source | 2.9448 | 2.9773 | 2.7429 | 3.3626 | +| FHIRPath 512 | prepared LR | 2.4565 | 1.9853 | 1.7413 | 2.1148 | +| FHIRPath 512 | full source | 3.2144 | 3.3384 | 3.4735 | 4.0724 | +| Arithmetic 512 | prepared LR | 0.9260 | 0.8371 | 0.8229 | 0.8247 | +| Arithmetic 512 | full source | 1.4478 | 1.2146 | 1.4126 | 1.2914 | + +Full-request allocation decreases reproducibly: GQL 512 from 4,330,163.2 to +4,231,795.2 bytes, FHIRPath 512 from 5,585,483.2 to 5,454,603.2 bytes, and +Arithmetic 512 from 3,696,539.2 to 3,565,659.2 bytes. Prepared LR allocation is +unchanged in every family. These are admitted no-GC intervals, not raw deltas +across collections. + +CPU does not provide a consistent complete-request benefit. Candidate A's +small GQL request and candidate B's large GQL and FHIRPath requests are slower +than both references. Prepared LR observations also vary despite unchanged +code, so this series cannot establish a causal regression percentage. It does +establish that retention as a CPU optimization is unsupported. The macro is +withdrawn and the original fold restored. Independent candidate-order and +Unicode controls remain useful and stay in the test suite. The earlier HCL +allocation receipt remains a historical observation, not a current runtime win. + +** Broader Gerbil source study and structural experiment + +The installed =2591dcd= sources add distinct representation contracts to the +previous LL1 and iterator study: + +| SDK source | Mechanism inspected | Engine consequence | +|------------+---------------------+--------------------| +| src/std/text/parser/ebnf/stream.ss | char-stream-getc and location-getc keep lookahead and location ownership together | Request position state must have an explicit owner; stream state is not a free replacement for source-bound artifacts | +| src/std/text/parser/ebnf/lexer.ss | token-stream-get removes ignored tokens in a named loop; token-stream-next caches its EOF token | Name the execution algorithm and retain stable prepared state without creating a new language interface | +| src/std/struct/queue.ss | Typed private fields and tail-cell insertion; queue->list publishes a copy | Mutable construction and published representation have different ownership and allocation costs | +| src/std/vector/extensible.ss | Geometric storage growth, fill-pointer contract and boundary conversion | Growth policy must be evaluated with construction and final publication; substituting vectors for lists alone is insufficient | +| src/gerbil/runtime/hash.ss | defhash-method stages typed access; hash-fold retains a dynamic callback | Hygienic staging and runtime composition both exist in the SDK; style alone cannot decide their request cost | + +The next retained implementation is a benchmark capability, not another lexical +loop substitution. =benchmark-lr-request-backends= compares the existing +recognition and event representations across the same three languages and two +scales. It records requested preference and prepared backend admission separately; an +ineligible grammar remains a recognition fallback. An untimed cost observer +records actual source stages together with installed generated drive/source +capabilities. Its complete artifact is checked before timing; the observer is +absent from measured calls. The reference public artifact is always +constructed with recognition enabled. Both the selected prepared root and the +independent materialized GLR root must publish that same canonical artifact. +Timed public requests must agree with it too. Derivation instrumentation stays +on recognition and outside measured intervals. + +This closes a previous measurement gap: selecting an event parameter does not +prove that a language admitted the event executor, and comparing roots directly +would confuse representation with semantic equivalence. Complete source parsing +and prepared recognition remain separate workloads, not additive parts of a +cost decomposition. Language declarations and public loader slots are unchanged. + +** Structural qualification and remaining boundaries + +The first native representation control uses twelve samples of ten calls per +route with the original scanner fold restored. Six cases pass in the +first-character and semantic-preparation suites against the fresh native LR +runtime. Both explicitly admitted runtime objects are =.o6=. All twenty-four +prepared/full-source routes pass complete product controls; the child exits zero +in 19.810 seconds with =TEST-SUITE-OK=, =LR-REQUEST-BACKENDS-OK= and +=BACKEND-CONTROL-OK=. The lexical module and benchmark caller build together in +20.535 seconds. The earlier fresh LR build completes in 30.484 seconds with +the unchanged compiler and guard, closing that local artifact gate. + +| Family, 512 units | Recognition prepared CPU ms | Requested-event prepared CPU ms | Recognition full CPU ms | Requested-event full CPU ms | Prepared event admitted? | +|-------------------+-----------------------------+---------------------------------+-------------------------+-----------------------------+--------------------------| +| GQL | 1.7837 | 1.7961 | 3.0839 | 2.9312 | no | +| FHIRPath | 1.8466 | 2.8944 | 3.3349 | 4.0219 | yes | +| Arithmetic | 0.9576 | 1.4696 | 1.2874 | 1.3076 | yes | + +This is a representation diagnostic in recognition-then-event order, not an +alternating performance qualification. The event executor saves some prepared +allocation but its CPU observations do not justify enabling it by default. +GQL's negative eligibility is a control, not an event execution result. +Arithmetic's installed generated drive constructs recognition fragments directly; +its full-source allocation remains 3,696,539.2 bytes under both preferences even +though its prepared event route drops from 1,081,000 to 851,992 bytes. FHIRPath's +ordinary checkpoint path admits the event executor and exhibits its cost in both +prepared and complete requests. Prepared backend admission therefore cannot be +reported as proof of full-source executor selection. + +The shared follow-up now has explicit falsifiers: generated execution must own +its semantic representation contract; prepared and generated paths must publish +the same canonical artifact; and a representation change must qualify complete +requests across these distinct routes before becoming a default. Neither a +macro style preference nor reduced allocation closes that obligation. Bash's +contextual source parser is a separate engine route and is not qualified by this +LR family measurement. + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/lr-request-stack/benchmark) +(benchmark-lr-request-backends 20 20) +#+end_src + +Compile the current runtime and benchmark before native qualification. Source +loading, native artifact identity, semantic tests and completed exact-head CI +remain separate evidence boundaries. No process supervisor or alternate engine +is added to the repository for this study. + +The final route-witness caller is rebuilt natively without the observer in timed +calls. It repeats all twenty-four product-controlled routes at twelve samples +of ten calls, explicitly admits both =.o6= runtime objects, and exits zero in +58.928 seconds with =LR-REQUEST-BACKENDS-OK= and =BACKEND-CONTROL-OK=. The +witness confirms Arithmetic's generated drive and the ordinary GQL/FHIRPath +source path at both scales. This is a local receipt; PR CI belongs to the +subsequent exact pushed head. + diff --git a/docs/research/record-engine-cost.org b/docs/research/record-engine-cost.org new file mode 100644 index 00000000..b3586ffd --- /dev/null +++ b/docs/research/record-engine-cost.org @@ -0,0 +1,108 @@ +#+TITLE: Prepared Scanner and LR Kernels for Complete Record Requests +#+DATE: 2026-10-11 + +* Scope and algorithm + +This change moves grammar-static work into engine preparation. It does not +change language grammars, the Loader schema, longest-match ranking, semantic +reductions, or the public C ABI. + +The contextual scanner prepares the result kind for each rule and declared +parser position. A single-position rule captures its kind; other rules retain +a position lookup. Missing dispatch still fails only when its rule wins, so +candidate ranking and ambiguity checks retain their meaning. The keep action +has a prepared identity kernel. Single-character literals capture their scalar +value, and fixed literals capture their UTF-8 byte width. Variable-width +matchers still count the actual matched text. + +The ordinary LR action selector prepares one kernel per state. Token-only +states skip literal and case-fold probes. Token indexes specialize empty, +singleton, small symbol alist, and table shapes once during preparation. Literal +states preserve exact-literal priority, case-folding, EOF, duplicate-entry +selection and original action-pair identity. Layout selection is unchanged. + +* Conformance and independent controls + +The scanner tests cover direct IR and prepared plans, including successive +1/2/3/4-byte scalar offsets. Existing tests cover modes, queued actions, +longest-match, ambiguity, missing dispatch and checkpoint ownership. LR tests +compare the prepared selector with the independent reference on EOF, literal +priority, case-folding, duplicate entries, forks and small/wide token indexes; +they require identical action pairs. + +The Scheme benchmark constructs two independent public C ABI handles. Its +scanner control restores the former dispatch body, action interpretation, +literal prefix matcher and per-lexeme UTF-8 counting. Its LR control uses the +existing independent action selector. Both handles retain the same generated +reduction step, source decoder, event writer and complete GPA1 publication. +Private accessor resolution and preparation occur outside timing. + +Every timed call includes C input ownership, source admission, contextual +scanning, LR parsing, semantic construction, GPA1 publication, C result ownership +and readback. The 64-record sources are 256 ASCII bytes or 320 Unicode bytes; +both produce the complete 15,488-byte payload. Accepted, rejected, empty and +Unicode products are compared outside timing as well. + +The controls are source-owned former-algorithm kernels in the current binary, +not a historical binary comparison. These measurements qualify this record +workload; they do not establish a speedup for every language or grammar. + +* Admission and retained results + +The existing Scheme sampler uses 20 groups, 256 calls per batch and eight +balanced legs per comparison. CPU includes GC. Before/after control P10 must +be at least 0.90 and P90 at most 1.10; production P90 must be below 0.95. No +sample deletion, forced GC or threshold adjustment is used. Linux CI inherits +one allowed processor's affinity; each child retains the 120-second total, +5-second idle and 3-GiB heap limits. + +Initial controls with additional dispatch wrappers were invalid and are not +admitted as performance evidence. After correcting call depth, the earlier +scanner-only candidate had P50 0.897780 but P90 0.966302 and failed admission. +The earlier LR-only candidate had P50 0.918003 but P90 0.992806 and also failed. +Those failures motivated fixed-width scanner staging and token-row LR kernels; +median-only improvements were not treated as proof. + +| Final local combined workload | Control before P10/P50/P90 | Production P10/P50/P90 | Control after P10/P50/P90 | Admission | +|-------------------------------+---------------------------+-----------------------+--------------------------+-----------| +| ASCII, 64 records | 0.943768 / 1.020346 / 1.081533 | 0.735750 / 0.814937 / 0.890024 | 0.907160 / 0.999437 / 1.087364 | Passed | + +The local Unicode combined run had production P50 0.832936 and P90 0.880413, +but before-control P10 0.898407 failed the 0.90 gate. It is rejected, not admitted +as a speedup. The final scanner-only run had stable controls and P50 0.904196, +but P90 0.967412 failed the strict benefit gate. These receipts are retained; +Linux affinity qualification is a separate environment, not a discarded-sample +retry. + +The final LR-only run had P50 0.907958 and P90 1.017276, with after-control +P10 0.899800. It failed both control stability and strict benefit admission. + +The ASCII complete-request median CPU reduction is 18.51%. Individual scanner +and LR isolation runs are diagnostic obligations and are reported separately; +the combined result is not a claim that each component meets the benefit gate. + +* Reproduction and qualification ownership + +The fixture is =t/fixtures/contextual-result-selection.ss=. The complete-request +sampler is =t/benchmarks/language-abi/utf8-cpu.ss= and the strict entrypoint is +=t/benchmarks/language-abi/engine-proof.ss=. =build-ffi-tests.ss= owns fixture +and sampler compilation. From a prepared repository environment: + +#+begin_src sh +for unicode in false true; do + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log ".data/record-engine-${unicode}-cpu.log" \ + --require ENGINE-CPU-PROOF-OK \ + -- gxi t/benchmarks/language-abi/engine-proof.ss combined "$unicode" +done +#+end_src + +The first remote qualification stopped before CPU sampling: the new ordinary +LR Suite was exported but omitted from =t/conformance-main.ss=. The ownership +gate rejected this omission. The Suite is now both imported and executed in the +existing conformance group, with no duplicate source execution. + +Use =scanner= or =lr= in place of =combined= for independent component controls. +The existing workflow dispatch runs both complete combined workloads on Linux, +with unchanged ranked-parser, FFI event-publication and UTF-8 obligations. Local +receipts and exact-head CI receipts remain distinct evidence. diff --git a/docs/research/request-lookahead-classification.org b/docs/research/request-lookahead-classification.org new file mode 100644 index 00000000..9ca46c39 --- /dev/null +++ b/docs/research/request-lookahead-classification.org @@ -0,0 +1,133 @@ +#+title: LR lookahead classification: request evidence and rejected cache + +* One request-cost obligation + +The [[file:lr-request-cost-boundaries.org][request-cost study]] identifies prepared LR execution as a major phase +of complete GQL requests. A lookahead survives several reductions. The candidate +classified it once per input tail, rather than selecting its literal independently +in every state. It changed neither recognition semantics nor the language DSL. +The [[file:unary-reduction-corridors.org][corridor experiments]] retain their separate negative results. + +The staging rationale remains the Scheme LR specialization and static-analysis +papers discussed in [[file:dsl-effective-grammar-admission.org][effective grammar admission]]. They motivate separating +grammar facts from invocation facts. They do not prove this implementation faster. +Complete requests rejected it, despite passed semantic controls. + +* Candidate ownership and invariant + +The immutable prepared vocabulary interned private copies of literal strings in +sparse length buckets. Rows mapped those identities to the original entries, +retaining duplicate order and fork identity. Small rows used association lists; +wider rows used identity tables. Each deterministic invocation owned a fresh +selector closure keyed by its remaining token list. Exact literal, folded +literal and token-kind precedence were preserved. Folding reused the grammar +trie lazily; Unicode retained whole-string uppercase semantics; EOF stayed a +row fact. GLR, layout and arbitrary installed reduction callbacks retained the +original selector, because an arbitrary callback can mutate a pending token. + +The rejected implementation and its controls are reproducible from +[[https://github.com/tao3k/gerbil-parser/tree/604ff1cb62b03586918a1eddcfc67ff719b1034e][the measured revision]]. The production cache and its preparation factory +were subsequently removed. There is one production selection path. + +#+begin_src scheme +;; Rejected candidate: grammar facts shared, classification state per request. +(def fresh-selector (prepare-lr-lookahead-selector rows index casefold?)) +(def select-left (fresh-selector)) +(def select-right (fresh-selector)) +(eq? (select-left state tokens) (original-select state tokens)) +(eq? (select-right other-state other-tokens) + (original-select other-state other-tokens)) +#+end_src + +** Question: why does this exploit Scheme beyond S-expression notation? + +Inspectable terminal data supports static projection; private records identify +representation ownership; lexical closures express invocation ownership without +an author configuration or a global cache. POO continues to own extensible +declarations, while fixed selection algorithms remain named procedures. +Those useful language mechanisms do not guarantee efficient machine behavior. +Here their additional request closure and projection costs failed admission. + +** Question: how is the design scientifically justified? + +Entry identity, exact/folded/kind precedence, duplicate order, EOF, Unicode, +interleaved invocations and callback mutation are falsifiable semantic obligations. +CPU is a separate hypothesis. Counting repeated selections does not measure their +cost: rows without literals already have a cheap negative path. Extra global +classification and identity lookup can cost more than the original row lookup. +The retained diagnostic therefore counts literal-bearing row observations +separately from all non-EOF observations. It makes no additive CPU attribution. + +* Exact-head native evidence and rejection + +All three jobs of [[https://github.com/tao3k/gerbil-parser/actions/runs/38110861774][the measured exact-head acceptance run]] passed: Rust AOT, +Linux Scheme and macOS Scheme. Complete semantic tests, language and ABI +conformance, generated products and the extended complete request matrix passed. +Passing CI establishes those gates; it does not establish a request speedup. + +The request matrix uses twenty alternating paired groups, validates full +source-bound artifacts and requires every leg to resolve one millisecond of +process CPU. Timing contains no observer and includes natural GC. Arithmetic +uses the identical generated executor as a negative control. Values below are +candidate/reference CPU ratios at P10, P50 and P90. + +| Host | Request | P10 | P50 | P90 | +|------+---------+-----+-----+-----| +| Linux | GQL 1 | 0.592 | 0.961 | 1.574 | +| Linux | FHIRPath 1 | 0.539 | 1.002 | 1.850 | +| Linux | Arithmetic 1, identical | 0.204 | 0.998 | 1.020 | +| Linux | GQL 64 | 0.701 | 1.393 | 1.401 | +| Linux | FHIRPath 64 | 0.753 | 1.012 | 1.372 | +| Linux | Arithmetic 64, identical | 0.557 | 1.000 | 1.797 | +| macOS | GQL 1 | 0.504 | 0.967 | 1.856 | +| macOS | FHIRPath 1 | 0.432 | 0.978 | 2.248 | +| macOS | Arithmetic 1, identical | 0.979 | 0.999 | 6.954 | +| macOS | GQL 64 | 0.606 | 1.624 | 1.646 | +| macOS | FHIRPath 64 | 0.481 | 1.018 | 1.591 | +| macOS | Arithmetic 64, identical | 0.454 | 0.999 | 2.213 | + +GQL 64 regresses at the median on both hosts; FHIRPath has no stable benefit. +Identical-executor controls exhibit wide batch tails, so these quantiles are not +confidence bounds or production request percentiles. No precise causal slowdown +percentage is admitted. The result is nevertheless insufficient to retain the +candidate. Production closure allocation and vocabulary projection are removed. + +Untimed work observations are identical across hosts: + +| Request | Non-EOF selections | Distinct input tails | EOF selections | +|---------+--------------------+----------------------+----------------| +| GQL 1 | 6 | 2 | 50 | +| FHIRPath 1 | 1 | 1 | 41 | +| GQL 64 | 1013 | 128 | 51 | +| FHIRPath 64 | 1071 | 127 | 42 | + +Repeated selection counts establish an opportunity, not its CPU share. Small +requests are dominated in count by EOF reductions, which cannot benefit from +lexeme classification. Future request work must measure literal-bearing rows, +reduction kernels and source execution separately, retaining complete products +and identical-executor controls. It must not reintroduce this rejected cache +based on counts alone. + +* Retained implementation and reproduction + +- [[file:../../t/benchmarks/lr-request-stack/lookahead.ss][Request action diagnostics]] retain executor witnesses, independent original + selector comparison, literal-bearing row counts and complete CPU controls. + After withdrawal the comparison is production versus original-reference + selection; it is a control, not the historical cache experiment. +- [[file:../../t/lr-stack-reduction-test.ss][Callback mutation control]] requires rejection after an installed reduction + procedure changes a pending literal. This protects the extension contract. +- [[file:dense-lr-item-space.org::*Small-grammar attribution controls][Small compiler controls]] separate preparation and resolved complete batches; + the earlier small-grammar regression did not reproduce. + +The existing optional benchmark entry remains: + +#+begin_src scheme +(benchmark-lr-request-stack 20 20) +#+end_src + +The measured revision is required to reproduce the rejected cache. Current HEAD +contains only the production baseline and retained controls. The existing build +catalog orders fixtures before consumers. No workflow or deadline was added. +Historical local source and standalone native controls are weaker evidence than +the completed exact-head run; interrupted and idle-failed local builds remain +excluded from native request admission. diff --git a/docs/research/request-owned-lr-stacks.org b/docs/research/request-owned-lr-stacks.org new file mode 100644 index 00000000..da682a83 --- /dev/null +++ b/docs/research/request-owned-lr-stacks.org @@ -0,0 +1,150 @@ +#+title: Request-owned LR stacks: measured rejection + +* Decision and shared engine scope + +The request-owned vector-stack experiment is rejected for production. It +removes substantial allocation, but does not establish a CPU improvement +across shared-engine workloads. The runtime implementation remains unchanged. +The retained deliverable is the reproducible Scheme benchmark +=t/benchmarks/lr-request-stack/benchmark.ss= and this decision record. + +This study extends [[file:engine-reduction-generation.org][shared reduction generation]] and +[[file:lr-request-cost-boundaries.org][complete-request cost boundaries]]. The target is the +deterministic LR executor, rather than a GQL grammar special case. Grammar +authors receive no new option. Persistent checkpoints and generated reduction +protocols have independent ownership requirements. + +* Choosing the larger cost + +An untimed derivation census counted executed reductions, canonical identity +unary reductions, and the longest consecutive identity-unary chain. The census +does not claim these counts for every valid source in a language. + +| Source family | Tokens | Reductions | Identity unary | Longest chain | +|---------------+--------+------------+----------------+---------------| +| GQL representative | 37 | 288 | 81 | 1 | +| GQL return 64 | 128 | 935 | 197 | 1 | +| FHIRPath invocation | 12 | 135 | 15 | 1 | +| FHIRPath addition 64 | 127 | 985 | 191 | 1 | +| Arithmetic addition 64 | 127 | 318 | 127 | 1 | + +These observations weaken the case for collapsing long identity chains in +these samples. They motivate examining storage and semantic-result construction +paid at every reduction. The vector experiment replaces per-action state/value +conses with reusable buffers for one uninterrupted request. It snapshots lists +only when a frontier escapes to a persistent checkpoint or GLR fallback. + +* Official Gerbil mechanism and ownership proof + +The installed Gerbil source was inspected through its toolchain source tree: +=std/io/bio/types.ss= and =std/io/bio/output.ss=. Typed slots and =using= make +private buffer access visible to the compiler. The experiment followed that +mechanism; it did not add a language-level mutable-stack declaration. + +#+begin_src scheme +;; Private engine experiment, not a grammar author interface. +(defstruct request-stack + ((states :- :vector) (values :- :vector) (top :- :fixnum)) + final: #t) + +(def (current-state stack) + (using (s stack :- request-stack) + (vector-ref s.states s.top))) +#+end_src + +Why is this DSL design preferable? Canonical rules describe syntax and closed +semantic actions. Scheme lexical scope, typed slots, and hygienic macros allow +the engine to change storage without changing those declarations. Responsibility +for bounds, growth, continuation ownership, and fallback belongs to the engine. + +Why is it scientifically testable? The implementation must preserve complete +artifacts, not merely accepted token counts. Independent GLR materialization, +both semantic backends, retained snapshots, epsilon growth, consumed-reference +clearing, and underflow rejection provide separate controls. Performance then +requires matched native complete requests as well as prepared recognition. +An elegant representation is not evidence of faster execution. + +The private invariant was states[0..top] and values[0..top-1]. Growth precedes +epsilon writes. Reduction validates width before access, clears consumed value +slots, and publishes the result and goto together. Callback consumers requiring +the complete immutable frontier keep persistent lists. The last candidate used +typed =using= accesses and a closed reduction transaction; scoped unchecked +accesses depended on these private bounds. It is not shipped. + +* Matched native experiment + +The checked-in benchmark reconstructs prepared tokens from each public parser's +source-valid artifact outside timing. It compares the complete artifact with +independent GLR materialization before measurement. Prepared recognition and +full-source parsing are measured separately; they are not additive stages of +the streaming parser. Workloads cover GQL, FHIRPath, and Arithmetic at 64 and +512 operands. Arithmetic's generated full-source backend is a control outside +the candidate arena path. + +Use the existing source import and qualification entrypoints: + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/lr-request-stack/benchmark) +(benchmark-lr-request-stack 20 20) +#+end_src + +The final native trial order was baseline, candidate, candidate, baseline. +Each trial used 20 samples of 20 calls per workload/route. Compilation was +outside timing. The initial GC per component is outside timing; naturally +occurring GC remains inside the samples. Allocation is admitted only for +single-caller intervals without GC. CPU P50 below is a batch median divided +by 20, not a distribution of individual-request latencies. + +| Workload and route | CPU A1 ms/call | B1 | B2 | A2 | Bytes A/call | Bytes B/call | +|--------------------+----------------+----+----+----+--------------+--------------| +| GQL 64 prepared | 0.20975 | 0.22465 | 0.22855 | 0.20980 | 230932 | 129508 | +| GQL 64 full | 0.34890 | 0.31690 | 0.31755 | 0.32395 | 542941.6 | 441613.6 | +| FHIRPath 64 prepared | 0.21905 | 0.22920 | 0.22850 | 0.22785 | 255540 | 149412 | +| FHIRPath 64 full | 0.35060 | 0.36065 | 0.35705 | 0.35030 | 695181.6 | 589149.6 | +| Arithmetic 64 prepared | 0.10285 | 0.10590 | 0.10425 | 0.09950 | 134804 | 92708 | +| Arithmetic 64 full | 0.14825 | 0.15995 | 0.15980 | 0.14965 | 462141.6 | 462189.6 | +| GQL 512 prepared | 1.78120 | 1.74195 | 1.75250 | 1.62140 | 1894004 | 1104452 | +| GQL 512 full | 2.68365 | 2.44415 | 2.44240 | 2.35910 | 4330101.6 | 3540645.6 | +| FHIRPath 512 prepared | 1.74330 | 1.78440 | 1.77340 | 1.71580 | 2104980 | 1267716 | +| FHIRPath 512 full | 3.76565 | 2.81500 | 2.79360 | 3.24585 | 5585421.6 | 4748253.6 | +| Arithmetic 512 prepared | 0.79720 | 0.83995 | 0.84050 | 0.79585 | 1080980 | 737828 | +| Arithmetic 512 full | 1.21860 | 1.16620 | 1.15970 | 1.15835 | 3696477.6 | 3696525.6 | + +Full-source allocation falls about 18.2% for GQL 512 and 15.0% for FHIRPath +512. Arithmetic full-source allocation increases by 48 bytes because the +candidate introduced an extra publication callback closure. Several prepared +CPU measurements regress despite the allocation reduction. Full-source CPU +does not show a consistent cross-workload improvement. GC and host scheduling +also vary in the generated Arithmetic control. No end-to-end P95 improvement, +live-heap reduction, or language-wide speedup is established. + +Earlier generic-buffer, typed-buffer, and typed-access candidates also failed +to establish the required CPU result. Their allocation reductions remained +similar; repeatedly changing access details did not turn the representation +into an admitted performance improvement. + +* Receipts and next engine question + +All four final native children returned exit zero with +=LR-REQUEST-STACK-OK= and no supervision reason. Receipts are the +unarchived local execution +files. They are local evidence, not portable CI artifacts. The last candidate +compiled in 18.746 seconds and passed 19 focused source cases in 9.409 seconds. +Earlier source and native controls covered checkpoint/contextual ownership and +all-language ABI products. These establish experiment validity, not adoption. + +The rejected runtime sources are retained locally under +unarchived local execution for diagnosis. The repository +contains neither an unused arena backend nor a compatibility switch. +The final retained benchmark also passed from source with one sample/call per +route, including equality of the prepared root and independent GLR root. Its +child returned exit zero in 9.800 seconds with the required completion marker; +receipt: unarchived local execution. +That source run qualifies the benchmark contract and is not a timing result. + +The next question is which semantic operations dominate each reduction: +closed operand actions, sequence assembly, node construction, or event +publication. Attribute those costs with a representative shared workload and +retain complete artifact controls. The census alone cannot identify the +dominant operation. Any change must improve the admitted complete request and +avoid transferring work into publication or generated-backend preparation. diff --git a/docs/research/scheme-bash-source-frontend.org b/docs/research/scheme-bash-source-frontend.org new file mode 100644 index 00000000..2aed6aaf --- /dev/null +++ b/docs/research/scheme-bash-source-frontend.org @@ -0,0 +1,412 @@ +#+title: Bash author grammar and source frontend +#+date: 2026-10-07 + +* Decision and implemented boundary + +The author syntax contract belongs to =deflanguage=: one =syntax= block and +one =rules= block. Release identity belongs to parser.ss metadata. There is no additional required SourceGrammar model. +[[file:../../languages/bash/grammar.ss][Bash grammar.ss]] declares the common =shell= vocabulary and semantic command +rules. The old profile declaration and its author entry have been removed; +no independently runnable legacy grammar or compatibility alias is retained. + +| Author interface | Responsibility | +|------------------+----------------| +| grammar.ss | Lexical policies, semantic syntax and named AST rules | +| parser.ss | Release metadata, checked Loader entry and parser services | +| parser-test.ss | Independent inputs, tree/field/span expectations and benchmarks | + +[[file:../../src/language/grammar.ss][deflanguage]] elaborates source vocabulary through native macros. +[[file:../../src/compiler/source-language.ss][Source syntax elaboration]] expands =defgrammar-syntax= vocabulary and checks +command rule structure before emitting the common compiler call. +[[file:../../language-support/shell-grammar.ss][The common source library]] assembles internal POO profiles and the admitted +source strategy. Profiles and execution recipes are compiler products. +=language-source-support= exposes checked source inspection and the parser +receipt declaration, not profile constructors. + +* Metadata value and native POO alternatives + +All eight production packages now declare release language/version/contract in +parser.ss. Compiler products are named =*-syntax= and have no release version or +contract until the parser entry binds them. Upstream source pins in grammar.ss +identify grammar inputs; they are distinct from parser release metadata. + +Ordinary values are the default. Native POO is optional when authors need +inheritance or effective slot overrides. Both satisfy the metadata domain +contract; POO objects are not required wrappers around ordinary data. + +#+begin_src scheme +;; Normal author value; no POO construction or prototype declaration required. +(deflanguage-parser-loader bash-language + (source bash-syntax) + (parse parse-bash) + (slots metadata: '((language . "bash") (version . "5.3") + (contract . "bash-5.3-structured-source.v1") + (grammar-format . source-parser)))) + +;; With inheritance, metadata may instead be a native POO object: +;; (slots metadata: (.o (:: self MetadataBase.) version: "v2")) +;; Its effective slots must supply valid language/version/contract values. + +;; Question 1: Why is this design better? +;; Ordinary users declare values directly. Extension users keep native POO; +;; release identity has one authoring location, separate from recognition rules. +;; Question 2: Why is this scientifically defensible? +;; Equivalent value and inherited-object inputs must produce identical identity, +;; parse events and receipts. Missing fields, duplicate keys and invalid values +;; must fail admission. Independent corpus tests must preserve parser behavior. +#+end_src + +The engine binds effective metadata once, snapshots identity strings, retains +native POO extension behavior or ordinary association-list representation, and +adds digest/digest-kind. =language-metadata-ref= reads either representation. +Source digests include release identity and the normalized source recipe; LR +Parser IR digests remain syntax-content identities. Changing a parser edition +does not manufacture a different LR table. + +* Complexity transferred to the common library + +| Former author obligation | Author states now | Common library owns | +|--------------------------+-------------------+---------------------| +| Scanner modes and queue transitions | Shell lexical semantics through the shell vocabulary | Scanner states and FIFO here-document actions | +| Numeric quote/pair widths | Delimiters, quote policies and contexts | Derived widths and matching actions | +| Capture frames and projection instructions | AST kinds and labelled fields | Capture programs, indexes and admission | +| Repeated command schemas | Command AST constructors | Publication from one internal schema authority | +| take/list/until/balance/call programs | seq/choice/optional/repeat/reference/balanced | Consuming plans and call-site continuations | +| Strategy assembly | Identity and syntax vocabulary | Native POO strategy construction and binding | + +The author grammar no longer contains scanner transitions, command VM +instructions, capture programs or duplicate result schemas. Changing a labelled +rule changes the compiler input; the library derives the execution plan. +Shell semantics remain specific to the common shell library. Execution modules +still reside under =src/runtime=; complete physical relocation is not claimed. + +* Scheme witness and the two questions + +This mechanism is implemented with native Scheme vocabulary expansion: + +#+begin_src scheme +(defgrammar-syntax (compound-loop kind opener) + (node kind + (field keyword opener) + (field condition command-list) + (field keyword "do") + (field body command-list) + (field keyword "done"))) + +;; These rows belong inside deflanguage's rules block: +(while-loop (compound-loop WhileCommand "while")) +(until-loop (compound-loop UntilCommand "until")) + +;; Question 1: Why is this design better? +;; Authors state grammar structure and AST meaning once. Native macro vocabulary +;; reuses that structure. Compilation derives termination and capture programs; +;; authors do not maintain backend opcodes or scanner state machines. +;; Question 2: Why is this scientifically defensible? +;; Reduced author obligations are a testable hypothesis, not proof from brevity. +;; Independent cases must preserve acceptance, fields, spans, catalogs and +;; receipts. Diagnostic quality and grammar-change effort remain measurable +;; obligations; a shorter file establishes neither safety nor speed. +#+end_src + +S-expressions preserve structured syntax for subsequent analysis. Native +=defsyntax=/=defrules= add contextual transformation and reusable vocabulary; +POO supplies admitted implementation objects. General effective-rule selection +with precise origin tracking is a separate research obligation, not established +by this Bash frontend. + +* Elaboration and admission + +#+begin_example +grammar.ss: deflanguage syntax / rules + -> native source-vocabulary expansion + -> checked named command expressions + -> common compiler: lexical policies, continuations, capture programs + -> admitted scanner / part / result / command plans and source descriptor + -> parser.ss: checked Loader and services + -> parser-test.ss: independent semantic observations +#+end_example + +Source plans remain distinct from CFG/LR GrammarIR. Bound source identity incorporates +the normalized recipe; a refactor may change its digest while preserving +observable behavior. Versioning and downstream artifact renewal remain explicit +release responsibilities. + +The frontend rejects duplicate sections/rules, unknown rule constructors, +unresolved references, FIRST cycles, non-consuming repetition and command-list +calls without a continuation. Lexical/profile values undergo existing admission +when the declaration is bound. Structural diagnostics retain expression or author +macro invocation blame. Complete parameter and expansion provenance remains open. + +Dispatch inference is scoped to the closed shell adapter. Special executor +positions are declared independently of private rule names; ordinary selectors +follow token-prefix structure. See the coordinated migration below for the +policy and its limits. + +* Native validation and retained failures + +The first new executable stalled during source binding and exceeded the unchanged +ten-second child bound. That failure is retained. POO slot initializers used +names identical to outer lexical values, causing recursive reads; distinct +lexical names corrected the binding. No timeout was increased. + +For the preceding Bash frontend commit, after correction, native translation completed with actual exit zero, and the +native executable linked in 15.25 seconds with actual exit zero. The Bash child +completed with actual exit zero and =DSL-SUITE-OK: bash=, covering author contract +rejection and the existing independent Bash semantic suite: nested expansions, +Unicode spans, FIFO here-documents, compound commands, empty-body rejection, +long chains, arrays, catalogs and receipts. + +The complete closure subsequently passed all 26 bounded children with actual +exit zero, including all eight language suites, topology, Loader/source services, +composition/provenance controls and the one-sample benchmark smoke. This smoke +is not a performance comparison. Incremental local native +qualification does not establish a clean build, exact-head GitHub CI, a downstream +installed consumer or a performance improvement. + +** Metadata migration qualification + +The subsequent metadata migration preserves the two admitted representations: +ordinary association-list values and native POO objects. Focused controls compare +identity, source digest, parse events and roundtrip output, exercise native slot +overrides, reject malformed metadata and verify copied release identity strings. + +Two incremental native crashes occurred in old test consumers: the entry boundary +module and the common language-strategy module. Debugger frames identified those +consumers; recompiling and relinking them removed the crashes. An intermediate +loadable-module build and a link attempt exceeded the unchanged five-second +output-idle fence and remain failed receipts. The final native link completed in +218.41 seconds with actual exit zero under the 240-second build fence. +After narrowing the expression research fixture import, the final incremental +link completed in 32.18 seconds with actual exit zero. + +Bash product inspection now derives internal plans from the compiled syntax in a +test-only fixture. The narrow grammar frontend avoids importing unrelated grammar +adapters. Binding-name generation passed 135 traces. Command/Source generation +initially exceeded ten seconds while loading an unused Bash test-suite import; +removing that import produced 54 controls in 9.01 seconds with actual exit zero. +These generator checks do not establish downstream Rust execution or a speedup. +The expression research child first timed out during import, then completed all +85 assertions but exceeded its deadline during process exit. It now uses the +existing disposable test-child exit helper after every assertion and output +flush; the focused child exited zero within the unchanged ten-second fence. + +The final complete closure passed all 26 children with actual exit zero under +unchanged ten-second fences: all eight language suites, ten production/development +entries, value/POO contracts, topology, services, composition/provenance controls +and the ten-entry one-sample benchmark smoke. This is incremental local evidence; +the smoke is not a performance comparison. + +The preceding GitHub head, =2a55f99=, passed Rust CI, but both native CI jobs +exceeded the 240-second conformance compilation fence. Local incremental checks +do not establish that clean CI gate. No timeout was widened. + +* Paper connection and transfer limit + +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]], section 1 and Figure 1, motivates +an extensible macro surface elaborated into a checked compiler-oriented core. +Our inference is that the removed Bash recipe exposed too much of that core. +The implementation transfers the boundary principle, not Racket's syntax-spec +API. The paper does not prove this compiler or a runtime speedup. + +The [[file:scheme-dsl-research-basis.org][research basis]] owns the broader paper study. This module owns the complete +Bash migration, implementation limits and independent admission evidence. + +* Coordinated contextual compiler migration + +The review of =2a55f99= identified a real semantic coupling: private rule names +selected entry positions in the compiler and in both execution backends. The +metadata migration at =14741fe= did not remove that coupling. This migration +changes the compiler, admission, Scheme execution, Rust emission and Rust +execution together; it is not a grammar-file shortening exercise. + +** Entry identity and structural selection + +Only rules invoked at a special executor position declare that capability. +Ordinary rules retain structural inference. The source vocabulary separates the +private rule identifier, semantic entry position, selector and output node kind: + +#+begin_src scheme +(assignment-elements + (at assignment-tail + (node ArrayAssignment + (field open (operator "(")) + (repeat (field element word)) + (field close (operator ")"))))) + +(pipeline-modifiers + (at pipeline-head + (node Pipeline + (optional (field negate "!"))))) + +;; Question 1: Why is this design better? +;; Entry capability is declared independently of private helper names. Ordinary +;; productions infer selectors; authors do not maintain dispatch integers. +;; Question 2: Why is this scientifically defensible? +;; Rename every private rule and call target while retaining entry capabilities. +;; Acceptance, fields, byte spans, events and deferred-body links must agree. +;; Missing, duplicate and foreign entry bindings must fail before execution. +#+end_src + +The compiler emits a distinct entry table. Admission requires exactly one +assignment-tail and one pipeline-head binding, checks their node/program +contracts and resolves their target references. Both Scheme and Rust execution +read that table, rather than interpreting =array-tail= or =pipeline-prefix= as +reserved private names. Direct command token prefixes determine lookahead up to +the first contextual operation; the field name =keyword= has no selector meaning. + +The current shell adapter orders selectors by specific token head and prefix +width. Its bounded numeric encoding belongs to the generated backend product, +not the author DSL. This is a documented adapter policy, not a proof of general +ambiguity resolution or a complete precedence partial-order implementation. + +** Source diagnosis and transfer limits + +Expansion retains an occurrence map to syntax objects before passing closed +rule data to lowering. A direct structural error blames its expression; an error +inside generated syntax blames the author macro invocation. Executable controls +check these two cases separately. This does not yet provide macro parameter +identity, a complete expansion lineage, or hygienic binding identities in the +canonical source product. Those remain part of the unified-core migration. + +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]] supplies the boundary principle: +precise extension positions and binding structure elaborate into a checked core. +It does not establish this implementation's correctness. Native POO composition +is optional for declaration inheritance; ordinary values remain valid inputs. +No extra required author file or public SourceGrammar model is introduced. + +** One remaining architecture objective + +| Boundary | Current implementation | Remaining acceptance obligation | +|----------+------------------------+---------------------------------| +| Entry positions / selectors | Explicit bindings, structural selection, both backends | General overlap witnesses and declared incomparable choices | +| Macro origins | Expression or author invocation blame | Parameter identity and complete lineage through canonical admission | +| Recognition / Context / Tree | Checked command, scanner, part and result products | Shared contextual Parser IR authority, rather than parallel source recipes | +| Incremental context | Canonical scanner checkpoint and same-source restore | Cross-edit boundary mapping, entry/exit state and pending-queue reuse evidence | +| Recovery | Existing lossless rejection and LR recovery paths | Context-aware partial trees, budgets and repair evidence without marking repaired source valid | +| Physical execution | Scheme command executor and generated Rust Source executor | Measured specialization from the unified product; no assumed speedup | + +This table is one migration plan. Entry-contract closure is not advertised as +completion of Contextual Parser IR, editor recovery, or incremental Bash parsing. +The existing canonical LR(1) reference tests and contextual scanner compiler are +inputs to that migration; adding duplicate implementations would not close it. + +** Coordinated migration qualification (2026-10-07) + +The final native link completed in 21.62 seconds with exit 0. The focused Bash +suite passed 46 cases and exited 0 in 1.28 seconds. All 26 children in +The historical Bash research runner, now replaced by =test-processes.ss research=, +exited 0 under the unchanged ten-second limit, +including every language suite and the all-language benchmark smoke control. +The benchmark smoke control has one sample per entry and establishes execution, +not a statistically supported latency improvement. + +The Scheme generator emitted 54 Command/Source controls. The Rust library suite +passed all 127 tests, including private-rule renaming across all 49 base controls. +Prepared Rust entry references and Scheme entry forms are resolved once, before +request execution. No end-to-end performance improvement is claimed. + +An earlier provenance control completed its assertions but failed the child exit +limit. Research children now use the existing native disposable-child exit +binding after assertions and flushed output. CI compiles that binding before +research execution. The original timeout remains a failure receipt; completion +markers alone are not acceptance. Local qualification does not establish remote +CI success or completion of the remaining architecture obligations above. + +** Single source compilation and Rust access boundary + +Source providers now classify declaration shape and compile once. Compilation +returns the identity recipe and executable engine together. Shell scanner, +result, part and command plans are compiled/admitted in that operation; the +recipe is projected from the plans used by the runtime. Line providers and +negative test providers use the same protocol. This removes the former separate +recipe-generation and engine-binding compilations. The coordinated +[[file:scheme-contextual-parser-ir.org][Contextual Parser IR migration]] now supplies a closed v2 envelope for LR, +command and line products. General command-selector overlap witnesses, cross-edit +source certificates and recovery remain separate implementation obligations. No +measured latency gain is claimed. + +The native convenience facade obtains release-bound descriptors from +=parser.ss=. Raw syntax from =grammar.ss= cannot substitute for a descriptor +whose metadata supplies language identity. The existing entry-boundary suite +now checks native publication for accepted and rejected GQL/Cypher input, +including UTF-8 and the binary event count/length contract. + +Rust consumers access Scheme AST results through the following primary path: + +#+begin_example +Rust caller -> C ABI -> Scheme parser engine -> canonical result + <- owned payload + borrowed AST/CST access +#+end_example + +The Rust access layer must retain node kinds, field associations, child order, +source spans and source text. It may index the result for navigation; it must +not re-run language recognition or construct another authoritative syntax tree. +An owned payload wrapper must keep the native runtime and language handle alive +until all borrowed views are released. Semantic AST wrappers should resolve +fields through the canonical result catalog, with no second grammar contract. + +The implemented native Rust wrapper now obtains a catalog from its own language +handle and borrows GPA1 through =NativeParsed::view()=. Node, token and named +field navigation require no generated =LanguageSpec= or =ContextualParserSpec=. +The native crate depends only on =gerbil-parser-artifact= and exposes borrowed +navigation directly over the admitted payload. The independent Rust AOT backend uses flat +source-range indexes in =gerbil-parser-runtime= and is not a dependency of the +primary Scheme FFI client. See [[../architecture/0006-rust-aot.org][the implementation and qualification contract]]. + +Benchmark obligations distinguish Scheme execution, ABI transport, artifact +admission/indexing, AST field traversal and optional extra tree materialization. +Architecture alone establishes no relative performance result. Equivalent outputs and downstream traversal must be compared before a +performance conclusion is admitted. + +Qualification: the final native link exited 0 in 36.06 seconds. All 26 DSL +research children exited 0 under the unchanged ten-second child limit. The +entry-boundary suite passed five cases, including native facade publication; +the source-services suite passed four cases, including single compilation. +The Rust library passed 127 tests after command product types were separated +from execution and the glob import was removed. + +The original full FFI test module could not load its record-parser fixture in +the temporary environment. A separate cold interpreted facade probe exceeded +the ten-second limit while loading dependencies, before assertions ran. Neither +is counted as a successful FFI receipt. Native entry-boundary tests provide the +bounded publication receipt above; the broader native ABI suite remains a CI +obligation. The ASP Rust quality gate was not qualified locally: its initial +cold run exceeded ten seconds, and separate dependency compilation later failed +the five-second output-idle fence. Required limits were not increased. + +* Implemented native representation separation + +The preceding qualification records belong to the prior published revision. +The current refactor implements the separation proposed above: an independent +=gerbil-parser-artifact= vocabulary and borrowed AST API, handle-bound catalog +admission, engine-derived =rootKind= publication, and field-owner validation. +The FFI client has no dependency on generated =LanguageSpec= values or on the +independent Rust AOT executor. Native consumers retain borrowed event navigation without materializing another tree. + +#+begin_src scheme +;; Parser products publish their actual root and result catalog together. +;; Native publication reads these engine-owned values; authors do not maintain +;; a second Rust grammar or positional catalog convention. +(source-language-root-kind descriptor) +(source-language-result-catalog descriptor) +#+end_src + +Why does this design preserve Scheme's advantage? The =deflanguage= declaration, +macro expansion and POO composition still select the effective parser product; +Rust consumes that product's vocabulary and events through one publication +contract. Language authors retain =grammar.ss=, =parser.ss= and +=parser-test.ss= as their interfaces. + +Why is this design technically justified? Source and grammar digests, vocabulary +admission, owner-specific fields and ordered scope indexes provide falsifiable +contracts. Tests compare independent execution with canonical Scheme artifacts +and reject damaged or mismatched results. Removing a redundant representation +does not establish a latency improvement; equivalent traversal benchmarks are +still required for that conclusion. + +The observed prior-head macOS generator failure came from applying a +LanguageDevelopmentLoader contract to an ordinary parser Loader with one added +slot. The generator now emits its bound BuildStrategy directly. The prior-head +Ubuntu conformance build exceeded the unchanged 240-second fence. Test entry +modules now skip whole-module Scheme optimizer analysis while linking the +optimized production parser; this does not change production build settings. diff --git a/docs/research/scheme-build-ownership.org b/docs/research/scheme-build-ownership.org new file mode 100644 index 00000000..8916c40a --- /dev/null +++ b/docs/research/scheme-build-ownership.org @@ -0,0 +1,511 @@ +#+title: Scheme native build ownership +#+startup: overview + +This module follows [[file:scheme-engine-library-boundary.org][the engine/library boundary]] and records native build +ownership, failed controls and qualification evidence. =std/make= owns library +builds; native =gxtest= owns source test evaluation; the closed conformance product +owns independently bounded native controls. + +* CI ownership and build pipeline + +At =0a4175f=, Linux exhausted the unchanged 240-second conformance-build limit; +macOS rejected fixture-writing output calls inside a =*-test.ss= file. The binary +fixture generator now lives in its own fixture-publication module. The compiled +conformance gate includes the test-style contract before other language controls. +Each static-build worker translates a module and immediately compiles its object, +removing the global translation barrier. GCC reports optimized inlining instead +of every rejected inlining candidate; optimization settings are unchanged. +Qualification retains the 240/5-second build fences and ten-second child fences. +A local cold-static build still reuses Scheme outputs and immutable SDK objects; +only exact-head remote CI qualifies the complete platform build. + +The local cold-static receipt removed 451 generated C/object files from the +disposable package environment. With four workers, static translation, compilation +and linking completed in 90.65 seconds with exit zero under the 240/5-second +fences. This reused Scheme compilation outputs and SDK objects. + +* Separate static and shared test artifact owners + +At =87972d8=, Linux still exhausted 240 seconds: shared test-module compilation +finished at about 80 seconds, then the runner compiled the project closure for +static linkage. macOS stopped at the five-second silence fence before that link +phase. These failed runs invalidate complete platform closure claims. + +A shared-module runner prototype avoided static compilation but failed the +unchanged ten-second all-language admission fence. It was discarded; the closed +native conformance entry remains the owner of runtime controls and binary fixture +publication. This negative result is not a reason to relax the runtime limit. + +=compile-static-conformance!= uses the official Gerbil =compile-module= API with +=invoke-gsc: #f= for test leaf declarations. Their interfaces and static Scheme +products feed the existing parallel native link owner, which compiles the closed +runtime once. Source-expansion research helpers retain their loadable native +products. The =0dd4391= revision added a =compile-shared-tests!= owner for loadable native test +modules for gxtest's source-interface imports. That revision qualified both artifact forms locally; +conformance did not manufacture an unused second form before its runtime gate. +The separate shared-test owner retained a 240/5-second build fence. Production +optimization settings, independent AOT ownership and all child fences are unchanged. + +The disposable-environment check quarantined 401 prior shared test outputs before +building the static product. That owner completed in 71.90 seconds, followed by +all 28 bounded children exiting zero and exact regeneration of the twenty native +ABI controls. The independent shared-test build completed in 30.06 seconds. +These receipts reused production/SDK objects; complete Linux/macOS cold closure +still requires exact-head CI. The combined static owner must run generation and +linkage in one compiler process, as in =build-conformance.ss=. +This retains compiler phase contexts while constructing the closed product. + +* Reusable production object closure and retained Linux failure + +At =1cf51d1= macOS passed the closed build, all 28 bounded children and native ABI +fixture comparison. Linux finished Scheme generation at about 37 seconds but +still exhausted 240 seconds while compiling production-library objects. The +reusable AOT library object closure is now a separate bounded product rooted in +all eight public parser interfaces and the generic artifact codec. It contains +112 project/dependency modules and no test corpus or language dispatcher. The +conformance consumer reuses these objects with the same compiler flags; it owns +its test objects and final link. Each build owner retains a 240/5-second fence. + +The subsequent macOS header generator failed its five-second idle fence during +cold imports. Real native dependency admission now precedes that unchanged +producer. Both generated Rust fixtures compare byte for byte; the local qualified +producer completed in 6.31 seconds under its unchanged 90/5-second fences. + +The local production-object check quarantined 224 generated C/object files from +that exact 112-module closure. Cold translation and object compilation completed +in 40.99 seconds with four workers; SDK objects and Scheme outputs were retained. +The real conformance consumer then completed in 39.75 seconds, followed by all +28 children exiting zero and identical native ABI fixture bytes. These are local +artifact-ownership receipts, not Linux timing or complete remote closure claims. + +* Native gxtest owns source tests + +At =0dd4391=, Linux and Rust CI passed. macOS passed conformance and both ABI +consumers but stopped while generating redundant loadable test modules. Reusing +=std/make= for twelve imported test services still failed the unchanged five-second +silence fence locally. This negative control rules out claiming that merely +moving a worker loop into a library solves the redundant artifact problem. + +The closed executable retains native conformance bodies and all 28 bounded +children. Native =gxtest= evaluates test source owners and imports corpus services +from source; their production parser dependencies remain compiled. No assertion, +benchmark threshold, production compiler flag or watchdog limit is removed. + +=prepare-gxtest!= delegates output cleanup to =std/make#make-clean= using its +native module output map plus =copy:= declarations for generated runtime wrappers. +The library directory is explicit. A wrapper without its =.ssi= is not admitted +as a complete native module. A matched control confirms qualified corpus imports +resolve with the ordinary source path after cleanup; no extra path configuration +is needed. This removes a second C build of conformance test bodies and the +project-owned shared-test compile queue. + +The local preparation completed in 0.54 seconds under the unchanged 240/5-second +fence. Source gxtest controls passed 57 TLA+ cases, 10 artifact cases and the +versioned language benchmark, with zero harness status and final GC completion. +The full local suite subsequently passed all 105 modules and 920 cases, with +zero child status and final GC completion. Exact-head platform qualification +remains a separate gate. + +The retained manual static closure/link builder is a separate remaining boundary; +it has not been replaced by =gerbil-scheme-rust= archive APIs in this change. +Those APIs require a compiler-owned program manifest and a static archive +consumer. A downstream parser should reuse that contract when migrating its AOT +product, rather than claiming that a different artifact format is interchangeable. + +The =4b1c6a6= macOS run stopped earlier, in a research helper compiled by +=std/make=: GCC's old inline-success-only diagnostic was quiet for 5.15 seconds +while compiling =list-language~0=. Native GCC =-fopt-info-optimized-missed= now +reports real optimization work for these helper products. No compiler optimization +setting or watchdog threshold changes. The focused cold helper completed in +11.67 seconds; the complete helper group plus static declarations completed in +24.63 seconds under the unchanged 240/5-second fence. Linux passed full gxtest +and reached the pinned TLA+ corpus gate on that revision. + +* Standard native workgroup + +The static object build now delegates workers, queueing and exception propagation +to =std/sync/wg= (=make-wg=, =wg-add!=, =wg-wait!=). The project no longer owns a +mutex or hand-written thread pool. Native =list-sort= orders large inputs first; +=std/wg= alone owns queueing and workers. Two workgroups share the existing +core budget: large objects use at most =cores - 1= workers and small objects use +one. The single-worker group receives approximately =1 / cores= of source bytes, +selected from the smallest inputs. Whole files remain indivisible. This retains real progress from independent compiler products; it does not +report a timer heartbeat. Native =dynamic-wind= drains the small group when the +large group raises. Module translation +and object compilation still retain the compiler-generated source inputs. + +An earlier warm native conformance control completed in 82.20 seconds under +the unchanged 240/5-second fence. Compiler diagnostics report actual optimization work; no +synthetic heartbeat or compiler optimization downgrade is introduced. + +Gambit link partitioning remains in the compiler adapter. Migrating it requires +the compiler-owned program manifest accepted by =gerbil-scheme-rust='s archive +API and preserving this executable's bootstrap, link order and runtime lifecycle. +A static archive is not yet this project's executable publication contract. + +At =92cb800= Linux rejected the FIFO object build at 62.17 seconds: GQL and +Cypher grammar objects were the final two unfinished products. Submission now +orders large inputs first through native =list-sort=. Both worker groups remain +=std/wg= instances; no project mutex or custom queue is restored. Input size is +a scheduling hint, not a bound on compiler cost or a proof of idle-fence compliance. + +The largest-first single-group control also failed its five-second idle fence +locally. The two standard-workgroup control then rebuilt all 112 native library +objects from the exact compiler manifest in 67.97 seconds; SDK objects and +generated C were retained. The full helper group and static declarations passed +in 47.19 seconds with native =-fopt-info-all= diagnostics. The isolated Gambit +multiple-host experiment failed in 6.16 seconds and is not production configuration. +No compiler optimization level, test assertion or timeout threshold changes. + +The first two-group conformance-only control failed at 47.06 seconds with three +unfinished test objects. It overlapped helper compilation, so it does not qualify +the CI pipeline. Repeating the complete helper-generation, static-declaration and +link pipeline sequentially, in one compiler process, completed in 123.74 seconds +with exit zero and =CONFORMANCE-LINKED= under the unchanged 240/5-second +fence. These local receipts retain immutable SDK and production objects. Exact-head +Linux/macOS CI must still qualify their own complete platform builds. + +After that sequential build, all 28 independently bounded children exited zero. +The twenty accepted/rejected ABI controls regenerated in 1.74 seconds and matched +the committed binary fixture byte for byte. Focused workgroup controls passed in +3.05 seconds: the one-core path executes, a worker exception reaches the caller, +and the independent small group drains before the caller handles that exception. +These controls verify build lifecycle and retained semantics; elapsed build times +are qualification receipts, not parser performance measurements. + +At =a16054e= Linux still rejected library objects after 26.94 seconds: the +count-based half split gave its single worker only 480,230 of 10,020,826 source +bytes (about five percent), leaving GQL, Cypher and TLA+ proof objects unfinished +when the short group drained. The split now uses input bytes rather than file +count. A cold rebuild of all 112 project/dependency objects completed in 88.01 +seconds under the unchanged 240/5-second fence. This is a scheduling control, +not a guarantee that input size predicts wall-clock compiler cost. + +Isolated GQL object controls with full GCC diagnostics failed the idle fence in +both the existing single-host mode (6.40 seconds) and Gambit's multiple-host mode +(6.80 seconds). Neither result justifies changing production compiler modes. + +The corrected partition also completed the full sequential conformance build in +54.80 seconds using existing helper/production products. Its single-core and +worker-failure controls passed in 0.81 seconds, and the regenerated twenty ABI +controls matched the committed fixture byte for byte (2.09 seconds). Runtime +assertions and compilation modes remain unchanged. + +Compiler diagnostics follow [[https://gcc.gnu.org/onlinedocs/gcc/Developer-Options.html][GCC's documented developer options]]: +=-Q= reports function compilation and completed pass statistics; +=-fopt-info-all= admits native optimization notes as well as successful and +missed transformations. The generated host procedure choice follows the +installed Gambit header and [[https://github.com/gambit/gambit/blob/master/INSTALL.txt][Gambit's build documentation]]. Scheduling does not +establish a performance equivalence between host modes. + +* Publication and compiler qualification records + +** Rebase evidence boundary + +The alignment commit was rebased onto the concurrent PR revision 7d0a86d, +preserving its resolved-repetition validation and research documents. The complete +language topology suite passed again in 2.95 seconds. The twenty artifact and +projection receipts above precede that rebase and are not exact-head CI receipts. +The updated LR sources emitted and compiled locally; an additional resolved- +grammar suite then terminated with status -11 during its nullable-prefix positive +control. That local environment combines refreshed and existing native products; +the cause is unconfirmed. This negative receipt does not establish a source-code +root cause, nor may it be called a successful complete LR regression run. The +rebased native image and exact-head CI remain required gates. + +** Closing the local LR crash + +The earlier status -11 receipt is superseded by a diagnosed build-product +mismatch, not by an algorithm workaround. The cached optimized lr-parser calls +compiler/lr#alist-ref__0. Re-emitting compiler/lr with optimize: #f defines only +compiler/lr#alist-ref; optimized native consumers still retain the former direct +call. Production and non-optimized products must not share this output owner. +Re-emitting LR and its compiler with optimize: #t and generate-ssxi: #t restored +the required entry and matching compiler declarations. Both native modules then +compiled successfully. All fifteen resolved-grammar cases, including the formerly +crashing positive case, passed in 36.86 seconds with exit status zero under the +existing source-owner fences. No grammar algorithm or runtime behavior was +changed to obtain that result. The earlier negative receipt remains part of the +investigation history; it is no longer an unexplained source-code defect. + +Production modules must keep the package build's optimization mode and refreshed +optimizer interfaces. Non-optimized semantic test modules have a distinct owner; +they must not overwrite a production dependency while retaining its optimized +consumers. A cold compiler-interface stall is still a separate failure and is +not repaired by this ABI correction. + +** Rust profile publication repair + +Exact-head CI at b0347f7 completed native conformance compilation, DSL/POO closure, +resolved-grammar tests and all twenty native artifact controls on both platforms. +It then rejected the checked-in FHIRPath Rust profile at its grammar_digest field. +A full production re-emission of the current FHIRPath grammar and parser followed +by the repository's Scheme generator reproduces the drift. After rustfmt, the +entire fixture delta is one digest string; kinds, terminals, lexical rules, +actions, gotos, productions and independent profile cases are unchanged. The +updated fixture is generated from Scheme, not patched by hand or admitted by +weakening the comparison gate. All 130 runtime-crate tests passed locally. The +same exact regeneration comparison remains mandatory in the next CI revision. + +** Complete helper products and bounded native ABI compilation + +Revision 11179d2 passed Rust AOT and the macOS FHIRPath comparison, then exposed +two separate build-owner defects. Linux exceeded the five-second silence fence +inside GCC while compiling the monolithic ABI codec. Increasing genuine GCC +optimization diagnostics alone also failed a local control, so it is not the +repair. Native language context/catalogs, binary event writing and ABI binding +now occupy separate Scheme modules. The public codec API and byte layout are +unchanged; three regenerated static C units compiled in 12.10 seconds with exit +zero under the existing 240-second/five-second owner fences. This is a build +receipt, not a parser runtime speedup claim. + +The macOS result-projection generator failed when it loaded bash-products~0.scm +as an incomplete native module. Conformance helpers also serve source-mode +generators: their compiler owner must publish complete loadable products, not +only static Scheme. All helper modules now use the public compiler's native +publication option; the special list of macro-only exceptions is removed. Test +bodies retain their separate source/native owners. With a completed Bash helper, +all thirteen result-projection controls regenerated in 32.59 seconds with exit +zero, and the formatted Rust output matched the checked-in fixture exactly. + +#+begin_src scheme +;; Helper modules are imported by both executable and generator owners. +(compile-module helper + [optimize: #t generate-ssxi: #t static: #t keep-scm: #t invoke-gsc: #t]) +#+end_src + +Why is this preferable? The compiler publishes its documented loadable product +without per-fixture loader exceptions. Why is it justified? The negative control +identifies a missing native body, while exact regeneration checks the resulting +consumer output. Neither elapsed nor silence limits are increased. Full CI on +the resulting revision remains required to qualify platform behavior. + +The existing independent-language native-admission suite also passed after the +split in 37.82 seconds with exit zero under source-owner fences. Its controls +include exact direct/canonical payload bytes, source descriptors through the +length-delimited C ABI, symbol indexing, rejected publications and released +handles. This receipt is not a closed-native ten-second execution claim. + +** Compiler-option admission and macro-location overhead + +Revision cd7b03f passed Rust AOT but remained unqualified natively. Linux stopped +while compiling the TLA+ syntax C units, before reporting their first host body. +macOS stopped during the native publication of list-language. These are distinct +from the earlier fixture digest and ABI codec failures. + +The pinned Gerbil compiler driver reads gsc-options: and separates compiler, C +and linker arguments. It does not read a cc-options: entry in compile-module's +option plist. The helper owner now supplies the documented gsc-options: list, +so real compiler diagnostics reach the existing progress gate. + +The Linux SDK compiler command also lacked the macOS SDK's +-ftrack-macro-expansion=0. GCC documents that its default full macro token-location +tracking consumes additional memory; this affects diagnostics, not the selected +optimization level. A matched local control over the same TLA+ generated C file +failed the five-second silence fence with tracking level two (5.87 seconds, exit +70), while the SDK's level zero completed with exit zero (9.40 seconds). The +native object owner now explicitly aligns this option across platforms. + +An exploratory Gambit SINGLE_HOST_LIMIT=256 control also completed, but it is +not the retained implementation: ordinary single-host compilation already passed +when the macro-location option matched. No host partition, optimization level, +grammar declaration, parser algorithm or time fence is changed by this repair. +C macro expansion stack detail is reduced; compiler errors and normal source +diagnostics remain enabled. Exact-head CI is still the platform qualification. + +#+begin_src scheme +(compile-module helper + [invoke-gsc: #t + gsc-options: ["-verbose" "-cc-options" + "-v -Q -fopt-info-all -ftrack-macro-expansion=0"]]) +#+end_src + +Why is this preferable? One compiler-owned argument list replaces an ignored +option and keeps build behavior inspectable. Why is it justified? The SDK driver +provides the admission contract, and the matched negative/positive C controls +isolate macro tracking without changing generated parser behavior. See GCC's +[[https://gcc.gnu.org/onlinedocs/cpp/Invocation.html][preprocessor option reference]] for the diagnostic-memory tradeoff. + +** Evidence forwarding must not become false child silence + +The macOS object owner at revision 4cd91ba failed at 43.55 seconds while the +console was forwarding queued compiler output. The runner checked its idle +deadline before polling the child pipe again. Forwarding one large chunk can +block behind a slow consumer longer than the idle interval; newly queued child +bytes must be read before deciding that the child was silent. + +The existing runner now polls readiness first. The absolute deadline still wins +even when the pipe is readable; an idle failure still occurs when no event is +available for the existing interval. Only actual child bytes update progress. +No timer heartbeat or timeout increase is introduced. A matched slow-consumer +control over 20,000 output records failed with the old runner (reason=output idle +timeout, status 124) and completed with the repair (all records, exit zero). The +existing inherited-pipe test now holds its consumer beyond the idle interval +and checks complete stdout/log delivery. All ten runner/compiler-proxy tests +passed, including true silence, hard total timeout, escaped-child cleanup and +error/exit propagation. These are runner controls, not parser performance data. + +Why is this preferable? Console throughput cannot decide parser admission. Why +is it justified? The negative/positive control isolates queued-byte handling, +while the unchanged timeout controls still reject stalled or over-budget work. +Full native CI remains the integration qualification. + +** Observe compiler work before changing artifact representation + +Revision 9dc3247 still failed the native object owner on both platforms. macOS +retained an unfinished HCL Scheme-to-C translation after the other workers +completed; Linux retained GQL and TLA+ C compilation. The existing verbose +translation trace did not expose all work on the large module expressions. + +Gambit's documented gc-report-set! reports real collections, allocation, heap +size and live data. The translation owner now enables this report and unbuffered +stdout in the compiler process. It does not force collections, change heap +settings, alter generated code or increase either deadline. A fresh optimized +HCL control completed and its C product matched the control byte for byte. + +#+begin_src scheme +(port-settings-set! (current-output-port) (list buffering: #f)) +(gc-report-set! #t) +#+end_src + +Why is this preferable? Compiler-owned reports expose work already taking place +without adding a timer or a second progress protocol. Why is it justified? +Identical generated C separates observation from parser semantics. The reports +also let allocation and collection costs be measured rather than treating GC +as evidence of poor parser performance. See the Gambit manual's +[[https://gambitscheme.org/4.9.7/manual/#Procedures-related-to-debugging][debugging procedures]] and +[[https://gambitscheme.org/4.9.7/manual/#Object_002dport-settings][object-port settings]]. + +An isolated GQL encoding control replaced the compressed Base64 literals with +raw homogeneous bytevector literals. Both single-unit translations exceeded +the existing silence fence without collection reports. With real collection +reports the bytevector control completed in 14.98 seconds, but emitted 6.6 MiB +of C versus the current image's 6.2 MiB. This encoding is not retained: shorter +runtime byte storage alone does not establish lower compiler cost. Platform CI +remains necessary; collection reports do not repair a silent C compiler phase. + +** Native image storage and complete worker capacity + +Revision 7c05c47 passed the complete macOS and Rust pipelines. Linux retained +unfinished GQL, TLA+ core and TLA+ layout C objects at the five-second silence +fence. Compiler reports repaired the hidden HCL translation work; they did not +repair this C frontend burden. + +The same GQL module has four compressed images totaling 697,184 bytes. Its +Base64 string constants generated 54,910,424 bytes of preprocessed C in the +local control. Direct bytevector literals generated 40,577,740 bytes and did +not satisfy the isolated C silence gate. Standard unsigned word-vector literals +generated 12,182,461 bytes; each of the four compressed images round-tripped +byte for byte through Gerbil's standard binary reader/writer. The word control +still failed the isolated C silence gate, including a sequential control; these +size measurements are not a standalone compilation or parser latency win. + +The engine now owns one native word image for every declaration producer. +The first u64 element carries byte length, subsequent elements carry network-order +words read and written with std/io, and the final word has checked zero padding. +No pointer reinterpretation, generated C rewriting or host-endian assumption is +needed. The compressed bytes, digest, locator, schema and complete datum admission +are unchanged. Base64 storage is removed from this internal path; old string +images are rejected. Both common assembly and the ANTLR catalog emitter quote +native data hygienically, because u64 literals require quotation in Gerbil. + +#+begin_src scheme +;; Engine-owned image, emitted by defsyntax/rules; grammar authors add no field. +(load-compiled-language-artifact/embedded + "gerbil-parser.grammar-ir.v1" 'content-locator 'native-word-image) +;; std/io performs the byte-order conversion: +(BufferedReader-read-u64 reader) +(BufferedWriter-write-u64 writer word) +#+end_src + +Why is this preferable? Homogeneous native values keep compressed data compact +in the compiled image, while the reader/writer and compiler own representation +conversion. The same hygienic emission applies to all eight language packs rather +than requiring a pack-specific initializer. Why is it justified? Exact recovered +bytes and unchanged admission semantics isolate storage from parser algorithms; +framing tests cover every tail length, all byte values, the maximum unsigned word, +invalid length, extra words and nonzero padding. Unicode, schema and malformed +UTF-8 controls exercise the real native loader. The isolated codec/loader control +completed in 16.66 seconds under the unchanged 240/5-second source fences. + +The fixed large/small queue partition also withheld worker capacity after its +small queue drained. One std/sync/wg queue now lets any worker take the next +remaining module; large inputs retain priority and each worker still owns a +translation/object pair. A controlled three-core workload requires all three +large jobs to enter before any finishes. The former partition failed at the +unchanged silence fence (5.38 seconds); the common queue completed (2.25 seconds). +This is a capacity witness, not a native compilation speed measurement. Runtime +assertions, compiler optimization, host mode and all deadlines remain unchanged. +Exact-head Linux/macOS CI must qualify the complete new publication pipeline. + +The current source-owned artifact suite completed in 12.23 seconds with exit 0, +including cache admission, concurrent publication and the new image framing +controls. A fresh arithmetic expansion emitted three quoted u64 images into its +optimized Scheme module. Single-module native publication still hit the silence +fence after GCC optimization reports; ANTLR source compilation also retained a +cold expansion timeout. These local failures are recorded rather than treated +as complete native publication. Whole-package CI is the remaining qualification. + +** Composition conformance compilation unit partition (2026-10-08) + +On revision =f21204f=, macOS CI failed the unchanged five-second silence gate +while GCC compiled =gerbil-parser__t__grammar-composition-test.c=. The last +diagnostics concern the module's large initializer, rather than a Scheme syntax +or test assertion failure. The bounded build ended after 68.09 seconds with +=reason=output idle timeout=. Rust CI passed; Ubuntu completed the native suite, +but a concurrent branch update cancelled its remaining reporting steps. + +The shared grammar declarations now live in +=t/fixtures/grammar-composition-models.ss= with explicit exports. The original +=grammar-composition-test.ss= imports those fixtures and retains all 31 cases. +The declarations and complete assertion body were compared byte for byte against +the prior file before commit. The native conformance build publishes the fixture +before the test module, through its existing helper compiler owner. This reduces +the combined module's source/initializer responsibility without changing compiler +optimization, concurrency, test coverage or any total/silence deadline. + +Local engine transfer tests passed independently. Local compilation of the split +composition modules encountered stale language/DSL interfaces, so native build +success is not claimed from that attempt. Exact-head CI must qualify both new +compiler units and the complete composition suite. This structural partition is +not evidence of an end-to-end parser performance improvement. + +The first partition was insufficient: on =4119de4= the remaining 31-case +assertion module again hit the five-second silence gate in its GCC initializer +(elapsed 73.25 seconds). The assertion owner is therefore replaced by three +native test modules: contracts, lowering, and execution. The old aggregate module +is removed, and both the static build inventory and conformance entry select the +three suites directly. Source gxtest discovers each new suite exactly once. + +An independent Scheme reader extracted the old and new =test-case= forms and +required exact structural equality, original ordering and count 31. This check +passed (=COMPOSITION-PARTITION-OK cases=31=). No assertion was deleted or weakened. +Compiler flags, worker scheduling and silence/total limits remain unchanged. +Fresh full CI must qualify the smaller compilation units; partitioning alone is +not a proof that every compiler phase stays below the silence limit. + +** Preparation input ownership and compiler diagnostics (2026-10-08) + +At PR #13 revision =007bf80=, both native jobs failed with +=native preparation inputs changed during build=. Their inventories each grew +by 98 files while compiling conformance helpers. The generated language artifact, +parser, declaration and program memo directories were incorrectly treated as +immutable preparation inputs. Their exact directory prefixes are now excluded +from that input inventory; project sources, dependency interfaces, SDK products +and the input-mutation rejection remain admitted and checked. + +The native Scheme preparation controls pass locally (1.41 seconds, exit 0), +including memo publication during preparation, cache hits after memo changes, +rebuilding after dependency interface changes, sibling-prefix rejection and +rejection of actual input mutation. Full native CI remains a separate gate. + +Gerbil compiler =verbose:= is disabled in the conformance expansion and link +owners; CI uses normal build verbosity instead of level 9. Internal +=declare-type= accessor/signature dumps are compiler optimization diagnostics, +not parser results. Explicit module progress and native compiler diagnostics +remain available under the unchanged total and idle limits. + +* Native process qualification + +The process contracts and the replacement of Bash coordination are specified in +[[file:scheme-test-process-ownership.org][Native ownership of process qualification]]. diff --git a/docs/research/scheme-component-provenance.org b/docs/research/scheme-component-provenance.org new file mode 100644 index 00000000..f0098ebb --- /dev/null +++ b/docs/research/scheme-component-provenance.org @@ -0,0 +1,201 @@ +#+title: Scheme component provenance: existing binding ownership and composition admission +#+date: 2026-10-06 + +* Question and scope + +How should an author declaration, generated helper rules and POO composition +remain explainable after macro expansion and artifact generation? This study +extends [[file:scheme-list-role-composition.org][the multi-rule component experiment]]. It uses the current committed +baseline =783381f33fa1b309ac4cc4b0012696f24c852d93= plus local research fixtures. +The initial study left compiler code unchanged. The local implementation now +connects occurrence selection, canonical publication and a checked composed entry; +its current contracts and qualification are described below. + +The objective is a coherent declaration-to-artifact contract. It is not to add +another grammar representation or a second parser implementation. + +* Existing mechanisms found in source + +| Existing owner | Capability | Design implication | +|----------------+------------+--------------------| +| [[file:../../src/compiler/language-artifact.ss][Language artifact elaboration]] | Verbose declarations accept =source-ownership= and =lineage= | Reuse these inputs for component origins | +| [[file:../../src/compiler/bound-ir.ss][Bound IR]] | Structured binding IDs, source records, declaration IDs, lineage, checked references and separate digests | Preserve grammar identity while adding declaration explanations | +| [[file:../../src/compiler/normalize.ss][POO normalization]] | Composition steps, role/section identities, composition digest and a grammar-body receipt | Retain the existing composition witness | +| [[file:../../src/compiler/parser-ir.ss][Parser compilation]] | Copies =compositionDigest= from canonical Grammar IR into Parser IR | The core already supports carrying this identity | +| [[file:../../src/runtime/language-artifact.ss][Artifact loading]] | Content-addressed materialization and checked loading | Test persisted origins through existing admission tools | + +The earlier broad suggestion to design a new origin mapping needs refinement: +these existing inputs already support much of the work. A production design +still needs precise origin generation and composition association, but it should +start from the current compiler contracts. + +Bound binding IDs are structured data: package, grammar, namespace and name. +The list fixture's =owner/tail= symbol spelling is only its deterministic helper +naming experiment; it is not a replacement binding namespace or a settled public +private-rule naming policy. + +* Executable identity and artifact probes + +[[file:../../t/fixtures/language-pack-research/list-origins.ss][list-origins.ss]] constructs a fixture-specific rule source map from the existing +composition receipt. Every rule in this unique merge/append example receives a +role owner and a truthful provider path. Component-produced rules are marked +=generated? #t=; base role rules are marked =#f=. + +The =location= is a descriptive role/rule label. It is not a verified line/column +range. =componentOwner= is a research source-record field carried by existing +Bound IR; it is not a newly standardized public provenance schema. + +[[file:../../t/fixtures/language-pack-research/provenance-study.ss][provenance-study.ss]] compares current compiler products with and without that +source map, materializes a Bound IR artifact and loads it through the existing +artifact loader. Its bounded native invocation exited 0 and reported +=PROVENANCE-STUDY-OK: 17 checks passed= (historical driver): + +#+begin_src sh +PARSER_RESEARCH_ARTIFACT_ROOT="$(mktemp -d)" \ + GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" GERBIL_PARSER_LR_TRACE=1 \ + gtimeout 10 gxi t/fixtures/language-pack-research/provenance-study.ss +#+end_src + +| Controlled change | Observed result in the fixture | +|-------------------+--------------------------------| +| Add source/owner metadata | Binding IDs, reference digest and grammar digest unchanged | +| Add source/owner metadata | Declaration IDs and Bound IR identity digest change | +| Materialize and reload enriched Bound IR | Complete value retained, including owner and generated flag | +| Remove only =compositionDigest= from Grammar IR | Parser's composition field becomes =#f= | +| Remove only =compositionDigest= from Grammar IR | LR specs are equal; all other Parser IR fields are equal | +| Remove only =compositionDigest= from Grammar IR | Bound identity digest unchanged, but its grammar digest changes | + +These checks distinguish recognition declarations, declaration provenance and +composition metadata. They do not prove invariance for arbitrary compiler +extensions or source transformations. + +Source inspection adds a digest distinction: normalization's receipt +=grammarIrDigest= hashes the grammar body before =compositionDigest= is inserted; +Bound IR's =grammarDigest= hashes the canonical grammar value it receives. +These fields must not be treated as interchangeable hash names when connecting +receipts to artifacts. This distinction is source-inspected, rather than a new +receipt-hash assertion in the probe. + +* Actual declaration-to-Bound-IR integration + +The local [[file:../../t/fixtures/language-pack-research/list-stage.ss][list-stage.ss]] adapter now calls compile-grammar/context, projects its +selected sources and enters expand-admitted-language-declaration-syntax directly. +It no longer rebuilds a deflanguage-grammar declaration. Shared binding, +compilation, publication and machine emission receive the exact normalized IR. + +[[file:../../t/fixtures/language-pack-research/provenance-language.ss][provenance-language.ss]] checks the actual generated Bound IR, complete canonical +Grammar IR and Parser IR compositionDigest, then compares call/array events to +the independent inline control. The earlier section-reification witness expected +a missing compositionDigest; the current witness requires it to equal the accepted +receipt. The separate flattening control in provenance-study.ss still demonstrates +that removing this metadata changes the parser composition field. + +The generated helper retains its selected component owner, provider path, +generated flag and declared lineage. Lineage remains a declared fixture chain, +not a recorded trace of every macro call. The ordinary and single-argument policy +variants use separate language identities and existing checked Loader entries in +[[file:../../t/fixtures/language-pack-research/list-parser.ss][list-parser.ss]]. Independent authored semantics remain in +[[file:../../t/fixtures/language-pack-research/list-parser-test.ss][list-parser-test.ss]], with complete-event and roundtrip controls in list-runtime.ss. + +* Source-map construction constraints + +The existing binding lookup chooses the first source-map section for a namespace. +If an explicit =rule= section only covers some names, a missing name falls back +to the module origin; it does not search a later auto-generated =rule= section. +The fixture therefore supplies all rule entries for this namespace. A production +collector must define merging before passing its final map to Bound IR. + +Receipt role names alone cannot identify the selected reader occurrence. The +engine now captures source associations during accepted row operations, rather +than reconstructing them from receipt names. Its effective-source policy covers +equal merges, append, override, remove and reappend; repeated names retain +distinct step indices. The old append-only source map remains an explicit stale +negative control. See [[file:scheme-dsl-engine-admission.org::*Effective origins follow composition before binding][the selection contract]] for history and coordinate limits. + +An author changes grammar semantics by changing declarations. Ordinary formatting +can change source locations and diagnostic identity without changing the grammar +meaning. This distinction matters for the earlier indentation discussion: source +metadata belongs in the explanatory binding product, while the target language's +recognition and layout policy remain explicit grammar decisions. + +* Three composition-admission directions + +| Direction | Strength | Remaining contract | +|-----------+----------+--------------------| +| Admit composed canonical IR through the existing language compiler owner | Retains its current composition field; avoids section reconstruction loss | Validate origin map, policy settings and receipt association together | +| Put composition association only beside Bound IR | Keeps explanatory data separate | Must define consistency with current Parser IR composition field and artifact admission | +| Persist an additional composition witness linked to the admitted products | Can retain detailed contribution history | Needs exact hash scopes, serialization and admission checks; do not invent it merely to duplicate the existing receipt | + +Working preference: extend the existing language compilation boundary to accept +an already composed canonical declaration plus the established receipt and +binding inputs. Bind, compile and materialize through the same existing owners. +The current list-stage implementation uses this direct route. The separate +located-source reification study remains a historical control. + +Canonical compilation and effective-source selection are locally implemented. +There is no new public author composition clause or additional artifact schema. Unknown policies or arbitrary +unverified metadata must not become a bypass of existing grammar admission. + +* Relation to the papers and next discussion + +Our interpretation of +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Ballantyne/Gamburg/Hemann]] is to keep an extensible author vocabulary and a +shared compilation core, with explicit boundaries that retain DSL invariants. +Our interpretation of +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Culpepper/Felleisen]] is to explain mistakes in the author's vocabulary and +resolve declaration dependencies with the appropriate context. Neither paper +specifies our artifact hashes or component-origin schema. + +The concrete Scheme/POO design now has evidence for contextual single-rule +macros, phase-qualified multi-rule construction, checked POO composition and +persisted declaration origins. The next discussion should settle the admission +contract for composed IR and provenance history before choosing a public author +surface. That will let syntax become simpler without making generated rules or +composition decisions opaque. + +* Follow-up: override and removal + +The [[file:scheme-composition-history.org][composition history study]] adds nineteen native checks for selected owners, +retained operations, dangling-reference rejection and closed override/removal. +The earlier controls preserve the append-only mapper's negative control. The +expanded engine study now passes 63 operation/source checks and the composed +package passes its checked-entry controls; see [[file:scheme-dsl-engine-admission.org::*Package closure criteria at this boundary][current receipts]]. Exact reader +spans for arbitrary extensions and a production public composition clause remain +separate design obligations. + +* Scheme design rationale: two required questions + +This Scheme datum illustrates the research source-record shape. It is +metadata, not a new declaration API; its descriptive location is not a reader +line/column span. + +#+begin_src scheme +'((path . "t/fixtures/language-pack-research/list-roles.ss") + (location . (role call-arguments rule call-arguments/tail)) + (generated? . #t) + (componentOwner . call-arguments)) + +;; Question 1: Why is this DSL design better? +;; Authors declare a component while generated helpers retain an explainable owner. +;; Concise syntax retains provenance, so diagnostics and tools need not guess origins. +;; Question 2: Why does this DSL design have a scientific basis? +;; Separate binding identity from provenance: source changes preserve bindingId/referenceDigest. +;; Evidence: 17 identity/artifact checks and 14 actual-language integration checks. +;; Falsifier: a source-only change alters reference identity or loaded Bound IR loses sources. +;; Limit: no public schema; reader-origin construction passes, actual-language loading times out. +#+end_src + +Source integrity and accurate source attribution are separate obligations. +Persisting a supplied record does not prove that its origin was automatically +collected or that its history is correct. + +* Follow-up: reader origins + +The [[file:scheme-source-origin-contract.org][reader source-origin study]] now checks actual author and definition token +coordinates through existing Bound IR construction. Artifact-backed language +loading remains a separate failed bounded receipt. + +The expanded 2026-10-06 driver requires a fresh =PARSER_RESEARCH_ARTIFACT_ROOT=. +Its 31-check exit-0 receipt and bounded interpretation are recorded in the +[[file:scheme-dsl-engine-admission.org][engine admission module]], under verified declaration-context and parser-cache +reuse. The earlier 17-check result remains historical evidence. diff --git a/docs/research/scheme-composition-history.org b/docs/research/scheme-composition-history.org new file mode 100644 index 00000000..4bf30029 --- /dev/null +++ b/docs/research/scheme-composition-history.org @@ -0,0 +1,191 @@ +#+title: Scheme component history: selected declarations, override and removal +#+date: 2026-10-06 + +* Question and boundary + +The [[file:scheme-component-provenance.org][component provenance study]] verified persisted origins through existing Bound IR +inputs for unique merge/append contributions. This follow-up asks how an effective +rule's provenance changes under explicit override or removal. + +This is local research on baseline =783381f33fa1b309ac4cc4b0012696f24c852d93=. +It adds an executable Scheme control and a design contract for discussion. +It does not implement a new normalizer, automatic history resolver or public +provenance schema. The existing POO normalization and compiler owners determine +all final grammar values used by the experiment. + +* Actual override and removal experiment + +[[file:../../t/fixtures/language-pack-research/composition-history.ss][composition-history.ss]] starts from the list grammar, then contributes: + +- An override of =arguments= by =single-argument-extension=, whose new rule is + =(field argument (reference name))=. This deliberately changes recognition + organization to a single required argument expression. +- A removal of =call-arguments/tail= by =retired-tail=. + +The driver invokes normalization, parser compilation and Bound IR binding. +It does not execute source parsing or claim parity with the earlier multi-item +language. Its successful invocation was: + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" GERBIL_PARSER_LR_TRACE=1 \ + gtimeout 10 gxi t/fixtures/language-pack-research/composition-history.ss +#+end_src + +The historical driver exited 0 and reported =COMPOSITION-HISTORY-OK: 19 checks passed=. +The first attempt exited 70 because a new fixture form was missing a closing +parenthesis; that driver syntax was repaired before any result was admitted. + +| Case | Checked outcome | +|------+-----------------| +| Override =arguments= only | Exact final row equals the replacement; the independent tail rule remains | +| Override receipt | Prior append and later override owners remain in operation order | +| Apply the old append-only source mapper | It selects the old owner, demonstrating its stated scope limitation | +| Supply an explicit selected-row source | Bound IR carries the override owner and the supplied history | +| Remove the tail while keeping its reference | Normalization removes it; Parser IR and Bound IR both reject the dangling reference | +| Override the reference-bearing row, then remove the tail | Final row is the replacement, deleted helper has no binding, parser and binding admission succeed | +| Override or remove an absent target | Existing typed messages reject the operation | + +The parser's dangling-target diagnostic includes =arguments= and +=call-arguments/tail=. The Bound IR diagnostic identifies the deleted rule through +its structured binding ID. The removal check demonstrates admission behavior; +it is not an input recovery test. + +The explicit history in the selected source is a manually supplied research +record with append step 1 and override step 3. Its successful persistence proves +that the existing source record can carry this data. It does not prove that the +compiler collected or validated that history automatically. + +* Design consequence: effective source and history are different + +There should be two related views: + +1. Effective declaration provenance explains the row that is actually bound and + compiled, including its current contribution origin. +2. Contribution history explains which operations introduced, replaced or removed + that identity, including earlier contributions that are no longer effective. + +The current Bound IR source lookup uses the first matching name in the selected +namespace map. An append-only list of every historical source is therefore not +an effective provenance projection. After override, it can select the original +contributor while the grammar contains the replacement. After removal, a history +record may legitimately remain, but it must not masquerade as an active binding. + +The experiment preserves this negative control instead of silently fixing the +old unique-append fixture. A production source collector must state the +projection contract before extending to general composition. + +* Composition operation matrix + +| Operation | Existing grammar meaning | Provenance requirement to design | +|-----------+--------------------------+----------------------------------| +| Merge new key | Admit its row | Record the contributing source | +| Merge equal existing row | Coalesce the row | Retain contributing origins without pretending a semantic override happened | +| Merge different existing row | Reject conflict | Explain both contributions | +| Append | Admit a new key; reject an existing key | Preserve owner and diagnose the colliding identity | +| Override | Replace an existing row | Select replacement provenance and retain earlier history | +| Remove | Delete an existing key | Exclude active provenance; retain the removal history if needed | + +Equal-row merge behavior is source-inspected in +[[file:../../src/utilities/lists.ss][merge-keyed-row]]; this follow-up does not add a new equal-merge runtime control. +Consequently, an automatic "last contributor wins" source collector would not +accurately describe every operation. Equal merges may have several contributors +for one effective row. A primary diagnostic location can be selected separately +from the complete contributor set. + +Override does not imply ownership of every helper from the original component. +Helper lifetime follows explicit final references and composition operations. +The successful override-plus-remove case provides a concrete control for that +boundary; no automatic helper garbage collection is proposed here. + +* History identity and association with final IR + +Existing composition steps carry index, operation, role and contributed keys per +section. They do not contain the original row bodies or exact source ranges. +The receipt alone cannot reconstruct an overridden rule's old definition. + +A history design can reference a step by composition identity and index, then +identify its section and key. It also needs admitted contribution content or a +reference to that content, plus author/template origin information. The final +projection must agree with the normalizer's actual final row set. History +processing should observe that result, not implement another grammar merger. + +The current =compositionDigest= hashes the step descriptors; row content belongs +to the separately recorded grammar-body identity. Neither hash alone is a complete +version of every author's historical contribution. Their scopes need explicit +association when the composed IR enters artifact admission. + +A proposed contract should require: + +- Every effective rule's selected provenance corresponds to its final declaration. +- Removed rules have no active Bound IR binding or reference target. +- Every surviving reference resolves in both canonical parser and binding admission. +- Equal-row contributors are distinguishable from override events. +- History references have a defined content/source owner, not just a role spelling. + +These are design requirements, not a new implemented validator. + +* Exact source origins remain a separate experiment + +The research source records use truthful provider paths and descriptive labels; +they do not assert exact source spans. A macro-owned declaration can preserve its +syntax object and capture its source. A Scheme factory constructing role data +needs an explicit origin input if its invocation site should be explained later. + +A future component declaration should retain both the author's instantiation +location and the template's definition location. Generated helper rules should +point back to that relationship. Merely giving reified rows the adapter's source +location would explain the mechanical expansion site rather than the author's +component choice. This design follows from the phase boundary; no source-range +propagation implementation has been added in this follow-up. + +* Connection to research and working recommendation + +Our reading of +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Culpepper/Felleisen]] supports explaining errors in the author's vocabulary and +separating dependency-sensitive checks from initial declaration recognition. +Our reading of +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Ballantyne/Gamburg/Hemann]] supports explicit extension positions and a common +compilation core. Neither supplies our composition history contract. + +Working recommendation: derive effective origins from the admitted final IR and +retain contribution history alongside it, while keeping the existing POO merger +as the grammar authority. Define this association before publishing a general +component import/override surface. The next research should use actual syntax +locations for one component declaration to test the author-to-generated-rule +explanation end to end. + +* Scheme design rationale: two required questions + +These are the canonical rule values used by the experiment. The operation +comments describe the existing composition operations; they introduce no new +surface syntax. + +#+begin_src scheme +;; Entry rule contributed by append: +'(arguments + (sequence (field argument (reference name)) + (reference call-arguments/tail))) +;; Replacement rule contributed by override: +'(arguments (field argument (reference name))) +;; Remove call-arguments/tail only after its references are gone for admission. + +;; Question 1: Why is this DSL design better? +;; Checked POO operations distinguish adding, replacing and deleting contributions. +;; Effective provenance identifies the selected replacement; history explains earlier steps. +;; Question 2: Why does this DSL design have a scientific basis? +;; Final rules, effective origins and history have explicit consistency obligations. +;; Evidence: 19 checks retain the append-only mapper's override mismatch as a negative control. +;; Falsifier: admitted dangling references or effective origins naming a replaced contribution. +;; Limit: history is supplied manually; no automatic collector or old-parser parity claim. +#+end_src + +The negative control limits the recommendation: a general collector must follow +each operation's semantics, rather than extrapolating an append-only mapper. + +The expanded 2026-10-06 driver passes 31 checks and exits 0 under +=gtimeout -k 2 10=. Its 12 additional rule/catalog/source controls and their +limits are owned by [[file:scheme-dsl-semantic-contracts.org][the semantic contract module]]. The historical receipt +above does not qualify those newer controls by implication. + +The later 37-check receipt includes six existing catalog-projection controls; +see [[file:scheme-dsl-semantic-contracts.org][the semantic contract module]] for inputs, failures and limits. diff --git a/docs/research/scheme-contextual-parser-ir.org b/docs/research/scheme-contextual-parser-ir.org new file mode 100644 index 00000000..0be2dacb --- /dev/null +++ b/docs/research/scheme-contextual-parser-ir.org @@ -0,0 +1,211 @@ +#+title: Contextual parser IR migration +#+startup: overview + +This design follows [[file:scheme-bash-source-frontend.org][the source frontend migration]] and +[[file:scheme-engine-library-boundary.org][the engine/library boundary]]. It is one coordinated +migration, with implementation and qualification listed separately below. + +* Objective and authority + +The language author continues to own =grammar.ss=, =parser.ss= and +=parser-test.ss=. Macro expansion and optional POO composition elaborate ordinary +values or inherited declarations into admitted compiler products. The engine owns +cross-reference checks, context selection, executable lowering and publication. + +A common result wire format does not establish common recognition semantics. +The migration must bind recognition, scanner context, result construction and +release identity through the same admitted product. LR and command execution may +remain different lowerings; neither gets a separate source of language truth. + +* Required compiler boundary + +An admitted product contains closed recognition data, context transitions, +construction operations, public vocabulary and an entrypoint. Backend-specific +instructions belong to explicit instruction dialects. No procedure-valued slot +may masquerade as portable recognition IR. Execution closures and navigation +indexes are derived runtime state, not canonical definitions. + +The following Scheme block is a contract sketch, not an implemented public DSL: + +#+begin_src scheme +;; These are engine sections produced by deflanguage expansion, not new author files. +((schema . "gerbil-parser.contextual-parser-ir.v2") + (recognition . ((dialect . command) (program . ...))) + (scanner . ...) + (context . ((positions . ...) (entries . ...) (transitions . ...))) + (construction . ...) + (root-kind . ...) + (digest . ...)) +#+end_src + +The =command= and =lr= dialects must each have executable semantics and checked +references. Putting existing plans into an outer record without these obligations +would not complete this migration. + +** Why is this design better? + +It removes repeated interpretation of author declarations. A declaration is +admitted once; the native executor, identity publisher and Rust generator consume +that exact product. Extensions modify declaration values before admission, rather +than injecting behavior into the recognition loop. A derived representation has +no authority to reinterpret the declaration. + +** Why is this design scientifically defensible? + +The claims have executable controls: equivalent canonical/contextual artifacts, +rejection of inconsistent identities and unknown references, selector-overlap +witnesses, publication from the admitted construction vocabulary, and unchanged +accepted/rejected controls. A syntax improvement or a passing benchmark alone +cannot discharge those semantic obligations. + +#+begin_src scheme +;; Acceptance controls must observe behavior, not merely an IR label. +(check (compile-count declaration) => 1) +(check (native-events admitted source) => (reference-events admitted source)) +(check-exception (admit-with-unknown-entry declaration) true) +(check-exception (admit-with-incomparable-selectors declaration) true) +(check (descriptor-digest admitted) => (artifact-digest admitted source)) +#+end_src + +These are acceptance descriptions; actual test names and implemented operations +must be recorded with the implementation receipts. + +* Paper-to-implementation mapping + +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]], section 1 and Figure 1, +separates extensible surface elaboration from a compiler core and checked host +boundaries. Our transfer is a single admitted recognition/context/construction +product; Scheme execution and Rust emission consume its owned lowerings. +The paper's Racket metalanguage is not assumed to exist in Gerbil. + +The benefit comes from hygienic expansion, =defsyntax=/=defrules= staging and POO +composition followed by checked admission, rather than the shape of S-expressions +alone. Extension authors retain Scheme abstraction; the engine verifies references +before executing or emitting a backend. This is an engineering application of the +paper, not a claim that its results prove this parser or its performance. + +#+begin_src scheme +;; Executed in t/source-strategy-test.ss: the provider returns an admitted product. +;; No original parts/commands/scanner declaration slots exist on `bound`. +(check (command-source-rust-module-source bound) => reference) +(check compilations => 1) +;; Consequently a generator cannot silently reinterpret a second declaration. +#+end_src + +* Recognition and selector obligations + +The existing LR position compiler selects maximal requirement/exclusion clauses +and rejects incomparable matches with different positions. The existing scanner +dispatch resolves finite mode/position/form cells with origin receipts. Those +implementations should be reused and extended, rather than replaced with a +second selector engine. + +Command lookahead and adjacency selectors require explicit token constraints and +source-adjacency constraints. Duplicate-selector rejection is insufficient: +nonidentical predicates may overlap. The compiler must retain a concrete overlap +witness and either prove specificity, apply an explicitly declared choice, or +reject unresolved alternatives. Input declaration order is not a general proof. + +* Context, incremental reuse and recovery + +Existing LR incremental and recovery implementations remain reference inputs. +The source path has scanner checkpoints and same-source restoration, but these +do not establish cross-edit subtree reuse. A reuse certificate must bind the +program digest, source boundary mapping, entry/exit context and pending scanner +obligations. A changed delimiter queue or lexical mode invalidates reuse unless +its equivalence is established. + +Recovery must retain repaired-source status, bounded search and repair evidence. +A partial tree is not permission to advertise repaired input as originally valid. +These requirements are independent of C ABI transport success. + +* Compilation staging + +Ordinary LR admission runs during declaration expansion. The existing immutable +artifact publisher stores the common product and embeds its compressed payload +in the generated module. Runtime assembly extracts the backend Parser IR from +that product; the descriptor and machine share that exact owned backend datum. +A compilation cache hit returns the product locator without loading or hashing +the backend instructions again. + +This placement matters: an initial implementation constructed, copied and hashed +the common LR envelope during module initialization. The native conformance +executable exceeded its unchanged ten-second completion fence before reaching +the first test. That failure is retained as a staging regression, not attributed +to GC or parsing throughput. The compile-time migration requires fresh native +qualification; no latency improvement is inferred from the code change alone. + +* Native consumption + +Descriptor metadata is read from the actual bound product. For contextual LR, +its digest differs from the base grammar digest; choosing the latter would break +identity admission. Descriptor lookup must not execute recognition. + +Rust owns an immutable foreign payload while borrowing its source and handle +catalog. First view access admits GPA1 and creates navigation indexes. Later views +borrow those indexes. The cache contains no references into its own payload owner, +so moving the owner does not create a self-reference. The artifact crate performs +no recognition and remains free of unsafe code. + +* Implementation status + +| Boundary | Implemented behavior | Remaining gate | +|----------+----------------------+----------------| +| Bound descriptor identity | Direct source/machine/contextual digest access; metadata spy and all 20 native language ABI controls pass | Exact-head CI qualification | +| Rust navigation | Immutable payload storage admits and indexes once; fresh 20-language fixture controls check shared indexes and exact source slices | Exact-head CI qualification | +| Source compilation | Compiler-owned command/line products supply identity, Scheme plans and Rust emission; single-compilation, absent-slot and native parity controls pass | General command semantic obligations below | +| Common contextual IR | Ordinary/contextual LR and command/line products use the v2 closed envelope; ordinary LR admission is published during expansion; native startup and 28 closure children pass | Exact-head CI and broader selector/reuse/recovery obligations | +| Selectors | LR clause specificity and finite scanner dispatch admission | General command selector overlap witnesses | +| Incremental and recovery | LR facilities and source checkpoints exist | Cross-edit source certificates and contextual partial-tree recovery | + +The complete migration is qualified only when these obligations are implemented +and the coordinated controls pass. Public type names and document diagrams are +not completion evidence. + +* Local qualification receipts + +The focused native suite passes 33 cases across contextual parsing (15), source +strategy admission (8), and native identity/event publication (10), including +final child exit and GC controls. The Rust workspace passes 151 tests with two +ignored performance cases; formatting and Clippy pass. The existing contextual +and native-artifact Rust fixtures match fresh Scheme generator output. + +Native conformance attempts retained output-idle and total-timeout failures. +Inspection found four orphaned conformance processes in this checkout after +PTY wrappers moved children into separate process groups. The existing bounded +runner now captures its owned descendant groups before terminating the wrapper; +a separate-session child control joins the existing ten runner/proxy tests. +This operational repair changes neither parser semantics nor completion fences. +The native entrypoint now directly executes the Gerbil compiler, preserving its +real output while retaining the bounded owner process group. A fresh conformance +executable linked in 77.93 seconds using existing objects; this is incremental +build evidence, not a cold-build improvement. All 28 existing DSL closure children +exited zero under their unchanged ten-second fences. Fresh native ABI publication +completed in 3.23 seconds and emitted all 20 accepted/rejected controls. + +The production fixture comparison preserves names, sources, statuses, every event +byte, source digests and descriptor fields except grammarDigest. Only the two +Bash controls update their product digest. Rust qualification was repeated after +that fixture update. The all-language benchmark child is a one-sample smoke +control, not a supported p50/p95 or end-to-end performance claim. Exact-head +remote CI remains a separate gate from these local receipts. + +* Author-contract qualification boundary + +The preceding receipts qualify engine products at the earlier revision. They do +not prove a closed author DSL: its flat =deflanguage= fallback still accepted +retired declarations. The coordinated migration now removes that fallback and +the old source-adapter exports, migrates every language to =syntax= and =rules=, +and moves backend admission to =parser.ss=. See the concrete Scheme contract and +its two design questions in [[file:scheme-dsl-author-interface.org][the author interface]]. + +Requalification requires actual macro rejection of formerly valid declarations, +all-language structure and recognition controls, fresh native and Rust artifacts, +and CI at the new head. Semantic expectations remain independently authored. +Compiler-generated catalog indices and digest changes are not parser speedups. + +The 2026-10-08 author migration passes all 28 native closure children and +preserves all 20 language controls after resolving catalog indices to event +names. These are author admission and recognition receipts. They do not qualify +cold compilation, deployed Rust consumers or the new GitHub head; those gates +retain separate outcomes in the author-interface migration evidence. diff --git a/docs/research/scheme-declaration-diagnostics.org b/docs/research/scheme-declaration-diagnostics.org new file mode 100644 index 00000000..5945a7af --- /dev/null +++ b/docs/research/scheme-declaration-diagnostics.org @@ -0,0 +1,162 @@ +#+title: Scheme declaration diagnostics: parameter blame and absent source +#+date: 2026-10-06 + +* Research question and local status + +How can a component macro report an invalid author parameter before expanding +into generated helper rules, while remaining correct when syntax has no reader +location? This extends the [[file:scheme-source-origin-contract.org][source-origin contract study]]. + +The local fixture modifies =deflocated-list=, an experimental component macro; +it does not migrate public =deflanguage= or publish a new diagnostic schema. +The committed baseline remains =783381f33fa1b309ac4cc4b0012696f24c852d93=. +Research documentation and Scheme comments are written in English. + +* Scheme rationale: the two required questions + +This is an excerpt from the existing local =syntax-case= branch in +[[file:../../t/fixtures/language-pack-research/located-component.ss][located-component.ss]], not a standalone module. Each identifier parameter +has an equivalent explicit check before the existing role construction. + +#+begin_src scheme +;; Inside the matched component declaration: +(unless (identifier? #'label) + (raise-syntax-error #f + "deflocated-list: field label must be an identifier" + #'label stx)) + +;; Invalid author declaration: +(deflocated-list component call-arguments arguments + (reference name) (literal ",") (field 45)) + +;; Question 1: Why is this DSL design better? +;; Compared with a generic failed pattern guard, the author learns which parameter +;; is invalid and its required kind before generated helper construction begins. +;; Scheme syntax objects retain the offending term and the whole author declaration. +;; Concise component syntax can therefore retain precise explanations of misuse. +;; Question 2: Why does this DSL design have a scientific basis? +;; The contract specifies exception type, message, blame term and reader coordinates. +;; Evidence: 48 real-reader checks and 6 source-free checks, each with exit 0. +;; Falsifier: an unrelated exception is accepted, the wrong term is blamed, +;; a coordinate points to different text, or an absent location is invented. +;; Limit: no editor-navigation or developer-task study; general grammar errors remain separate. +#+end_src + +The four parameter kinds checked are the Scheme result binding, component owner, +public rule entry and per-item field label. An identifier-shaped owner is a +valid declaration parameter; its uniqueness remains a POO composition question. +Similarly, a reference's existence is a whole-grammar binding question. +These different checks must not be collapsed into the same parameter diagnostic. + +* Read the paper through concrete design obligations + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Culpepper and Felleisen, Fortifying Macros (ICFP 2010)]], section 2, distinguishes +pattern guards from explanations of failed syntax constraints. The local change +uses explicit checks because a failed boolean guard alone gives no reason. +Section 7.2 studies a parser macro using two passes to resolve dependencies +between declaration clauses. We take that as a design principle: local parameter +shape and whole-grammar name resolution have different required information. +The paper's Racket syntax classes are not assumed to exist in Gerbil, and its +code-size results are not evidence about our implementation. + +| Obligation | Existing owner / local design | Evidence boundary | +|------------+-------------------------------+-------------------| +| Declaration shape | Component syntax-case and explicit checks | Four identifier parameter categories checked | +| Local blame | Offending syntax first, declaration as supporting detail | Actual parameter token coordinates checked | +| Grammar reference resolution | Existing Bound IR / compiler | Existing composition-history controls cover dangling references | +| Contribution collision | Existing checked POO composition | Earlier list collision controls; no new collector here | +| General diagnostic ordering | Define which failed contract to report first | Current order is binding, owner, entry, label; multi-error controls pending | + +This division allows author vocabulary to improve feedback while reusing the +parser engine's existing semantic checks. It does not justify implementing a +second independent grammar validator in the macro layer. + +* Reader input and transformer templates are different controls + +The first diagnostic driver placed invalid forms directly inside the collecting +transformer with =#'= templates. Their blame locations were absent. The driver +then incorrectly assumed a location existed and exited 70. A second attempt +also found that the template's whole declaration had no source location. Neither +result establishes that numeric literals read from a file lack locations. + +The corrected driver accepts actual invalid declarations as its own macro +arguments. It traverses their preserved syntax using Gerbil's =stx-map= and +invokes the component expander without evaluating generated definitions. +Both numeric and compound invalid parameters retain real reader locations. +A mistaken expectation that numeric reader parameters lacked locations was +rejected by the control and corrected. An intermediate attempt used nonexistent +=stx->list= and exited 70; the corrected fixture uses the existing Gerbil utility. + +For all eight declarations, the driver checks the actual SyntaxError type, +exact message, blame datum, presence of a parameter location, source text at +that coordinate, and preserved declaration context. The coordinate check reads +the source file; it does not compare against a hand-authored line table. + +The expander's installed =raise-syntax-error= constructs SyntaxError with +=stx= followed by detail arguments as its irritants. Installed runtime +=SyntaxError= extends Exception rather than Error. The driver consequently reads +its =irritants= slot, instead of assuming =error-irritants= admits this type. +No claim about rendered UI highlighting follows from inspecting these slots. + +* Explicit source-free control + +[[file:../../t/fixtures/language-pack-research/source-free-diagnostics.ss][source-free-diagnostics.ss]] deliberately reconstructs syntax with retained +lexical context and no source: + +#+begin_src scheme +;; Inside the collecting transformer: +(datum->syntax #'source-free-cases (syntax->datum form) #f #f) +#+end_src + +For a valid reconstructed component, its inert expansion preserves owner and +known template origin, while authorOrigin, path and location remain =#f=. +For an invalid owner, the exact SyntaxError, message and blame value remain +available while its source remains =#f=. These are expansion controls; they do +not execute the role construction or claim source-free artifact integration. + +The working presentation contract is to use an offending parameter's location +when present, otherwise the author declaration's location when available, +otherwise an explicit unknown location. No runtime or editor renderer for this +fallback protocol is implemented here. Independent origins must not be replaced +by a template location and presented as an author invocation. + +* Native receipts and retained failures + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout 10 gxi t/fixtures/language-pack-research/reader-diagnostics.ss + +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout 10 gxi t/fixtures/language-pack-research/source-free-diagnostics.ss +#+end_src + +- [[file:../../t/fixtures/language-pack-research/reader-diagnostics.ss][reader-diagnostics.ss]] reported =READER-DIAGNOSTICS-OK: 48 checks passed=, + exit 0. +- [[file:../../t/fixtures/language-pack-research/source-free-diagnostics.ss][source-free-diagnostics.ss]] reported =SOURCE-FREE-DIAGNOSTICS-OK: 6 checks passed=, + exit 0. +- The retained [[file:../../t/fixtures/language-pack-research/declaration-diagnostics.ss][combined driver]] completed all 54 assertions and its summary but + exited 124. It remains a timeout. The passing receipts cover the two independently + run responsibilities; they do not change the combined invocation's status. + +No timeout was widened. Earlier actual-language source-loading timeouts also +remain open. These controls establish bounded macro diagnostic behavior, not +runtime speed, rendered diagnostic quality or general language-package usability. + +* Working recommendation and next research questions + +A component declaration should validate its own parameter contracts and attach +blame to preserved author syntax before generating its role. POO composition and +existing compiler binding should continue to validate grammar relationships. + +Before settling a public declaration protocol, study imported/renamed macro +presentation, nested wrappers retaining original invocation context, multiple +simultaneous errors, and operation-aware attribution after override/removal. +A developer-task study should measure whether authors can locate and repair an +invalid declaration from the diagnostic. Correct exception metadata alone does +not establish that usability benefit. + +* Follow-up: nested vocabulary + +The [[file:scheme-nested-declaration-context.org][nested context study]] supplies passing forwarding and renamed-import +controls. It retains the negative result that the inner diagnostic context does +not automatically become the author's outer declaration. diff --git a/docs/research/scheme-deflanguage-design-synthesis.org b/docs/research/scheme-deflanguage-design-synthesis.org new file mode 100644 index 00000000..2ebf7c27 --- /dev/null +++ b/docs/research/scheme-deflanguage-design-synthesis.org @@ -0,0 +1,340 @@ +#+title: Scheme parser DSL design overview +#+date: 2026-10-06 + +* Research objective + +Develop the syntax vocabulary underneath =deflanguage= so that language authors +can express intent, compose recurring grammar patterns and give the parser +compiler useful structural information. This is a design study of the existing +Scheme interface. Its questions concern expressiveness, semantics, authoring +quality and compiler analysis. A defect hypothesis is not the research premise. + +Historical source baseline: =29dcb79dafcde202fc9818def3ac536cc20f47c4=. +Executable-study baseline: =783381f33fa1b309ac4cc4b0012696f24c852d93= plus +uncommitted local research fixtures. Receipts qualify their stated boundaries; +no single source-aware package implementing every proposal has been qualified. +The current =deflanguage=, =defgrammar-syntax=, canonical IR and three language +pack interfaces are the starting architecture. Proposed vocabulary examples +in the modules use current constructors. The local GQL implementation and its +open compiler-product gate are recorded separately; broader proposals remain +design work. Scheme remains the authoring and executable research language. + +* Research modules and reading order + +This overview owns the shared objective, fixed user boundary, evidence scope, +open decisions and closure sequence. Detailed explanations live in the modules +below. The call/array semantic control and actual GQL package anchor the same engine +contract. The modules are connected responsibilities, not independent features. + +| Module | Question it owns | Connected output | +|--------+------------------+------------------| +| [[file:scheme-dsl-author-interface.org][Author interfaces]] | What does the user declare in the three files? | Complete current/candidate package and test DSL | +| [[file:scheme-dsl-engine-admission.org][Engine admission]] | How does the engine elaborate, bind and publish it? | Canonical/context/cache/descriptor contract | +| [[file:scheme-dsl-semantic-contracts.org][Semantic contracts]] | What must remain true through composition and execution? | Recognition/tree/field/source expectations and controls | +| [[file:scheme-dsl-complexity-ownership.org][Complexity ownership]] | Does the boundary reduce author change and diagnosis effort? | Package tasks and engine maintenance obligations | +| [[file:scheme-dsl-research-basis.org][Research basis]] | What do the papers support, and where does transfer stop? | Primary readings, design method and bounded hypotheses | + +The [[file:scheme-bash-source-frontend.org][complete Bash source frontend study]] owns the implemented syntax/rules +migration and removal of legacy author recipes. It records common compiler +ownership, native semantic evidence and remaining analysis/publication limits; +three-file ownership alone is not treated as DSL qualification. + +The [[file:shared-scanner-staging-and-fusion.org][shared engine specialization study]] connects Gerbil compiler and runtime +source patterns with staging/fusion research, source lifetime and complete-product +controls. It keeps optimization below the same three author interfaces. + +Focused experiment documents remain evidence references. New conclusions should +update their owning module and this overview's decision/evidence summary when +needed. Do not append detailed experiments to this overview. Module boundaries +should be revised when a topic no longer has one coherent responsibility. + +* Fixed user boundary: three DSL interfaces + +The agreed author-facing architecture is =grammar.ss=, =parser.ss= and +=parser-test.ss=. DSL capabilities converge within these three interfaces; +implementation complexity belongs to the engine. This is the design constraint +for the integrated proposal, not an additional package interface to select. + +| User interface | Author expresses | Engine implements and validates | +|----------------+------------------+---------------------------------| +| grammar.ss | Grammar, lexical/tree policy and declarative reusable vocabulary | Macro elaboration, helper identities, checked POO composition, canonical admission, binding and analysis | +| parser.ss | Public checked entry and declared services/products | Loader admission, dispatch, artifact loading, scanners and backend/FFI mechanisms | +| parser-test.ss | Independent semantic expectations, fixtures and benchmark cases | Test execution, event/span comparison, reporting and measurement infrastructure | + +Reusable vocabulary is available through these DSL interfaces. A supported +Scheme declaration of new vocabulary can remain in grammar.ss; users are not +required to write a component factory module, source-map adapter or compiler +closure. Engine implementation modules may export facilities internally, but +that does not create a fourth required user interface. Optional library reuse +must preserve the same three-interface authoring model. + +Native POO slot composition is the preferred author substrate; the existing +merge/append/override/remove normalization labels are not four new author +keywords. See [[file:scheme-poo-language-components.org::*Native POO substrate and parser operation labels][the native POO contract and matched controls]]. Parser-specific +reference, field and provenance constraints remain engine responsibilities. + +The lower-level POO construction, canonical compiler and manual probe blocks in +this research explain engine behavior and provide experimental controls. They +are not mandatory steps for defining a parser. Public composition intent must +be expressible by supported DSL declarations; underlying row keys, receipts and +helper lifetime are engine responsibilities. Their inspectability supports +explanation and diagnosis without requiring authors to maintain them. + +Scientific evaluation must therefore include whether a task can be completed +through these three interfaces. If a routine grammar change requires direct IR +assembly, a hand-maintained source map or knowledge of generated helper names, +the proposed DSL boundary is incomplete even when the resulting parser is correct. + +* One objective and one representative language package + +The objective is to let authors state a language's intentional choices once, +reuse grammar vocabulary through Scheme modules, and compile it through the +existing parser engine with explainable origins. Checked POO composition remains +an engine capability, not a required author protocol. + +Use the existing small call/array language as the common running case: +=f(a,b)=, =[a,b]= and =f()=. It connects lexical declarations, component reuse, +public tree fields, composition, source origins and native qualification. The +expression-tier studies provide an additional binding-policy control; they do +not replace this package-wide narrative with another proposed entry API. + +The complete explicit control is in +[[file:../../t/fixtures/language-pack-research/list-language.ss][list-language.ss]]. Its author choices include identifier rules, punctuation, +whitespace, the root, empty-container policy, rejection of trailing commas, +per-item fields, conflicts and case sensitivity. A shorter DSL must preserve +those choices or document a chosen default; a shorter form alone is not a +semantic justification for hiding them. + +* The author-to-engine path + +| Stage | Input and responsibility | Output / current status | +|-------+--------------------------+-------------------------| +| Package declaration | Identity, root, lexical policy, intended public tree and target products | Existing language declarations; combined component author syntax pending | +| Scheme elaboration | defrules forwards syntax; defsyntax handles context; ordinary Scheme functions share local contracts | Existing primitives or checked role construction; experimental vocabulary | +| POO composition | Combine named contributions using checked merge/append/override/remove | Canonical Grammar IR plus composition receipt; existing mechanism | +| Binding and analysis | Resolve references; use existing nullable/FIRST facts; keep tree and parsing policy explicit | Existing Bound IR and parser compiler; proposed author-facing obligations pending | +| Artifact publication | Admit canonical input through the existing compiler owner and preserve admitted metadata | Shared canonical compiler and emission now retain compositionDigest in the composed fixture | +| Execution and qualification | Parse through existing runtime; compare events, fields, spans, roundtrip and errors | Selected passing controls; combined source-aware package closure pending | + +This is one compilation architecture. It does not require a parser per component, +a second Scheme compiler, or a new macro-specific grammar analysis engine. +The author syntax, checked role values and canonical IR are different interfaces +within that path; their phase and ownership must be specified together. + +* Integrated implementation decision review + +This review consolidates the paper-to-code argument. It prioritizes a complete +author diagnosis task over adding an explanation mechanism without a consumer. +The three-interface boundary is agreed. The first source-diagnostic slice below +is now implemented locally; broader internal choices remain recommendations with +explicit qualification work. + +| Decision | Present basis | Recommendation and remaining qualification | +|----------+---------------+--------------------------------------------| +| Author vocabulary | Existing Scheme macros and complete package controls | Retain ordinary Scheme abstraction; add no contribution/owner protocol | +| Semantic admission | Existing canonical, POO, binding and catalog owners | Preserve one effective grammar and its origins through current owners | +| First diagnostic integration | Existing checks plus transient expression associations implemented locally | Twelve admission/diagnostic cases plus 47 existing semantic cases pass with fresh native owners; selected package IR parity checked; full provenance route remains open | +| Recursive fact explanations | Existing analyses plus a researched finite-cause model | Defer integration until a concrete existing consumer needs an explanation; no new nullable-item rejection | +| Evidence lifetime | Current callbacks can be skipped on cache hits | Prefer current-input, request-scoped capture if needed; qualify replay cost and source identity | +| Portable source evidence | Current persistent Bound IR sources are declaration-level data | Keep initial live syntax transient; richer artifact projection requires a tooling task | +| Optimization admission | Recognition is only one observable contract | Require tree/field/span/policy parity and matched phase benchmarks | + +The implemented diagnostic task now also includes root shape and node/token kind +obligations, with direct/macro, multiple-body and source-free controls. See the +[[file:scheme-dsl-semantic-contracts.org::*Declaration obligations explain the authored term][declaration contract]] for Scheme rationale, falsifiers and limits; its +[[file:scheme-dsl-engine-admission.org::*Declaration obligations retain author syntax][engine receipt]] retains the failing regression and matched native product control. + +The current code task is the integrated [[file:scheme-dsl-semantic-contracts.org::*One constructor contract across declaration entries][constructor-admission boundary]], spanning +canonical algebra, concise author projection and canonical declaration lowering. +Its [[file:scheme-dsl-engine-admission.org::*Integrated constructor admission and native qualification][native qualification]] includes composition/LR controls and the complete package, +not just individual diagnostic examples. Shared canonical constraints and retained +syntax absorb this complexity in existing engine owners; the three author files +and native Scheme vocabulary remain the agreed interface. + +** First complete task: diagnose an existing invalid declaration + +This task has been integrated into the existing compiler owners. Its current +scope is the [[file:scheme-dsl-semantic-contracts.org::*One constructor contract across declaration entries][constructor-admission contract]], including direct/vocabulary origins, +forward helper validation, root/kind obligations and both declaration entries. +The [[file:scheme-dsl-engine-admission.org::*Integrated constructor admission and native qualification][native qualification]] records 59 passing cases, three compiled owners and +selected complete package IR parity. Earlier per-slice experiments remain evidence +in that module; they are not separate delivery milestones. + +The POO normalizer now qualifies effective-source selection; general author capture and LR explanations remain dependent +research tasks. They must serve an actual package consumer before becoming new +implementation work. Do not expand the public author interface to accommodate +unqualified mechanisms. + +* What the papers contribute to this one design + +| Research input | Role in the integrated design | Transfer boundary | +|----------------+-------------------------------+-------------------| +| Syntactic Abstraction in Scheme | Preserve syntax objects and contextual transformations | Actual Gerbil behavior needs native controls | +| Compiled, Extensible, Multi-language DSLs | Shared core/compiler; pinned implementation responsibilities inspected | Its backend and performance results do not predict this parser | +| Fortifying Macros | Specify local declaration contracts and resolve contextual dependencies in stages | No automatic Gerbil syntax-class implementation assumed | +| Binding as Sets of Scopes | Separate host/module binding from grammar-domain identity | Racket's model is not claimed as Gerbil's implementation | +| Keeping it Clean with Syntax Parameters | Carry scoped contextual syntax across cooperating macros | Declaration versus expression scope and nested policy need separate checks | +| PEG semantics | State the algebra and conditions of transformations | Ordered choice and PEG restrictions are not CFG/LR rules | +| Abstract interpretation | Bound conclusions by the resolved facts and reuse fixed points | Selected full-text sections studied; engine soundness proof remains open | +| Object-algebra research | Consider composition and interpretation of reusable contributions | Existing checked POO is the concrete mechanism, not a new Java-derived engine | + +The [[file:scheme-dsl-research-basis.org][research basis module]] provides primary references and reading +boundaries, including the integrated reading matrix and two incomplete reading +areas. Their common contribution is a disciplined author-to-core design. The GQL +implementation is one witness in that argument; it does not replace research on +hygiene, scoped context, diagnosis, parsing algebra, analyses or POO extensibility. + +* Evidence map: what has and has not been integrated + +| Responsibility | Supporting study | Current bounded result | +|----------------+------------------+------------------------| +| Whole small list language | [[file:scheme-list-role-composition.org][List role composition]] | 62 runtime/composition checks and 2 collision checks, exit 0 | +| Binding policy vocabulary | [[file:scheme-poo-language-components.org][Named tiers]] and [[file:scheme-rule-context-design.org][rule context]] | 65 and 59 native checks, exit 0; separate expression fixture | +| Origins and persisted identities | [[file:scheme-component-provenance.org][Component provenance]] | Historical 17 construction/artifact checks; expanded control has 31 checks, exit 0; separate language receipt and timeout retained | +| Effective contribution under mutation | [[file:scheme-dsl-semantic-contracts.org][Composition/catalog control]] | Expanded 37 checks, exit 0; includes projection controls; selected sources supplied manually | +| Exact reader origins | [[file:scheme-source-origin-contract.org][Source origins]] | 12 construction checks, exit 0; actual-language drivers timed out | +| Local and absent-source diagnostics | [[file:scheme-declaration-diagnostics.org][Declaration diagnostics]] | 48 and 6 checks, exit 0; combined driver timeout retained | +| Wrapper/alias preservation | [[file:scheme-nested-declaration-context.org][Nested vocabulary]] | 24 forwarding and 28 author-contract checks, exit 0 | +| Scoped delegated context | [[file:scheme-syntax-parameter-context.org][Syntax parameters]] | 9 scope, 28 diagnostic, 12 real-role checks, exit 0 | +| Resolved consumption facts | [[file:scheme-list-role-composition.org][List semantic contract follow-ups]] | 35 item/FIRST and 29 repetition checks, exit 0 | +| Expression-vocabulary package through checked Loader | [[file:../../t/fixtures/language-pack-research/package-expression-runtime.ss][Integrated local control]] | 85 checks, exit 0; selected events/fields/rejections and identity | +| Same package through declarative test DSL | [[file:../../t/fixtures/language-pack-research/package-expression-parser-test.ss][Conformance declarations]] | 11 cases, suite OK and exit 0; overlaps manual control | +| Inferred versus retained public fields | [[file:scheme-dsl-semantic-contracts.org][Matched list package DSL variants]] | 5 cases each, exit 0; initial combined invocation timeout retained | +| Entire package/product lifecycle | Local composed canonical, published machine and checked entry controls | Production public composition syntax, general author capture and full native target gates remain open | + +Do not add these counts into a package pass total. Some fixtures overlap, some +cover a different language, and successful local checks do not close failed +combined invocations. Correctness, developer usability, compilation cost and +runtime performance are different evidence obligations. + +* Integrated architecture decision and remaining boundary + +The research question is now singular: can a developer declare a composed +language using native Scheme abstraction in grammar.ss, publish it through +parser.ss and state its expectations +in parser-test.ss, while the engine preserves semantics, effective ownership and +artifact identity without requiring manual metadata or compiler knowledge? + +The working architecture is an extensible Scheme front end over one canonical +parser compiler and runtime. Hygienic syntax preserves host binding/context; +checked POO selects grammar contributions; existing inference, binding and +analysis establish effective grammar facts; existing publication supplies the +checked Loader. These responsibilities are one dependency chain. None is a +separate public feature justified by its own passing fixture. + +| Connected design obligation | Why the next stage depends on it | Present boundary | +|-----------------------------+---------------------------------+------------------| +| Preserve authored syntax during elaboration | Diagnostics and contribution origins need the original declaration | Local syntax/origin controls; combined collection unresolved | +| Select effective contributions and ownership | Catalog derivation and binding must describe the selected rules | POO operations qualified; automatic effective-source integration unresolved | +| Derive catalogs and include explicit public obligations | Canonical publication must agree with observable trees | DSL variants and projection controls qualified separately | +| Admit the canonical grammar intact | Binding, Parser IR and caches must consume the same grammar | Canonical publication, direct composed emission and checked Loader controls qualified locally | +| Publish the descriptor/machine through the checked entry | User tests and runtime must exercise the admitted products | Expression package qualified; composed source-aware route not closed | + +The combined rationale uses the papers together: Scheme syntactic abstraction +explains contextual elaboration; Fortifying Macros informs declaration checking; +scoped-context research informs cooperative diagnostics; compiled extensible DSL +research connects elaboration to a shared compiler. The pinned implementation +reading supplies concrete ownership examples. Actual Gerbil controls establish +only their stated boundaries. This chain is detailed in the +[[file:scheme-dsl-research-basis.org][research basis]], rather than used to justify unrelated vocabulary features. + +The fixed decisions are the three user interfaces and engine ownership of +mechanisms. The contribution/owner/compose candidate is withdrawn: it imposed a +new author model without establishing a required language-package task. The +[[file:scheme-dsl-author-interface.org::*Native Scheme abstraction is the author baseline][author baseline]] is the existing complete declaration with native Scheme +abstraction. Existing POO roles and operations stay engine capabilities or their +current lower-level API; they are not a new required concise DSL protocol. +The [[file:scheme-dsl-engine-admission.org::*Source-grounded implementation route: one admitted package][source-grounded implementation route]] remains useful for consistent effective +facts, receipt sealing and runtime publication. Any future public extension must +follow a demonstrated limitation of the complete author package. + +#+begin_src scheme +;; Existing author vocabulary under the agreed grammar.ss interface. +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; Question 1: Why is this DSL design better? +;; Scheme vocabulary expresses reusable intent; the engine owns the whole admitted path. +;; One language edit should flow to canonical grammar, products and independent tests. +;; Question 2: Why does this DSL design have a scientific basis? +;; Paper principles, pinned source responsibilities and native controls inform one chain. +;; Verify the same composed package through all three interfaces, not summed fixture counts. +;; Falsifier: manual catalogs/sources/IR are required, or different stages describe +;; different effective grammars while the package is reported as qualified. +;; Limit: the complete source-aware composition path is still not implemented/qualified. +#+end_src + +The [[file:scheme-dsl-author-interface.org::*Complete package review: the actual GQL declaration and its consumers][actual GQL package review]] now grounds the author-complexity discussion in a +production declaration and all its consumers. Its common-source emission refactor +is now implemented locally: GQL retains its catalog export, derives metadata from +the accepted products and uses one source-processing declaration. Existing source +and GQL suites pass under a 10-second bound with exit 0; a wrong digest is rejected +in expansion. Independent expected identities remain unchanged. The subsequent +[[file:scheme-dsl-author-interface.org::*Whole-package gate: compiler products must match the source contract][compiler-product gate]] built the changed native macro, but GQL optimization found +stale 9-field machine metadata against the current 12-field source; bounded +dependency refreshes and separate Scheme emission did not complete. This is local +source conformance with an open build gate, not installed or Rust/AOT closure. +The call/array package remains the small semantic control; its receipts cannot +establish author-task improvement for the GQL package. + +* Architectural delivery: thin author entry and focused engine owners + +The implementation sequence now refactors the engine ownership boundaries as +one unit. The [[file:scheme-dsl-complexity-ownership.org::*Implemented internal owners: one author surface, explicit engine stages][implemented owner map]] records the extracted concise elaborator, +lexical expression compiler, lexer generator and retained nominal machine owner. +The public deflanguage entry delegates using real hygienic syntax bindings. +Both DSL surfaces enter one compile-time declaration record and direct compiler +function, without whole-declaration macro replay. Runtime assembly receives the +same canonical grammar used for binding, compilation and publication. Analyses +and cache/publication mechanisms retain their existing owners. Admission, +compilation/publication and hygienic emission are now separate named phases. +Normalized POO grammar and supplied context can use the same publication phase; +effective-origin selection and composed runtime/Loader controls are locally +qualified. A production public composition clause and full native runtime linkage +remain open. +No additional author clause or parallel analysis engine is introduced. + +The actual GQL grammar.ss, parser.ss and parser-test.ss route remains the large +package control; call/array remains the small complete semantic control. Native +qualification is an acceptance gate for this architecture, not the research +agenda. Build troubleshooting must not displace the connected paper argument, +engine refactoring and complete author-package evaluation. + +| Acceptance obligation | Evidence boundary | Required observation | +|-----------------------+-------------------+----------------------| +| Author semantics and hygiene | Existing complete controls run with current compiled declaration owners and source consumers | Same identity, catalogs, macro context, diagnostics, trees, fields and spans | +| Canonical products | Complete call/array Grammar IR and Parser IR comparison | Same serialized products; no second compilation authority | +| Native owner compilation | Three lexical/machine owners and four declaration owners compile in isolated output directories, exit 0 | Standard build discovery includes the new owners | +| Native consumers | Existing installed LR/machine closure is inconsistent; GQL native compilation still exceeds the genuine-output watchdog | Rebuild and qualify the complete dependency and consumer closure | +| Performance and application service | No speedup or concurrent-service closure admitted by this refactor | Matched stage measurements and queue/execute/cancel tests with qualified products | + +Parallel cold verification attempts exceeded the existing process bounds; those +attempts are failures, even where assertion markers appeared. Source checks are +rerun sequentially to avoid conflating resource contention with semantic results. +Native module compilation, native consumer execution and GQL/Rust/AOT closure +remain separate claims. + +* One package-level research and implementation sequence + +1. Use the complete existing grammar.ss declaration, parser.ss entry and + parser-test.ss expectations as the baseline. Identify a concrete author change + or diagnosis task before proposing any additional public syntax. +2. Trace that declaration through existing elaboration, effective POO selection, + origin association, catalog inference/projection, binding and canonical publication. + Record the existing code owner at each boundary and any concrete extraction needed. +3. Qualify that same route with one intentional contribution replacement: selected + rules, retained optional fields, effective origins, artifact reload and checked + Loader observations must agree. Reuse the explicit language as semantic control. +4. Close the package's declared native products and bounded process-completion gates. + Then measure author change/diagnosis tasks and separate compile/load/parse phases. + +Progress means closing an edge in this complete path or resolving its public +contract. More passing local cases, new paper lists or new wrapper macros are +not independently package closure. Existing focused experiments remain supporting +controls; add an experiment only when it answers a named unresolved edge in this +sequence and state how its result changes the architecture decision. + +Keep detailed work in its existing module. Update this overview only when the +architecture decision, evidence boundary or next package-level step changes. +Research drafts remain local until design and review boundaries are agreed. diff --git a/docs/research/scheme-dsl-author-interface.org b/docs/research/scheme-dsl-author-interface.org new file mode 100644 index 00000000..6d49e63d --- /dev/null +++ b/docs/research/scheme-dsl-author-interface.org @@ -0,0 +1,800 @@ +#+title: Scheme DSL author interfaces and complete package examples +#+date: 2026-10-08 + +Scope: complete grammar.ss, parser.ss and parser-test.ss declarations for the same +call/array package. The [[file:scheme-deflanguage-design-synthesis.org][design overview]] owns the fixed interface boundary and decisions. +Compiler responsibilities are specified in [[file:scheme-dsl-engine-admission.org][engine admission]]; observable contracts +and field evolution are specified in [[file:scheme-dsl-semantic-contracts.org][semantic contracts]]. + +* Closed author contract and migration gate + +The author grammar has exactly two blocks: =syntax= selects a vocabulary and +its recognition policy; =rules= supplies local productions. Release metadata and +validated execution backends belong to =parser.ss=. The former flat declaration, +its release =identity= clause, the old source-adapter macro exports and authored +canonical =alias/sequence/precedence= constructors are removed from this API. +Engine IR still has its own closed encoding; it is not an alternative author DSL. + +#+begin_src scheme +;; grammar.ss: names and fields are collected through hygienic expansion. +(defgrammar-syntax (named kind item) + (node kind (field value item))) +(deflanguage example + (syntax (lexical + (root name) + (lex (identifier Identifier (identifier))))) + (rules (name (named Name identifier)))) + +;; parser.ss: release identity is owned by the public parser entry. +(deflanguage-parser-loader example-language + (grammar example-syntax) + (parse parse-example) + (slots metadata: '((language . "example") (version . "1") + (contract . "example.v1")))) + +;; Question 1: Why is this design better? +;; The author declares token recognition, node construction and release identity +;; once under their respective owners. Defsyntax/rules express reusable grammar +;; abstractions; the compiler derives consistent catalogs and admitted products. +;; POO extends values where inheritance is needed; ordinary values need no wrapper. +;; Question 2: Why is this scientifically defensible? +;; Old declarations must fail before publication. The new declarations must keep +;; independently specified acceptance, fields, spans, diagnostics and roundtrip. +;; A shared wire format or a successful import alone cannot establish alignment. +#+end_src + +The strict macro boundary, all production grammar declarations, native source +vocabularies and test declarations are migrated together. Qualification of the +previous contextual IR commit establishes engine-product admission only; it does +not qualify this new author-contract migration. Current migration receipts are +recorded only after their actual tests finish. No timing improvement is inferred. + +** Migration evidence, 2026-10-08 + +The complete DSL closure passed all 28 child processes under the existing +10-second completion fence. The four runtime research controls now expose native +Scheme procedures in the same conformance executable; their assertions remain +unchanged. Their earlier interpreted-import timeout is retained as a failed run. +The source rejection controls cover flat declarations with and without identity, +retired constructors, duplicate blocks and author-supplied execution recipes. + +The existing Scheme producer regenerated all 20 native language controls. A +separate Rust comparison resolved every event symbol through its catalog before +comparing node/field/terminal names, event identifiers, spans, root kinds, source +bytes and statuses. All controls retain those semantics. GQL catalog ordering +and product digests can change; bytewise equality alone would obscure the +distinction between a catalog index change and an event behavior change. + +Native compilation is a separate gate. An isolated Cypher C diagnostic took +24.71 seconds, including 22.29 seconds in parsing/preprocessing and 2.41 seconds +in optimization/code generation. Its preceding five-second silence-gate failures +remain failures. A successful independent link and semantic closure do not turn +that diagnostic into cold-build qualification or parser latency evidence. + +The existing full conformance build entry subsequently passed in 101.58 seconds +with its unchanged 240-second completion and five-second silence gates. This was +an incremental run reusing admitted production objects. All 28 closure children +passed again after that build. The refreshed Rust fixtures pass 151 workspace +tests (two ignored), strict Clippy and formatting checks. No cold-build speedup +is claimed. + +Ten additional native gxtest source owners pass 40 cases with propagated zero +exits and final OK. They cover header delimiters, catalog admission, layout, +first-character dispatch, lexical plans, empty LR actions, module source, quoted +profiles and structured lexical/text profiles. The catalog rejection probe +expands the invalid grammar directly; a fixture-release wrapper must not mask +the intended rejection with an unrelated expansion-context error. + +* Historical public API audit before the closed author migration + +The three-file topology is implemented, but it is not evidence that the entire +public DSL is minimal. On 2026-10-07 the eight grammar.ss modules contain neither +=setq= nor =set!=. Their =(seq ...)= forms express grammatical sequence, not +Scheme assignment. Sequence is a necessary semantic distinction; requiring every +author to spell every lowered sequence, reference and token wrapper is a separate +interface decision that must be justified with complete examples. + +The current public boundary has concrete remaining leaks: + +| Current surface | Responsibility | Design direction | +|-----------------+----------------+------------------| +| Grammar exports of grammar IR, parser IR and generated parser bindings | Compiler products | Keep diagnostic inspection explicit; ordinary parsing must not require these bindings | +| Whole grammar re-export from parser.ss | Consumer API | Publish the checked language entry, parse operation and deliberate metadata; keep compiler products out of that ordinary entry | +| Arithmetic/HCL imports of generated direct drivers and digest bindings | Backend admission | Derive or bind validated backend products behind the compiler/library boundary | +| GQL lexical recognition and token bindings | Source adapter and lexical semantics | Catalogs are derived from one lexical declaration; token policy stays explicit | +| Repeated literal/reference/token/sequence wrappers | Grammar vocabulary | Use existing hygienic grammar syntax to express recurring constructs and preserve their source origins | + +These are design obligations, not changes already shipped. Removing an export +without tracing generators, semantic tests and artifact inspection would merely +hide a dependency. Moving language-specific semantics into the generic engine +would also violate the engine/library boundary. The source adapter owns source +format translation; the engine owns canonical admission and compilation. + +The compiler already implements these shorthand rules in +[[file:../../src/compiler/concise-language.ss::175][surface-body and surface-expression]]: +strings lower to literals; unambiguous declared names lower to token/reference +forms; multiple rule/node body expressions lower to a sequence. Explicit token +or reference forms remain required for a token/rule name collision. Therefore +removing those redundant wrappers needs no new DSL constructor or parser runtime. +The complete call/array package below now uses these existing facilities; its +inline control, accepted/rejected expectations, field counts and exact AST/span +test remain independently authored. Language-specific lexical and conflict +policies remain explicit. + +The existing =defgrammar-syntax= example below is an executable starting point: + +#+begin_src scheme +;; Existing author vocabulary: express a required separated list once. +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; Rule use states language intent. The macro owns the repeated lowered shape. +(required-items argument name ",") + +;; Question 1: Why is this interface better? +;; It removes duplicated implementation of the separated-list policy while +;; keeping the author-visible field, item and separator choices explicit. +;; Hygienic expansion preserves binding and origin; parentheses alone do not. +;; Question 2: Why is this scientifically defensible? +;; The inline and macro forms must agree on accepted/rejected strings, exact +;; AST fields and spans, source roundtrip, diagnostics and published identity +;; rules. Those independent observations can falsify the proposed abstraction. +#+end_src + +The updated whole package completed its native eleven-case suite and the +independent inline-control program's 85 checks with actual exit zero, each under +the unchanged ten-second fence. The native test build completed in 32.44 seconds +under its existing 240/5-second fences. These receipts cover the list package; +they do not qualify rewriting every language declaration or removing public IR +exports. Such migration must preserve each language's independent expectations. + +This example qualifies one reusable construct, not a universal syntax redesign. +The next design comparison must cover complete arithmetic and GQL packages, +including their exports, backend admission and source adaptation. It must show +what ordinary callers import and what grammar authors write, alongside macro +expansion, native POO selection and canonical IR. A shorter surface is admitted +only when these observables remain equivalent and the number of concepts authors +must understand actually decreases. The cited macro and language-implementation +research in [[file:scheme-dsl-research-basis.org][the research basis]] supplies mechanisms and proof obligations; +it does not by itself establish that every current API choice is elegant. + +* Parser metadata ownership + +Release identity is authored in parser.ss and bound to syntax products by the +engine. Metadata accepts ordinary association-list values or native POO objects; +only extension users need object construction/inheritance syntax. All eight +production packages use the value form. The complete +[[file:scheme-bash-source-frontend.org][source frontend study]] owns the Scheme witness, domain contract and +value/object equivalence controls. Upstream grammar provenance remains an input +identity in grammar.ss, not a second parser release declaration. + +* Bash source frontend + +The complete [[file:scheme-bash-source-frontend.org][Bash author grammar and source frontend study]] owns the implemented +=deflanguage= syntax/rules migration, common compiler boundary and independent +admission evidence. Legacy author recipes have been removed. Native Bash tests +pass; precise macro origins, general ambiguity analysis and clean-build/downstream +qualification remain explicit limits. + +* Complete authoring comparison across the three package interfaces + +A coherent package keeps =grammar.ss=, =parser.ss= and =parser-test.ss= as +intentional interfaces. Their shared descriptor connects author intent, checked +entry and qualification. This proposal does not collapse them into a single +macro. The existing engine already derives terminal, lexical, node and field +catalogs; eliminating their duplication is an existing capability, not a new +research contribution. See [[file:../../languages/README.org][the language package contract]]. + +** Complete current grammar declaration + +The following is the complete inline control from the running list fixture. +The import shown here isolates its grammar dependency from that fixture's +additional POO study. This is a declaration comparison, not a newly qualified +standalone package. + +#+include: "../../t/fixtures/language-pack-research/list-language.ss" src scheme + +** Complete candidate grammar module using existing Scheme vocabulary + +The candidate below expresses the same list policy with the existing +=defgrammar-syntax= interface. It introduces no new constructor. Both list +instances remain optional at their container boundary; trailing commas remain +excluded. Explicit policy sections are retained to make the comparison inspectable. +The new package identity names this candidate; it is not a claim that its +persisted artifact identity equals the control's identity. + +#+include: "../../t/fixtures/language-pack-research/package-expression-grammar.ss" src scheme + +At each use, expansion produces =seq(field(item), repeat(seq(separator, +field(item))))=, the same expression as the control. It does not create helper +rules or an additional AST node. The exact candidate grammar has now been exercised +through a checked Loader and the local package control described below. The earlier +POO receipts remain separate evidence; they do not qualify this path by implication. + +The local executable package is +[[file:../../t/fixtures/language-pack-research/package-expression-grammar.ss][package-expression-grammar.ss]], +[[file:../../t/fixtures/language-pack-research/package-expression-parser.ss][package-expression-parser.ss]] and +[[file:../../t/fixtures/language-pack-research/package-expression-runtime.ss][package-expression-runtime.ss]]. On 2026-10-06 its driver ran with +=GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" gtimeout -k 2 10 gxi t/fixtures/language-pack-research/package-expression-runtime.ss=: +85 checks passed, the final =PACKAGE-EXPRESSION-OK= marker was emitted and the +process exited 0. It checks five Loader identity/dispatch expectations, eight +accepted inputs with complete event parity and repeated field counts, and eight +rejected inputs with independent rejection expectations, valid artifacts and +lossless source roundtrip. Full event comparison includes the event offsets. + +This is an integrated local expression-vocabulary/Loader control. Its manual +probe implementation is research infrastructure, not the intended author test DSL. +The later declarative-suite receipt below separately qualifies selected test DSL +clauses. Neither receipt qualifies canonical POO metadata admission, exact origin +propagation, cold-start timing, +Rust/AOT products or a performance improvement. The preceding timeout receipt +from provenance-language remains a failed invocation of a different driver. +The earlier timeout is not cleared by this successful package invocation. + +A shared vocabulary module may export =required-items= instead of defining it +locally. Ordinary Gerbil imports then select the vocabulary. The simple example +needs only =defrules= through =defgrammar-syntax=. Procedural =defsyntax= becomes +useful when parameter contracts or context need elaboration. Checked POO roles +become useful when a contribution owns multiple rules or participates in explicit +merge/append/override/remove. These are different responsibilities in the same +architecture; a simple expression idiom need not generate multi-rule roles. + +** Native Scheme abstraction is the author baseline + +The previous contribution/owner/compose candidate is withdrawn. It introduced +public bookkeeping without establishing a language-author task that required it. +Checked POO roles are existing engine capabilities; they do not justify making +ordinary authors learn a contribution protocol. No new author form is selected. + +The design starts from the complete existing package above: grammar.ss declares +language meaning, parser.ss declares the checked entry, and parser-test.ss declares +observable expectations. A reusable grammar idiom can use the existing +=defgrammar-syntax= template or ordinary Gerbil =defrules/defsyntax=. More powerful +macro facilities are useful only when they make a concrete declaration or change +clearer. A new name or shorter expansion is not a sufficient improvement. + +#+begin_src scheme +;; Existing author vocabulary: an expression abstraction with language meaning. +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; Existing grammar expression. No contribution owner or compiler metadata. +(node Call + (seq (field callee (token identifier)) (literal "(") + (optional (required-items argument (reference name) (literal ","))) + (literal ")"))) + +;; Question 1: Why is this DSL design better? +;; Compared with repeated list productions, a named grammar idiom centralizes +;; list policy while leaving the node, fields and delimiters visible to the author. +;; The engine derives catalogs and compiles the result through the existing entry. +;; Question 2: Why does this DSL design have a scientific basis? +;; Compare the complete explicit and macro-based packages using independent tree, +;; event, span and rejection expectations, then evaluate real edit/diagnosis tasks. +;; Falsifiers: hidden policy changes, worse diagnosis or manual metadata required. +;; Limit: native package controls establish stated semantics, not measured author +;; effort, universal superiority or a completed source-aware POO admission path. +#+end_src + +Keep the engine dependency order explicit: preserve syntax during elaboration; +resolve effective declarations; derive and validate catalogs; finalize canonical +data and receipt; compile and publish consistent runtime inputs. These are engine +obligations, not a sequence users must configure. Host macro availability and +language-domain reference validation remain distinct. The engine must diagnose +errors in terms of authored rules and parameters rather than internal role names. + +Existing keyed POO operations remain available at their current lower-level API. +The concise DSL does not acquire new composition forms by implication. If a +complete language-package task later needs multi-rule reuse, first express its +before/after declarations and observable contract with normal Scheme facilities. +Only a demonstrated limitation can justify another public form. The same rule +applies to helper identity, registration and inspection protocols. + +The paper-to-source rationale is in [[file:scheme-dsl-research-basis.org][the research basis]]. The corresponding +selection/finalization/publication responsibilities are in +[[file:scheme-dsl-engine-admission.org][engine admission]]. The checked entry and test declarations below complete the +same author package; they do not depend on the withdrawn candidate. + +** Complete checked entry and declarative conformance module + +The same candidate now exercises all three DSL interfaces. The two modules below +use existing loader and test declarations. Research filenames are mapped to +=./grammar= and =./parser= here to show the normal package layout; no fourth user +interface is required. The runner is test infrastructure, not authoring syntax. + +#+begin_src scheme +;;; -*- Gerbil -*- +;;; Complete research package entry using the existing checked Loader. +(import (only-in :gerbil-parser/language-support/entry + deflanguage-parser-loader LanguageLoader.) + ./grammar) +(export list-study-language parse-list-study) + +(deflanguage-parser-loader (list-study-language :: self LanguageLoader.) + (grammar list-study-language-grammar) + (parse parse-list-study)) +#+end_src + +#+begin_src scheme +;;; -*- Gerbil -*- +;;; User-facing test declarations for the same expression-vocabulary package. +(import :gerbil-parser/language-test-support ./parser) + +(deflanguage-parser-tests package-expression-parser-test "expression vocabulary package DSL" + (loader list-study-language) + (identity "package identity" + (language "list-study") (version "v1") (contract "list-study.local.v1") + (schema "gerbil-parser.language-entry.v2")) + (accepted "empty call" "f()" + (root SourceFile) (counts (Call 1) (Name 0)) + (field-counts Call argument (0))) + (accepted "empty array" "[]" + (root SourceFile) (counts (Array 1) (Name 0)) + (field-counts Array element (0))) + (accepted "single call argument" "f(a)" + (counts (Call 1) (Name 1)) (field-counts Call argument (1))) + (accepted "three call arguments" "f(a,b,c)" + (counts (Call 1) (Name 3)) (field-counts Call argument (3))) + (accepted "call whitespace" "f( a , b )" + (counts (Call 1) (Name 2)) (field-counts Call argument (2))) + (accepted "single array element" "[a]" + (counts (Array 1) (Name 1)) (field-counts Array element (1))) + (accepted "three array elements" "[a,b,c]" + (counts (Array 1) (Name 3)) (field-counts Array element (3))) + (accepted "array whitespace and newline" "[ a ,\n b ]" + (counts (Array 1) (Name 2)) (field-counts Array element (2))) + (rejected-many "separator and closing policies" + '("f(,)" "f(a,)" "f(a b)" "f(a" "[,]" "[a,]" "[a b]" "[a")) + (ast "exact empty call tree and spans" "f()" + (node SourceFile 0 3 + (field form 0 3 + (node Call 0 3 + (field callee 0 1 (token identifier "f" 0 1)) + (token punctuation "(" 1 2) + (token punctuation ")" 2 3)))))) +#+end_src + +The executable test declaration is +[[file:../../t/fixtures/language-pack-research/package-expression-parser-test.ss][package-expression-parser-test.ss]]. Its existing engine support is +[[file:../../src/testing/language-parser.ss][language-parser.ss]] and [[file:../../src/testing/parser-ast.ss][parser-ast.ss]]. Accepted/rejected clauses automatically +check artifact validity, intended status, lossless source roundtrip and source +digest. Field-count and exact AST expectations then add independent author intent; +they do not replace those shared invariants with per-package traversal code. + +On 2026-10-06 the +[[file:../../t/fixtures/language-pack-research/package-expression-test-runner.ss][bounded research runner]] emitted all 11 =CASE-OK= results, =SUITE-OK= and +=PACKAGE-EXPRESSION-DSL-OK=, then exited 0 under =gtimeout -k 2 10=. +It explicitly tests =test-result-ok?= and exits 1 on a failed suite, so the receipt +does not rely on a printed suite sentinel alone. The 11 cases comprise identity, +eight accepted examples, one rejection group containing eight sources, and one +independently specified exact AST/span example. These counts are not added to the +85 manual-control checks: the two drivers intentionally overlap. + +Two earlier runner invocations exited 70 before suite execution: the first imported +execution bindings from =:std/test= instead of their defining =:std/test/base= +module; the second called =TestConfig= positionally instead of using its required +=verbosity:= and =capture-output?:= keywords. The installed Gerbil source guided +both corrections. They were research infrastructure mistakes, not failures of +the package's declarative Grammar or test clauses. The corrected run preserves +failure propagation and the same execution limit. + +The loader binds its parse method once and derives identity and capabilities +from the grammar descriptor. Tooling slots, corpus fixtures and build strategies +are explicit DSL extensions when needed. Repeating identity in the test is an +independent expectation; production metadata remains descriptor-owned. +The shown loader does not declare Rust or AOT qualification. Backend capability, +artifact loading and performance gates remain separate obligations. + +*** Two questions for the test interface + +#+begin_src scheme +;; Author intent inside deflanguage-parser-tests: +(accepted "three array elements" "[a,b,c]" + (counts (Array 1) (Name 3)) + (field-counts Array element (3))) + +;; Question 1: Why is this DSL design better? +;; The author states observable language expectations within parser-test.ss. +;; Engine support performs artifact, roundtrip, digest and field comparison. +;; Grammar changes do not require copied traversal or custom assertion reporting. +;; Question 2: Why does this DSL design have a scientific basis? +;; Independent accepted/rejected expectations supplement the matched control. +;; Exact AST nodes, children, tokens and byte spans supplement field counts. +;; Native suite result and process exit jointly establish this local receipt. +;; Falsifiers: accepted malformed input, mismatched tree/spans, swallowed test errors, +;; or a required routine expectation that needs per-language engine reimplementation. +;; Limit: selected examples do not prove all-input equivalence or faster parsing. +;; Full event parity remains supported by the separate matched-control driver. +#+end_src + +** What the author writes, what the engine derives, what still needs evidence + +| Responsibility | Author interface | Derivation or qualification owner | +|----------------+------------------+-----------------------------------| +| Identity, root, lexical and tree policy | grammar.ss | Existing deflanguage validates and expands | +| Reusable expression intent | Imported or local Scheme vocabulary | Hygienic expression expansion into existing core | +| Existing lower-level grammar composition | No new concise author protocol | Existing checked POO API; source-aware admission remains an engine study | +| Catalogs and reference resolution | No duplicate terminal/field catalog | Existing compiler, canonical IR and Bound IR | +| Runtime entry and tooling extensions | parser.ss | Existing descriptor-bound POO Loader | +| Corpus and exact semantic expectations | parser-test.ss and fixture values | Existing test support plus explicit tree/event/span controls | +| Target products | Explicit existing build-strategy declarations | Backend capability and native artifact gates | +| Performance evidence | Matched language/package benchmark cases | Separate elaboration, compilation, loading and parse measurements | + +The local expression package now covers selected acceptance/rejection, full event +parity, field counts, lossless roundtrip and an independent exact tree/span example. +Complete closure still needs canonical composition metadata, effective origins and +product-loading gates for the same package. Finite example coverage is not a proof +of all-input equivalence. +Developer-facing benchmarks must identify the measured stage: macro expansion +and grammar compilation are not request-time parsing. A concurrent application +service can add queue/execution/cancellation measurements without changing the +library grammar or treating actor throughput as single-request parser latency. + +** Two questions applied to the complete authoring path + +#+begin_src scheme +;; Question 1: Why is this DSL design better? +;; The comparison preserves the complete lexical/root/tree/policy declaration. +;; required-items names one reusable intention; its uses state field, item and separator. +;; Host modules reuse the vocabulary, syntax objects preserve binding context, +;; and defsyntax can elaborate contracts before the existing compiler takes ownership. +;; Checked POO adds explicit multi-rule composition when that responsibility exists. +;; The descriptor connects grammar, loader and tests without another configuration model. +;; This is a proposed reduction in duplicated author intent, not measured usability. +;; Question 2: Why does this DSL design have a scientific basis? +;; Its expansion is inspectable against the explicit control, with recognition and +;; full public tree events as separate equivalence obligations. +;; Hygienic macro research informs binding preservation; staged DSL research informs +;; elaboration to a shared core; composition studies inform explicit contribution ownership. +;; These transfers are hypotheses bounded by actual Gerbil and parser-engine checks. +;; Falsifiers include changed accepted language, extra tree wrappers, captured host names, +;; dropped composition metadata, misleading source attribution or slower author tasks. +;; Limit: the expression package and selected test DSL clauses have local checks; +;; POO admission, developer tasks, target products and performance need separate qualification. +#+end_src + +Review the complete author package first: reusable grammar intent, explicit +language policy, existing compiler admission and declared product expectations. +Internal POO integration cannot independently justify an additional public author +model. The actual GQL review below checks that this discipline reaches a real pack. + +* Complete package review: the actual GQL declaration and its consumers + +The call/array package isolates semantic preservation. It cannot establish that +an entire production language pack is easy to author. The +[[file:../../languages/gql/grammar.ss][GQL grammar]], [[file:../../languages/gql/parser.ss][checked parser entry]] and +[[file:../../languages/gql/parser-test.ss][parser-test declarations]] provide the real package review here. The baseline audit below led to the local +implementation and native receipts recorded at the end of this section. It does +not establish measured author effort or standalone Rust/AOT qualification. + +** Follow one declaration through all three interfaces + +| Boundary | GQL implementation route | Review obligation | +|----------+----------------------------+-------------------| +| grammar.ss | Pins edition, upstream version/commit/digest, source path, root and parsing policy; exports admitted products and source metadata | Distinguish intentional language policy from repeated derivable metadata | +| Expansion | antlr4 vocabulary reads/checks the pinned source, derives catalogs/rules/origins and submits a language-declaration record directly | Keep the accepted source identity attached to every generated product | +| Canonical compilation | Existing binding, Parser IR and LR owners compile the lowered data | Reuse these owners; expansion templates must not implement another parser | +| parser.ss | Binds gql-language-grammar through the production Loader, supplies metadata, exports parse-gql; parser-test.ss owns the corpus | Preserve entry identity, immutable products and admission guarantees | +| parser-test.ss | Checks independent identity literals, Parser/Bound IR structure, accepted/rejected syntax, precedence and corpus fixtures | Preserve independent expectations rather than derive the oracle from implementation | +| Source catalog tests | t/antlr4-source-test.ss consumes the exported typed source catalog | Preserve that consumer contract while reviewing repeated source processing | + +GQL's source resource is ANTLR4 text. The implementation processing it is Scheme; +this source-import path differs from a grammar authored directly in Scheme. +Research on concise native grammar templates and research on imported grammar +admission share compiler obligations but must not be presented as the same author +workflow. No source-format migration is proposed by this audit. + +** Three files are responsibility boundaries, not frozen implementations + +The complete migration changes the contents of all three author modules. Keeping +three interfaces does not require preserving the old declarations, assembly or +test layout. The compiler route is: + +#+begin_src scheme +;; Architecture, not a new author syntax: +;; grammar.ss -> hygienic expansion and POO composition -> canonical IR +;; -> binding, compilation and publication -> generated machine +;; parser.ss -> checked public entry over the accepted descriptor and machine +;; parser-test.ss -> independent obligations across the complete route +#+end_src + +The arrows express product dependencies. parser-test.ss is authored conformance, +not a generated final compiler stage. It must independently test semantics, +fields, effective origins, immutable products and public entry behavior. Likewise, +parser.ss consumes the machine but need not implement parsing algorithms. + +The GQL migration applies this split to an actual language package: + +| File or engine owner | Implemented change | Contract retained | +|----------------------+--------------------+-------------------| +| languages/gql/grammar.ss | Contains the pinned source declaration and metadata derived from accepted products; corpus leaves this module | Standard edition, source pins, root, policy and exported grammar products | +| language-support/antlr4-language.ss | Builds language-declaration directly from the accepted source catalog; no canonical author replay | Canonical admission, original author syntax, source ownership and lineage | +| src/language/assembly.ss | Shares hygienic emission between native and imported declarations | Existing immutable artifact loading and machine constructor owner | +| languages/gql/parser.ss | Owns the corpus-free production entry; corpus and development services move to parser-test.ss | Public parse-gql and descriptor identity | +| languages/gql/parser-test.ss | Adds shared-product and entry-service identity checks alongside independent semantic expectations | 574 rules, 1176 bindings, 14 official fixtures, source locations, precedence and rejection checks | + +#+begin_src scheme +;; Current imported-source compiler handoff, inside the existing source macro. +(expand-language-declaration-syntax + declaration compile-parser bind-grammar-ir + #'assemble-language-parser #'make-language-grammar #'begin #'def) + +;; Current independent package test, abbreviated from parser-test.ss. +(property "published products and checked entry share one declaration" (bindings) + (equal (eq? (language-grammar-grammar gql-language-grammar) gql-grammar) #t) + (equal (eq? (language-grammar-ir gql-language-grammar) gql-parser-ir) #t) + (equal (eq? (language-grammar-machine gql-language-grammar) gql-parser) #t) + (equal (language-parser-entry-ref gql-language 'contract) + "iso-iec-39075-2024.opengql-1.9.0-syntax.v1") + (equal (language-loader-fixture-count gql-test-language) 14)) + +;; Question 1: Why is this design better? +;; Scheme syntax remains contextual until canonical admission. Native macros +;; and imported-source elaboration use the same compiler without reconstructing +;; another complete DSL declaration. Runtime entry and corpus policy live in +;; parser.ss; parser-test.ss holds independent obligations for those products. +;; S-expression shape enables structured handoffs; hygiene and phase ownership +;; make them safe to compose. Shape alone does not establish the advantage. +;; Question 2: Why does this design have a scientific basis? +;; Compiler-oriented elaboration and context-preserving macros motivate this +;; separation. Shared-product assertions and independent semantic controls can +;; falsify accidental divergence. Neither these assertions nor the cited papers +;; prove optimal author effort, parser speed or the unfinished POO route. +#+end_src + +Local qualification on 2026-10-07 compiles six declaration/assembly/source +compiler modules through sequential std/make in isolated output. The unchanged +240-second total and five-second genuine-output watchdog admits the build at +17.24 seconds with exit 0. Against those newly compiled owners, the six bounded +verification processes pass 87 cases (17 source/GQL, 5 surface, 13 diagnostics, +11 complete expression package, 31 composition, 10 publication), all exit 0. +The representative expression package's complete 6661-byte Grammar/Parser IR +snapshot remains byte-identical to its earlier canonical control, also exit 0. +This comparison concerns the expression fixture, not a complete GQL/Bound IR +byte comparison. GQL retains its independent counts, source and semantic controls. + +These receipts compile declaration owners and exercise runtime consumers from +source. They do not close full GQL native machine/runtime linkage, installed +consumer qualification or Rust/AOT publication. Test verbosity markers alone +are not sufficient evidence; the recorded process exits are part of admission. + +This realizes the imported-source edge and migrates all three GQL modules. It +does not by itself complete the native Scheme/POO author route. The subsequent +[[file:scheme-dsl-engine-admission.org::*Effective origins follow composition before binding][composed canonical implementation]] now qualifies effective source selection, +shared emission and checked Loader variants in the existing Scheme fixture. A +production public composition clause and full native runtime closure remain required. Completion means migrating a representative composed Scheme +package through all three interfaces, then validating those exact products. +The [[file:scheme-dsl-research-basis.org][research basis]] explains the paper mechanisms and their limits; +[[file:scheme-dsl-engine-admission.org][engine admission]] owns the unresolved composition and source-selection gates. + +** A concrete duplication with a concrete owner + +At baseline =783381f33fa1b309ac4cc4b0012696f24c852d93=, the grammar module repeats upstream version, commit and digest in exported +metadata constants, the =antlr4 vocabulary= declaration and the separate +=defsyntax-antlr4-source= declaration. Both macros resolve/read the source and call +=parse-antlr4-source/expected=. Source inspection therefore identifies two source +processing sites when both declarations expand. It does not establish two parses +per request or per module load: compiled products and cached expansion can avoid +executing transformer code on those paths. + +The separate catalog has a real consumer. The +[[file:../../t/antlr4-source-test.ss][source catalog tests]] assert total/parser/lexer rule counts, known rules and +canonical lowering. Deleting it simply to shorten grammar.ss would remove an +existing contract. Likewise, parser-test.ss deliberately repeats expected identity +literals: changing implementation identity must not silently change its oracle. + +Both declaration macros require literal string syntax for identity and digest. +Replacing those literals with existing runtime constants does not satisfy their +current transformer contract. Ordinary Scheme values and transformer inputs are +different stages; an apparent deduplication must specify how it crosses that +boundary before claiming the complete module remains valid. + +#+begin_src scheme +;; GQL uses (deflanguage gql (syntax (antlr4 ...)) (rules)). +;; The complete checked declaration is languages/gql/grammar.ss. +;; Question 1: Why is this DSL design better? +;; A declaration expresses source identity and language policy; Scheme expansion +;; can derive consistent definitions and compiled products from that accepted input. +;; Compared with synchronized declarations, an engine-owned common source result +;; removes repeated source pins while retaining the Loader, catalog and test contracts. +;; Question 2: Why does this DSL design have a scientific basis? +;; Trace one source identity into source catalog, Grammar/Bound/Parser IR and Loader. +;; Keep independent identity, structure, precedence and rejection expectations. +;; Test mismatched digests before publication and compare complete package behavior. +;; Falsifiers: a removed consumer, stale metadata/origins, changed recognition/tree +;; policy, or tests whose expected values are regenerated from the changed input. +;; Limit: bounded local conformance does not measure author effort, compilation +;; speedup, standalone generated products or parse-latency improvement. +#+end_src + +** Local implementation through the existing declaration + +The refactor is now implemented locally in +[[file:../../language-support/antlr4-language.ss][antlr4 vocabulary]] and [[file:../../languages/gql/grammar.ss][the GQL declaration]]. The macro serializes its already +validated catalog and emits =prefix-antlr4-source= alongside the existing grammar, +Parser IR, machine and descriptor. The runtime constructor binding is imported +by the macro module, preserving hygiene; the public generated name uses the prefix +syntax context, following the existing generated binding convention. + +GQL keeps its existing =gql-antlr4-source= export and removes its separate +=defsyntax-antlr4-source= invocation. Its source-version, commit and digest metadata +now come from that catalog; edition and syntax contract come from the descriptor. +The pin is declared once. Its parser.ss entry and parser-test.ss declarations retain +their existing contracts and independent expected identity literals. The standalone +source macro remains available for separate resources such as FHIRPath's normative +catalog. No contribution, owner or compose interface was added. + +| Before | After | Semantic obligation | +|--------+-------+---------------------| +| Two GQL source-processing macro invocations | One accepted catalog in the grammar macro | Digest validation occurs before canonical emission | +| Independent typed catalog materialization | Typed catalog emitted from that accepted catalog | Existing source export and consumers remain usable | +| Metadata pins repeated in runtime constants | Constants derived from catalog/descriptor | Independent literal expectations retain their values | +| Existing checked Loader and grammar compiler | Same Loader and compiler owners | Recognition, precedence, corpus, origins and rejection remain qualified locally | + +The existing [[file:../../t/antlr4-source-test.ss][source suite]] now additionally compares all lowered catalog rule rows +with the published Grammar IR's rule rows. This is a cross-product consistency +assertion, not an independent recognition oracle; it supplements the unchanged +independent source identities, rule counts and language semantics. + +On 2026-10-06, a temporary Scheme runner invoked =test-suite!= for the existing +=antlr4-source-test= and =gql-parser-test= with case verbosity and capture disabled. +It checked =test-result-ok?= and propagated a failed suite as exit 1. The invocation +used =GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" gtimeout -k 2 10 gxi=. +Both runs completed within that bound with all 6 source cases and all 10 GQL cases +passing, =GQL-DECLARATION-REVIEW-OK= and exit 0. The second run included the new +rule-row consistency assertion. The corpus check includes 14 official fixtures; +these are not 14 additional independently counted suite cases. + +A separate expansion control used the real GQL source and an intentionally wrong +all-zero digest with the same 10-second bound. It exited 70 with +=ANTLR4 source digest mismatch= from =parse-antlr4-source/expected= inside the +language transformer. This is the expected negative rejection, not a passing +process invocation. The call precedes catalog serialization and parser emission. +It establishes neither artifact-corruption recovery nor a complete negative-source +matrix. + +This implements a concrete use of Scheme macros: one accepted declaration expands +into consistent definitions and compiled products. The common-core principle in +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]] motivates that boundary; dependent +validation in [[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros]] motivates rejecting invalid pins before downstream +emission. Their Racket mechanisms and miniKanren algorithms are not runtime +dependencies or GQL performance evidence. + +** Whole-package gate: compiler products must match the source contract + +The implementation is local and uncommitted. The next check on 2026-10-06 used +an isolated output directory, the project GERBIL_PATH, and the same 10-second bound +for each compiler invocation. SDKROOT and Nix C/linker flags were cleared for the +native compiler. This gate adds compiler-product evidence to the preceding +semantic receipts; it does not replace or broaden them. + +| Stage | Result | Evidence boundary | +|-------+--------+-------------------| +| Existing source and GQL semantic suites | exit 0; 6 + 10 cases pass | Catalog, published rules, checked entry, identities, sources, corpus and selected parse semantics | +| Wrong source pin | expected exit 70, digest mismatch in expansion | Invalid identity is rejected before downstream emission | +| Changed macro, gxc -O with Gambit compilation | exit 0 | Native phase products for the changed transformer were built | +| GQL grammar, gxc -O against that output | exit 70, constructor arity mismatch | Whole native package compilation is not qualified | +| Rebuild current machine dependency, gxc -O | exit 124 | Optimized dependency refresh did not complete within the bound | +| Same dependency, standard gxc | exit 124 | Standard compilation is also unqualified; partial outputs are not a pass | +| GQL grammar, separate gxc -S stage | exit 124, no completed interface | Scheme emission alone is not qualified by this invocation | + +The typed failure is specific: the source =parser-machine= structure in +[[file:../../src/compiler/machine.ss][machine.ss]] has 12 fields, while the installed optimization metadata at +=.gerbil/lib/gerbil-parser/src/compiler/machine.ssxi.ss= declares 9, ending in +=contextual-product=. The optimized GQL expansion calls the source's current +constructor shape; the optimizer reports an illegal struct constructor application. +This identifies a concrete source/optimizer-metadata mismatch. It does not establish +a root cause for the subsequent refresh timeouts or the reported parse P50/P95. + +Partial machine interfaces and objects appeared in the isolated directory after +the standard-build timeout. They were not installed or admitted as successful +products. A process check found no surviving compiler child belonging to this +temporary verification. The isolated output is evidence, not a replacement +for a complete installed package. + +The next implementation obligation is a consistent compiler-product closure for +this exact source, followed by a consumer outside the checkout. Refresh the +identified dependency and its required closure through the existing build owner; +do not patch generated optimizer metadata by hand, consume partial outputs, +relax timeouts or report source tests as installed/AOT success. Rust/AOT products +and measured author/expansion costs remain separate gates. + +This distinction completes the research argument: the declaration and macro can +be simpler only if the compiler and published products retain the same contract. +The semantic controls answer whether the current source behaves correctly; the +compiler gate answers whether that source can produce a consistent reusable +package. Neither establishes a quantitative author-effort or performance result. +No LR or runtime parser algorithm was changed by the refactor. + +* Engine and common-library responsibility boundary + +The follow-up [[file:scheme-engine-library-boundary.org][engine/library boundary contract]] moves the GQL lexical +profile into grammar.ss, separates production entry from optional development +services, and records semantic limits and executable falsifiers. The earlier +corpus-in-parser implementation above is superseded by that contract. + +* Complete parser-entry alignment + +All eight shipped language packages and ten public parser entries use the same +slot declaration. The old metadata section and three-argument descriptor alias +form are removed from the production macro. Public descriptor accessors remain +ordinary definitions after entry admission. Arithmetic and HCL install their +backends through the canonical slots declaration, without a separate entry DSL. + +| Package | Public entries | Configuration | +|---------+----------------+---------------| +| arithmetic | arithmetic | ordinary metadata value | +| bash | bash | ordinary metadata value | +| cypher | opencypher | ordinary metadata and source-schema values | +| fhirpath | fhirpath | ordinary metadata value | +| gql | gql | ordinary metadata value | +| hcl | hcl | ordinary metadata value | +| hl7 | hl7v2 | ordinary metadata value; parser-owned projection | +| tla-plus | core, layout, SANY candidate | inherited POO metadata | + +The HL7 patient projection is a parser-side product adapter. Its columns, message +identity, schema and source digest do not define token recognition or grammar +construction. The grammar no longer imports POO or RecordProjection, and the +three-interface authoring contract rejects projection declarations in grammar.ss. +The public parser continues to export the same patient projection. + +#+begin_src scheme +;; Shared configuration only needs POO when variants inherit and override it. +(def source-metadata + (.o language: "example" grammar-format: 'concise-dsl + source-catalog: pinned-source)) +(deflanguage-parser-loader example-language + (grammar example-syntax) + (parse parse-example) + (slots metadata: (.o (:: @ source-metadata) + version: "v2" contract: "example.syntax.v2"))) +;; Question 1: Why is this design better? +;; The same slot interface covers normal values and native inheritance. Shared +;; fields have one owner; variants declare overrides. Recognition stays in +;; grammar.ss, configuration and product adapters in parser.ss, witnesses in +;; parser-test.ss. Removed forms cannot continue to publish competing entries. +;; Question 2: Why is this scientifically defensible? +;; Expansion rejects old forms. Independent topology checks cover every pack. +;; Matched accepted/rejected controls preserve descriptors and named parse events; +;; projection fields and source identity have their own independent witnesses. +#+end_src + +** Qualification + +The complete topology suite passed in 2.69 seconds. All eight parser modules +expanded successfully against the current macro phase, and all current parser +runtime objects compiled. Semantic emission used optimize: #f; these are +correctness receipts, not production optimization or latency measurements. +The existing native alignment publisher exercised twenty accepted/rejected cases +across all ten entries. Its complete output is byte-identical to the preceding +literal-construction baseline; the independent Rust decoder also reported +ALL-20-NAMED-EVENT-SEMANTICS-UNCHANGED. These controls do not cover every input in +each language standard. The relocated HL7 projection independently preserved its +schema, patient identifier, family name, given names and artifact source digest. + +Four expansion controls reject development-service slots, the removed metadata +section and the removed descriptor alias form. The prior exact-head CI failure +also exposed a missing native phase owner for fixture-release and list-stage: +emitted Gambit Scheme was imported as Gerbil source. The conformance compiler now +builds those exported macro phases through compile-module. A fresh output directory +compiled and imported both helpers successfully. This repair does not establish +full conformance-image linking or exact-head CI success; those gates remain +separate. All receipts retain the existing 240-second source/build and 5-second +silence fences. The closed native child still has its separate 10-second fence. + +The migrated source-entry fixtures, production-entry boundary suite and topology +suite also compile against the current expander. One separate cold macro import +attempt hit the existing silence fence before completion; its result is retained +as a negative receipt rather than replaced by the successful build/import run. +A further separate process admitted the fresh native helper dependencies and +imported both newly built macro phases successfully in 30.20 seconds, with exit +status zero. Its real module boundaries supplied progress; no synthetic output +or wider silence allowance was used. + +** Native publication qualification + +The compiler, artifact representation, rebase and build-product receipts are +maintained together in [[file:scheme-native-build-ownership.org][native build ownership]]. Author interface examples +remain here; native backend changes do not add a language-pack interface. diff --git a/docs/research/scheme-dsl-complexity-ownership.org b/docs/research/scheme-dsl-complexity-ownership.org new file mode 100644 index 00000000..921eaaf7 --- /dev/null +++ b/docs/research/scheme-dsl-complexity-ownership.org @@ -0,0 +1,587 @@ +#+title: Scheme DSL complexity ownership and author change tasks +#+date: 2026-10-06 + +Scope: assess total author and engine complexity under the fixed three-interface +DSL boundary. The [[file:scheme-deflanguage-design-synthesis.org][overview]] owns that boundary. Apply these change tasks to the +[[file:scheme-dsl-author-interface.org][complete author package]], with [[file:scheme-dsl-semantic-contracts.org][semantic contracts]] and [[file:scheme-dsl-engine-admission.org][engine admission]] as +separate, connected responsibilities. + +* Architectural reflection: concentrating complexity in the engine + +This is a package-wide design criterion, not a request to hide every decision +behind =deflanguage=. The objective is to reduce the knowledge and coordinated +edits needed to author and evolve a language, while keeping engine behavior +inspectable and language semantics explicit. Fewer declarations alone do not +establish that objective. A hidden dependency can shorten grammar.ss while +increasing the effort needed to diagnose or change the package. + +** Implemented internal owners: one author surface, explicit engine stages + +The local refactor follows the architecture already developed across the paper +readings. It keeps grammar.ss, parser.ss and parser-test.ss as the author boundary +and moves mechanisms to focused engine owners. These are responsibility boundaries, +not promises to retain the contents of the three author files. The +[[file:scheme-dsl-author-interface.org::*Three files are responsibility boundaries, not frozen implementations][whole-package migration]] changes GQL grammar, entry and tests together. Declaration handoff now uses a +compile-time record and direct compiler invocation; this is more than file extraction. +The runtime assembly takes its sections from the same admitted canonical grammar. +This does not establish a parse-speed improvement or add language syntax. + +| Owner | Responsibility after refactoring | Preserved boundary | +|-------+----------------------------------+--------------------| +| [[file:../../src/language/grammar.ss][language/grammar.ss]] | Public hygienic grammar entry | Closed deflanguage syntax/rules contract and hygienic defgrammar-syntax fragments | +| [[file:../../src/language/assembly.ss][language/assembly.ss]] | Emit admitted products into runtime bindings for native and imported declarations | Definition-context machine and artifact loader bindings | +| [[file:../../language-support/antlr4-language.ss][antlr4-language.ss]] | Elaborate one checked source catalog into the shared declaration record | Original source identity, origin map and author syntax | +| [[file:../../src/compiler/concise-language.ss][compiler/concise-language.ss]] | Collect syntax, expand vocabulary, validate rules, derive catalogs and submit a compile-time declaration record | Declaration-local source associations; validation before field inference | +| [[file:../../src/compiler/language-artifact.ss][compiler/language-artifact.ss]] | Read the explicit surface or accept the concise record; admit, bind, compile, publish and emit from one canonical grammar | Existing Grammar IR, Bound IR and Parser IR authority | +| [[file:../../src/compiler/lexical-expression.ss][compiler/lexical-expression.ss]] | Expand the closed lexical algebra and ranked dispatch | Existing matching, rank and byte-boundary semantics | +| [[file:../../src/compiler/lexical-lexer.ss][compiler/lexical-lexer.ss]] | Generate traversal and prepare lexical certificates | Existing scanner and lexical-plan owners | +| [[file:../../src/compiler/machine.ss][compiler/machine.ss]] | Own the nominal machine type, prepared entry and backend installation | Same twelve fields, type identity and public exports | + +The public macro supplies real syntax bindings to the compiler helper. The +helper does not guess the public macro's identifiers from symbols, import the +public language module recursively, or retain syntax in runtime parser objects. +The lexical expander is passed explicitly from its existing phase export. +Private lexical helpers remain definition-context bindings in generated syntax. + +The native concise, explicit and imported-source entries now converge on expand-language-declaration-syntax. The explicit +surface reads its clauses once; the concise surface constructs the same internal +language-declaration record directly. Optional concise clauses normalize by an +internal call carrying the original author syntax, rather than another macro +expansion. The record retains syntax sections and context until admission. It is +neither a new author protocol nor a serialized IR/backend contract. A named +compiled-language-declaration result ties the admitted grammar to its three +artifact locators; publication and runtime emission do not exchange a positional +seven-value tuple. The compiler reads section and policy fields from its input +record instead of forwarding fifteen positional arguments. Existing POO +normalization and nullable/FIRST/LR owners retain their responsibilities. + +The orchestrator now connects explicit engine phases: + +| Phase | Input and responsibility | Side-effect boundary | +|-------+--------------------------+----------------------| +| derive-language-declaration-bindings | Derive hygienic generated names once | No compilation or publication | +| admit-language-declaration | Build canonical grammar with origin, lineage and source map | No artifact publication | +| compile-admitted-language-declaration | Bind and compile that exact grammar through existing owners | Existing immutable publication and caches | +| emit-compiled-language-declaration | Emit assembly and descriptor syntax from compiled products and named bindings | No binding/compiler invocation | + +Normalized POO grammar can enter compile-admitted-language-declaration directly, +carrying the supplied effective context in an admitted-language-declaration. +The extended existing provenance study checks input-object retention, reloaded +Grammar/Bound/Parser equality, composition digest and supplied component ownership. +It does not reconstruct a deflanguage declaration. Its output-dirs keyword belongs +to engine artifact isolation, not to author configuration. The normalizer now captures effective occurrence sources during row acceptance, +including override/remove. The composed fixture directly enters shared emission +and qualifies both ordinary and override/remove policies through checked Loader +entries. See [[file:scheme-dsl-engine-admission.org::*Effective origins follow composition before binding][the selection contract]]. The production author composition clause, +full native runtime closure and measured author effort remain open. + +#+begin_src scheme +;; Engine-only compile-time handoffs; runtime receives immutable artifacts. +(defstruct language-declaration + (syntax prefix identities sections conflict-policy case-insensitive? lineage source-map)) +(defstruct admitted-language-declaration (grammar origin lineage source-map)) +(defstruct compiled-language-declaration + (grammar grammar-locator bound-locator parser-locator)) +(defstruct language-declaration-bindings (grammar bound parser machine language)) + +;; Implemented public entry; compilation mechanisms belong to the engine. +(defsyntax (deflanguage stx) + (expand-concise-language-syntax + stx expand-text-lexical-rows compile-parser bind-grammar-ir + #'assemble-language-parser #'make-language-grammar #'install-parser-machine-backends! + #'deflanguage #'begin #'def #'list)) + +;; Question 1: Why is this DSL design better? +;; Authors can reuse native defrules vocabulary while the engine owns derived +;; catalogs, checks and products. The same authored syntax reaches the existing +;; compiler through a thin entry, so changing an engine mechanism does not add +;; author configuration. Direct record handoff removes whole-declaration macro +;; replay; runtime emission shares the admitted grammar instead of author rows. +;; Named stages keep canonical input and context together during publication; +;; normalized POO data reuses that stage without another declaration expander. +;; Named records keep stage context and all products tied to the same grammar. +;; The nominal machine contract remains in one owner. +;; Question 2: Why does this DSL design have a scientific basis? +;; Context-preserving expansion, declaration dependency checking and compilation +;; to one core are explicit constraints from the connected paper readings. +;; Test their application through renamed/generated macros, source blame, +;; complete Grammar IR / Parser IR equality and recognition/tree/span controls. +;; A mismatch falsifies preservation; fewer lines alone do not prove superiority. +;; Limit: native dependency closure and measured author-task gains remain gates. +#+end_src + +This applies the joint argument from +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]] and +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros]] together with the +[[file:scheme-dsl-research-basis.org::*Syntactic abstraction in Scheme: how should authors define vocabulary?][Scheme syntax-object reading]]. The transfer is an architectural inference: +we retain Gerbil's own defsyntax/defrules and the existing POO/compiler contracts; +we do not claim to reproduce Racket's metalanguage or its optimization results. +The [[file:scheme-dsl-engine-admission.org][admission document]] owns semantic integration; this section owns code boundaries. + +*** Qualification of the implemented boundaries + +The first ownership extraction passed 75 source cases. Qualification of the +subsequent direct-admission refactor uses freshly compiled declaration owners +with source runtime consumers; that is not a fully native parser qualification. +The final record and canonical-emission owners compile through sequential +std/make, exit 0 (16.42 s). The three lexical/machine owners retain their separate +native compilation receipt (14.56 s); build times do not establish a speedup. +Standard PackageSpec discovery includes all three new compiler modules. + +The final contract set passes 86 cases with freshly compiled declaration owners +and source runtime consumers: 13 declaration diagnostics (including defaulted +context/source-free input), 5 language-surface cases, 31 POO composition cases, +11 complete call/array package cases, 16 source/GQL cases and 10 immutable +publication/cache cases. Every runner checks assertions and exits 0 within the +unchanged ten-second bound. The complete Grammar IR and Parser IR remain +byte-identical to the pre-extraction 6661-byte control, with its snapshot process +also exiting 0. These are preservation observations, not a proof of universal +DSL superiority or improved developer task times. The subsequent named-phase +refactor reruns those same 86 cases and the 6661-byte product comparison with +freshly compiled declaration owners; every process exits 0. The direct POO +publication/provenance control separately passes 38 checks, including seven +checks on the new canonical stage, and exits 0 under the same ten-second fence. +The loaded Parser IR's existing materialization=aot-expansion metadata is checked +explicitly; after removing that checked publication-only row, every remaining +field equals a fresh compile-parser result. + +The initial POO stage witness failed (exit 70) because it compared a materialized +Parser IR directly with the compiler result while omitting the existing +publication tag from its expectation. The corrected witness asserts that tag +independently and preserves full equality for the remaining fields; the failed +attempt is not admitted as successful evidence. + +Keep failures visible. Whole-spec native preparation exceeded the genuine-output +watchdog twice before compilation output; sequential dependency builds passed +without relaxing it. Cold source tests and earlier parallel attempts exceeded the existing process +bound. A final source lexical aggregate emitted all 19 passing case markers but +exited 124, so it is not a passing process receipt; a separate cold lexical runner +also exceeded the bound during imports. Loading the compiled lexical owners with +a source machine likewise completed the 19 assertions but exited 124; it does +not repair that process gate. A native-machine consumer reported an +undefined lr-parse/prepared binding and duplicate installed step backend. GQL +native builds still exceed the genuine-output watchdog. One intermediate GQL +consumer run exceeded its process fence (exit 137); the same-bound retry and the +final named-record run pass. The failed attempt is not a passing receipt. Module compilation, +assertion completion and a qualified native dependency/consumer closure are three +different observations. This refactor does not close GQL/Rust/AOT or justify a +parse-latency improvement. + +** Absorb mechanism; preserve intentional language choices + +| Responsibility in the call/array package | Appropriate owner | Reason and boundary | +|-----------------------------------------+-------------------+---------------------| +| Whether containers may be empty or allow trailing commas | Author grammar or selected vocabulary policy | Defines the accepted language; the engine cannot infer the intended answer | +| Public Call/Array nodes and argument/element fields | Author syntax contract | Defines observations consumed by application code | +| Repeated field and terminal catalogs | Existing engine inference | Derived facts should have one computation owner | +| List idiom expansion | Reusable Scheme vocabulary | Shares author intent without requiring every idiom to become an engine primitive | +| Multi-rule composition and reference resolution | Existing checked POO and compiler owners | Enforce common invariants after effective contributions are selected | +| Nullable/FIRST facts, LR construction, source offsets | Existing engine analyses/runtime | Language authors should not implement parallel versions | +| Conflict handling | Explicit policy; engine enforcement | Default rejection must not silently choose unspecified language semantics | +| Artifact storage, cache mechanics, scanners and FFI generation | Existing engine/product owners | Mechanisms shared across packs; target capabilities remain explicit | +| Runtime service scheduling and cancellation | Optional application service around the parser | Service policy must not become required knowledge for a library parse call | +| Native tests and benchmark infrastructure | Shared test owners with package expectations | Infrastructure is shared; semantic assertions remain independent author evidence | + +The existing [[file:../../languages/README.org][language pack contract]] already assigns many mechanisms to the engine. +The research contribution should make that boundary easier to use and verify, +not add another layer that merely forwards the same sections. The canonical +[[file:scheme-dsl-engine-admission.org][admission design]] is a concrete example: preserve the composed value at +one compiler boundary instead of making authors reconstruct its metadata. + +** Concentration must not create a monolithic engine interface + +A single semantic owner does not require a single implementation module. +Elaboration, composition, binding, analysis, artifact publication and execution +can retain focused internal responsibilities connected by explicit IR contracts. +Authors should not need to know their internal call order to declare a language. +Engine maintainers must be able to inspect each boundary independently. + +Keep three kinds of information accessible: the authored policy, the effective +expanded/composed grammar, and the derived engine facts used to compile it. +An explanation should connect them through real origins and explicit absent-source +cases. Inspectable output is a design obligation, not a new implemented debug API. +Avoid requiring authors to select LR allocation strategies, cache internals or +backend representations to obtain a normal checked parse entry. + +Extensibility also needs a boundary. Expression vocabulary elaborates into the +supported core; POO contributions enter checked composition; tooling uses checked +Loader extensions. An arbitrary host callback does not acquire portable parser +IR or native backend capability by being placed in an extension slot. Advanced +extensions should state their capabilities and obligations without changing the +common authoring path into a collection of escape hatches. + +** Evaluate the total change and diagnosis effort + +Use the same package and explicit control for representative developer tasks: +allow empty containers, change the separator, add another list-bearing construct, +rename a public field, override a contribution and diagnose a rejected declaration. +Record the intentional choices, coordinated edits, engine details the author must +understand, and effort needed to identify a failure. Separately record engine +maintenance cost, regression scope and phase-specific benchmark changes. + +*** Concrete change tasks on the same package + +This is a source-based task analysis, not a timed usability experiment. Apply each +change independently to the [[file:scheme-dsl-author-interface.org][complete call/array package]]; state the intended +scope before counting edits. Shared vocabulary does not imply that every use +must change together. + +| Author task | Intentional declaration changes | Engine obligation / unresolved boundary | +|-------------+---------------------------------+------------------------------------------| +| Use semicolons for array elements only | Array vocabulary argument becomes literal semicolon; punctuation lex includes semicolon; call keeps comma | Validate recognition through the declared lexical policy; do not silently rewrite all separator uses | +| Permit a trailing comma in arrays only | Array selects an explicit trailing-separator pattern; call stays unchanged | Compile that pattern through existing analysis; verify ambiguity, fields and rejects | +| Add a brace-delimited collection | Add braces to lex, one root alternative and a node/field declaration using shared list vocabulary | Infer catalogs and compile; syntax-contract evolution remains explicit | +| Rename argument to parameter | Change the call field argument plus the independent expected tree contract | Derive the field catalog; retain intentional changes to consumer-visible AST identity | +| Replace the argument-list entry with a single item | Explicit POO override of arguments; decide whether its old helper remains | Current normalizer replaces one keyed row, not the whole component | + +The separator task is a useful test of concentration. A literal in a grammar +expression and the lexical recognizer are related declarations with different +roles. Automatically adding every referenced literal to a scanner could affect +priority, fallback, keywords or an imported scanner. The current explicit lexical +contract remains the baseline. A future engine inference proposal must state its +lexical semantics and validate them before claiming to eliminate an author edit. + +For the trailing-separator task, one inspectable candidate is: + +#+begin_src scheme +;; Candidate vocabulary, using existing constructors; not a selected public API. +(defgrammar-syntax (required-items/with-trailing label item separator) + (seq (required-items label item separator) + (optional separator))) + +;; Replace only the array's existing required-items expression. +;; The outer optional still permits an empty array; separator-only stays excluded +;; for this study's nonnullable Name item. Calls retain their original policy. +(optional + (required-items/with-trailing element (reference name) (literal ","))) + +;; Question 1: Why is this DSL design better? +;; A distinct vocabulary states a semantic policy change at the array use site. +;; It reuses the common list intention without adding an opaque Boolean option. +;; Existing engine inference and analysis handle catalogs and parser construction. +;; Question 2: Why does this DSL design have a scientific basis? +;; Compare independent accepted/rejected expectations and complete public events. +;; Required examples include [], [a], [a,b], [a,] and [a,b,]. +;; Required rejects include [,], [,a], [a,,] and [a,,b]; f(a,) stays rejected. +;; Nullable items/separators and overlapping recognizers need separate analysis; +;; this expansion must not be generalized from the concrete Name/comma control. +;; Falsifier: a changed call policy, separator-only acceptance, conflict, duplicate +;; field occurrence or a new public wrapper caused by this vocabulary expansion. +;; Limit: these are proposed expectations; this candidate has not been executed. +#+end_src + +The POO replacement task exposes an additional ownership choice. The existing +[[file:../../t/fixtures/language-pack-research/composition-history.ss][composition-history control]] checks that overriding =arguments= retains +=call-arguments/tail=. It also checks that removing a still-referenced helper +rejects, and that overriding the entry then removing the unused helper admits. +The existing driver was rerun on 2026-10-06 under =gtimeout -k 2 10=: +the historical 19-check driver completed and the process exited 0. This qualifies the tested +composition/binding/compiler controls, not a whole-component replacement API or +the proposed trailing-separator package. + +The engine therefore supplies checked row operations; the author still needs to +know whether helper names are supported composition targets. + +*** Component interface: distinguish host exports, grammar targets and public trees + +The complete package has three naming contracts. They interact but are not +interchangeable. Gerbil's official +[[https://gerbil.scheme.org/reference/gerbil/prelude/macros.html][Prelude Macros reference]] specifies =only-in=, =rename-in= and phase-shifting +imports for host identifiers. These forms do not automatically rename symbols +inside a canonical grammar value or change its advertised extension targets. +The actual [[file:../../src/compiler/bound-ir.ss][Bound IR]] uses structured package/grammar/namespace/name identities; +field identity additionally includes its syntax-kind scope. + +| Contract | Running example | Change or hiding boundary | +|----------+-----------------+---------------------------| +| Scheme module binding | make-nonempty-list-role or required-items | Module export/import governs host access and macro phase | +| Grammar composition target | arguments and call-arguments/tail | Canonical row keys govern references and keyed operations | +| Public tree contract | Call.argument and Array.element | Nodes/fields govern application-visible parse observations | + +Not exporting a helper's host constructor does not make its emitted grammar row +inaccessible to composition. Likewise, syntax hygiene for a temporary host binding +does not prevent collision between equal grammar-domain keys. The existing +[[file:../../t/fixtures/language-pack-research/list-collisions.ss][collision control]] distinguishes a shared public entry from a shared generated +helper key. Both controls were rerun on 2026-10-06 under =gtimeout -k 2 10=; +2 checks passed and the process exited 0. This qualifies the existing collision +checks, not the candidate host-alias illustration or a new visibility contract. +Current composition checks both; no private-rule visibility mechanism +has been implemented by this research. + +#+begin_src scheme +;; Caller context in the existing list-role study. +;; A renamed factory binding still receives explicit grammar-domain symbols. +(import (rename-in "list-roles" + (make-nonempty-list-role make-item-list))) + +(def call-contribution + (make-item-list 'call-arguments 'arguments + '(reference name) '(literal ",") 'argument)) +(def array-contribution + (make-item-list 'array-elements 'elements + '(reference name) '(literal ",") 'element)) + +;; The factory derives call-arguments/tail and array-elements/tail. +;; The host alias make-item-list does not rename either entry or helper. +;; The public field argument remains distinct from the rule entry arguments. +;; This illustration has not been executed as a new combined package. + +;; Question 1: Why is this DSL design better? +;; Ordinary Scheme modules govern vocabulary access; checked composition governs rows. +;; A component contract advertises supported targets without making every helper stable API. +;; Authors can distinguish host renaming, grammar replacement and AST evolution. +;; The engine retains reference checks and inspectable generated rows. +;; Question 2: Why does this DSL design have a scientific basis? +;; Test host aliases, entry/helper collisions and exact public tree observations separately. +;; Compare a helper implementation change with a public field or entry-target change. +;; Falsifiers: host aliases unexpectedly rename grammar data, helper changes alter promised +;; observations, or a supported override requires undisclosed implementation knowledge. +;; Limit: documenting supported targets does not enforce visibility in canonical IR. +#+end_src + +The working contract should advertise three things for each contribution: +its author parameters and language policy; its intended public tree observations; +and its supported composition targets/dependencies. Implementation helpers may +remain inspectable for diagnostics without a promise that their names are stable +extension API. This is an API stability recommendation, not a claim of enforced +access control, helper deletion or a selected new metadata schema. + +For the list package, prefer expression vocabulary for ordinary policy changes +when no multi-rule extension is needed. Use checked POO contributions when named +multi-rule ownership or explicit override is the actual requirement. Replacing an +advertised entry should preserve or intentionally change its public field contract; +it does not imply removal of all rows originally emitted by its provider. + +If the public DSL later promises whole-component replacement, specify contribution +membership, shared rows, dependency checks, source selection and receipt behavior. +This is a larger contract than current keyed override. Neither cascading deletion +nor automatic helper garbage collection follows from module privacy. Generated +rule identities also need reproducible artifact and diagnostic behavior; arbitrary +fresh host symbols are not automatically a portable persistent identity scheme. + +Within the fixed three-interface DSL boundary, the engine design has two +candidates: retain existing +checked row operations with explicitly documented stable targets, or add a checked +component-level interface if complete package tasks demonstrate a need. The first +reuses current mechanisms; the second needs membership and evolution semantics. +No choice is settled solely by shorter syntax. Qualify both against the same +consumer tree contract and DSL editing tasks before selecting composition syntax. +Authors express supported composition intent in grammar.ss; the engine implements +keyed operations, helper ownership and canonical admission. + +These tasks narrow the next discussion: lexical intent should remain inspectable; +list policy can be shared through named Scheme vocabulary; component extension +visibility requires an explicit contract. The engine absorbs derivation and +validation, while vocabulary captures recurring intent. No per-language engine +branch is required by the changes proposed here. + +A shorter declaration accompanied by more hidden defaults, weaker diagnosis, +per-language engine branches or duplicated analysis does not meet this criterion. +No numerical complexity score or developer-task improvement is established here. +The current artifact timeout is also a reminder that successful assertions do +not explain loading/process costs; those costs require their own bounded evidence. + +#+begin_src scheme +;; The same declared intention can serve both containers. +(required-items argument (reference name) (literal ",")) +(required-items element (reference name) (literal ",")) + +;; Question 1: Why is this DSL design better? +;; Authors name list policy and public fields; shared vocabulary handles repetition. +;; The engine derives catalogs, resolves references and compiles the admitted grammar. +;; A policy change should not require knowledge of caches, LR tables or FFI layouts. +;; Complexity reduction is conditional on easier changes and accurate diagnosis. +;; Question 2: Why does this DSL design have a scientific basis? +;; Evaluate explicit author tasks against the complete control, preserving semantics. +;; Observe coordinated edits, required knowledge and diagnosis, not only source length. +;; Inspect expanded grammar, origins and compiler facts to test hidden dependencies. +;; Falsifiers: silent policy changes, extra per-pack engine code, opaque expansion, +;; duplicated analysis, broader regressions or increased effort for ordinary changes. +;; Limit: centralized mechanisms still impose engine maintenance and qualification cost. +;; This is an evaluation plan, not proof that concentrating complexity always helps. +#+end_src + +The primary design reading here is Ousterhout's +[[https://web.stanford.edu/~ouster/cgi-bin/cs190-winter18/lecture.php?topic=complexity][The Nature of Complexity]]: it identifies change amplification, cognitive load and +unknown dependencies as developer-facing symptoms. His +[[https://web.stanford.edu/~ouster/CS349W/lectures/abstraction.html][Designing Abstractions]] discusses interfaces that isolate implementation decisions. +These are author lecture notes, not an empirical proof of our DSL design. +Our inference is to assess engine concentration through package change and +diagnosis tasks, while the staged DSL papers inform the elaboration/core boundary. + +The working recommendation is therefore to centralize derivable mechanisms and +shared invariants, keep language intent and product capabilities explicit, and +make the resulting decisions explainable. Review the author and engine costs +together before introducing a new primitive, extension point or configuration. + +* Account for explanation work and retained data + +Apply the [[file:scheme-dsl-engine-admission.org::*Evidence lifecycle across cache hits and misses][evidence lifecycle]] to the same call/array and GQL author tasks. +Let N be the lowered nonterminals, T the terminals, P the productions and L the +maximum rhs length. Recording one first cause per nullable fact and FIRST +membership gives at most |N| + |N| * |T| fact nodes. This is our design bound, +not a measured memory result. A naive record also copying every dependency prefix +can use O(|N| * L + |N| * |T| * (L + 1)) edges; fact-count bounds alone do not +bound those copied prefixes, syntax retention or rendered text. + +Production associations require current grammar identity and occurrence data. +Live syntax can retain source containers and other context; counting associations +is not a heap measurement. Shared dependency graphs avoid duplicating each proof +subtree in storage, but rendering without shared references can still expand the +output. Shortest-witness selection is not included in first-cause capture. + +| Complete task/control | Measure separately | Required semantic observation | +|-----------------------+--------------------+-------------------------------| +| Cold compile, explanation not requested | Expansion, admission, lowering/facts, LR construction, publication and allocation | Existing production/fact/product parity | +| Cold compile with a fact diagnostic | Cause capture and rendering, witness nodes/edges, peak retained data | Finite checked cause and accurate author text | +| Current-source declaration with parser cache hit | Cache lookup/load versus requested lowering/analysis replay | Same grammar/facts; current coordinates | +| Edited source with unchanged grammar semantics | Contextual identity and retained syntax lifecycle | Parser facts may match; diagnostic locations follow current source | +| Long nullable prefix and shared recursive dependencies | Prefix evidence size, capture work and bounded rendering | No self-dependent cause or fabricated completion | +| Normal parser invocation | Single-request P50/P95 and concurrent throughput under existing harness | Same recognition, events, fields and spans | + +Compile-time explanation work is not parse latency. Track actual phase durations, +allocation and request counts before making an optimization claim. A warm parser +cache cannot stand in for an uncached declaration benchmark, and a failing +process-completion gate cannot be counted as a benchmark pass. Reuse the existing +language corpus and benchmark owners; do not add a new author benchmark protocol. +No numerical overhead, memory improvement or end-to-end speedup is established +by this cost model. + +* Coordinated syntax, metadata and backend boundary: Cypher, 2026-10-08 + +The grammar module declares recognition and construction. The parser entry owns +release metadata and parser adapters. The test module owns independent expected +sources and behavior. An upstream grammar pin is a compilation input: changing +it may change recognition. A release label or audit description is entry metadata: +changing it must not silently replace recognition. This distinction determines +ownership by meaning, rather than by the fact that both are Scheme data. + +The Cypher migration moves public version/source descriptions and overlay receipt +values into parser.ss. Its grammar retains the checked BNF input and the rules +that affect recognition. Repeated provenance fields are derived by the compiler +from that input. This does not move Cypher-specific rules into the generic engine. + +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]], Figure 1 and sections 3--5, +provides the architectural distinction between extensible elaboration, checked +core boundaries and compilation. Our transfer is explicit ownership of grammar +elaboration and parser publication. Its Racket implementation and measured +miniKanren gains do not establish Gerbil behavior or parser performance. + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros]], its local-contract discussion and parser-generator case +study, motivates rejecting malformed authored terms at their syntax boundary. +Our additional obligation is that a test must reach the intended boundary: +wrapping an invalid grammar in an invalid expansion context cannot prove its +catalog contract. The corrected catalog control preserves all twelve independently +specified malformed catalogs and checks their actual rejection category. + +#+begin_src scheme +;; grammar.ss: recognition policy, not release publication. +(deflanguage query + (syntax (iso-bnf + (reference "upstream-version" "immutable-commit") + (digest "sha256:verified-source") (source "grammar-source/query.bnf") + (rule-precedences + (identifier (disambiguation keyword-preference) left -1)) + (entrypoint program) (conflicts selective-glr) (case-insensitive #t))) + (rules)) +;; parser.ss: a normal metadata value needs no POO wrapper. +(deflanguage-parser-loader query-language + (grammar query-syntax) (parse parse-query) + (slots metadata: '((language . "query") (version . "release") + (contract . "query.syntax.v1")))) +;; Question 1: Why is this design better? +;; Authors edit recognition and publication under distinct, predictable owners. +;; Hygienic syntax abstractions compose rules; the compiler derives catalogs and +;; provenance. POO adds inherited values when needed, without duplicating a DSL. +;; Question 2: Why is this scientifically defensible? +;; Controls must reject retired forms, preserve independent event semantics and +;; bind generated products to their admitted identity. Source relocation alone +;; proves neither recognition equivalence nor compilation/runtime improvements. +#+end_src + +The example describes the author contract; its illustrative upstream pin is not +an executable source fixture. Native source admission, publication, Rust emission +and exact-head CI remain separately reproducible obligations. + +The source catalog now uses the existing compressed, content-addressed artifact +transport. Its decoded value retains the original typed catalog and digest +validation. With the same output filename, completed generated GQL C decreased +from 8,626,218 to 6,428,075 bytes (25.5 percent); this source-size result is +separate from build-time and runtime qualification. Literal-rule construction now shares one Scheme procedure rather +than repeating its capability and first-character branches in each expansion; +the ranked literal trie and token traversal are unchanged. + +A raw bytevector transport experiment increased generated C to +8,947,917 bytes and failed the unchanged object output-silence gate, so it was +reverted rather than admitted as an optimization. + +Qualification for this coordinated change: all 28 native declaration/POO/language +children completed under their existing ten-second fences. All 20 regenerated +native language artifacts were byte-identical to the tracked corpus, and an +independent comparison resolving catalog IDs to names preserved event semantics. +The artifact generator completed in 3.31 seconds with the engine's existing +actual decode/verification/admission trace enabled. This trace exposes startup +work; it does not synthesize progress or change the timeout policy. + +Build admission remains open: the complete conformance attempt linked its image +but failed the five-second output-silence fence before process completion. Earlier +cold/incremental object attempts also failed that fence, and one complete build +exceeded 240 seconds. The semantic receipts and completed C-size comparison do +not supersede those negative build receipts or establish exact-head CI closure. + +** Declarative source schemas at the parser entry + +Moving twelve expanded provenance records into parser.ss preserved duplication. +The entry now declares one source-schema slot, using the existing slots interface. +The common source-schema library admits either an ordinary value or a native POO +object; object inheritance supplies effective fields without a second composition +language. Each receipt supplies only its independent name and optional kind. +The parser's sealed schema slot remains its engine identity. source-schema is a +separate extension describing provenance records, not AST recognition or dispatch. + +#+begin_src scheme +(deflanguage-parser-loader query-entry + (grammar query-syntax) + (parse parse-query) + (slots + source-schema: '((schema . "gerbil-parser.iso-bnf-rule-overlay.v1") + (namespace . query) + (sourceVersion . "pinned-source-version") + (upstreamCommit . "immutable-source-commit")) + metadata: '((language . "query") (version . "release-version") + (contract . "query.syntax.v1")))) +(language-parser-source-receipt query-entry '(normalization factoring)) +;; Question 1: Why is this design better? +;; Shared record fields are declared once. Normal values need no object wrapper; +;; POO inheritance is available for schema extension. Language-specific rule +;; transformations remain authored in grammar.ss, with no engine special case. +;; Question 2: Why is this scientifically defensible? +;; Entry admission checks field types, completeness and distinct value keys. +;; Independent witnesses check exact output for ordinary and inherited values, +;; reject malformed configurations, and preserve parser behavior. Cypher still +;; compares declared expected receipts with the compiler's admitted Bound IR. +#+end_src + +The source version in this configuration is independent of the parser release +version. Common construction does not infer upstream identity from a release +label. No retired receipt constants or compatibility export aliases remain. + +The schema library depends only on standard list operations and native POO +access. It does not import loader identity, parser dispatch, or a language +catalog. Configuration validation therefore does not require parser startup. +The receipt renderer constructs an expected record; it does not authenticate a +compiler transformation or replace Bound IR source evidence. + +*** Qualification boundary + +Focused Scheme controls for the current schema library completed in 5.76 seconds +with exit status zero under the existing 240-second source-owner and 5-second +silence fences. Independently specified records matched for ordinary values and +inherited POO values; five malformed configurations and an unknown provenance +kind were rejected. This receipt does not qualify the native conformance child +or establish a parser throughput improvement. Production-entry admission and +Cypher Bound IR assertions are committed tests whose full native execution +remains gated by the unresolved conformance build. Earlier attempts reached the +assertions but failed during process exit; their final markers alone were not +accepted as successful runs. The focused run uses the existing disposable test +child exit contract after all assertions and buffered results complete. diff --git a/docs/research/scheme-dsl-engine-admission.org b/docs/research/scheme-dsl-engine-admission.org new file mode 100644 index 00000000..3a22c544 --- /dev/null +++ b/docs/research/scheme-dsl-engine-admission.org @@ -0,0 +1,918 @@ +#+title: Scheme DSL engine admission and artifact publication +#+date: 2026-10-06 + +Scope: elaboration, canonical admission, contextual identities, publication and +cache reuse. These are engine responsibilities beneath the [[file:scheme-dsl-author-interface.org][three author interfaces]]. +Read the [[file:scheme-deflanguage-design-synthesis.org][design overview]] for decisions and [[file:scheme-dsl-semantic-contracts.org][semantic contracts]] for effective +field/source obligations. Exact source studies remain supporting evidence. + +* Canonical admission contract for the same complete package + +The next design boundary is the transition from Scheme elaboration and checked +POO composition into artifact compilation. Source inspection confirms that +[[file:../../src/compiler/parser-ir.ss][compile-parser]] already accepts canonical Grammar IR v1 and copies its +=compositionDigest= into Parser IR. The research +[[file:../../t/fixtures/language-pack-research/list-stage.ss][list-stage adapter]] instead reifies selected sections into a new +=deflanguage-grammar= declaration. In +[[file:../../src/compiler/language-artifact.ss][expand-language-grammar-syntax]], that declaration builds a fresh Grammar IR +without the composition digest. This locates the missing connection in the +adapter/declaration boundary, rather than in LR compilation. + +** One admitted grammar, several deliberately different outputs + +For the list package, the proposed compiler contract is to accept the composed +canonical grammar together with declaration context, then derive Bound IR and +Parser IR from that same admitted grammar. Existing +=compile-language-declaration-artifacts= already takes grammar plus compiler +closures. An internal extraction/reuse of that path is a design option; a new +public declaration clause has not been selected or implemented. + +#+begin_src scheme +;; Design witness using existing functions; not the final author-facing API. +;; The caller supplies the package grammar and effective source map. +;; This function illustrates the common input boundary, not artifact publication. +(def (study-canonical-package grammar origin lineage source-map) + (let-values (((canonical receipt) (compile-grammar/receipt grammar))) + (values canonical + receipt + (bind-grammar-ir canonical origin lineage source-map) + (compile-parser canonical)))) + +;; Question 1: Why is this DSL design better? +;; Scheme vocabulary and POO contributions converge on one admitted grammar value. +;; The compiler keeps responsibility for binding, LR construction and artifacts. +;; A macro need not replay the package declaration or reconstruct compiler metadata. +;; Its author-facing freedom is constrained by the same canonical grammar contract. +;; Question 2: Why does this DSL design have a scientific basis? +;; Compare the original canonical input with each derived or reloaded output. +;; Distinguish preserved metadata, resolved bindings and observable parse semantics. +;; The staged DSL architecture motivates extensibility before compilation; +;; the actual preservation claim must pass this engine's native controls. +;; Falsifiers: a lost digest, stale effective source, changed events or an artifact +;; cache that returns output compiled from a different admitted/contextual input. +;; Limit: this witness is not an executed complete package or a selected public API. +#+end_src + +Passing an already canonical value to =compile-grammar/receipt= returns that value +and =#f= for the receipt. It does not recover prior composition history. Therefore +an earlier receipt must travel explicitly if the package promises to expose it; +re-normalizing canonical data cannot synthesize verified history. + +** Identity, context and publication are separate promises + +| Data | Existing meaning | Proposed preservation obligation | +|------+------------------+----------------------------------| +| Canonical grammar | Effective normalized rows and grammar policy | Bind and compile the exact admitted value; do not reconstruct selected sections | +| compositionDigest | Digest of composition step descriptions | Preserve through Parser IR and reload; not proof of full contribution contents | +| Receipt grammarIrDigest | Digest of normalized body before compositionDigest insertion | Verify against that body convention; do not equate it with the complete enriched IR digest | +| Bound IR sources | Source records selected by namespace/name | Use effective row ownership after mutation; supplied records are not automatically authenticated | +| Expansion lineage | Explicit declaration-context input | Preserve its stated meaning without claiming a complete macro expansion chain | +| Declaration cache key | Generator contract, declaration identity and grammar | Include every contextual input that affects cached bound/parser outputs | +| Artifact locator/digest | Identity of serialized published content | Verify load/schema/content under existing artifact machinery | +| Runtime observations | Recognition, events, fields, spans and roundtrip | Compare the same package with the explicit semantic control | + +The current declaration cache keys =declaration-identity= and =grammar=; +=declaration-identity= contains origin, lineage and source map. A canonical +admission path should preserve that dependency when reusing the cache owner. +Receipt persistence is a separate decision: retaining a digest in Parser IR does +not mean the full receipt is stored in the published language descriptor. +A receipt excluded from executable compilation may need its own artifact identity; +that choice should not be hidden inside the grammar digest. + +** Verified engine reuse: declaration context and parser caching + +The existing [[file:../../t/fixtures/language-pack-research/provenance-study.ss][provenance-study control]] now exercises +=compile-language-declaration-artifacts/output-dirs= for the same POO-composed +call/array grammar. This is an engine integration study: compiler closures and +source records are infrastructure inputs, not additional author declarations. +No production API or cache algorithm was changed. + +A new temporary output root makes the first invocation a declaration/parser +cache miss. Counter checks distinguish actual Bound IR and Parser IR compiler +calls from locator reuse. Changing only the declaration origin from +=cache-owner-A= to =cache-owner-B= leaves the canonical grammar, lineage and +supplied descriptive source map unchanged. + +| Invocation | Declaration result | Cumulative bound/parser compiler calls | Artifact observations | +|------------+--------------------+----------------------------------------+-----------------------| +| First origin A | miss | 1 / 1 | Reloaded Grammar IR equals admitted canonical input; Parser IR retains compositionDigest | +| Repeat origin A | hit | 1 / 1 | All three locators equal the first invocation | +| Change origin to B | miss | 2 / 1 | Bound locator changes; grammar/parser locators remain equal; reloaded binding originModule becomes B | + +The componentOwner source facet remains =call-arguments= after that origin change. +This distinguishes declaration context from the effective component source; +one is not a substitute for the other. It does not test changed reader locations, +source-map rows, lineage, grammar rules, target products or artifact corruption. + +The bounded invocation on 2026-10-06 was: + +#+begin_src sh +PARSER_RESEARCH_ARTIFACT_ROOT="$(mktemp -d)" \ + GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout -k 2 10 gxi t/fixtures/language-pack-research/provenance-study.ss +#+end_src + +The expanded driver emitted =PROVENANCE-STUDY-OK: 31 checks passed= and exited 0. +This consists of the previous 17 identity/artifact controls plus 14 cache and +reload controls, not 31 new independent package observations. A fresh output root +is required to reproduce the cache-miss expectations; it is not a cold-start +runtime benchmark. The earlier actual-language driver timeout remains separate. + +#+begin_src scheme +;; Existing engine boundary, with canonical input and contextual binding input. +;; This is a compiler-side witness, not code required in grammar.ss or parser.ss. +(compile-language-declaration-artifacts/output-dirs + (list origin lineage source-map) + canonical-grammar + (lambda () (bind-grammar-ir canonical-grammar origin lineage source-map)) + (lambda () (compile-parser canonical-grammar)) + output-roots) + +;; Question 1: Why is this DSL design better? +;; Three user interfaces can delegate publication and invalidation to one engine owner. +;; A contextual change refreshes the bound product without requiring a user cache switch. +;; Preserving canonical input avoids manual reconstruction of composition metadata. +;; Question 2: Why does this DSL design have a scientific basis? +;; A fresh-root miss, repeated hit and origin-only change form matched controls. +;; Compiler-call counters and reloaded identities test distinct cache responsibilities. +;; Falsifiers: stale binding origin, lost composition digest, changed canonical payload +;; or an incorrect parser reuse after an input affecting parser compilation changes. +;; Limit: only origin-context changes were tested; no speedup or full DSL composition +;; admission is established by compiler-call counts. +#+end_src + +The implementation recommendation can now be more precise: reuse the existing +publication/cache owner when admitting composed canonical grammar. The unresolved +work is the DSL-to-compiler connection and descriptor/machine publication, +including effective source collection. Creating another cache manager or asking +language authors to supply those compiler closures is not justified by this study. +The low-level success does not make the section-reifying adapter metadata-complete. + +** DSL admission plan within the fixed three user interfaces + +The verified expression package and the verified canonical publication control +establish two parts of one architecture. The remaining connection must be owned +by the engine. Keep =deflanguage= as the package declaration, the checked Loader +as the parser entry and the existing declarative test interface as the expectation +surface. Low-level compiler closures, source-map records and helper factories +must not become required user declarations. + +Source inspection of [[file:../../src/compiler/concise-language.ss][the concise compiler owner]] locates two boundaries: +=expand-surface-syntax= recursively elaborates vocabulary while retaining author +occurrence associations; later code validates rules, derives catalogs and submits +a compile-time =language-declaration= record directly. The explicit grammar surface +reads its clauses into the same record. In [[file:../../src/compiler/language-artifact.ss][the declaration compiler]], +=expand-language-declaration-syntax= collects source rows, constructs one canonical +grammar, binds/compiles/publishes it and emits runtime assembly from that same value. +A named compilation-result record retains the grammar and all three artifact +locators; source associations are consumed before runtime emission. +This inspection identifies where original syntax can be retained. It does not +establish that every current location is absent or wrong; exact reader-location +qualification remains a separate requirement. + +| Engine stage | Proposed responsibility | Reuse and qualification boundary | +|--------------+-------------------------+----------------------------------| +| Read and elaborate author DSL | Retain author syntax and bad-parameter context while expanding vocabulary | Existing Gerbil macros and scoped-context studies; no full expansion-chain promise | +| Collect declaration contributions | Associate authored/generated rules with origins and supported targets | Existing checked roles; combined DSL contribution surface pending | +| Select effective contributions | Apply explicit operations; resolve ownership under override/remove | POO normalizer now captures effective occurrence/source association; production author clause pending | +| Establish effective catalogs | Derive or validate catalogs against selected rules and explicit public tree contract | Existing inference/projection; ordering for combined components needs design | +| Admit canonical grammar/context | Supply one canonical grammar plus effective origin/lineage/source information | Existing binding, parser compiler and verified publication/cache owner | +| Publish runtime descriptor and machine | Reuse current machine/descriptor assembly with admitted artifacts | Existing mechanism; composed-input bridge not yet qualified | +| Explain and test | Show authored intent, effective rows and analysis facts; execute test expectations | Existing selected DSL checks; integrated origin/product explanations pending | + +The engine should collect origins before lossy representation conversion, and +associate them with effective declaration identities after composition. A generated +helper can point to its author invocation and template definition without inventing +a reader token for the helper itself. A source-free input must keep an explicit +absent location. Host syntax context and persistent source records serve different +purposes; arbitrary syntax objects need not become serialized grammar payloads. + +Catalog derivation is also an ownership decision. An override can change the +fields emitted by a rule, while an explicit public tree contract may intentionally +retain optional fields. Neither unconditional union of all historical contributions +nor unconditional pruning to currently emitted fields is sufficient. The design +must distinguish inferred effective fields from explicitly retained public fields, +using the existing =node-fields= mechanism where applicable. The declaration +compiler then validates their relationship once at its admission boundary. + +See [[file:scheme-dsl-semantic-contracts.org][the semantic contract module]] for matched inferred/retained field controls. + +*** Reuse catalog projection after effective contribution selection + +Source inspection and the expanded [[file:scheme-dsl-semantic-contracts.org][semantic controls]] identify an existing +admission mechanism: =project-language-catalog= validates a published catalog +over supplied inferred kinds and terminals. It is a validator/projector, not a +field inference pass. The DSL engine must supply the appropriate effective inputs. + +The proposed dependency order is: + +1. Elaborate author vocabulary while retaining origins. +2. Select effective rule contributions and effective ownership through checked composition. +3. Derive effective kinds/fields/terminals using existing concise inference. +4. Apply explicit optional-field obligations such as =node-fields=. +5. Validate any published projection with the existing catalog owner. +6. Bind and compile the admitted canonical grammar, then publish through existing caches. + +This is a design order for the combined DSL path. Current raw POO catalogs remain +explicit canonical data; this plan does not claim that composition automatically +re-runs concise inference today. Exact provenance selection and integration of the +currently local inference code still need implementation and qualification. + +#+begin_src scheme +;; Existing engine API; inputs must already represent effective declarations. +(project-language-catalog + effective-kinds-with-explicit-field-obligations + effective-terminals + published-catalog) + +;; Question 1: Why is this DSL design better? +;; Authors keep language intent in grammar.ss; engine owns derivation and admission. +;; Existing projection checks remove the need for another catalog validator or manual +;; metadata synchronization in each language package. +;; Question 2: Why does this DSL design have a scientific basis? +;; Matched supplied catalogs test retained fields and stale/dropped-field negatives. +;; Separate tests enforce terminal mappings and duplicate-field rejection. +;; Falsifier: old contribution catalogs enter publication after a semantic override, +;; or explicit public field obligations disappear during projection. +;; Limit: supplied-catalog controls do not implement effective-rule inference or +;; qualify the source-aware DSL-to-descriptor connection. +#+end_src + +*** Source-grounded implementation route: one admitted package + +The architecture must connect declaration, compilation and executable publication +in one route. This source audit identifies extraction boundaries. The concise +elaborator and lexical owners have now been separated as recorded in the +[[file:scheme-dsl-complexity-ownership.org::*Implemented internal owners: one author surface, explicit engine stages][implemented owner map]]. Direct canonical publication does not implement the proposed combined +composition clause or close its effective-origin integration obligations. + +| Current owner | Observed responsibility | Required integration change | +|---------------+-------------------------+-----------------------------| +| [[file:../../src/compiler/concise-language.ss][expand-concise-language-syntax]] | Expands vocabulary, infers fields, adds node-fields, projects catalogs and submits a compile-time record directly | Retain original syntax before conversion; reuse inference over selected contributions and explicit public obligations | +| [[file:../../src/compiler/normalize.ss][compile-grammar/receipt]] | Normalizes checked POO operations and produces canonical grammar plus a receipt | Implemented compile-grammar/context captures effective sources in the accepting fold; full receipt remains unchanged | +| [[file:../../src/compiler/language-artifact.ss][expand-language-declaration-syntax]] | Shared direct record admission constructs grammar, captures context, publishes artifacts and emits from that grammar | Preserve effective composition and origin information when extending this admitted input; direct handoff alone does not implement combined selection | +| [[file:../../src/compiler/bound-ir.ss][bind-grammar-ir]] and [[file:../../src/compiler/parser-ir.ss][compile-parser]] | Bind and compile canonical input | Invoke on the same admitted value with effective context; preserve their existing analyses | +| =compile-language-declaration-artifacts= | Owns declaration identity, cache reuse and immutable artifact publication | Reuse with origin/lineage/source-map dependencies; do not install another cache | +| =assemble-language-parser= and [[file:../../src/compiler/machine.ss][defgeneral-parser-machine]] | Load artifacts and generate the runtime machine | Emit lexical/extras inputs from the same admitted grammar used to compile Parser IR | +| [[file:../../src/language/descriptor.ss][make-language-grammar]] and [[file:../../language-support/entry.ss][common checked entry]] | Connect grammar, Parser IR, machine and protected Loader operations | Preserve existing descriptor and entry contracts; qualify the complete composed route | + +The declaration compiler now separates canonical construction/context capture, +compilation/publication and hygienic emission into named phases. Its orchestrator +passes an admitted-language-declaration to compile-admitted-language-declaration, +then a compiled-language-declaration and named bindings to emission. The canonical +POO publication control supplies the normalized grammar and its source context +directly to that same compilation phase, retaining composition metadata and all +three reloaded products. The normalizer now selects effective occurrence sources +under merge/append/override/remove. The existing composed fixture reaches shared +canonical emission and checked runtime entry, preserving the complete grammar +and composition identity. A production combined author clause and full native +runtime linkage remain open gates. + +Emission still needs structured sections. Deriving these *from* the admitted +canonical grammar is different from rebuilding canonical grammar *from* selected +sections: the former keeps the authoritative value intact. The engine must validate +and adapt the canonical expression representation for the existing macro consumers; +it must not pass the pre-composition declaration as an independent runtime input. + +**** The machine is a second consumer of the admission result + +=assemble-language-parser= supplies both the reloaded Parser IR and declaration +sections to =defgeneral-parser-machine=. In that macro, =lr-prepare= reads =lr-spec= +from Parser IR, while =generated-lexer= consumes the supplied lexical rows and +extras. The trivia predicate also consumes those extras; the lexical source plan +reads lexical rules from Parser IR. Rule rows and entry rows appear in the macro +pattern, but this implementation does not evaluate their expression bodies. + +Consequently, a metadata-preserving compiler bridge could still create an +inconsistent runtime if it emitted stale lexical rows or extras alongside the new +Parser IR. This is a design hazard identified by the consumer audit, not evidence +that an existing package currently exhibits that failure. The machine's +=grammar-digest= argument is taken from the encoded Parser IR artifact locator; +it must not be replaced with =compositionDigest= or the receipt's body digest. + +The expander also receives syntax bindings for assembly, descriptor construction, +=begin= and =def=. Reuse those bindings when extracting emission. Generated public +names use the declaration's syntax context; implementation helpers must retain +their actual bindings. Replacing this with guessed symbol names would discard a +concrete Scheme advantage while apparently simplifying the emitted S-expressions. + +#+begin_src scheme +;; Existing grammar.ss vocabulary; there is no fourth author interface. +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; Existing engine calls illustrate the admission-to-publication boundary. +;; canonical and effective-sources are engine results, not author bookkeeping. +(compile-language-declaration-artifacts + (list origin lineage effective-sources) + canonical + (lambda () (bind-grammar-ir canonical origin lineage effective-sources)) + (lambda () (compile-parser canonical))) + +;; Question 1: Why is this DSL design better? +;; Compared with section replay and manually synchronized component metadata, +;; Scheme vocabulary reuses syntax-aware elaboration and one compiler admission. +;; The engine owns effective catalogs, origins, artifact identity and machine inputs. +;; Authors express reusable grammar intent and retain the same checked entry/tests. +;; Question 2: Why does this DSL design have a scientific basis? +;; Staged DSL papers motivate extension-to-core-to-compiler separation; this source +;; audit identifies each actual producer and consumer in the Gerbil implementation. +;; Qualify the complete package using independent semantics, reload and contextual +;; controls, including a lexical/extras change that reaches the running machine. +;; Falsifiers: stale lexer/trivia, lost composition identity, wrong effective owner, +;; changed tree contract or required author knowledge of internal compiler closures. +;; Limit: the illustrated publication call is not a completed composed DSL path, +;; and neither papers nor source inspection establish a performance improvement. +#+end_src + +**** Finalize effective facts before sealing canonical identity + +The current =compile-grammar/receipt= normalizes every keyed section, forms the +body, computes =grammarIrDigest= over that body, and only then inserts +=compositionDigest=. Passing an already canonical list returns no new receipt. +The concise front end currently derives catalogs before its separate canonical +constructor. Combining those two paths creates a dependency that the design must +make explicit: effective rule selection must precede derived field catalogs, but +receipt/body sealing must follow the final catalogs. + +Do not normalize into a sealed canonical value, patch its catalogs afterwards and +carry the earlier receipt as if it described the patched body. Parser compilation +may succeed while the receipt describes a different admitted input. This is an +integration hazard, not an observed failure of the existing raw POO path. + +The required internal extraction is a selection/finalization boundary in the +existing normalizer: reuse its ordered keyed operations to select rows; derive +concise catalog facts using the extracted existing inference owner; then finalize +and hash the complete body under the existing receipt convention. Raw POO catalog +rows remain explicit checked data and must keep their existing operation behavior. +This is a proposed internal refactor, not a new public normalizer callback or a +second implementation of row selection. Whether derived catalogs need additional +receipt explanation is a design decision; step descriptions must not masquerade +as a verified history of inferred facts. + +The same selection must also determine effective source ownership. A source-map +pass that independently reimplements override/remove would create another state +machine to maintain. Record effective ownership where the existing row operation +selects, replaces or removes a row, then project descriptive persistent sources. +Specify the identical-row merge ownership convention explicitly before exposing +it; equal row contents alone do not select one of several author locations. + +#+begin_src scheme +;; Existing normalizer result, followed by the existing publication owner. +;; The proposed DSL integration must finalize derived catalogs before producing +;; this result. It must not mutate canonical after receiving receipt. +(let-values (((canonical receipt) (compile-grammar/receipt checked-grammar))) + (compile-language-declaration-artifacts + (list origin lineage effective-sources) + canonical + (lambda () (bind-grammar-ir canonical origin lineage effective-sources)) + (lambda () (compile-parser canonical)))) + +;; Question 1: Why is this DSL design better? +;; One ordered selection determines grammar facts and owners; one finalized input +;; determines receipt, artifacts and machine, avoiding synchronized parallel passes. +;; Question 2: Why does this DSL design have a scientific basis? +;; Check the receipt against the actual final body's digest convention and reload +;; the published canonical input; pair those checks with independent parse events. +;; Falsifier: post-seal catalog mutation or ownership derived from different steps. +;; Limit: this block shows existing calls, not the proposed extraction implementation. +#+end_src + +**** One acceptance sequence, with explicit design decisions + +Start with the existing complete declaration in =grammar.ss= and its entry/test +surfaces. The contribution/owner/compose candidate is withdrawn; this engine +route does not require its adoption. Specify any required internal normalization +and publication extraction against a concrete package change, preserving the +existing author vocabulary and interfaces. + +Exercise one complete package through that route: replace a contribution, preserve +an explicitly optional public field, select the effective source, change lexical +or trivia policy, reload artifacts and parse through the checked Loader. Its +assertions must connect the canonical input, contextual Bound IR, Parser IR, +emitted machine and independent tree/events expectations. Existing local controls +remain supporting evidence; their check counts cannot substitute for this route. +A bounded native invocation must finish with exit 0. Performance is a later, +separate measurement of expansion/compilation, loading and actual parse execution. + +This implementation route follows the extension/core/compiler boundary in +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]] and the declaration-dependency treatment +in section 7.2 of [[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros]]. The pinned implementation reading in +[[file:scheme-dsl-research-basis.org][the research basis]] supplies a concrete comparison of thin entries and compiler +owners. These are one architecture argument; Racket-specific binding machinery +and miniKanren optimization passes are not proposed parser dependencies. + +** Effective origins follow composition before binding + +The normalizer now captures effective occurrence context in the same fold that +accepts keyed rows. compile-grammar/context returns a named normalized-grammar +result containing canonical IR, the unchanged composition receipt and effective +selections. Existing compile-grammar/receipt callers omit source capture. There +is no second operation validator or receipt-replay normalizer. + +| Accepted operation | Effective source policy | History policy | +|--------------------+-------------------------+----------------| +| merge of a new key | Select that occurrence | Start effective history | +| merge of an equal row | Keep the first selected occurrence and its source | Append the accepted duplicate to history | +| append | Select the new occurrence | Start effective history | +| override | Select the replacement occurrence without changing row position | Retain prior effective history and append replacement | +| remove | Remove row and its effective source | The complete receipt still retains the removal | +| remove followed by append | Select the newly appended occurrence | Start new effective history; the receipt retains the earlier lifecycle | + +These are the existing parser contract's internal operation labels, not a +recommended new author operation DSL. Native .o/inherited-slot author controls +now use the same canonical emission and checked entry route; see +[[file:scheme-poo-language-components.org::*Native POO substrate and parser operation labels][native substrate qualification]]. Their prototype semantics are supplied upstream; +parser admission still validates the resulting grammar and source associations. + +A metadata callback receives the actual role object, flattened step index, +section and row, after that row operation is accepted. Repeated role names cannot +collapse distinct occurrences. Removed and rejected rows do not acquire an +effective source. This policy is a local engine decision, not an algorithm +prescribed by the papers. The existing row operation owner still rejects missing +override/remove targets, conflicting merges and duplicate appends. + +normalized-grammar-source-map projects the selected rows into existing Bound IR +namespaces. Fields inherit the selected explicit syntax-kind catalog source with +the existing qualified key order (kind field); this does not infer a new field +catalog from effective rules. The engine supplies selected owner, operation, +step and effective history. Reader coordinates and generated status come from +supplied metadata. Missing coordinates fall back to origin; generated status is +not invented. Fixture labels are declared fixture locations, not reader spans. + +#+begin_src scheme +;; Implemented engine handoff; this is not an additional author DSL. +(def normalized (compile-grammar/context composed-grammar source-at-occurrence)) +(def canonical (normalized-grammar-ir normalized)) +(def sources (normalized-grammar-source-map normalized origin)) +(def admitted + (make-admitted-language-declaration canonical origin lineage sources)) +(expand-admitted-language-declaration-syntax + declaration admitted compile-parser bind-grammar-ir + #'assemble-language-parser #'make-language-grammar #'begin #'def) + +;; Question 1: Why is this design better? +;; Selection, source ownership and canonical rows share one acceptance fold. +;; Native macro elaboration and POO construction can feed one canonical compiler; +;; neither authors nor source adapters replay a whole declaration to publish it. +;; The engine derives binding sources and machine input from accepted products. +;; Question 2: Why does this design have a scientific basis? +;; The extensible-surface/core compiler architecture motivates the shared route; +;; declaration dependency checking motivates selection before binding. Matched +;; IR/receipt equality, stale-source controls and full-event comparisons make +;; the implementation claims falsifiable. They do not prove optimal syntax, +;; improved author task times or faster parsing. +#+end_src + +The existing list-stage fixture now enters that route directly. Its grammar +module is list-language.ss, checked entry is list-parser.ss and independent +conformance is list-parser-test.ss. list-runtime.ss is verification infrastructure, +not a fourth required author interface. Two independently identified policies +qualify the ordinary list and an override/remove variant through checked Loader +entries. The full canonical Grammar IR and compositionDigest survive publication +and machine emission; no section-reification bridge reconstructs canonical input. + +The joint catalog override controls retain explicit optional fields and qualify +their selected origins. The append-only fixture source map remains a negative +control: it demonstrably selects stale metadata after override. Whole native +runtime linkage, a production public composition clause and arbitrary extension +source capture remain separate gates. Gerbil syntax utilities provide actual +syntax/source access; they do not recover reader locations from canonical data. + +** Connect analysis evidence to author syntax before publication + +Read this with [[file:scheme-dsl-semantic-contracts.org::*Derived facts after expansion: nullable, FIRST and completion][the derived-fact contract]]. A nullable result or FIRST +membership describes a lowered grammar. A useful author diagnostic additionally +needs the failed author promise, its dependency evidence and a truthful source. +A generic nullable item is not itself an error. No new promise or rejection is +introduced merely to exercise diagnostics. + +*** Existing seams and the information they retain + +| Source seam | Current observation | Consequence for the integrated route | +|-------------+---------------------+--------------------------------------| +| =expand-surface-syntax= in [[file:../../src/compiler/concise-language.ss][concise-language.ss]] | Recursively expands a nonprimitive with =core-expand1=, then emits datum; =macro-lineage= records names | Retain invocation syntax and primitive occurrence paths before datum conversion; the unique lineage list is not an expansion trace | +| Concise compiler record | Submits normalized rules and derived catalogs without macro replay | Keep original declaration syntax across defaults; extend effective source associations without fabricating parameter origins | +| [[file:../../src/compiler/language-artifact.ss][compile-language-declaration]] | Collects declaration-name locations and supplied source ownership | Row origin is useful fallback; it is not a subexpression or dependency explanation | +| [[file:../../src/compiler/bound-ir.ss][bind-grammar-ir]] | Selects a source record per namespace/name and incorporates source/lineage in identity | Reuse this persistent declaration owner; keep richer transient syntax outside portable IR unless its contract is deliberately extended | +| [[file:../../src/compiler/lr.ss][lower-rules]] | Generates production ids and synthetic nonterminals from expression structure | Associate emitted productions with the effective rule and expression occurrence at this owner; do not reverse-engineer a synthetic name into an author location | + +These are source-inspection conclusions, not implemented diagnostic propagation. +The existing source-location fallback may contain an origin description rather +than reader coordinates. A renderer must distinguish that description from a +verified coordinate and preserve explicit unknown locations. + +*** Native Gerbil mechanisms and the minimum engine seam + +Source inspection used the installed Homebrew distribution +=gerbil-scheme@0.19/0.19.2591dcd.patchf5cedd8168cb=, bundled source directory +=2591dcd=. This identifies the inspected installation, not a claim that all +upstream revisions or package patches behave identically. + +In =src/gerbil/expander/core.ss=, =core-expand1= resolves the head binding and +applies its expander. =core-apply-expander= wraps a procedure's result using the +input source when available. In =stx.ss=, =stx-wrap-source= returns an already +wrapped AST unchanged; otherwise it supplies an outer AST when a source exists. +This agrees with the [[https://gerbil.scheme.org/reference/gerbil/expander/utilities.html][official source utility contract]]. It does not recursively +annotate all generated children or produce a complete author/definition chain. +=core-apply-user-macro= separately applies expansion marks for lexical context. +Hygiene and source attribution are distinct mechanisms. + +Therefore the parser should reuse those syntax objects and record invocation +associations at its existing expansion boundary. It should not patch Gerbil's +expander, overwrite template coordinates with call coordinates, or equate matching +datum with matching occurrence. Direct primitives, substituted argument syntax +and generated wrappers can have different available origins. Preserve those +origins and describe the precision actually known. + +For lowering, =make-production= currently publishes five elements: id, lhs, rhs, +action and precedence. =production-table= indexes those productions by id; +=compile-lr-spec/selected= lowers once and feeds the same products through analysis +and construction. The minimum design seam is a transient association keyed by +production id for that exact invocation, populated at the existing =emit= sites +from threaded expression occurrence information. It is not a globally stable +production identity. Rule edits can renumber productions and synthetic names. +Do not extend every runtime production or portable artifact merely to carry live +syntax. Persistence, if required by an actual tooling consumer, needs a separately +specified source projection and contextual cache dependency. + +The [[file:scheme-dsl-semantic-contracts.org::*Finite explanations belong to the existing fact owner][finite-explanation design]] specifies when analysis causes can be recorded +without a second solver. A checked transient association connects those production +causes to retained syntax; Bound IR remains the declaration-level persistent +source owner. Both associations must be derived from the same effective grammar. +These seams are researched implementation choices, not current capabilities. + +On 2026-10-06 the existing =source-free-diagnostics.ss= control was rerun +under =gtimeout -k 2 10=. All six assertions and +=SOURCE-FREE-DIAGNOSTICS-OK: 6 checks passed= completed with exit 0. It preserves +absent author/path/location data while retaining typed parameter blame and known +template detail. This qualifies the missing-source boundary, not production +associations or analysis explanation generation. + +*** Implemented first slice: concise author blame + +On 2026-10-06, [[file:../../src/language/grammar.ss][deflanguage]] gained declaration-local source associations while +expanding existing vocabulary. Fresh compound datum identity distinguishes equal +expressions at different occurrences; rule and operand associations retain reader +syntax for atomic parameters. Existing expression validation consumes those +associations and keeps the whole declaration as supporting context. Unknown +reference/token parameters and unknown constructor heads retain their existing +error messages and rejection policy. + +Inside macro expansions the retained outer author invocation is conservative +blame. The implementation does not infer arbitrary transformer parameter origins +from equal datums or template coordinates. Source-free input retains absent +coordinates. Associations are local expansion data; they are not serialized into +Grammar/Bound/Parser IR, caches or runtime productions. Existing datum lowering, +field inference and artifact owners remain the output path. + +The [[file:../../t/language-diagnostics-test.ss][diagnostic suite]] uses existing admission failures and checks messages, +blame data, declaration context and source text at actual reader coordinates. +The [[file:../../t/fixtures/language-diagnostics-vocabulary.ss][vocabulary fixture]] includes a deliberately invalid library template to +check that a generated unknown constructor points to its author call. Non-syntax +exceptions and unexpected acceptance are test failures, not accepted diagnostics. + +| Bounded local qualification | Receipt | +|----------------------------+---------| +| Focused author diagnostics | Five test cases, SUITE-OK, LANGUAGE-DIAGNOSTICS-OK, exit 0 | +| Existing language-surface suite | Five cases, SUITE-OK, LANGUAGE-SURFACE-OK, exit 0 | +| Complete expression call/array package | Eleven cases, SUITE-OK, PACKAGE-EXPRESSION-DSL-OK, exit 0 | +| Changed grammar module native compilation | gxc -O -s to isolated output; phase-zero/one .o1, .ssi and optimizer metadata produced; exit 0 | + +Every invocation used =gtimeout -k 2 10= with the project load path. Native +compilation writes only to a temporary output directory. A subsequent diagnostic +run put that fresh output first on GERBIL_LOADPATH and passed the same five +cases, SUITE-OK and LANGUAGE-DIAGNOSTICS-OK with exit 0. The suites qualify their selected semantic observations, not all-input +product equivalence or a benchmark improvement. No compilation overhead has been +measured. Tests do not qualify source-aware POO selection, lowered-production +associations, recursive witnesses or portable tooling projections. + +This implements the first task from [[file:scheme-deflanguage-design-synthesis.org::*First complete task: diagnose an existing invalid declaration][the decision review]]. The earlier seam audit +is historical for concise source collection: expression associations now exist, +while declaration publication and LR explanation seams remain open. The broader +route below keeps those distinctions explicit. + +*** Implemented follow-up: validate rules before deriving fields + +The next complete diagnosis task exposed a remaining dependency seam: a root node +references a helper declared later, and that helper contains =(field argument)=. +The original inference walked raw helper expressions before their validation and +reported the whole declaration. The added regression observed that wrong blame; +its first invocation emitted a failed check and exited 124. A later SUITE-OK in +that failing transcript is not a pass; neither deadline nor acceptance expectation +was relaxed. + +The implementation now normalizes and validates all rule bodies with their retained +syntax before deriving node fields. Existing field inference consumes that same +normalized rule set. Node registration remains postorder, references retain their +visited-name guard, named nodes stop field propagation and alternatives retain the +existing ordered field union. This is a phase adjustment inside the existing owner, +not a new field algorithm, reference resolver or author configuration. Invalid +helpers are diagnosed by existing shape checks before inference can dereference +their operands. Diagnostic priority now puts rule validation ahead of dependent +field inference; no general multi-error ranking contract is claimed. + +The focused suite adds the forward malformed field and malformed unary/reference +helpers. All seven cases pass with SUITE-OK, LANGUAGE-DIAGNOSTICS-OK and exit 0, +both from source and with fresh native grammar output first on GERBIL_LOADPATH. +The existing surface suite passes five cases; the complete expression package +passes eleven. All runs use =gtimeout -k 2 10=. The changed grammar module compiles +with =gxc -O -s= to isolated phase-zero/one products and exits 0. + +An additional native before/after control uses the previous first-slice output +and the new output separately. Its malformed-helper blame head is =deflanguage= +before and =field= after, confirming the distinct macro implementations. It also +serializes the complete call/array Grammar IR and Parser IR: both snapshots contain +6661 bytes and =cmp= exits 0. Snapshot capture exits 0 on both sides. This is exact +product parity for that package, not every possible grammar or runtime backend. +The initial temporary snapshot runner selected its own script as the output due +to an incorrect command-line index; it was repaired before admitting these receipts. +That harness failure is not a parser or IR failure. + +See [[file:scheme-dsl-semantic-contracts.org::*Field derivation uses the validated grammar][the field-derivation contract]] for the Scheme rationale and semantic limits. +No compilation performance or memory improvement has been measured. General +source-aware POO selection, LR production origins and recursive witnesses remain +outside this implemented slice. + +*** Declaration obligations retain author syntax + +The same admission path now uses retained syntax for two cross-expression checks: +the root body must construct one node, and a syntax kind cannot be both node and +token. Existing acceptance policy, messages and checking order are preserved. +The root check selects its single authored body or the whole authored rule row +for multiple bodies. The kind collision selects the first encountered node-kind +parameter; vocabulary-generated nodes retain the conservative invocation fallback. +Declaration-local rule/kind associations do not enter any serialized product. + +The regression first failed both blame checks with exit 1, despite SUITE-OK. +An earlier harness mistakenly wrapped the non-node root in a valid node and was +correctly admitted; that invocation exited 70 and is not implementation evidence. +After correcting the author task and changing the existing owner, all nine +=language-diagnostics-test= cases pass from source and fresh native output, exit 0. +The controls include direct/macro roots, an authored multiple-body row, +direct/generated kind collisions and source-free input. Existing five surface +and eleven complete package cases also pass; =gxc -O -s= exits 0. + +Matched native snapshots identify the old root blame as the complete declaration +and the new blame as =identifier=. Both package product snapshots are 6661 bytes; +=cmp= exits 0. This qualifies the call/array Grammar IR and Parser IR only. +The [[file:scheme-dsl-semantic-contracts.org::*Declaration obligations explain the authored term][semantic rationale]] connects this result to Fortifying Macros section 4.2. +Arbitrary transformer parameter origins, two-sided collision locations and POO/LR +provenance remain open; no performance improvement is claimed. + +*** Integrated constructor admission and native qualification + +The current implementation closes one expression-admission boundary across +=deflanguage= and =deflanguage-grammar=. The canonical algebra's shared header +predicate owns arity/scalar constraints; its recursive membership predicate and +source-aware compiler traversal consume the same constraint definition. The latter +retains syntax through rule lowering and rejects malformed headers before list +access. Concise projection preserves its existing shape messages and delegates +remaining literal/layout/precedence scalar constraints to the canonical owner. +Low-level malformed constructors now produce =invalid GrammarExpr declaration= +with the offending expression and declaration context, rather than downstream +list-access or parser-production errors. Accepted package schemas remain unchanged. + +| Integrated native qualification | Cases / result | +|---------------------------------+----------------| +| Author diagnostics and canonical admission matrix | 12 cases, LANGUAGE-DIAGNOSTICS-OK, exit 0 | +| Existing grammar composition, precedence and LR suite | 31 cases, GRAMMAR-COMPOSITION-OK, exit 0 | +| Existing concise/explicit language surface | 5 cases, LANGUAGE-SURFACE-OK, exit 0 | +| Complete call/array package, including exact tree/spans | 11 cases, PACKAGE-EXPRESSION-DSL-OK, exit 0 | +| Algebra, language-artifact and grammar native compilation | Three gxc -O -s invocations in dependency order, exit 0 | +| Selected package product comparison | 6661-byte Grammar/Parser IR snapshot equals previous qualified snapshot; cmp exit 0 | + +Each compilation/test invocation used =gtimeout -k 2 10=. Fresh native outputs +were first on GERBIL_LOADPATH. The first native matrix failed with exit 70: +its generated macro replaced the newly added =list?= condition with true and +malformed dotted input reached =length=. The repaired boundary uses existing +Gerbil syntax-list traversal; the generated macro retains =gx#stx-list?= and +all 59 native cases pass. This observes one failing guard, not a general optimizer +defect. The earlier source matrix alone was insufficient qualification. + +The [[file:scheme-dsl-semantic-contracts.org::*One constructor contract across declaration entries][constructor contract]] includes the full constraint table, Scheme rationale, +negative controls and analysis boundaries. This is a coherent compiler-entry task; +it does not add grammar vocabulary, a second nullable/FIRST solver or runtime +origin storage. Full declaration-section contracts, arbitrary transformer +parameter origins and performance qualification remain open. + +*** Evidence lifecycle across cache hits and misses + +The preferred research direction is request-scoped explanation from an immutable +snapshot of the current admitted grammar and current retained syntax associations. +Ordinary compilation need not retain a full explanation graph for every fact. +This is a recommendation for discussion and implementation qualification, not a +new public mode or an implemented capability. + +The current [[file:../../src/compiler/language-artifact.ss][artifact owner]] has two distinct caches. Parser artifacts use generator +contract plus grammar; declaration artifacts additionally use declaration identity +(origin, lineage and source map). A declaration hit returns three locators without +calling binding or compilation. A parser hit loads Parser IR without invoking its +compile callback. The current [[file:../../src/compiler/lr-compiler.ss][LR spec]] contains productions, nullable and FIRST, +but no first-admission evidence or expression-to-syntax association. + +| Situation | Proposed evidence work | Identity and lifetime obligation | +|-----------+------------------------+----------------------------------| +| Successful ordinary declaration | Preserve normal admission/publication; no full fact-witness graph required | Release live syntax after its compile-time consumers finish | +| Local shape/binding failure | Use retained offending syntax and the existing typed macro diagnostic | No analysis replay for a grammar that was not admitted | +| Explicit fact-dependent diagnostic during admission | Enable cause capture in the existing analysis owner over current admitted input | Keep immutable grammar, source associations and lowering policy together until rendering completes | +| Current declaration with cached parser, explanation requested | Use current snapshot; invoke existing lowering/analysis with capture if cached facts have no evidence | Compare production ordering and facts if relating the explanation to cached Parser IR; do not reuse old live syntax | +| Artifact-only tool with no current syntax snapshot | Provide available persistent declaration sources at their actual precision | Full macro/parameter explanation unavailable unless a separately qualified persisted projection exists | + +A fact-dependent admission promise must be checked for the current declaration, +not only inside a compile callback skipped on a cache hit. Validated cached facts +can answer the predicate; absent witness data may require capture to explain a +failure. Changes to acceptance or compiled-product guarantees require the existing +generator-contract/cache invalidation policy. A source-only edit cannot justify +reusing old author coordinates simply because parser semantics are unchanged. + +When explanation requires replay, replay the existing deterministic lowering and +fact analyses on the already elaborated, checked grammar. Do not re-expand author +macros: arbitrary transformers can have effects, and a second invocation need not +produce the same declaration. Do not rebuild LR states merely to explain nullable +or FIRST. Match root, conflict-dependent lowering policy and ordered rules. If a +cached product and replay disagree, report the inconsistency rather than silently +attribute one product's facts to another grammar. + +This clarifies the earlier single-pass requirement: populate associations at the +existing lowering owner, not by another lowering algorithm or a parallel source +reconstruction. A requested replay can execute that same owner again; it has a +real additional cost. It is not promised to require zero passes on cache hits. +During a cold compilation, capture can use the existing run when requested before +analysis; a request after completion may need replay. Persistence of witnesses +would change the cost and cache contract and is not selected by this recommendation. + +#+begin_src scheme +;; Existing analysis calls on an already admitted rules/root snapshot. +;; This is an engine observation, not an API authors configure in grammar.ss. +(let ((productions (lower-rules rules root allow-dynamic?))) + (let-values (((nullable nullable-index) (compute-nullable productions))) + (let-values (((first first-index) (compute-first productions nullable-index))) + (list productions nullable first)))) +;; Proposed capture belongs inside these existing owners, as specified in the +;; finite-explanation contract. The calls above do not currently return witnesses. + +;; Question 1: Why is this DSL design better? +;; The author obtains explanations without managing compiler/cache phases, while +;; normal products stay under their existing owners and identities. +;; Question 2: Why does this DSL design have a scientific basis? +;; Compare cold/cache-hit paths on the same ordered grammar and current source. +;; Check facts, finite causes and reader text separately from cache receipts. +;; Falsifiers: a hit needs unavailable callback state, replay re-executes a macro, +;; old coordinates are used for edited source, or stale facts receive fresh blame. +;; Limit: demand-scoped capture can repeat analysis; its cost must be measured. +#+end_src + +The [[file:scheme-dsl-complexity-ownership.org::*Account for explanation work and retained data][cost evaluation]] includes cache-hit requests, long prefixes and shared +proof dependencies. This recommendation reuses fact owners and artifact caches; +no second cache, mandatory author declaration or runtime actor is introduced. + +*** One route, with bounded attribution + +1. During expansion, retain the author rule syntax and each macro invocation. + Keep primitive occurrence paths through sequence/choice/wrapper traversal. + Hygiene identifies Scheme helpers; grammar labels remain domain labels. +2. During effective rule selection, select the associated origins with the rule. + A replaced or removed rule cannot supply the active diagnostic's source. +3. During lowering, associate each production with its originating occurrence. + Multiple generated productions can point to one occurrence; the association + must not change LR production semantics or become a second lowering pass. +4. Consume existing nullable/FIRST results. When an explicit promise needs an + explanation, inspect the existing productions for a finite dependency witness. + Nullable witnesses justify an empty derivation; FIRST witnesses justify a + sentential prefix and never imply completion. Evidence capture at first fact admission is a + proposed consumer within those existing owners, not a second solver or current API. +5. Report the violated promise using a known offending parameter when available, + otherwise the macro invocation, otherwise the effective rule, otherwise an + explicit unknown location. Show supporting dependency locations separately. + Retain the generating definition for developer detail, not as author blame. +6. Finish admission before publication and caching. If explanatory data persists, + include its actual dependency/identity policy; do not attach unkeyed mutable + context to an already published artifact. + +Arbitrary =defsyntax= transformers need not preserve a one-to-one parameter map. +A generated primitive therefore cannot automatically be blamed on a parameter +merely because its datum equals that parameter. Identical repeated terms are +separate occurrences. Invocation-level attribution is the honest fallback when +no finer syntax association survives. No author-maintained source table or new +configuration in grammar.ss is required by this design direction. + +#+begin_src scheme +;; Existing author vocabulary; the engine retains the invocation before expansion. +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; Existing primitive input to the same analysis owner: +(item (reference peer)) +(peer (optional (token identifier))) +;; Nullable(item) depends on peer's optional branch. If a consumer explicitly +;; requires a nonnullable item, explain that dependency at the author invocation. +;; Without that consumer promise, these rules are valid and need no rejection. + +;; Question 1: Why is this DSL design better? +;; Authors keep grammar intent and semantic expectations in the three interfaces. +;; The engine explains inferred facts in author vocabulary without asking authors +;; to know generated LR names or maintain syntax-origin bookkeeping. +;; Question 2: Why does this DSL design have a scientific basis? +;; Compare the same direct and macro-authored grammar under the same promise. +;; Check equal facts, the actual reader text at reported coordinates, and a finite +;; production witness for the explanation. Distinguish scope from source identity. +;; Falsifiers: stale overridden origin, blame on a template as if it were a call, +;; fabricated parameter precision, or a FIRST prefix presented as completion. +;; Limit: current scoped checks qualify local macro blame, not this entire route. +#+end_src + +*** Paper connection and qualification order + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros]], sections 4.1–4.2 and 7.2, distinguishes matching progress, +local descriptions and declaration dependencies. Its parser example gathers +clauses before dependent checks. Apply that separation alongside syntax-object +and scope readings: preserve context early, resolve dependencies in the engine, +and describe errors at the author's abstraction. The paper's alternative-pattern +failure ranking does not define an ordering for all global grammar diagnostics. +We do not propose importing its Racket matcher into Gerbil. + +The abstract-interpretation contract adds a different obligation: the explanation +must substantiate the particular analysis observation. Hosted-DSL compilation +adds the phase and admission boundary; POO normalization adds effective-source +selection. Together they constrain one implementation route rather than separate +paper-specific APIs. + +On 2026-10-06 the existing =scoped-author-diagnostics.ss= control was rerun +under =gtimeout -k 2 10=. It emitted =SCOPED-AUTHOR-DIAGNOSTICS-OK: 28 checks +passed= and exited 0. The checks cover typed errors, messages, offending parameter +text, reader coordinates, renamed import context and a direct call after failed +scoped expansion. They do not run the proposed analysis-witness integration. + +Integrate expansion and lowering associations in their current owners next. +Compare direct/macro declarations, +indirect nullable dependencies, duplicate identical occurrences, changed effective +rules, renamed imports and missing reader source. Verify both explanation facts +and actual source text, then the complete package and compiler-product gates. +Existing focused receipts do not discharge those remaining integration gates. + +** Package closure criteria at this boundary + +Use the same call/array package for the complete preservation check: + +1. Admit its composed canonical grammar without section reconstruction. +2. Verify the same composition digest at compiler input, Parser IR and reloaded artifact. +3. Verify effective sources after append/override/remove; distinguish receipt history + from the selected row's source and preserve explicit absent-source cases. +4. Exercise cache hits and contextual changes, showing which artifact identities + must change and which semantic observations must remain equal. +5. Compare accepted/rejected inputs, full events, spans and roundtrip with the + inline control through the checked Loader and bounded artifact-loading driver. + +The current list-stage implementation qualifies the composed canonical route +through publication, machine emission and checked Loader. provenance-language.ss +now requires the complete canonical input and compositionDigest to survive. +The selection policy and qualified ordinary/override-remove controls are described +[[file:scheme-dsl-engine-admission.org::*Effective origins follow composition before binding][above]]. A production author composition clause and full native target closure +remain distinct from this local compiler/entry qualification. + +Historical rerun on 2026-10-06 used +=GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" gtimeout -k 2 10 gxi t/fixtures/language-pack-research/provenance-language.ss=. +All 14 assertion markers and =PROVENANCE-LANGUAGE-OK= were emitted, including +=composition-remains-unpropagated=, but the process exited with =124=. +This invocation failed its bounded process-completion gate. It does not replace +the earlier exit-0 receipt with a new pass, and no cause of the timeout has been +established by this rerun. Keep assertion completion and process completion distinct. + +The 2026-10-07 qualification builds seven compiler/declaration owners natively +through std/make (32.64 s, exit 0), then rebuilds the corrected normalizer alone +(5.02 s, exit 0). Three composed fixture phase modules also compile natively +(16.43 s, exit 0). Each build keeps the 240-second total and five-second genuine +output watchdog. Runtime consumers still load the current machine/runtime from +source; these receipts are not full native machine linkage or installed consumer +qualification. + +| Controlled boundary | Passing checks or cases | Process boundary | +|---------------------+-------------------------+------------------| +| Operation/history/source selection | 63 checks | Serial native-owner consumer, exit 0 | +| Ordinary and override/remove checked entries | 79 checks | Complete events, policy distinction and roundtrip, exit 0 | +| Actual published Bound/Grammar/Parser provenance | 15 checks | Complete canonical input and compositionDigest retained, exit 0 | +| Composed package declarative conformance | 11 cases | Native phase adapter, source runtime, exit 0 | +| Existing concise package declarative conformance | 11 cases | Same shared emission owner, exit 0 | +| Existing surface, diagnostics, composition, publication and source/GQL regression | 76 cases | Five bounded processes, each exit 0 | +| Concise package complete Grammar/Parser product control | 6661 bytes, byte-identical | Snapshot and comparison exit 0 | + +The 255 assertions/cases above are separate controls, not one full native +end-to-end suite or a parse benchmark. The first selection witness failed with +exit 70 because the implementation used field/owner rather than the existing +owner/field Bound IR source key. The fix changes the implementation to the +existing contract; original assertions remain. Later five-process concurrent +qualification attempts exited 124/137, including one that printed 63 successful +assertions before exit 124. They remain failed attempts. Running the same final +native-owner consumers serially under the unchanged ten-second bound exits 0. +No root cause is established for the concurrent failures; these observations do +not establish GC behavior or concurrent parser throughput. + +The architecture reading is tied to this boundary: section 1 and Figure 1 of +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]] describe extensible surface elaboration +to a compiler-oriented core. Our design inference is to converge Gerbil vocabulary +and POO contributions on existing canonical IR. The paper uses Racket and a +miniKanren compiler; its optimizations, implementation and performance results +are not qualifications of our parser engine. diff --git a/docs/research/scheme-dsl-research-basis.org b/docs/research/scheme-dsl-research-basis.org new file mode 100644 index 00000000..6e0f7974 --- /dev/null +++ b/docs/research/scheme-dsl-research-basis.org @@ -0,0 +1,632 @@ +#+title: Scheme DSL research basis and design method +#+date: 2026-10-06 + +Scope: primary-paper readings, transfer boundaries and historical candidate +vocabulary. The [[file:scheme-deflanguage-design-synthesis.org][overview]] records current decisions; [[file:scheme-dsl-author-interface.org][author examples]], +[[file:scheme-dsl-engine-admission.org][engine admission]] and [[file:scheme-dsl-semantic-contracts.org][semantic contracts]] apply these readings to one package. +Historical follow-ups below explain supporting studies; they do not add user interfaces. + +* Read the sources as one architecture argument + +The research program includes eight primary lines and an additional embedding +paper currently known only from its publication record/abstract. All must feed +one design argument. The GQL refactor is a concrete implementation witness for +shared admission; it cannot stand in for the whole paper program. + +| Primary source | Recorded reading boundary | Question answered in the same architecture | Implementation or qualification obligation | +|----------------+---------------------------+---------------------------------------------+--------------------------------------------| +| Syntactic Abstraction in Scheme | Sections 3–4 of TR355 | How do patterns, templates and procedural expansion preserve context? | Retain syntax objects and real helper bindings; do not reduce everything to datum before diagnosis | +| Binding as Sets of Scopes | Extended paper, especially section 3 | Which names are host bindings, domain labels and generated public identifiers? | Test module/import renaming and phases; do not treat grammar keys as Scheme variable bindings | +| Keeping it Clean with Syntax Parameters | Sections 4, 4.1 and 6; supporting Gerbil study | How do cooperating macros carry scoped context without accidental capture? | Use scoped context only where needed; qualify nested declaration versus expression behavior | +| Fortifying Macros | Sections 3–4 and parser case 7.2 | How are declarations specified and checked at the author's abstraction level? | Collect dependent declarations before checking references; preserve author blame and parameter roles | +| Compiled, Extensible, Multi-language DSLs | Sections 3–5; pinned hosted-miniKanren source | How do extensions receive existing compiler analyses and products? | Elaborate to one admitted core; reuse compiler owners and validate host/domain boundaries | +| Parsing Expression Grammars | Sections 3.5–3.7 | Which algebra and side conditions make a transformation valid? | State CFG/LR semantics separately; test tree/field/span preservation beyond recognition | +| Abstract Interpretation | Scanned paper sections 5.1–5.3, 6–7 and 8.1–8.3; appendix proofs not reproduced | What facts can safely be derived, and what do approximations mean? | Reuse nullable/FIRST owners; specify domains and soundness obligations for any additional fact | +| Extensibility for the Masses | Sections 2–3 and modular-combination discussion in supporting study | How are construct extension and operation extension separated? | Reuse POO/IR consumers; require a concrete task before selecting another algebra or public protocol | + +[[https://www.mathematik.uni-marburg.de/~rendel/hofer08polymorphic.pdf][Polymorphic Embedding of DSLs]] is additional background, now read in sections +3.1–3.6 and 4.4–4.7 of the author paper. Listings were inspected; no Scala artifact +was reproduced. Its phase and recursive-analysis limits are recorded below. +Ousterhout's author lecture notes in [[file:scheme-dsl-complexity-ownership.org][complexity ownership]] supply evaluation +criteria; they are not additional peer-reviewed evidence of this DSL's superiority. +Reading sections, inspecting code, reproducing an artifact and proving a property +are separate activities; none of the table's boundaries implies completion of all. + +The implemented [[file:scheme-dsl-complexity-ownership.org::*Implemented internal owners: one author surface, explicit engine stages][engine ownership refactor]] applies these readings jointly: the public +macro passes hygienic bindings to a compile-time elaborator, that elaborator +retains declaration checks and submits an internal record directly to the shared +canonical compiler, and focused +lexical owners generate execution mechanisms without new author configuration. +Its [[file:scheme-dsl-complexity-ownership.org::*Qualification of the implemented boundaries][qualification record]] distinguishes semantic/product preservation from open +native consumer gates. Default normalization keeps original author context, and +runtime emission shares the admitted canonical grammar. Named admission, +publication and emission phases now expose a direct normalized-POO/context +publication edge, checked in the existing provenance control. They do not +infer override/remove source ownership or establish composed Loader closure. +This is our application of the papers, not a result +proved by their implementations or an established performance improvement. + +** Joint constraints on the design + +These sources constrain each other. Contextual syntax makes reusable vocabulary +possible, while binding/phase distinctions restrict how that vocabulary can be +combined. Declaration contracts determine when invalid forms must be rejected. +Core admission gives the compiler one value to analyze. Parsing semantics limits +valid rewrites; abstract-interpretation reasoning limits claims based on derived +facts. POO extensibility concerns then determine whether another observation can +reuse that core. A gain in one dimension does not waive any of the other contracts. + +Apply these constraints to the same three interfaces: + +1. grammar.ss states language intent and reusable Scheme vocabulary. Study a real + edit and diagnosis task before adding any public form. Keep original syntax + available until binding and diagnostic decisions have been made. +2. The engine elaborates, resolves/checks domain data, derives existing facts and + publishes a canonical input consistently. Syntax parameters are an optional + internal context mechanism, not obligatory author configuration. +3. parser.ss publishes the checked entry and declared services/products. Multiple + observations of one IR must keep explicit contracts; a new interpreter or + callback cannot silently become a portable backend. +4. parser-test.ss specifies independent recognition/tree/source expectations and + benchmarks. Each paper-informed change must be checked against those same + observations and the compiler-product gate. + +#+begin_src scheme +;; Existing grammar.ss vocabulary and existing parser-test.ss expectation. +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; Inside the existing test declaration for that package: +(accepted "three call arguments" "f(a,b,c)" + (counts (Call 1) (Name 3)) + (field-counts Call argument (3))) + +;; Question 1: Why is this DSL design better? +;; A grammar idiom reuses contextual Scheme syntax; the engine owns validation, +;; analyses and publication, while independent expectations stay with the author. +;; Evaluate the same complete edit/diagnosis task against the explicit declaration. +;; Question 2: Why does this DSL design have a scientific basis? +;; The papers jointly identify abstraction, binding, validation, core and semantic +;; obligations. Exact expansion, independent parse observations and consistent +;; compiled products are distinct evidence for those obligations in this engine. +;; Falsifiers: captured helpers, wrong blame, invalid accepted grammar, changed tree +;; policy, unsound analysis facts, inconsistent products or worse author effort. +;; Limit: current controls support selected contracts; neither a paper inventory nor +;; the GQL witness proves universal superiority or closes the full research program. +#+end_src + +** Complete the readings in dependency order + +The next reading output is an integrated design contract, not a fixture per paper. +First combine syntactic abstraction, scope and syntax-parameter readings into a +binding/context/phase specification. Combine that with declaration checking and +hosted-DSL implementation to specify admission and diagnosis. Then connect the +parsing algebra and existing analysis code to transformation/fact obligations. +Finally review POO and polymorphic embedding against concrete IR consumers and +three-interface author tasks. The selected full-text readings below replace the +earlier abstract-only boundaries. Appendix proof reproduction, implementation +artifact reproduction and our own transfer/lowering proofs remain separate open +obligations before claiming a new algorithm or implementation protocol qualified. + +The small call/array controls and real GQL package answer different parts of this +same contract: semantic preservation and actual author/product integration. The +GQL compiler-product mismatch remains a qualification gate in [[file:scheme-dsl-author-interface.org][author interfaces]]; +it must not redirect the entire research program into build troubleshooting. + +* Study each paper through an explicit design question + +* Syntactic abstraction in Scheme: how should authors define vocabulary? + +Dybvig, Hieb and Bruggeman, 1993; study text: +[[https://scholarworks.iu.edu/dspace/bitstreams/3f729d08-c698-4430-9d04-6a782c5b9300/download][Indiana University TR355, revised July 1992]]. Section 3 explains syntax +objects, pattern matching, template construction and procedural transformations; +section 4 includes source annotation handling. This reading extends the earlier +abstract-only annotation in our initial research document. + +Design conclusion: support a progression from a small template to a richer +compile-time transformation. Simple grammar idioms can use =defgrammar-syntax=; +parameter validation or operator-table elaboration can use Gerbil's procedural +macro facilities. Callers should see the meaning of arguments, node labels and +fields in either form. Keep expansion explanations traceable to author syntax. + +Study task: express the same binary production as explicit constructors, a +parameterized template and an operator-table elaboration. Compare semantic +clarity, expansion size and effort to change an operator tier. More powerful +macros should improve the author task, not require a new Scheme compiler. + +* Compiled, Extensible, Multi-language DSLs: paper and implementation + +Ballantyne, Gamburg and Hemann, ICFP 2024, +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][author paper]], sections 3–5 and Figures 5, 9–11. The design elaborates extensions +into a compiler-oriented core with specified host boundaries. Compiler analyses +can then serve both primitive forms and extensions. This motivates a common +admitted Grammar IR here; it does not justify transferring miniKanren algorithms +or performance results to CFG/LR parsing. + +** Pinned implementation reading + +The authors' [[https://github.com/michaelballantyne/hosted-minikanren][Hosted miniKanren repository]] was inspected on 2026-10-06 at +=867755c61acd064a07dfcfd0849b01a70f315bed=. The paper describes the architecture; +this repository revision is a separate implementation snapshot, not a verified +reproduction of the paper artifact. No Racket installation or benchmark was run. + +| Pinned source | Observed implementation responsibility | Gerbil parser design inference | +|---------------+-----------------------------------------+--------------------------------| +| [[https://github.com/michaelballantyne/hosted-minikanren/blob/867755c61acd064a07dfcfd0849b01a70f315bed/private/spec.rkt][private/spec.rkt]] | Binding classes, permitted term/goal extensions, scoped fresh variables and host-expression positions | Distinguish syntax extension from grammar-domain identity and explicit host positions | +| [[https://github.com/michaelballantyne/hosted-minikanren/blob/867755c61acd064a07dfcfd0849b01a70f315bed/private/interface-macros.rkt][private/interface-macros.rkt]] | Definition/expression entries call compiler functions; expanded and compiled relation code is recorded | Keep DSL declarations thin over engine owners; make stages inspectable without author-maintained metadata | +| [[https://github.com/michaelballantyne/hosted-minikanren/blob/867755c61acd064a07dfcfd0849b01a70f315bed/private/compile.rkt][private/compile.rkt]] | compile-goal organizes optimization and generation; reference compilers handle term-variable access; named passes record intermediate code | Reuse parser analyses under one compiler entry; separate inspection from a second execution path | + +The code's identifier fallback and host conversions serve miniKanren semantics. +Our grammar retains explicit token/reference disambiguation and backend capability +contracts. Racket binding classes, syntax-spec and reference compilers are not +assumed to exist in Gerbil. Existing Gerbil syntax objects, defrules/defsyntax, +syntax parameters and checked POO provide the concrete mechanisms to evaluate. + +** Integrated review: reuse mechanisms, justify any new boundary + +The pinned implementation distinguishes ordinary templates from declared extension +classes: =define-goal-macro= and =define-term-macro= call =define-dsl-syntax= with +the corresponding class, while =spec.rkt= permits each class at particular grammar +positions. Those wrappers enforce an extension boundary; they are not evidence +that every reusable template needs a new public macro declaration. + +For the language-author interface, no task requiring a comparable contribution +binding class or registration contract has been established. The recommendation is therefore ordinary +Gerbil =defrules= for a simple declaration template, =defsyntax= when parameter +validation requires computation, and validation of elaborated grammar at the existing declaration boundary. +The contribution/owner/compose candidate is withdrawn from the author design. Consider a dedicated form only after a complete author task establishes a +limitation of ordinary Scheme abstraction. This conclusion compares actual responsibilities; +it does not transfer Racket's binding-class implementation to Gerbil. + +Gerbil's [[https://gerbil.scheme.org/reference/gerbil/prelude/macros.html][prelude macros]] supply those author mechanisms. Its +[[https://gerbil.scheme.org/reference/gerbil/expander/][expander reference]] distinguishes syntax-time API access from explicit runtime +imports, and the [[https://gerbil.scheme.org/reference/gerbil/prelude/core-expander-syntax.html][core syntax reference]] specifies differential-phase imports. +These support distinguishing host macro availability from domain validation. The current parser's +=expand-surface-syntax= already calls =core-expand1= on vocabulary invocations; +that source establishes an existing expansion mechanism, not implementation of +multi-section contribution admission. + +| Decision in the same package | Source basis | Concrete implementation obligation | +|------------------------------+--------------+------------------------------------| +| Template definition | Gerbil defrules/defsyntax; pinned extension wrappers | Reuse standard macros; validate contribution output instead of inventing a wrapper without a contract | +| Collection before dependent checking | Fortifying Macros section 7.2 | Collect domain declarations, select effective rows, then check references/catalog obligations | +| One compiler input | Compiled extensible DSL architecture; existing parser compiler | Finalize canonical data once and reuse binding, analysis and publication | +| Identity after finalization | Current normalizer and artifact cache source | Seal receipt/body after derived catalogs; keep runtime inputs consistent with that value | +| Effective ownership | Current ordered row operations | Associate sources with the same selection, including an explicit equal-row merge convention | + +The final two obligations are deductions from our implementation, not claims made +by the papers. They explain why a syntax proposal cannot be approved independently +of its engine route. The concrete declaration is in [[file:scheme-dsl-author-interface.org][author interfaces]] and the +selection/finalization seam is in [[file:scheme-dsl-engine-admission.org][engine admission]]. No new algorithm, empirical +complexity reduction or complete composed runtime is established by this review. + +** Transfer into the three-interface implementation plan + +Use =grammar.ss= for declared syntax and supported vocabulary, =parser.ss= for +checked entry/services, and =parser-test.ss= for independent expectations. Engine +elaboration should converge on the canonical publication owner already studied. +The source-reading comparison supports extracting/reusing engine responsibilities; +it does not select a new public component clause or persistent metadata schema. + +Stage inspection could expose authored/elaborated syntax, effective contributions, +canonical/Bound/Parser IR and selected existing analysis receipts. Such inspection +must preserve their meanings and absent origins. It should not require authors to +invoke passes, rewrite ASTs or maintain compiler snapshots. This remains a design +obligation, not an implemented inspection API or automatic expansion-history log. + +#+begin_src scheme +;; Existing Scheme extension mechanism in grammar.ss: +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; Question 1: Why is this DSL design better? +;; Vocabulary elaborates into the same core as explicit rules; engine owners handle +;; binding, analysis, publication and runtime while users keep three DSL interfaces. +;; Question 2: Why does this DSL design have a scientific basis? +;; Paper architecture and pinned code identify concrete boundaries and compiler consumers. +;; Our native expression/Loader/test and publication controls qualify separate Gerbil paths. +;; Falsifiers: unsupported host execution enters portable Grammar IR, extensions bypass +;; admission, or diagnostics require authors to understand internal pass scheduling. +;; Limit: source inspection is not a port, performance result or complete POO DSL bridge. +#+end_src + +The next code-facing decision is where to factor existing concise elaboration and +canonical publication so both explicit and composed declarations use them. Validate +that seam against the same tree/origin/product contracts in +[[file:scheme-dsl-engine-admission.org][engine admission]] and [[file:scheme-dsl-semantic-contracts.org][semantic contracts]]. Do not reproduce miniKanren's +optimizer as a new parser analysis layer. + +* Fortifying Macros: how should a declaration explain itself? + +Culpepper and Felleisen, ICFP 2010, +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][author paper]], sections 3–4 and parser-generator case study 7.2. Syntax +classes describe structured inputs and expose attributes. The parser case uses +two stages to resolve dependencies between declarations and productions. + +Design conclusion: write down the language of declarations as a specification. +First collect sections, names and policies; then elaborate rules against that +context. Associate each constructor with argument roles, cardinality, output +shape and constraints. The same specification can guide documentation, checking +and expansion explanations. Use Gerbil facilities to implement the useful ideas; +Racket's syntax-parse API is not a required dependency. + +Study task: document =prec=, =field=, =choice= and a list abstraction with the same +rubric. Evaluate how easily an author understands valid combinations and locates +an incorrect argument. Diagnostic quality is part of a mature author interface. + +* Parsing Expression Grammars: which algebra are authors actually using? + +Ford, POPL 2004, +[[https://pdos.csail.mit.edu/papers/parsing:popl04.pdf][author paper]], sections 3.5–3.7. Abstract simulation supports conservative +structural reasoning; well-formedness and transformation identities depend on +PEG recognition semantics, including prioritized choice. + +Design conclusion: make the semantics of our CFG/LR constructors explicit before +building higher abstractions. Borrow the practice of stating an algebra and its +conditions. Our arithmetic grammar deliberately uses left recursion, so PEG's +left-recursion restriction cannot be copied as our grammar rule. An equivalence +useful for recognition also needs a separate check for nodes, fields, precedence +and offsets when used as a compiler rewrite. + +Study task: give =seq=, =choice=, =optional= and =repeat= semantic contracts, then +specify exactly which transformations preserve recognition and which additionally +preserve the observable tree. Keep grammar choice and lexical priority distinct. + +* Binding as Sets of Scopes: how can independently written fragments compose? + +Flatt, POPL 2016, +[[https://www-old.cs.utah.edu/plt/scope-sets/][author extended paper]], especially section 3 on procedural macros, aliases, +modules and phases. The model makes scope and name roles explicit. + +Design conclusion: a grammar vocabulary can live in ordinary Scheme modules. +Its interface should identify caller-supplied grammar references, public labels +and implementation helpers. Begin with fragments whose dependencies are explicit +parameters. Larger fragments with private rules and exported entries require a +separate design justification. Avoid conflating symbolic Grammar IR identities +with Scheme variable bindings. + +Study task: compose two independently authored fragments, rename imports and +change local spellings. Assess whether authors can explain the resulting grammar +without knowing the fragments' implementation. Existing host binding experiments +provide controls for that study, rather than a defect investigation. + +* Syntax parameters: what context should cooperating macros share? + +Barzilay, Culpepper and Flatt, +[[https://www.barzilay.org/misc/stxparam.pdf][Keeping it Clean with Syntax Parameters]], sections 4, 4.1 and 6. +The paper studies adjustable syntactic bindings that compose with hygiene; +section 6 distinguishes context captured by an ordinary function from a macro +that expands under a later parameterization. This is a constraint on contextual +elaboration, not a universal solution for every generated identifier. + +Our [[file:scheme-syntax-parameter-context.org][Gerbil study]] records the installed mechanism and selected nested-scope +controls. Its design role is engine-side declaration context and precise blame. +Do not infer full expansion history, public syntax-parameter configuration or +semantic runtime behavior from those construction/diagnostic controls. Read this +paper together with syntactic abstraction and scopes when defining the one phase +and diagnostic contract. + +* Object algebras: which extension task actually needs another protocol? + +Oliveira and Cook, ECOOP 2012, +[[https://www.cs.utexas.edu/~wcook/Drafts/2012/ecoop2012.pdf][Extensibility for the Masses: Practical Extensibility with Object Algebras]]. +The [[file:scheme-poo-language-components.org][supporting POO study]] reads construct/operation extension in sections 2–3 and +modular combination in section 7. Java factories and generics are its concrete +mechanism. Our POO grammar roles are not thereby object algebras with the paper's +static guarantees. + +Use the distinction to ask whether adding a grammar construct and adding an +analysis/product require coordinated author edits. Prefer the existing canonical +IR and receiver/compiler owners. A contribution interface is not warranted simply +because a paper describes extensibility. The rejected author candidate remains +withdrawn; any extra protocol requires a complete author task and a demonstrated +limitation of the existing Scheme/POO mechanisms. + +* Abstract interpretation: what can declarations teach the compiler? + +Cousot and Cousot, POPL 1977, +[[https://www.di.ens.fr/~cousot/COUSOTpapers/publications.www/CousotCousot-POPL-77-ACM-p238--252-1977.pdf][author scanned paper]], sections 5.1–5.3, 6–7 and 8.1–8.3 (printed pages +241–244). Section 6 requires a correspondence between concrete and abstract +interpretations and local consistency of transfer operations. Section 8 separates +fixed-point correctness, termination and efficiency. Reading these sections does +not reproduce the appendix proofs or prove our compiler correct. + +Our application of this framework is specified in +[[file:scheme-dsl-semantic-contracts.org::*Derived facts after expansion: nullable, FIRST and completion][the derived-fact contract]]. The current =compute-nullable=, =compute-first= and =compute-productive= +compute finite-domain fixed points in the existing LR owner. Nullable +now uses [[file:dsl-effective-grammar-admission.org::*Dependency-driven nullable closure (2026-10-09)][occurrence dependency propagation]]; FIRST uses +[[file:dsl-effective-grammar-admission.org::*FIRST propagation: contract and implementation][nullable-prefix dependencies and coalesced terminal deltas]]. +This changes evaluation order and work, not the meaning of the nullable fact. +The design obligation is +to state what each result observes after lowering, rather than create a second +analyzer. Finite monotone growth establishes termination; semantic correspondence +and transfer correctness are additional obligations. Nonempty FIRST does not +establish a completed derivation, as the existing dead-tail control demonstrates. + +[[file:dsl-effective-grammar-admission.org::*Productivity: finite derivation and LR entry admission][Productivity now has a finite-word contract and LR entry-admission consumer]]. +[[file:dsl-productive-entry.org::*Joint completion solver: one graph, two meanings][The joint completion solver]] now shares occurrence registration and scheduling +with nullable while preserving the separate terminal policies. +Field multiplicity and child-category analysis still require their own consumer, +concrete meaning and transfer contract. Any grammar rewrite must +also preserve the tree, field and source contracts; recognition facts alone do +not justify changing an author's declared grammar. + +The existing LR lookahead consumer now also uses +[[file:dsl-effective-grammar-admission.org::*Unified suffix facts across LR consumers][shared compositional suffix facts for LR closure and FOLLOW/partition consumers]], +avoiding their repeated forward suffix scans. This is an evaluation improvement with independent +forward-scan and complete LRSpec controls, not adoption of a predictive-parser +type theorem for arbitrary CFGs. + +* Polymorphic embedding: which interpretations share the admitted grammar? + +Hofer, Ostermann, Rendel and Moors, GPCE 2008, +[[https://www.mathematik.uni-marburg.de/~rendel/hofer08polymorphic.pdf][Polymorphic Embedding of DSLs]], selected full-text reading: sections 3.1–3.6 +and 4.4–4.7. The paper separates language operations from semantic domains and +illustrates evaluation, printing and a semantic transformer with Regions. +Its section 3.6 places optimization during denotation construction, before repeated +execution; this is not necessarily host-language compilation. Section 4.4 leaves +side-effect discipline to the programmer. Section 4.7 uses transfer functions as +a domain to accommodate dataflow analysis, rather than eliminating fixed points. +These are paper arguments and inspected listings, not reproduced parser results. + +Combined with object algebras, this suggests asking which new observation can +consume our existing admitted IR. Combined with abstract interpretation, it +requires recursive grammar facts to retain their global analysis contract. +Combined with hosted DSL compilation, it requires a named phase and a checked +boundary before an observation becomes a published product. This is our design +inference; the papers do not prove Gerbil POO roles equivalent to Scala type +families or Java object algebras. + +The concrete direction is one checked grammar feeding existing parser, catalog +and diagnostic consumers. =grammar.ss= should not configure an interpretation +framework; =parser.ss= declares supported products; =parser-test.ss= supplies +independent observations. Additional consumers must demonstrate a complete task +and preserve admitted grammar identity. No new public algebra protocol follows +from the paper alone. The GQL compiler-product gate remains independent of these +conceptual conclusions. + +The [[file:scheme-dsl-engine-admission.org::*Connect analysis evidence to author syntax before publication][integrated diagnostic route]] now applies these readings to the actual +expansion, declaration-source, Bound IR and LR lowering seams. It separates +author promises, fact witnesses and source precision; it remains a proposed +integration with existing owners, not a new author protocol. + +The implementation comparison also preserves an important observable boundary: +section 5.2 of the hosted-DSL paper retains otherwise trivial =fresh= nodes because +they affect search interleaving and answer order. Our corresponding obligation is +to state the parser's recognition/tree/field/span observations before simplifying +a macro expansion. The [[file:scheme-dsl-engine-admission.org::*Native Gerbil mechanisms and the minimum engine seam][native Gerbil source audit]] and +[[file:scheme-dsl-semantic-contracts.org::*Finite explanations belong to the existing fact owner][finite explanation design]] connect syntax preservation and recursive fact +justification to the existing owners. Neither transfer establishes a parser +optimization or a new portable metadata format. + +Section 5.3 of the hosted-DSL paper makes the host interaction boundary part of +optimization correctness: opaque host code limits transformations while checked +boundary operations retain useful invariants. The pinned =compile.rkt= separates +core optimization from generation, and its =save-optimized-step= retains pass +snapshots in an =optimized-relation-code= symbol table. This is inspected tooling +state, not a cost-free mechanism or our parser's cache model. Polymorphic +embedding section 4.4 also makes side-effect discipline an implementer obligation. +Our joint inference is to avoid re-executing arbitrary Scheme macros to recreate +an explanation. Reuse current admitted input and specify evidence lifetime, +cache-hit behavior and cost in [[file:scheme-dsl-engine-admission.org::*Evidence lifecycle across cache hits and misses][the integrated lifecycle contract]]. + +The hosted-DSL paper's sections 5.4.1–5.4.2 also constrain how we evaluate a +transfer. Its benchmark families exercise different workloads, and Table 1's +columns cumulatively add optimizations. Those ratios are not isolated causal +measurements of each pass. The results discussion includes code-size-related +slowdowns from specialization; an optimization architecture is not a universal +performance guarantee. This selected reading does not reproduce the artifact or +transfer miniKanren speedups to parsing. Our corresponding method is matched +semantic controls and separate phase/cache/request benchmarks, preserving negative +results. The [[file:scheme-deflanguage-design-synthesis.org::*Integrated implementation decision review][integrated implementation review]] uses existing source-diagnostic failures +as the first task; recursive evidence and optimization remain separate hypotheses. + +The first implementation now applies Fortifying Macros section 4.2's author-level +error principle at the existing Gerbil validation owner. It keeps current messages +and selects retained syntax instead of importing syntax-parse or adding a matcher. +The [[file:scheme-dsl-engine-admission.org::*Implemented first slice: concise author blame][implemented slice]] checks actual reader text, direct/macro controls, import +renaming, generated invalid constructors and absent-source behavior. Its five +focused cases are evidence for these boundaries, not the paper's complete error +selection semantics. The next research/implementation step must respond to a +remaining author task rather than expand diagnostic mechanisms without a consumer. + +The next implementation applies Fortifying Macros section 7.2's dependency-staging +principle to field derivation: check/normalize the declarations before consuming +referenced helper bodies. The paper's two-pass parser resolves clause dependencies; +our Gerbil change keeps existing validation and inference, using normalized rules +as their shared semantic representation. This is a design transfer, not adoption +of Racket syntax classes. The [[file:scheme-dsl-semantic-contracts.org::*Field derivation uses the validated grammar][field contract]] ties the forward-reference +counterexample, postorder catalog preservation and native product comparison to +one complete author task. The implementation evidence supports that task and +leaves global inference proofs and performance effects open. + +The same section 4.2 criterion now covers existing root and node/token-kind +obligations. Retained syntax connects a cross-expression constraint to an authored +body or kind parameter; generated terms use invocation blame. The +[[file:scheme-dsl-semantic-contracts.org::*Declaration obligations explain the authored term][declaration contract]] records negative controls and native product parity. +This remains a targeted Gerbil implementation, not the paper's full contextual +failure selection algorithm or an empirical usability result. + +The integrated implementation now covers the full constructor-header contract +at both existing declaration entries. The [[file:scheme-dsl-semantic-contracts.org::*One constructor contract across declaration entries][constructor contract]] separates +shared canonical constraints, source-aware syntax traversal and resolved LR policy. +Fortifying Macros sections 3–4 and 7.2 inform that separation; its syntax classes +are not copied into a second validator. Native controls exposed a proper-list +boundary absent from interpreted controls, reinforcing that implementation evidence +must include the actual compilation phase before admitting a diagnostic claim. + +The composed implementation applies the same paper argument across the engine. +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]], section 1 and Figure 1, separates an +extensible surface from a compiler core. Our local application is direct canonical +POO admission and shared machine emission, rather than whole-declaration replay. +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros]], section 7.2, motivates collecting declarations before checking +dependent references. Our application selects effective rows and sources before +Bound IR binding. The particular merge/override/remove source policy is our +explicit engine contract, not a policy extracted from either paper. Matched +composition, provenance and checked-entry controls qualify these implementations; +the papers do not establish our engine's performance or author-effort advantage. + +* Proposed vocabulary structure + +| Layer | Author intent | Current foundation | Design work | +|-------+---------------+--------------------+-------------| +| Grammar primitives | Sequence, alternatives, repetition and references | =seq/choice/optional/repeat/token/reference= | Uniform semantic contracts | +| Tree construction | Public nodes, fields and spans | =node/alias/field= | Explain output shape, including fragments | +| Parsing policy | Associativity, precedence, conflicts, layout and recovery | =prec= and existing declaration sections | Keep policy visible in higher forms | +| Grammar idioms | Infix tier, separated list, delimiter and optional clause | =defgrammar-syntax= | Small parameterized vocabulary with inspectable expansion | +| Package products | Catalog, loader, tooling, tests and target builds | Existing compiler and POO interfaces | Derive repeated facts; retain intentional choices | + +These are explanatory layers, not five new public DSLs. Authors should be able +to use an idiom and inspect its expansion into familiar primitives. Advanced +users retain direct access to those primitives. + +* Candidate infix abstraction + +This template uses the existing public macro mechanism. Its name and parameter +list are study vocabulary, not a newly published standard library function: + +#+begin_src scheme +(import (only-in :gerbil-parser/language-support defgrammar-syntax)) + +(defgrammar-syntax (infix-node associativity level kind lhs operators rhs) + (prec associativity level + (node kind + (seq (field left lhs) + (field operator operators) + (field right rhs))))) + +;; Production alternatives in an arithmetic study: +(infix-node left 10 Expression expression (choice "+" "-") expression) +(infix-node left 20 Expression expression (choice "*" "/") expression) +#+end_src + +This abstracts repeated tree structure while keeping associativity, level, +operator set and recursive operands visible. It does not decide that every +language's binary expression should have the same fields or tree shape. +A later operator-table form could generate these alternatives, provided its +expansion retains their individual policy and source origins. + +* Candidate list abstraction + +#+begin_src scheme +(defgrammar-syntax (nonempty-field-list label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; A study body for a call's existing argument fields: +(optional (nonempty-field-list argument expression ",")) +#+end_src + +This makes empty/nonempty intent explicit and retains repeated field instances. +Trailing separators require a deliberate additional policy. Applying one outer +=field= to the whole list would produce a different structure; the library should +explain both choices rather than hide their distinction behind one convenience +name. A delimiter abstraction can compose around this body without automatically +introducing another public node. + +* Give each constructor a scientific contract + +Use one rubric for primitives and derived vocabulary: + +1. Argument roles: expression, declared reference, public label, host expression + or parsing policy; state the allowed multiplicity of each argument. +2. Recognition meaning: language of accepted tokens and interactions with choice, + precedence, layout and recovery. +3. Tree meaning: emitted nodes, field instances, fragments and spans. +4. Expansion: the existing primitives/core forms produced, with an explanation + connecting parameters to the result. +5. Static facts: derivable properties, conservative unknown cases and consumer. +6. Cost: authoring burden, expansion growth, compiler analysis and runtime work + measured separately. + +For example, =optional= may make a field absent, but the exact tree depends on +where the field wrapper sits. An operator tier's precedence number is a semantic +choice, while its derived catalog entries should be compiler-owned. + +* Engine improvements enabled by this design + +| Design information | Potential engine use | Qualification required | +|--------------------+----------------------+------------------------| +| Explicit constructor contracts | Consistent elaboration and explanatory diagnostics | Same grammar admission and source meaning | +| Shared elaborated core | Apply existing compilation to user vocabulary | Core/product equivalence on matched cases | +| Productivity/nullability | Useful declaration feedback and analysis scheduling | Match current CFG/LR semantics and reuse existing facts | +| Field/shape summaries | Richer tooling and generated schema | Correspondence with actual CST/field events | +| Operator/list intent | Candidate specialization and expansion explanation | Preserve conflicts, associativity, trees and spans | +| Qualified target capabilities | Share declared semantics across generated products | Existing native backend and artifact qualification | + +Structured declarations can make optimization possible; they do not establish +that an optimization is necessary or beneficial. Keep host/compiler improvements +and parser-domain analysis in their existing owners. Macro vocabulary itself can +be valuable through comprehension and maintenance even when runtime is unchanged. + +* Concrete next research deliverables + +1. A constructor reference using the six-part rubric for the current primitives. +2. An arithmetic/HCL/FHIRPath comparison using infix, list and optional-clause + idioms, with explicit expansions and preserved semantic choices. +3. A declaration-to-analysis map identifying facts already computed by the engine + and facts that genuinely need new analysis. +4. A Scheme prototype of the most useful vocabulary, admitted through the current + compiler and tested against the explicit original rules. +5. Author-task and compile/runtime measurements that answer separate questions. + +The previous count-model checks and host macro controls remain supporting studies. +The main research sequence now follows these authoring and engine deliverables. + +* Required Scheme rationale for every design conclusion + +Every accepted recommendation must answer both questions in a Scheme block, +with an author form, an explicit semantic contract, evidence and a remaining +boundary. A paper supplies principles; it does not automatically establish the +superiority of our particular DSL. “Better” must name its comparator and task. + +The local list vocabulary provides a concrete example (within the current +=defgrammar-syntax= extension context): + +#+begin_src scheme +(defgrammar-syntax (nonempty-field-list label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; In grammar syntax: the container chooses optionality, the component per-item fields. +(optional (nonempty-field-list argument expression ",")) + +;; Question 1: Why is this DSL design better? +;; Compared with explicit seq/repeat/field, item and label each require one author occurrence. +;; Macros share intent and tree contracts, while core expansion and language policy stay inspectable. +;; Scheme supplies syntax abstraction and module boundaries; multi-rule versions use checked POO roles. +;; Question 2: Why does this DSL design have a scientific basis? +;; Recognition is item (separator item)*; optional admits zero items; fields occur per item. +;; Compare acceptance, rejection, full tree events, fields and source spans against explicit rules. +;; The multi-rule role has 62+2 checks; these do not replace this surface macro's integration checks. +;; Falsifier: admitted trailing separators, altered field shape or disagreement with explicit rules. +;; Limit: no author-task experiment; shorter declarations imply neither speed nor universal optimality. +#+end_src + +For each paper-derived conclusion, record the chain: +=paper principle -> Scheme mechanism -> explicit contract -> executable control +-> bounded conclusion=. The focused studies now include their own two-question +blocks for named tiers, rule context, list roles, origins and composition history. +Author-task studies should compare editing errors and completion time with the +explicit form; compiler/runtime benchmarks must measure their own phases. Those +future measurements remain open rather than being inferred from semantic tests. + +* Supporting implementation studies + +These studies qualify distinct parts of the common design. Consult their full +receipts and limits; their successful counts are not one package pass total. + +| Responsibility | Evidence owner | +|----------------+----------------| +| Checked expression/multi-rule contributions | [[file:scheme-poo-language-components.org][POO component study]] | +| Reader invocation and template origins | [[file:scheme-source-origin-contract.org][Source-origin study]] | +| Typed local errors and source-free inputs | [[file:scheme-declaration-diagnostics.org][Diagnostic study]] | +| Wrapper forwarding and shared transformer-phase contracts | [[file:scheme-nested-declaration-context.org][Nested context study]] | +| Official scoped syntax parameters, nearest/retained context and scope isolation | [[file:scheme-syntax-parameter-context.org][Syntax-parameter study]] | +| Required items, resolved nullable/FIRST facts and separator repetition | [[file:scheme-list-role-composition.org][List contract study]] | + +Keep local syntax checks separate from resolved grammar obligations. Reuse the +existing engine analyses; consumption, completion and tree contracts are different +questions. The [[file:scheme-deflanguage-design-synthesis.org][overview]] owns current decisions and the closure sequence. diff --git a/docs/research/scheme-dsl-semantic-contracts.org b/docs/research/scheme-dsl-semantic-contracts.org new file mode 100644 index 00000000..b2d1c7a9 --- /dev/null +++ b/docs/research/scheme-dsl-semantic-contracts.org @@ -0,0 +1,451 @@ +#+title: Scheme DSL semantic contracts and composition evolution +#+date: 2026-10-06 + +Scope: recognition, public trees, fields, composition and source consistency for +the common call/array package. Read the [[file:scheme-deflanguage-design-synthesis.org][design overview]] first. These contracts +connect [[file:scheme-dsl-author-interface.org][author declarations]] to [[file:scheme-dsl-engine-admission.org][engine admission]]; local receipts have separate +scopes and are not a combined public-package qualification. + +* Unified semantic and diagnostic contract + +| Contract | Meaning in the running language | Owner and evidence boundary | +|----------+---------------------------------+-----------------------------| +| Recognition | Containers may be empty; an item follows each comma | Existing parser; selected native event controls | +| Tree | Call/Array expose repeated argument/element fields without a list wrapper | Existing tree construction; compare complete events and spans | +| Required items | One required item expression in the list entry | Component shape; does not imply token consumption | +| Repetition consumption | Nullable body equals nullable(separator) AND nullable(item) | Existing resolved LR facts; no general runtime liveness proof | +| Completion | A finite item derivation exists | Neither nonnullable nor nonempty FIRST alone proves this | +| Composition | Operations have different admission and replacement meanings | Existing POO normalization; effective provenance must follow the selected row | +| Author context | Keep author entry, offending parameter and validating declaration distinct | Scoped syntax controls; presentation and general chains pending | +| Persistent identity | Metadata-sensitive declaration identities differ from reference identity | Existing Bound IR; arbitrary supplied source records are not verified history | + +Constructor-local validation can reject immediate syntax or visibly nullable +repetition. Reference-dependent facts belong after composition and reference +validation. Stronger author contracts must consume existing engine facts and +retain the relevant component origin. This is a proposed diagnostic integration, +not an implemented new global gate. + +* Derived facts after expansion: nullable, FIRST and completion + +This contract connects contextual Scheme expansion, canonical admission and +analysis. Reusable vocabulary expands to existing grammar primitives; resolved +rules are lowered once for the existing [[file:../../src/compiler/lr.ss][LR analysis owner]]. An author-level +idiom name is not an additional semantic guarantee. Facts must refer to the same +effective grammar used by the published parser and carry its diagnostic origin. +The full source-aware publication path remains an integration obligation. + +| Existing fact | Domain and transfer | Meaning and limit | +|---------------+---------------------+-------------------| +| Nullable | Subsets of finite nonterminal set N; add a production's lhs when every rhs symbol is nullable | A finite empty derivation exists in the lowered grammar; terminals are not nullable | +| FIRST | One subset of finite terminal set T per nonterminal; union each rhs prefix through nullable symbols | A terminal can begin a derived sentential form; the remaining symbols need not complete | +| Completion/productivity | Finite nonterminal membership; terminals/epsilon satisfy clauses and all nonterminal occurrences must complete | =compute-productive= establishes a finite terminal derivation; the LR target rejects an unproductive selected entry | + +All three algorithms start with empty facts and grow monotonically. Nullable +and productivity each have at most |N| newly admitted membership facts; FIRST has at most |N| * |T|. +These are bounds on strict fact additions, not sweep counts, compilation time or +parser latency. FIRST uses existing indexed terminal masks and decodes them to +its public table; epsilon is represented by nullable, not a FIRST terminal. +No widening or duplicate analyzer is needed for these finite domains. +[[file:dsl-productive-entry.org::*Joint completion solver: one graph, two meanings][Nullable/productivity now share one completion graph]] and publish both completed +observations; shared scheduling does not identify their two meanings. +[[file:dsl-effective-grammar-admission.org::*Productivity: finite derivation and LR entry admission][The productivity implementation contract]] records the LR policy and its controls. + +The proof obligations are distinct: lowering preserves the intended grammar +observation; each transfer implements that observation; iteration reaches the +least solution of those equations; consumers use the result with its stated +meaning. Finite monotone iteration alone does not discharge the first two. +The [[file:scheme-dsl-research-basis.org::*Abstract interpretation: what can declarations teach the compiler?][abstract-interpretation reading]] supplies this reasoning framework, not a +completed proof of this engine's lowering or tree construction. + +#+begin_src scheme +;; Engine observation using existing APIs, not additional author configuration. +;; The same grammar can derive an identifier prefix, but no complete terminal word. +(let* ((rules '((item (sequence (token identifier) (reference blocked))) + (blocked (reference blocked)))) + (productions (lower-rules rules 'item))) + (let-values (((nullable nullable-index) (compute-nullable productions))) + (let-values (((first first-index) (compute-first productions nullable-index))) + (let-values (((productive productive-index) (compute-productive productions))) + (list (table-ref nullable-index 'item #f) + (table-ref first-index 'item '()) + (table-ref productive-index 'item #f)))))) +;; Expected observations: #f, ((terminal token identifier)), #f. +;; blocked has no terminating production, so FIRST is not a completion witness. + +;; Question 1: Why is this DSL design better? +;; Authors write reusable Scheme vocabulary and independent semantic expectations. +;; The existing engine derives recursive facts after expansion and resolution; +;; authors do not duplicate FIRST tables or manually traverse macro output. +;; Compare the same edit and diagnosis task with explicit primitive declarations. +;; Question 2: Why does this DSL design have a scientific basis? +;; Finite domains and monotone transfer explain termination; the observation +;; contract distinguishes a sentential prefix from a completed derivation. +;; Falsifier: using nonempty FIRST to promise completion for the dead-tail grammar, +;; or reporting an analysis fact from a grammar different from the published one. +;; Limit: these facts do not prove lexer realizability, tree equivalence or runtime speed. +#+end_src + +On 2026-10-06, this existing control was rerun under =gtimeout -k 2 10= +with the project load path. It emitted all 35 case observations, +=LIST-NULLABILITY-CONTRACT-OK: 35 checks passed= and exited 0. This receipt +qualifies these selected analysis observations, not a new compiler proof or +runtime performance claim. + +The existing [[file:../../t/fixtures/language-pack-research/list-nullability-contract.ss][list-nullability control]] covers token, optional/empty item, +recursive base, indirect nullability, closed cycle and token-prefix/dead-tail +cases. Its required-items entry is nullable when its item is nullable; a nullable +item also permits the comma to appear in FIRST. An idiom's required first +expression therefore does not imply token consumption. A stronger author promise +needs an explicit contract and an origin-aware diagnostic consuming existing +facts. It must not silently rewrite the grammar or reject all nullable items. + +For the call/array package, test acceptance, fields and spans independently of +these facts. For GQL, retain independent source/corpus expectations and the +compiler-product gate. The polymorphic-embedding reading motivates distinct +observations of one admitted grammar, while these recursive facts retain the +existing fixed-point owner. This is one admission/analysis/product contract +across the three interfaces, not an additional author DSL. + +** Finite explanations belong to the existing fact owner + +A proposed explanation consumer must not choose arbitrary matching productions +from the final fact tables. For =A -> A | empty=, choosing the recursive production +would never finish explaining nullable(A), although the empty derivation is finite. +The existing recursive-base and closed-cycle controls distinguish these facts; +they do not yet qualify explanation generation. + +The code-facing design direction is to record evidence when the existing analysis +first admits a new fact. This is a proposal for the current =compute-nullable= and +=compute-first= owners, not a new solver or published callback API: + +| New fact | Evidence needed at its first admission | Decreasing dependency condition | +|----------+----------------------------------------+---------------------------------| +| Nullable(A) | Production id and nullable facts for every rhs symbol; empty rhs is a base | Each referenced fact was already available before admitting Nullable(A) | +| FIRST(A,t) | Production id, selected rhs position, nullable facts for its preceding prefix, and terminal t or FIRST(B,t) at that position | Referenced FIRST fact predates FIRST(A,t); nullable evidence belongs to the already completed nullable phase | + +Assign admission order to new facts, not iterations: the current loops update +facts within a sweep. A later update in the same sweep can depend on an earlier +one. A self-dependent fact cannot bootstrap its own first admission. For each +new FIRST mask bit, retain a supported cause before publishing that bit; a bulk +union alone loses which rhs occurrence justified it. Existing terminal indexing +and ordered decoding remain the fact implementation. + +This supplies a finite directed acyclic explanation graph, subject to checking +each recorded cause against the actual production. Store shared evidence once; +rendering it as a fully duplicated tree could grow much larger. A deterministic +first cause is not necessarily the shortest or most helpful explanation. Grammar +reordering can change that cause while leaving the fact unchanged. A stable +minimal explanation is a separate problem, not an implied guarantee. + +#+begin_src scheme +;; Existing grammar primitives illustrating the evidence obligation. +(a (choice (reference a) (empty))) +(b (reference a)) +;; nullable(a) is first justified by empty, not by a -> a. +;; nullable(b) can refer to that already admitted fact. + +;; Question 1: Why is this DSL design better? +;; The engine can explain recursive facts through existing analysis, while authors +;; keep reusable grammar vocabulary and independent parser-test.ss expectations. +;; No author needs to reconstruct the compiler's recursion or proof traversal. +;; Question 2: Why does this DSL design have a scientific basis? +;; Earlier-admission dependencies give a decreasing order for proof traversal; +;; each leaf and production is checked against the same lowered grammar. +;; Falsifiers: a cause cites itself or a later fact, references another grammar, +;; or presents a FIRST prefix as a complete parse. +;; Limit: a valid finite witness is not a proof of lowering, optimal diagnostics, +;; runtime performance or preservation of trees and spans. +#+end_src + +Implementation qualification must compare evidence enabled/disabled for equal +productions, nullable/FIRST values and parser observations. Check recursive base, +closed cycle, indirect nullable prefix and token-prefix/dead-tail cases against +the existing controls. Separately measure compilation work and evidence size; +do not infer zero overhead from sharing the fixed-point owner. The [[file:scheme-dsl-engine-admission.org::*Evidence lifecycle across cache hits and misses][lifecycle recommendation]] prefers request-scoped capture from current admitted +input, with explicit replay cost when needed. This is not implemented or yet +qualified; the current engine does not publish these evidence records. + +* Field derivation uses the validated grammar + +The [[file:scheme-dsl-engine-admission.org::*Implemented follow-up: validate rules before deriving fields][implemented phase adjustment]] connects existing author diagnostics and inferred +catalogs. Normalize/check every rule before field derivation traverses transparent +references. Reuse the resulting normalized rules for both inference and publication; +raw surface syntax remains diagnostic context rather than a second semantic input. + +#+begin_src scheme +;; Excerpt from the same grammar.ss rules section. +(source-file (node SourceFile (reference helper))) +(helper (field argument)) +;; The field arity is invalid. Validation reports (field argument) before the +;; root's field inference follows this forward reference. + +;; Valid recursive helper control from the existing surface suite: +(source-file (node Root helper)) +(helper (choice (field item word) + (seq "(" helper ")" (field tail word)))) +;; Existing expected catalog: (Root node (item tail)). + +;; Question 1: Why is this DSL design better? +;; Authors can order reusable rules naturally while the engine validates their +;; shapes and derives fields from one normalized grammar. They need no helper +;; ordering workaround or manually copied catalog to obtain an accurate error. +;; Question 2: Why does this DSL design have a scientific basis? +;; The forward-helper negative control isolates inference-before-validation blame. +;; Old/new native macro outputs differ in blame but the complete call/array Grammar +;; IR and Parser IR match byte for byte. Recursive/alternative catalogs and actual +;; AST/span expectations are additional independent observations. +;; Falsifiers: raw malformed helpers reach inference, node/field order changes, +;; a named child leaks its fields into a parent, or recursion loses sibling fields. +;; Limit: selected controls do not prove all-input field inference or faster parsing. +#+end_src + +The validated representation eliminates the need for inference to interpret bare +identifiers or repeat local shape validation. Postorder node collection preserves +its earlier catalog registration order. The visited-name mechanism is unchanged; +this work does not introduce a new fixed-point field analysis. Multi-error priority +can differ because validation now precedes inference, and is a diagnostic behavior +separate from accepted-language/tree parity. + +* One constructor contract across declaration entries + +This is the integrated expression-admission task, rather than another vocabulary +proposal. The canonical algebra owns constructor arity and scalar metadata through +=grammar-expression-header?=. Recursive =grammar-expression?= consumes that same +predicate. Source-aware compiler traversal checks each header before indexing its +children; the concise projection reuses the predicate for its missing scalar checks. +Grammar/Parser IR publication and resolved LR analyses retain their existing owners. + +| Contract | Canonical owner | Declaration behavior | +|----------+-----------------+----------------------| +| empty | No operands | Reject extra operands | +| literal / layout-start / layout-next | One nonempty string | Reject empty/non-string metadata before parser compilation | +| layout-end | Zero or more nonempty strings | Reject invalid boundary metadata | +| token / reference | One symbol | Canonical shape check; concise entry also resolves declared names | +| sequence / choice | One or more children | Validate shape before child traversal | +| optional / repeat / repeat1 | One child | Arity precedes indexing; resolved consumption remains separate | +| field / alias | Symbol plus one child | Concise node/alias implicit bodies normalize first | +| precedence | Direction in none/left/right/dynamic; integer rank; one child | Shared canonical metadata check, not a new associativity algorithm | + +#+begin_src scheme +;; grammar.ss excerpts; declaration sections and catalog ownership stay unchanged. +(source-file (node SourceFile (prec sideways 10 identifier))) +;; Invalid direction is a syntax error at this authored expression. +(source-file (node SourceFile (literal ""))) +;; Empty literal is invalid; use (empty) for an explicit empty expression. +(source-file (alias SourceFile (optional))) +;; The canonical entry also rejects arity before reading a missing child. +(source-file (alias SourceFile (optional . identifier))) +;; Proper-list structure is checked through Gerbil's existing syntax traversal. + +;; Question 1: Why is this DSL design better? +;; Authors keep native Scheme vocabulary and the three package interfaces. Both +;; declaration entries explain constructor misuse at syntax time without requiring +;; users to know which list operation or parser validation would fail downstream. +;; Question 2: Why does this DSL design have a scientific basis? +;; Constructor constraints are factored out of the existing canonical predicate, +;; not invented by a second validator. Independent invalid-input and membership +;; controls distinguish arity, metadata, child validity and resolved consumption. +;; Native compilation qualifies behavior that interpreted expansion alone missed. +;; Falsifiers: either entry accesses malformed children before rejecting shape, +;; canonical recursive membership changes, or complete accepted package IR changes. +;; Limit: selected controls are not a full language-equivalence proof, performance +;; qualification, usability study or complete declaration-section validation. +#+end_src + +Fortifying Macros sections 3–4 and 7.2 motivate reusable constraints, author-level +errors and separation of collection from dependent checking. The Gerbil transfer +uses ordinary predicates, syntax objects and existing compiler phases, without a +syntax-parse port. Constructor structure is distinct from resolved reference +semantics and nullable repetition. In particular, structural membership still +accepts =(repeat (empty))=; construction/resolved analysis owns consumption policy. + +The first native attempt exposed an important phase boundary: its generated macro +replaced the new datum =list?= condition with true, and dotted input reached =length=. +That run exited 70. The repaired boundary reuses Gerbil's =stx-list?= and runtime +=__AST-list?= traversal; the installed runtime implementation checks the terminal +tail rather than merely observing an outer pair. This is an observed native failure +and bounded repair, not a diagnosis of the optimizer's general correctness. + +* Declaration obligations explain the authored term + +Root shape and node/token kind separation are already engine obligations. Source +associations extend their explanations without asking grammar.ss authors to build +catalogs or diagnostic metadata. The [[file:scheme-dsl-engine-admission.org::*Declaration obligations retain author syntax][implementation receipt]] distinguishes authored +parameters, entire multiple-body rows and conservative vocabulary invocation blame. + +#+begin_src scheme +;; Excerpts from a declaration with (lex (identifier Identifier (identifier))). +(rules (source-file identifier)) +;; Root obligation: blame identifier, which does not construct the required node. +(rules (source-file identifier identifier)) +;; The implicit sequence has no single authored expression; blame this rule row. +(rules (source-file (node Identifier identifier))) +;; Kind obligation: blame the node-kind parameter Identifier, retaining the whole +;; declaration as supporting context for the conflicting lexical kind. +(defrules token-node () ((_ item) (node Identifier item))) +(rules (source-file (node SourceFile (token-node identifier)))) +;; Generated kind obligation: blame (token-node identifier), not template text. + +;; Question 1: Why is this DSL design better? +;; The engine explains existing declaration constraints using the syntax authors +;; supplied, including ordinary Scheme vocabulary calls. No source tables or extra +;; public forms are required in the language package's three interfaces. +;; Question 2: Why does this DSL design have a scientific basis? +;; The pre-change negative controls fail on blame; direct/macro, multiple-body and +;; source-free controls isolate distinct origin boundaries. Fresh native diagnostics +;; pass while the complete selected package's Grammar IR and Parser IR stay equal. +;; Falsifiers: template text is blamed for a generated kind, source-free input gains +;; coordinates, or accepted package products change merely to attach diagnostics. +;; Limit: first node-kind blame is not a two-location collision explanation, and +;; arbitrary transformer parameter correspondence remains unspecified. +#+end_src + +Fortifying Macros section 4.2 motivates locating the faulty authored term and +stating the expectation at its language level. This implementation transfers that +criterion to existing Gerbil checks; it does not reproduce syntax-parse's failure +progress ordering or establish usability superiority through a user study. + +* Verified field-contract controls through the three DSL interfaces + +Two complete variants of the same call/array package change the call's emitted +field from =argument= to =parameter=. The second additionally declares +=(node-fields (Call argument))=. All other grammar recognition policy remains +as in the baseline. Each variant uses its own grammar, checked parser entry and +declarative test module; no source-map or catalog assembly is required of the author. + +| Variant | Exact Call field catalog | Selected actual AST observations | +|---------+--------------------------+----------------------------------| +| [[file:../../t/fixtures/language-pack-research/package-parameter-inferred-grammar.ss][Inferred fields]] | callee, parameter | f() has no argument/parameter fields; f(a,b,c) has three parameter fields and no argument fields | +| [[file:../../t/fixtures/language-pack-research/package-parameter-retained-grammar.ss][Explicit retained field]] | argument, callee, parameter | The same selected observations; retained argument does not fabricate a field occurrence | + +The [[file:../../t/fixtures/language-pack-research/package-parameter-inferred-parser-test.ss][inferred variant tests]] and +[[file:../../t/fixtures/language-pack-research/package-parameter-retained-parser-test.ss][retained variant tests]] each use existing =syntax-kind=, =accepted=, +=rejected-many= and =ast= clauses. Each suite has five cases: exact catalog, +empty-call fields, three-parameter fields, four rejected call sources and an +independently authored exact tree/span expectation for =f(a)=. Both isolated +runners emitted five =CASE-OK= results, =SUITE-OK= and their success marker, +then exited 0 under =gtimeout -k 2 10= on 2026-10-06. + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout -k 2 10 gxi t/fixtures/language-pack-research/package-parameter-inferred-runner.ss +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout -k 2 10 gxi t/fixtures/language-pack-research/package-parameter-retained-runner.ss +#+end_src + +The initial invocation put the baseline and both new variants in one import path. +It emitted no output and exited 124 at the same 10-second limit. The fixtures +were then separated so each variant could be qualified without changing baseline +imports or extending the execution limit. The individual passes do not establish +why the combined invocation timed out or qualify that combined startup path. +These are selected semantic controls, not throughput or compilation measurements. + +#+begin_src scheme +;; In the complete retained variant's grammar.ss: +(optional (required-items parameter (reference name) (literal ","))) +;; At the language declaration boundary: +(node-fields (Call argument)) + +;; In its parser-test.ss, using existing engine-owned assertions: +(syntax-kind "exact effective Call field catalog" + Call node (argument callee parameter)) +(accepted "three parameters without historical argument fields" "f(a,b,c)" + (field-counts Call parameter (3)) + (field-counts Call argument (0))) + +;; Question 1: Why is this DSL design better? +;; The author states emitted fields and explicitly retained optional fields. +;; The engine derives the full catalog and checks independent tree expectations. +;; No copied field catalog, AST walker or manual artifact metadata is needed. +;; Question 2: Why does this DSL design have a scientific basis? +;; The matched variants isolate node-fields while keeping emitted-field policy equal. +;; Catalog equality and actual occurrence counts are different observations; +;; an independent exact tree/span expectation detects fabricated optional fields. +;; Falsifier: retained metadata creates occurrences or erases emitted parameter fields. +;; Limit: retention does not make a renamed emitted field compatible with old consumers. +;; The POO override/source-aware publication path remains unqualified by these variants. +#+end_src + +The engine admission plan can therefore reuse existing =node-fields= behavior +and the test DSL's catalog/AST checks. The open integration obligation is to +preserve that distinction when selecting effective POO contributions and their +sources, not invent another field-retention configuration. Optional catalog +availability and consumer-observed field compatibility must remain distinct +claims in the eventual component contract. + +* POO override and field catalogs: current engine control + +The existing [[file:../../t/fixtures/language-pack-research/composition-history.ss][composition-history control]] was extended for the same list package. +A rule-only override changes both the argument entry and its tail to emit +=parameter=. Its explicit canonical Call catalog remains =(callee argument)=. +A coordinated override supplies the same effective rules plus the explicit catalog +=(argument callee parameter)=. These are low-level engine controls, not author +requirements or a new public component DSL. + +| Observation | Rule-only override | Rule/catalog override | +|-------------+--------------------+-----------------------| +| Effective entry and tail field | parameter | parameter | +| Catalog retained by Parser IR | callee, argument | argument, callee, parameter | +| Bound parameter field | Absent from supplied catalog | Present | +| Bound optional argument field | Present | Present | +| Selected rule source | No automatic source collection asserted | Supplied owner parameter-extension preserved for entry and tail | + +On 2026-10-06 the expanded driver passed 31 checks and exited 0 under +=gtimeout -k 2 10=. This includes the earlier 19 composition controls and 12 +new field/catalog/source checks. The historical 19-check receipt remains useful; +counts must not be summed as separate complete-package passes. + +The observation confirms a responsibility boundary: canonical POO role data +already includes catalogs, whereas the concise author DSL derives catalogs. +The low-level normalizer/compiler does not replay concise inference merely +because a rule is overridden. The DSL admission owner must connect effective +rules, retained public fields and their sources before canonical publication. +The test supplies selected source records manually; no automatic origin resolver, +parser machine execution or full event parity is established for these POO variants. + +#+begin_src scheme +;; Engine control data; the user continues to declare intent through grammar.ss. +(Call node (argument callee parameter)) +(arguments + (sequence (field parameter (reference name)) + (reference call-arguments/tail))) + +;; Question 1: Why is this DSL design better? +;; Author DSL can delegate effective catalog reconciliation to the engine owner. +;; Public optional fields stay explicit without hand-maintained canonical catalogs. +;; Question 2: Why does this DSL design have a scientific basis? +;; Matched overrides hold effective rules equal and vary only explicit catalog data. +;; Bound fields, Parser IR catalogs and selected sources are independently checked. +;; Falsifier: DSL publication retains a stale inferred catalog or attributes a replaced +;; rule to its previous owner, while claiming automatic reconciliation. +;; Limit: these canonical controls do not qualify source-aware DSL runtime publication. +#+end_src + +Use [[file:scheme-dsl-engine-admission.org][the engine admission module]] to integrate this contract with canonical +publication. Use [[file:scheme-dsl-author-interface.org][the author interface module]] for the complete running declarations. + +* Existing catalog projection: bounded admission controls + +The same composition driver now also calls the existing +[[file:../../src/compiler/language-artifact.ss][project-language-catalog]] with explicit known catalog inputs. It does not +implement or claim a new derivation algorithm. + +| Control | Existing outcome | +|---------+------------------| +| Extra optional argument field over inferred parameter fields | Accepted | +| Historical catalog missing emitted parameter | Rejected | +| Projection drops argument after it is included as an explicit field obligation | Rejected | +| Terminal mapping changes identifier to Punctuation | Rejected | +| Duplicate parameter field | Rejected | +| No explicit projection | Supplied inferred kinds/terminals retained | + +On 2026-10-06, =gtimeout -k 2 10 gxi= completed the expanded driver with +=COMPOSITION-HISTORY-OK: 37 checks passed= and exit 0. This includes the earlier +31 controls plus six projection controls. An initial invocation exited 70 with a +reader error caused by an extra closing parenthesis in the new research block; +that fixture syntax was repaired before the successful bounded rerun. + +These controls qualify the existing validator on supplied inputs. They do not +prove automatic field inference after POO composition, automatic origin selection, +or all-input runtime tree equivalence. The [[file:scheme-dsl-engine-admission.org][engine module]] owns the resulting +integration order; authors continue to use the same three DSL interfaces. diff --git a/docs/research/scheme-engine-library-boundary.org b/docs/research/scheme-engine-library-boundary.org new file mode 100644 index 00000000..789d7568 --- /dev/null +++ b/docs/research/scheme-engine-library-boundary.org @@ -0,0 +1,412 @@ +#+title: Engine, Common Library, and Language Interface Boundaries + +* Decision and scope + +The parser engine accepts language-independent declarations and executes generated +machines. Language-specific token mappings, source-format adaptation, corpus +contracts, and development services belong to language authors or common libraries. +All eight shipped language packages now expose ten production entries independent +of their corpora. +It does not establish complete ISO GQL conformance or a general ANTLR lexer compiler. + +* Ownership and dependency direction + +| Owner | Responsibility | Excluded responsibility | +|-------+----------------+-------------------------| +| src/compiler, src/grammar | Canonical IR validation, binding, compilation, generated machine | GQL token names and standard examples | +| src/runtime | Recognition, lossless events, generic diagnostics and budgets | ANTLR source-file parsing and corpus execution | +| src/language/entry.ss | Descriptor dispatch and generic policy execution | Loader service registries and fixture admission | +| language-support/entry.ss | Public POO entry declaration over an admitted descriptor | Corpus, test and build requirements | +| language-support/development.ss | Optional descriptor-bound corpus, scanner and build services | A separate recognition algorithm | +| language-support/antlr4-source.ss | Source catalog and explicit terminal lowering | Inferring a token category from a GQL naming convention | +| languages/gql/grammar.ss | Pinned source identity, explicit lexical profile, generated products | Runtime parsing implementation | +| languages/*/parser.ss | Production descriptor and language parse function | Embedded examples or development service registries | +| languages/*/parser-test.ss | Corpus, development entry and independent semantic expectations | Reimplementing the grammar compiler | + +The common library depends on the engine. Engine dispatch does not depend on the +common fixture or development libraries. All shipped language packages follow this production/corpus split. +Bash keeps its declared source-parser receipt, HCL keeps its generated native +backend, and TLA+ keeps its layout and structured-proof policy descriptors. +Alignment does not replace these distinct language semantics. + +* Explicit imported-source semantics + +The ANTLR adapter formerly classified nonliteral lexer symbols by names containing +IDENTIFIER, CHARACTER_SEQUENCE, or UNSIGNED_DECIMAL. It also supplied fixed quote, +comment, and numeric scanner forms. Those assumptions described a GQL lexical +profile, rather than a language-independent ANTLR format contract. + +GQL now declares exact lexical bindings and scanner expressions in grammar.ss. +The format macro receives that profile and lowers it to the same canonical +language-declaration route as other grammar surfaces. Literal derivation receives +the same profile; it cannot silently use an implicit second mapping. + +#+begin_src scheme +;; A profile is language data consumed by the existing format adapter. +;; The full GQL profile also supplies its syntax kinds, terminals and extras. +(lexical-profile + (token-bindings ("REGULAR_IDENTIFIER" token identifier)) + (syntax-kinds (LexicalIdentifier token (text))) + (terminals (identifier LexicalIdentifier)) + (lexical-rules (identifier (identifier))) + (extras)) + +;; Question 1: Why is this design better? +;; The token's language meaning has an explicit language-package owner. +;; Changing its spelling cannot silently change the imported grammar semantics. +;; Scheme lists preserve the expression structure and feed the shared compiler. +;; Native POO remains the entry and service composition substrate. +;; Question 2: Why does this design have a scientific basis? +;; Extensible surfaces lower into one compilation core with checked boundaries. +;; Missing bindings fail closed; an independently supplied binding determines +;; the nonliteral terminal, and duplicate keys fail before grammar lowering. +;; These are executable falsifiers, not a proof of full ANTLR/GQL equivalence. +#+end_src + +The current profile deliberately preserves the previous scanner categories. +Several upstream numeric and quoted token classes still share engine token kinds. +Their full language equivalence is not proved by moving the mappings. Distinct +class acceptance and upstream lexer equivalence require further semantic controls. +Likewise, this refactor does not prove that the adapter's existing general +alternative-precedence projection reproduces all ANTLR adaptive decisions. + +* Production and development entry contracts + +#+begin_src scheme +;; parser.ss: production use requires no test corpus or build registry. +(deflanguage-parser-loader (gql-language :: self LanguageLoader.) + (grammar gql-language-grammar) + (parse parse-gql)) + +;; parser-test.ss: optional development facilities over the exact descriptor. +(deflanguage-development-loader + (gql-test-language :: self LanguageDevelopmentLoader.) + (grammar gql-language-grammar) + (parse parse-gql-test) + (slots fixtures: gql-official-fixtures)) + +;; Question 1: Why is this design better? +;; Parsing can load without fixture admission, hashing or fixture copies. +;; Development checks retain identity, source integrity and conformance guards. +;; Both entries execute the same admitted descriptor and engine dispatcher. +;; Question 2: Why does this design have a scientific basis? +;; The boundary test requires identical complete event streams for both entries. +;; It also checks that production has no fixture/test/build service slots. +;; Corpus expectations remain independently authored and can reject regressions. +;; The module split itself is our engineering decision, not a paper theorem. +#+end_src + +The optional development contract retains owned fixture copies, effective POO slot +validation, descriptor-bound scanner/build capabilities and sealed dispatch. +The production declaration checks parsing identity, metadata and descriptor +products, and rejects development service slots in both keyword and method +forms. Binding also checks effective inherited slots: a POO parent cannot +introduce fixtures, tests, scanner workers or build strategies into production. +The development test dispatcher admits its development contract before reading +service slots. Invalid calls therefore fail at the contract boundary. Shared +private loader-identity utilities avoid duplicate identity rules. +No compatibility forwarding module remains in src/language/entry.ss. + +The independent records consumer follows the same split. Its production parser +imports only its grammar; its test file binds an optional development entry over +the exact production descriptor and owns the corpus dependency. The shared +native conformance executable now includes this downstream sample under the +same ten-second child fence. The previous exact-head CI run passed the shipped +language gates on Linux and macOS, then exposed the records sample's missing test +service. That failure is retained as the regression motivating this extension. +The corrected local gate passes all twenty-seven bounded children and reproduces +all twenty native artifact controls byte for byte. The separate CI-shaped records +owner passes with explicit zero harness status, final GC and =OK=. These receipts +do not qualify a new remote head or establish a latency improvement. + +* Paper-to-code relationship + +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs]], section 1 and Figure 1, supports +an extensible elaboration surface feeding a compilation core with explicit +boundaries. Our adaptation is source-format and Scheme declaration front ends +feeding canonical admission, with runtime consumers using generated products. +The paper does not prescribe our directory names or prove our implementation. + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros]], section 7.2, demonstrates phased declaration processing +for dependent grammar/token clauses. Our profile and source catalog are collected +before terminal lowering and canonical validation. The paper motivates separating +these responsibilities; it does not establish equivalence of the imported lexer. + +* Reproducible verification + +Build =gxi build-conformance.ss=, then run +=gxi test-processes.ss research= in the project-qualified Gerbil environment. +The build installs =gerbil-parser-conformance= as a native Gerbil executable. +Suite processes call compiled runtime bindings without re-admitting the full +interpreter macro-interface graph. The source-expansion research programs retain +their explicit interpreter ownership. +The same gate now includes production/development boundary controls, the complete +three-interface topology test, Loader extension tests, explicit ANTLR profile +counterexamples, GQL conformance, and the prior DSL/POO/source-origin controls. +Each child has the unchanged ten-second completion fence and must exit zero. +Native conformance compilation retains the existing 240-second total and five-second +output-idle watchdog. The terminal wrapper preserves actual compiler output and +child exit status; GCC function/inlining diagnostics do not alter optimization +levels. A timed-out attempt is not a successful receipt. + +GQL performance controls remain under =t/benchmarks/gql/runtime=. This change does +not claim a speedup, corpus-wide latency percentile, or complete native runtime +closure. Moving corpus admission outside production parsing is a dependency +improvement; its timing impact needs matched measurements before any claim. + +* Qualification receipts and CI repair + +On 2026-10-07 the extended local gate passed 350 cases/checks in seventeen +processes, each exiting zero. Its cross-language metadata suite initially reached +the ten-second fence while creating compiler artifacts; the subsequent bounded +run completed using the real generated cache. The failed cold attempt is retained +and is not a successful cold-start latency receipt. + +The final ANTLR source/profile native compilation completed in 9.26 seconds, +and the final source-literals ordering macro in 4.15 seconds, both with exit zero +under the unchanged watchdog. Earlier attempts using a Nix cc selected from PATH +hit the output-idle fence; selecting the system C compiler and clearing inherited +SDK/Nix flags produced the admitted native receipts. This does not establish a +compiler performance comparison or qualify the complete native runtime graph. + +Independent old/new GQL product dumps completed with normal exits. A Scheme +comparison requires exact Grammar IR and Parser IR equality, plus matching 1176 +Bound IR identities, namespaces, references, reference digests, and pinned source +coordinates. Complete Bound IR differs in author-origin coordinates and consequent +identity digests; those locations must remain truthful rather than be normalized +away to manufacture byte equality. + +The prior PR CI failures had concrete harness causes: macOS lacked GNU timeout, +and Linux discovered package-expression-test.ss as a gxtest suite although it was +an executable research program. CI now installs coreutils on macOS; the program +is renamed package-expression-runtime.ss and its 85 controls execute explicitly +through the same bounded gate. The native no-cases guard is retained unchanged. +New exact-head CI results must be checked separately after publication. + +The final publication gate includes the renamed executable and retains all of +its 85 checks, for 435 total cases/checks across eighteen bounded processes. +Final child exit records, rather than assertion markers alone, determine success. + +* Complete language alignment + +The package inventory is arithmetic, Bash, openCypher, FHIRPath, GQL, HCL, +HL7 and TLA+. TLA+ publishes core, layout and SANY-candidate entries; this +accounts for ten descriptors. Every package still has exactly three Scheme +interfaces. Corpus declarations now live exclusively in parser-test.ss. + +| Entry | Corpus count | Optional development capability | +|-------+--------------+---------------------------------| +| arithmetic | 2 | Fused reduction and independent Rust AOT generation | +| Bash | 2 | Descriptor-bound command scanner | +| openCypher | 5 | Corpus conformance | +| FHIRPath | 4 | Corpus conformance | +| GQL | 15 | Corpus conformance | +| HCL | 15 | Generated backend parity and build strategies | +| HL7 | 2 | Corpus conformance | +| TLA+ core | 6 | Corpus conformance | +| TLA+ layout | 2 | Corpus conformance | +| TLA+ SANY candidate | 2 | Policy-bound corpus conformance | + +The HCL representative fixture remains an independently exported test fixture; +it is not an additional member of the fifteen-item official corpus. +Production C/Rust generators continue to consume grammar descriptors. Tests and +benchmarks obtain fixture data from the development entry and time production +parse functions. They do not restore fixture aliases in production modules. + +#+begin_src scheme +;; Question 1: Why is this design better? +;; A grammar declaration has one owner and one generated descriptor. +;; The optional test object composes native POO services over that same value. +;; Scheme syntax retains lexical binding and source provenance while macros +;; derive dispatch and contracts; authors supply data rather than new executors. +;; Question 2: Why does this design have a scientific basis? +;; The separation produces falsifiable obligations for every shipped entry: +;; descriptor identity, absent production services, unchanged grammar data, +;; independent corpus acceptance, complete event equality and exact roundtrip. +(eq? (.ref hcl-test-language 'descriptor) (.ref hcl-language 'descriptor)) +(run-language-test hcl-test-language 'fixtures) +#+end_src + +The topology guard rejects corpus declarations in grammar.ss and parser.ss, +and rejects development Loader declarations in parser.ss. The boundary suite +compares production and development dispatch over all 53 accepted and rejected +corpus members. Existing language-specific AST, layout, Unicode, concurrency, +recovery and generated-backend controls remain independent expectations. + +CI compiles language conformance modules before entering the existing bounded +execution gate. Compilation and test execution have distinct completion fences; +precompilation does not increase the ten-second test limit or admit a timed-out +process merely because it printed a success marker. + +Local qualification retains failed cold/source and incomplete native attempts. +An interrupted native build can leave an interface and wrapper without its +runtime phase object; a timestamp-only rerun is insufficient evidence. Valid +suite modules are compiled before the runtime gate, and their child exit status +remains mandatory even when every assertion printed a success marker. +The language boundary controls passed all 53 corpus dispatch comparisons in the +source-overlay environment. Rust all-target/all-feature Clippy passed for the +affected package. These receipts do not replace exact-head CI native execution. + +* Complete all-language native receipt + +On 2026-10-07 the native conformance executable linked successfully in 52.80 +seconds under the unchanged 240-second completion and five-second output-idle +fences. The local gate then completed all 26 child processes with exit zero, +each under its unchanged ten-second fence. This includes all eight packages, +all three TLA+ scopes, production/development comparison of all 53 corpus entries, +independent language expectations, DSL/POO/source-origin controls, and all ten +benchmark entry receipts. + +The native executable removes repeated interpreter admission of the combined +macro-interface graph; it does not change parser algorithms or weaken assertions. +Four explicit source-expansion research programs keep their interpreter execution. +Earlier source and mixed-artifact timeouts remain failed attempts. This receipt +uses existing compiled dependencies and is not a fresh-toolchain build claim. +The benchmark check uses one sample for semantic execution coverage; it does not +qualify latency percentiles. Remote platform CI must be assessed at the published +commit separately from this successful local receipt. + +* Native test-build partition and retained failures + +The first published native executable passed the local execution gate, but both +remote platforms subsequently failed its combined build's five-second idle fence. +On Linux the quiet stage was Gambit's sequential Scheme-to-C translation of the +GQL static module. A separate translation attempt then exposed a second quiet +stage: C compilation of a roughly twelve-megabyte combined link metadata unit. +Neither failed attempt is admitted as a successful qualification. + +The test build now uses the public Gerbil =compile-exe= API to generate the native +bootstrap and its admitted module identities. Project Scheme-to-C and object +compilation run as independent native jobs; immutable SDK objects are reused. +Gambit's documented incremental links form an ordered base chain with sixteen +modules per generated C unit. Intermediate bases use =___LIBRARY=; the final link +owns C main. This preserves bootstrap order and native module identities without +modifying the SDK compiler or parser algorithms. The partition is test-build +infrastructure and adds no author-facing grammar declaration. + +The rationale follows the official [[https://gambitscheme.org/4.9.8/manual/#Link-files][Gambit link-file documentation]] and its +separate compilation examples. A two-module native probe checked execution order +and the intermediate library flag before applying the build partition. Actual +compiler process failures propagate through worker joins; output reports only +real import, translation, compilation and link boundaries. + +The first local incremental build completed in 20.04 seconds, followed by all 26 +child processes exiting zero, with the existing 240/5-second build and ten-second +child fences unchanged. This reused project objects and is not a cold-build or +latency improvement claim. Remote qualification remains a separate gate. +The author-interface audit is in [[file:scheme-dsl-author-interface.org][the existing author-interface study]]: complete +file alignment does not establish that generated products belong in the public API. + +The subsequent Linux run at =712c65f= completed project Scheme-to-C translation, +but failed when all four workers began the largest C frontend jobs together. +The last real output preceded the five-second idle fence; the compiler remained +in those large units. The native work queue now starts large jobs early while +reserving one worker for short independent units. Its output still consists of +actual compiler diagnostics and completed jobs, without synthetic heartbeats or +a changed timeout. This scheduler change requires a fresh platform receipt. + +The local scheduler check quarantined only this disposable environment's project +static objects, retained existing Scheme-to-C outputs and SDK objects, and +rebuilt all project static objects. It completed in 64.06 seconds with exit zero +under the unchanged 240-second completion and five-second output-idle fences. +This is a cold project-object check, not a fresh dependency/source-toolchain build. + +* Language-wide native publication and root obligations + +The production descriptor is the native publication authority for all ten +entries in the eight language packages. =parser-test.ss= supplies accepted and +rejected corpus values; the shared conformance owner generates GPA1 and descriptor +binary fixtures from these values. The artifact crate admits those fixtures without +language-specific Rust parser tables. Native CI regenerates the same bytes. + +The alignment run exposed three independent contract gaps: HL7 rejection emitted +an engine-owned opaque token outside its published terminal directory; the shell +provider hardcoded a root that differed from its compiled command projection; +and the TLA+ proof product selected its root from catalog ordering instead of its +aliased entry rule. Publication now includes the engine rejection class, source +binding validates its derived root, and Parser IR compilation uses the entry +alias when present. Both generator cache contracts advance together, preventing\nunchanged grammar bytes from admitting outdated compiler semantics. No\nlanguage-specific ABI dispatcher is added. + +** Why does this DSL reduce authoring complexity? + +#+begin_src scheme +(deflanguage example + (root document) + (lex (name Identifier (identifier))) + (rules + (document (node Document (repeat (field name name))))) + (syntax-kinds + (Auxiliary node ()) + (Document node (name)) + (Identifier token (text)))) +#+end_src + +The root rule constructs =Document= even though =Auxiliary= appears first in the +published catalog. The language declares recognition and field associations +once. Macro expansion lowers the S-expression to admitted Grammar/Parser IR; +compiled plans supply runtime behavior and publication metadata. POO inheritance +can change an admitted declaration before binding, while ordinary values need +no object wrapper. These responsibilities belong to the engine and common DSL +implementation, rather than a language author's duplicate encoder or Rust grammar. + +** Which executable invariants support the design? + +For every accepted fixture, the borrowed root kind equals the same handle's +published root, and its text equals the original source. For rejection, there +is no accepted root, but ordered token spans cover the source exactly. Declared +node fields, product/source identity and UTF-8 boundaries are admitted before +navigation. The twenty cross-language controls independently check these rules; +the source-provider negative test rejects an undeclared root at binding time. +These are falsifiable semantic and ownership contracts. They establish neither +complete language-standard conformance nor a performance advantage. + +The integrated local receipt covers all twenty native controls, all twenty-six +bounded Scheme DSL/development child processes and 130 Rust runtime tests. +The native admission test reads binary fixture data and has no language-specific +recognition dependency. The unchanged fences and rejected controls are retained; +one accepted and one rejected source per entry establish publication alignment, +not exhaustive grammar coverage. Native CI reproduces the fixture bytes and +qualifies each pushed revision independently. + +* TLA+ declarative interface migration + +The previous TLA+ interface retained generated IR aliases, manual syntax catalogs, +low-level reference wrappers and release constants in a 1,599-line grammar file. +The public =grammar.ss= now exports syntax descriptors and lexical profiles only. +Four private =syntax/= modules hold core, layout, proof and lexical declarations; +=parser.ss= owns pinned SANY source identity and release metadata. Consumers obtain +IR and machines through descriptor accessors. The topology contract permits only +declarative syntax modules behind the three public interfaces. + +#+begin_src scheme +;; Representative declaration: recognition and publication share one source. +(module + (node Module module-border "MODULE" (field name identifier) module-border + (repeat (field item module-item)) module-end)) + +;; Question 1: Why is this design better? +;; Bare rule identifiers and literals preserve readable Scheme structure. +;; The engine derives node kinds, fields and terminals from recognition clauses; +;; authors no longer maintain a parallel publication catalog or generated IR API. + +;; Question 2: Why does this design have a scientific basis? +;; Compare canonical recognition before and after elaboration, then independently +;; test accepted/rejected sources, field publication and source identity. +;; A shorter surface is justified by these invariants, not by its line count. +#+end_src + +The migration compared seven canonical sections for each of the three TLA+ +products: rules, lexical rules, extras, keywords, recoveries, conflict policy and +case sensitivity. All 21 comparisons passed. The proof token is explicitly +renamed =proof-step-name= to distinguish it from the =proof-step= rule. The old +manual catalog advertised =fact= and =definition= fields on =TerminalProof= that +recognition did not emit; the inferred contract correctly publishes =item=. +This is an intentional publication correction, not unchanged artifact identity. +The existing corpus and native lexical/source/snapshot tests qualify this change; +they do not establish complete SANY compatibility or a performance gain. + +** Native build and qualification ownership + +The build pipeline, failed controls and artifact-owner receipts are maintained in +[[file:scheme-native-build-ownership.org][Scheme native build ownership]]. They describe build-time responsibilities +without enlarging the language interface contract. diff --git a/docs/research/scheme-language-pack-design.org b/docs/research/scheme-language-pack-design.org new file mode 100644 index 00000000..648c10aa --- /dev/null +++ b/docs/research/scheme-language-pack-design.org @@ -0,0 +1,277 @@ +#+title: Scheme language pack design: research and experiments +#+date: 2026-10-06 + +* Status and scope + +This is an independent research proposal, not an accepted API specification or +an implementation receipt. The source baseline examined is PR #13 revision +=d4a99efa558625fd4053cb6dcc1a0ba36c484e77=. Later revisions must recheck the +inventory before treating a question here as a current implementation gap. + +The research target is the complete authoring experience of a language pack: +syntax, source provenance, public products, loading, fixtures and qualification. +Shorter grammar expressions are one part of that experience. Our design basis +is Scheme's syntax objects, lexical scope, macros, modules and compilation. +Tree-sitter supplies a comparison sample. It does not determine the surface +language, parser algorithm or extension model. + +This document changes no parser runtime, GC policy, actor service, AOT compiler +or FFI contract. Performance work remains in +[[file:scheme-parser-performance-framework.org][the parser performance research]]. +The current author contract is [[file:../../languages/README.org][the language pack boundary]]. + +* Current evidence and corrected baseline + +| Owner | Examined behavior | Research question | +|-------+-------------------+-------------------| +| =languages/README.org= | Three interfaces: grammar, loader and tests | How much knowledge does a complete author task require? | +| =src/language/grammar.ss= | =deflanguage= lowers concise forms; =defgrammar-syntax= permits templates | Which binding and origin information must survive elaboration? | +| =src/compiler/language-artifact.ss= | Canonical grammar expansion and published catalog projection | Which inferred products can share one semantic analysis? | +| =languages/arithmetic/parser.ss= | POO loader, fixture values and declared build strategies | Which target choices require an explicit author decision? | +| =languages/arithmetic/parser-test.ss= | Declarative tests plus a native property | How can tests retain precise semantic assertions with less ceremony? | +| =languages/hcl/grammar.ss= | Public =deflanguage= with fielded grammar rules | Can reusable grammar abstractions reduce maintenance without hiding meaning? | +| =languages/fhirpath/grammar.ss= | =deflanguage=, text profiles and admitted source metadata | How can imported specifications and native rules share useful diagnostics? | + +The latest baseline already infers kinds, fields and terminals. Alternatives +with the same node kind contribute a union of fields. Transparent helper rules +propagate fields to their nearest named node; recursion is bounded by visited +rule names. Explicit =catalog= projection retains published identities and +ordering. Explicit =token= or =reference= disambiguates overlapping namespaces. + +An earlier assessment of versioned language directories described a same-node +field-list rejection. That assessment is superseded: =record-node!= now merges +fields. Richer field cardinality/type analysis is a separate research question; +the existing field union must not be reported as missing. + +=expand-surface-syntax= currently expands extension forms and converts their +syntax into datum expressions. Several later products are constructed with +=datum->syntax=. This is a useful seam for specifying the roles of grammar data and host syntax. +The current error helper raises against the complete =deflanguage= form. Study +whether diagnostics can retain the offending subform and macro expansion origin. + +* Papers and primary references + +These annotations distinguish a source's contribution from our proposed use. +They do not transfer a paper's performance results to this parser. + +** Syntactic abstraction in Scheme + +R. Kent Dybvig, Robert Hieb and Carl Bruggeman, /Lisp and Symbolic Computation/ +5(4), 1993, pp. 295–326. The +[[https://scholarworks.iu.edu/dspace/bitstreams/3f729d08-c698-4430-9d04-6a782c5b9300/download][Indiana University technical report (1992)]] +is an earlier version; its abstract motivates both avoiding identifier capture +and correlating transformed code with its source. The follow-up synthesis studies +section 3 and the source annotation treatment +in section 4 using the university-hosted text. + +Design use: grammar fragments should preserve the distinction between names +supplied by callers and names introduced by an abstraction. Source correlation +is part of authoring quality, not a cosmetic addition after code generation. +Experiment: compose a fragment across modules while shadowing its helper names; +check both binding resolution and error locations. The paper does not establish +that our current expander passes this experiment. + +** Compiled, Extensible, Multi-language DSLs (Functional Pearl) + +Michael Ballantyne, Mitch Gamburg and Jason Hemann, ICFP 2024, +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][author paper]], +[[https://doi.org/10.1145/3674627][DOI: 10.1145/3674627]]. Sections 1–3 describe an extensible +surface elaborated into a compiler-oriented core, with explicit boundaries +between a hosted DSL and its host. The case study is miniKanren in Racket. + +Design use: user grammar macros elaborate into the existing canonical Grammar +IR, where compiler validation and optimization remain shared. Any host escape +must expose a boundary with defined invariants and backend capabilities. +Experiment: two surface formulations generate equivalent core IR and artifacts; +a host callback cannot silently acquire a Rust backend capability. Racket's +syntax-spec implementation and miniKanren speedups are not Gerbil receipts. + +** Fortifying Macros + +Ryan Culpepper and Matthias Felleisen, ICFP 2010, +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][author paper]]. Its syntax classes and validation approach address +macro inputs, constraints and useful error reports. Section 3 is especially +relevant to checking malformed forms and reporting the responsible syntax. + +Design use: specify grammar constructor shapes and semantic constraints once, +then associate validation failures with precise source syntax. Use Gerbil's +existing facilities; adoption of Racket's syntax-parse is not a prerequisite. +Experiment: unresolved names, duplicates, malformed precedence and invalid +fragment arguments identify their offending subforms, including imported macros. + +** Parsing Expression Grammars: A Recognition-Based Syntactic Foundation + +Bryan Ford, POPL 2004, +[[https://pdos.csail.mit.edu/papers/parsing:popl04.pdf][author paper]]. PEG uses prioritized choice; that semantic distinction +matters when borrowing an apparently similar combinator notation. + +Design use: document our =choice=, precedence, ambiguity and recovery semantics +before creating convenience forms. A compact expression must not introduce PEG +ordered choice into a CFG/LR grammar. Experiment: alternatives with overlapping +prefixes retain the baseline recognition and artifact behavior. This reference +does not justify replacing the current parser engine or claiming linear runtime +for arbitrary grammars. + +** Official Gerbil facilities and comparison references + +[[https://gerbil.scheme.org/tutorials/languages.html][Custom Languages in Gerbil]] demonstrates contextual expansion and language +modules over the existing S-expression reader. +[[https://gerbil.scheme.org/reference/gerbil/prelude/runtime-bindings.html][Gerbil runtime/expander bindings]] documents syntax-object operations, +identifier comparisons and source utilities. These are the starting toolbox; +research should establish our binding policy before adding machinery. + +[[https://spoofax.dev/references/sdf3/][Official SDF3 documentation]] is a practical comparison for declarations that +feed multiple language products. It is documentation, not a paper result. +[[https://tree-sitter.github.io/tree-sitter/creating-parsers/2-the-grammar-dsl.html][Tree-sitter's grammar DSL]] is JavaScript, while +[[https://tree-sitter.github.io/tree-sitter/using-parsers/queries/1-syntax.html][its query language]] uses S-expressions. Its +[[https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types.html][static node types]] provide a comparison for field presence, multiplicity and +possible child types. These are useful product questions, without adopting its +external scanner interface or treating its grammar syntax as Scheme's model. + +* Design hypotheses: expand Scheme's advantages + +** Preserve binding information until names have meaning + +Distinguish grammar rule names, lexical token names, public node/field labels, +macro bindings, generated helpers and Scheme expressions. Public labels may +intentionally be symbolic data; references and introduced helpers need a stated +resolution policy. Not every identifier should be treated as a Scheme variable. + +Proposed elaboration order: + +1. Read syntax objects and establish declaration namespaces. +2. Expand authorized grammar fragments with Gerbil's macro facilities. +3. Resolve DSL references and retain origin information for each core form. +4. Validate and lower into the existing canonical IR. +5. Generate products through existing compiler and capability boundaries. + +An origin map can be compile-time metadata; its representation, persistence and +cost remain open. It must not silently change runtime IR schemas or published +digests. Decide explicitly which origin details belong in source provenance and +which are excluded from semantic identity. + +** Build a grammar vocabulary through ordinary modules + +The existing =defgrammar-syntax= is a starting point. Candidate abstractions +include separated lists, delimiters and repeated operator tiers. Each abstraction +must explain its expansion, fields, empty cases, precedence and conflict behavior. + +Prefer small composable forms that lower to existing constructors. Test module +imports, renaming, nested composition and caller-supplied references before +promoting a shared vocabulary. Do not add a parallel parser interpreter for +convenience forms. Syntax shortening alone does not prove better authoring. + +** Infer products once, preserve deliberate public contracts + +The field union already provides a catalog. A proposed next analysis asks: + +- Is a field present on every productive alternative or only some? +- Can a field occur more than once on a node? +- Which node/token categories can it contain? +- How do transparent rules, recursion and recovery affect those answers? + +A candidate analysis uses a fixed point over recursive grammar references. +Presence, multiplicity and possible types need conservative domains and defined +join rules. Unknown results remain unknown; union alone cannot establish required +presence. Repetition and nullable productions require explicit treatment. + +The first prototype should produce an internal explanation alongside inferred +facts. Publishing a new typed AST or metadata schema is a later contract decision. +Retain =catalog= projections for stable kind/field/terminal identities and order; +keep =node-fields= for deliberate public reservations. Never infer away a public +compatibility requirement merely because a fixture does not exercise it. + +** Reduce author knowledge across the complete package + +| Information | Proposed authority | Reason | +|-------------+--------------------+--------| +| Recognition, precedence, conflicts, node boundaries | Author grammar | These are language semantics | +| Derived kinds, fields, terminals and analysis facts | Compiler | Avoid independently maintained copies | +| Source revision, digests and corpus identity | Explicit provenance declarations | Reproducibility cannot be inferred from syntax alone | +| Parser entry, exports and tooling extensions | POO loader | Preserve effective-object admission and extension contracts | +| Backend selection and qualification | Explicit build strategy/capability boundary | Target support requires executable evidence | +| Acceptance, AST, offsets and rejection expectations | Test owner | The implementation cannot supply its own semantic oracle | +| Workloads, samples and thresholds | Existing benchmark owner | Keep performance admission reproducible | +| Actor queue limits, cancellation and application policy | Runtime service configuration | Independent of grammar meaning | + +Three files can be coherent if each has one clear role. A single large macro can +still require excessive knowledge. Count independent concepts and repeated facts, +then evaluate real maintenance tasks. Common defaults should be documented and +stable; semantic choices and target restrictions should remain visible. + +** Keep the closed core useful for optimization + +Macros can expose domain patterns early enough for shared compiler analysis. +However, template expansion alone does not prove faster parsing. If two surfaces +produce the same core, runtime improvement needs a separate measured compiler or +execution change. Investigate compile-time cost and authoring quality first. +Reuse Scheme/Gerbil compiler facilities for host code; this proposal introduces +no substitute optimizer, GC tuning or parallel runtime implementation. + +* Concrete study example + +This example uses the current documented surface, not a proposed new API: + +#+begin_src scheme +(import (only-in :gerbil-parser/language-support deflanguage)) + +(deflanguage example + (identity "example" "v1" "example.syntax.v1") + (root document) + (lex (word Identifier (identifier))) + (rules (document (node SourceFile (field name word))))) +#+end_src + +Study a change that adds a second document alternative, then extract a shared +fragment into another module. Record which declarations the author must edit, +what the compiler infers, whether public ordering changes, and where a deliberate +unresolved reference is reported. Keep the loader and native test declarations +in the study: a grammar-only comparison would miss the requested package scope. + +* Experiments and admission criteria + +No experiment below is claimed to have run as part of this documentation change. +Use repository native tests and benchmark contracts when prototypes exist. + +| Study | Inputs | Required evidence | +|-------+--------+-------------------| +| Complete authoring inventory | Arithmetic, HCL, FHIRPath and one downstream pack | Task steps, concept count, duplicated declarations, intentional choices | +| Fragment composition and source explanation | Imported/renamed fragments, shadowed helpers, nested expansions | Expected name resolution and offending source locations; negative cases | +| Surface/core equivalence | Existing rules and macro abstractions | Normalized canonical IR plus accepted/rejected inputs, CST/AST and offsets | +| Catalog stability | Existing published pack and declaration reorder | Stable required identities/order; deliberate migrations stated separately | +| Schema inference | Alternatives, optional/repeated/nullable forms, recursive helpers | Sound conservative presence/multiplicity/type facts and counterexamples | +| Backend preservation | Qualified Scheme and generated Rust/C products | Matching semantics and capability rejection; no inferred native qualification | +| Expansion scaling | Increasing rules, fragment depth and recursive analysis | Compile latency/allocation and termination; runtime results reported separately | +| Author comprehension | Matched add-rule, extract-fragment and diagnose-error tasks | Completion/correctness evidence; LOC reduction alone is insufficient | + +For equivalence, normalize only explicitly nonsemantic metadata; retain public +IDs, precedence, source offsets, conflict behavior and recovery meaning. Define +that normalization before collecting results. Tests must include ambiguity and +rejection cases, not just successful examples. + +Native performance claims require matched inputs, artifact semantics, warmup and +phase attribution through existing benchmarks. An authoring study or a paper's +results cannot close a parser P50/P95 performance gate. + +* Research sequence and unresolved decisions + +1. Complete the authoring inventory and choose representative maintenance tasks. +2. Specify namespaces, macro scopes, source origins and identity normalization. +3. Add a narrowly scoped research prototype for origin-aware elaboration, with + cross-module and negative binding examples. +4. Prototype conservative field analysis and explain its results on real packs. +5. Compare reusable macro vocabularies against the current author interface. +6. Propose a reviewed API/contract change only after semantic and author evidence. + +Open decisions include whether rule references are pack-local symbolic identities +or module-qualified bindings; how imported fragments declare dependencies; +which information survives elaboration; how recovery widens inferred schemas; +and whether inferred products justify any new public API. This document records +hypotheses and experiments so those decisions can be made from evidence. + +* Follow-up semantic study + +[[file:scheme-language-pack-semantics.org][Binding, field semantics and author tasks]] examines existing Bound IR and source +maps, compares binding policies, and develops count analysis against actual field +reduction behavior. It records fragment and empty-result counterexamples before +proposing any generated schema. diff --git a/docs/research/scheme-language-pack-semantics.org b/docs/research/scheme-language-pack-semantics.org new file mode 100644 index 00000000..33909d4b --- /dev/null +++ b/docs/research/scheme-language-pack-semantics.org @@ -0,0 +1,399 @@ +#+title: Scheme language pack semantics: binding, fields and author tasks +#+date: 2026-10-06 + +* Research status + +This study develops the questions in [[file:scheme-language-pack-design.org][Scheme language pack design]]. +Source baseline: =b455d402281aa1e0d9616a3acc210ba6c6b52cca=, whose runtime +and authoring implementation is unchanged from =d4a99ef=. The rules below are +candidate analyses and paper/design conclusions, not new public APIs or a +completed correctness proof. Source inspection and small mathematical checks +are distinguished from native parser execution. + +* Additional literature and its precise role + +** Binding as Sets of Scopes + +Matthew Flatt, POPL 2016; [[https://www-old.cs.utah.edu/plt/scope-sets/][author's extended paper]], especially +[[https://www-old.cs.utah.edu/plt/scope-sets/general-macros.html][section 3: procedural macros, modules and phases]]. The model attaches scope +information to syntax and explains binding comparisons, expansion contexts and +module phases. It provides a way to reason about composition beyond comparing +identifier spellings. + +Our use is a model for independently composable grammar vocabulary. Gerbil need not adopt Racket's expander +implementation or scope representation. We must preserve the host's established +binding behavior where host references occur, and independently specify the +meaning of symbolic grammar names. The paper's comparisons are not a guarantee +that the corresponding Gerbil operations have identical edge-case behavior. + +** Abstract interpretation + +Patrick Cousot and Radhia Cousot, POPL 1977, +[[https://www.di.ens.fr/~cousot/COUSOTpapers/POPL77.shtml][author publication and summary]]. The source describes abstract domains, +monotone equations, fixpoints and consistency with concrete semantics. It +motivates finite conservative analysis of recursive grammar products. + +Our field-count domain and transfers below are our proposed specialization, +not algorithms supplied by this paper. Their correctness must be checked +against this repository's reduction semantics. A finite domain can guarantee +termination of monotone iteration while still being too expensive or too +imprecise for a useful compiler pass. + +Official [[https://gerbil.scheme.org/reference/gerbil/expander/utilities.html][Gerbil syntax utilities]] document source access and syntax traversal. +They support an implementation direction using existing facilities. Retaining +syntax objects does not itself establish correct DSL binding or useful errors. + +* Binding model: distinguish six kinds of names + +| Name | Current role | Candidate policy | +|------+--------------+------------------| +| Rule reference | Pack-local symbolic Grammar IR reference | Resolve in the grammar's rule namespace | +| Token reference | Pack-local terminal reference | Resolve in the terminal namespace; ambiguity remains an error | +| Node/field label | Public tree/catalog identity | Preserve intentional labels and published ordering | +| Fragment macro | Gerbil expansion binding | Honor module imports, aliases and definition context | +| Generated private helper | Implementation detail | Keep private host bindings distinct from caller bindings | +| Host expression | Identity values, backend digest/procedure expressions | Preserve the caller's original syntax context | + +The existing Bound IR already creates structured identities containing package, +grammar, namespace and name. Fields are qualified by their node owner. +=src/compiler/bound-ir.ss= rejects unresolved references during admission. +This is a real DSL binding product, not evidence that every intermediate syntax +context survives expansion. Conversely, symbolic rule resolution is deliberate; +replacing it wholesale with host-variable lookup would change the language. + +** Candidate designs and tradeoffs + +A. Keep pack-local symbolic rule names; fragments explicitly receive all rule + dependencies as arguments. This retains the current core identity model and + makes reusable fragments easy to inspect. Large fragments may need many + parameters. Implicit free rule names remain a documented use-site dependency. + +B. Introduce scoped fragment-private rules and explicit exports. This can support + independently composed larger fragments, but requires a new definition and + dependency model. Private IDs must lower deterministically without leaking + generated names into public catalogs. No such author API is proposed yet. + +C. Treat all rule names as ordinary Scheme bindings. This enables host-style + imports but changes existing resolution and identity assumptions. It needs + substantially stronger evidence than a shorter syntax example. + +Research preference: evaluate A with ordinary macro/module composition first; +consider B only when real author tasks demonstrate an unmet need. Preserve the +canonical IR and current public identities throughout the comparison. + +** Syntax roles in composed declarations + +=src/language/grammar.ss= expands extensions using =core-expand1= and then +constructs canonical datum expressions. A grammar vocabulary specification should +classify which arguments are DSL data and which carry host syntax context. This +makes fragment parameters, introduced labels and generated products explainable +to an author using the existing implementation. + +The lower-level compiler in =src/compiler/language-artifact.ss= already derives +source maps from syntax rows and accepts an explicit source map. The concise +surface records macro names in =expansionLineage=; this is not a complete map of +every macro invocation to its original argument location. The current surface +test checks a rule's origin path, not exact imported argument spans. + +| Case | Expected observation | What would falsify the design | +|------+----------------------+------------------------------| +| Imported fragment renamed at import | Expansion uses the imported transformer | Behavior changes merely because its local spelling changes | +| Fragment parameter names a caller rule | Reference resolves to that declared rule | Parameter is confused with a macro-private helper | +| Fragment introduces a public field label | Label stays intentional symbolic data | Host binding/shadowing accidentally renames the public label | +| Two fragments reuse a private helper spelling | Independent private bindings | Composition silently joins their implementations | +| Macro supplies private backend expressions | Host references retain their original context | Reconstruction resolves them in the consumer module | +| Invalid nested fragment argument | Error points to the argument with expansion context | Only the outer language declaration can be identified | +| Same spelling in rule and token namespaces | Bare use rejects; explicit use resolves | New shorthand silently changes namespace precedence | + +These cases require executable expansion tests before any implementation defect +is reported. =t/language-surface-test.ss= already exercises a local fragment, +Bound IR lineage, namespace disambiguation and field union. Extend its evidence +with cross-module and precise-location cases; do not recreate its existing tests. + +* Field semantics: catalog membership is not cardinality + +** Concrete reduction observations + +=src/runtime/reduce.ss:recognition-children-field= has three cases: + +- Zero children: emit no fielded child. +- One unfielded child: attach the field directly to its value. +- One already fielded child, or multiple children: attach the outer field to + a fragment containing those children. + +=src/runtime/event-reduce.ss:event-children-field= implements corresponding +empty, direct and fragment cases. Named aliases wrap their children in a node, +even when the body is empty. These observations require analysis of emitted +structure rather than occurrences of constructor names. + +Two deliberately different forms illustrate the issue: + +#+begin_src scheme +;; Study expressions only, not complete language declarations. +(repeat (field item word)) +(field items (repeat word)) +#+end_src + +The first can emit many =item= field instances. The second emits zero or one +=items= field instance; when it wraps several words, that field contains a +fragment. A proposed typed product must define how fragments are exposed before +claiming a field has a single node/token type. + +The existing =surface-fields= gathers declared field names through transparent +helpers and stops at named nodes. It supplies a catalog. It does not claim to +compute exact instance counts or correlated field presence. + +** Define the observation before choosing the analysis + +Candidate observation: the sequence of immediate recognition children before a +named-node wrapper, and the labels attached to that sequence. Named nodes and +fragments also carry their own nested summaries. A field inside a fragment is +not automatically another immediate field of the enclosing named node. + +An alternative public API might flatten fragments. Its counts and possible types +would be a different projection. Keep both questions explicit; do not publish +Tree-sitter-style node types from catalog membership alone. + +Recovery is a separate observation class. The initial analysis covers ordinary +successful finite derivations. A recovery-enabled artifact may widen or omit +fields; required-field claims must either exclude that class in the contract or +include its behavior conservatively. + +** A finite candidate count domain + +For a fixed finite set of declared field labels F, let =cap(n)= be 0, 1 or 2, +where 2 means two or more. A summary is a set of reachable abstract tuples: + +#+begin_example +(arity, count[f1], ..., count[fk]) with every component in {0, 1, 2} +bottom = {} ; no finite productive derivation yet +epsilon = {(0, 0, ..., 0)} ; productive derivation emitting no children +join = set union +sequence = pairwise componentwise capped addition +#+end_example + +Do not replace bottom with epsilon. Recursive rules without a productive base +must not manufacture successful empty parses. Keep arity and field counts in +joint tuples; independent unions can lose the relation needed by field wrapping. + +Candidate transfers for immediate children: + +| Form | Transfer | +|------+----------| +| Token or recognized literal | One unfielded child | +| Empty production | Epsilon | +| Sequence | Capped sums across productive child tuples | +| Choice | Union of alternative summaries | +| Optional | Union with epsilon | +| Repeat | Least fixpoint X = epsilon union sequence(body, X) | +| Repeat1 | Sequence(body, repeat(body)) | +| Reference | Read the referenced rule's current summary | +| Precedence | Preserve this structural summary; actual conflict resolution remains separate | +| Field f around body | Arity 0 remains empty; nonzero arity emits one child labeled f | +| Named node/alias K | Record body's owned facts for K; emit one unfielded node to its parent | + +For =field=, outer immediate counts become one for f and zero for other labels; +existing inner labels remain in nested values/fragments, not in this immediate +projection. This transfer only addresses counts. A useful type analysis must also +track whether the single child is already fielded, its node/token identity and +whether wrapping produces a fragment. That shape component is not formalized +here, so these rules do not yet generate a complete typed schema. + +Resolve recursive equations by monotone worklist iteration from bottom. +With fixed F there are at most =3^(|F|+1)= tuple values per sequence summary; +the powerset lattice has finite height. This proves termination for these +monotone count equations, not practical efficiency or exactness. Partition by +reachable owner/field sets, track iterations, and consider a cheaper sound +product approximation if relational tuples grow too large. + +For a reachable node kind, join facts across its productive occurrences. A field +is potentially present if any tuple has a positive count, required if all tuples +have positive counts, and potentially repeated if any tuple has count 2. An empty +node-summary set means unreachable/unknown admission status; never use vacuous +truth to mark all fields required. + +Choice over all grammar derivations can overapproximate actual parses selected +by conflicts, precedence and lexical constraints. Such extra alternatives reduce +precision; they must not be discarded solely because a few fixtures omit them. +A soundness proof must relate summaries to actual reduction output, separately +from a precision assessment on reachable language inputs. + +** Worked source-level predictions + +The following are manual predictions for ordinary successful derivations, not +native artifact receipts or claims of an implemented analysis. + +| Current case | Expected immediate field counts | Remaining qualification | +|--------------+---------------------------------+-------------------------| +| HCL ConditionalExpression | condition: 1; consequent/alternative: 0 or 1 | Their presence is correlated; expression must produce a child | +| HCL BinaryExpression | left: 1; operator/right: 0, 1 or many | operator and right counts are correlated; grammar semantics stay unchanged | +| HCL CallExpression | argument: 0, 1 or many | Parentheses do not make argument required | +| Arithmetic Expression | left/operator/right: 1 per binary node | Recursion builds child nodes; it does not repeat these fields on that node | +| Transparent recursive helper witness | item: 1; tail: 0, 1 or many | Named-node barriers and recursion must both be respected | +| Field around empty | no emitted field | Catalog reservation is still a distinct public declaration | +| Field around a repeated token | zero or one outer field | Its value may be a fragment, unlike repeated field instances | + +The transparent witness is the grammar in =t/language-surface-test.ss=: +=helper= chooses a fielded word or a parenthesized helper followed by a fielded +word. Existing tests check the inferred =item/tail= catalog and roundtrip. +They do not assert these cardinality facts. Correlations such as both HCL +conditional branches appearing together belong to a richer schema than separate +per-field flags; defer them unless an actual consumer needs that precision. + +** Reproducible checks of the proposed equations + +This Gerbil Scheme block checks the mathematical candidate only. It does not import the +parser, qualify a backend, or serve as the semantic oracle for native tests. +Tuple layout is =(arity, item, tail, other)=; nonempty field wrapping resets the +outer immediate counts. Seven checks passed under Gerbil when this block was revised (exit status 0). +Save the block to a =.ss= file and run =gerbil env gxi =. + +#+begin_src scheme +(def count-zero '(0 0 0 0)) +(def count-epsilon (list count-zero)) +(def count-word '((1 0 0 0))) + +(def (count-set values) + (let loop ((rest values) (result '())) + (cond + ((null? rest) result) + ((member (car rest) result) (loop (cdr rest) result)) + (else (loop (cdr rest) (cons (car rest) result)))))) + +(def (count-subset? left right) + (let loop ((rest left)) + (or (null? rest) + (and (member (car rest) right) (loop (cdr rest)))))) + +(def (count-set=? left right) + (and (count-subset? left right) (count-subset? right left))) + +(def (count-join left right) + (count-set (append left right))) + +(def (count-sequence left right) + (count-set + (apply append + (map (lambda (x) + (map (lambda (y) (map (lambda (a b) (min 2 (+ a b))) x y)) + right)) + left)))) + +(def (count-field index body) + (count-set + (map (lambda (row) + (if (zero? (car row)) count-zero + (list 1 + (if (= index 1) 1 0) + (if (= index 2) 1 0) + (if (= index 3) 1 0)))) + body))) + +(def (count-fixpoint step) + (let loop ((current '()) (iterations 0)) + ;; Four capped components give 81 tuples and at most 81 strict additions. + (when (> iterations 81) (error "count model did not converge")) + (let (updated (step current)) + (unless (count-subset? current updated) + (error "count model is not ascending")) + (if (count-set=? updated current) current + (loop updated (+ iterations 1)))))) + +(def (count-check label actual expected) + (unless (count-set=? actual expected) + (error "count model mismatch" label actual expected)) + (display "COUNT-CASE-OK: ") (display label) (newline)) + +(def count-items (count-field 1 count-word)) +(def count-repeated-fields + (count-fixpoint + (lambda (x) (count-join count-epsilon (count-sequence count-items x))))) +(count-check 'repeated-fields (map cadr count-repeated-fields) '(0 1 2)) + +(def count-repeated-words + (count-fixpoint + (lambda (x) (count-join count-epsilon (count-sequence count-word x))))) +(count-check 'wrapped-repeat + (map cadr (count-field 1 count-repeated-words)) '(0 1)) +(count-check 'empty-field (count-field 1 count-epsilon) count-epsilon) +(count-check 'nonproductive-recursion + (count-fixpoint (lambda (x) (count-sequence x count-word))) '()) +(count-check 'nullable-repeat + (count-fixpoint + (lambda (x) (count-join count-epsilon (count-sequence count-epsilon x)))) + count-epsilon) + +(def count-helper + (count-fixpoint + (lambda (x) + (count-join count-items + (count-sequence count-word + (count-sequence x + (count-sequence count-word (count-field 2 count-word)))))))) +(count-check 'transparent-helper + (map cdr count-helper) '((1 0 0) (1 1 0) (1 2 0))) + +(def count-branches + (count-join count-epsilon + (count-sequence (count-field 2 count-word) (count-field 3 count-word)))) +(def count-conditional (count-sequence count-items count-branches)) +(count-check 'conditional-correlation + (map cdr count-conditional) '((1 0 0) (1 1 1))) +(display "COUNT-MODEL: 7 checks passed") (newline) +#+end_src + +The recursive helper uses =word= for punctuation because both emit one unfielded +child in this count projection. Nullable alias bodies, fragment value types, +recovery and precedence need additional native witnesses; these checks establish +only the stated count examples and do not prove the full abstraction sound. + +* Whole-package author task study + +The current three interfaces are the starting point. Inspection shows an +important ownership exception: FHIRPath declares =defsyntax-corpus= in +=parser.ss=, while arithmetic/HCL fixture declarations are in =grammar.ss=. +This is an organizational choice to describe accurately in author guidance. +The overview should describe allowed placement accurately before further +consolidation is considered. + +| Task | Current required decisions | Potential derivation | Must remain explicit | +|------+----------------------------+----------------------+----------------------| +| Add an arithmetic operator | Rule, precedence, node/fields, acceptance and AST tests | Catalog membership | Associativity and expected tree | +| Add an HCL optional clause | Field placement, nullability, accepted/rejected examples | Presence/count facts | Recognition and recovery policy | +| Share a fragment | Macro parameters/imports, reference dependencies | Repeated structure | Public node/field meaning and conflicts | +| Update normative FHIRPath input | Source identity/digest, grammar/profile changes, fixtures | Mechanical source checks | Version provenance and intentional divergence | +| Publish a native product | Loader exports and qualified target/build strategy | Product assembly from admitted IR | Target capability and semantic qualification | +| Add concurrent application parsing | Runtime service policy and workload tests | Existing parser reuse | Queueing, cancellation and resource limits | + +Separate three cost measurements: concepts the author must understand, facts +they maintain in more than one place, and steps to finish a real change. Count +source edits as supporting evidence, not as a substitute for comprehension. +Use matched tasks and a declared rubric: correctness, omitted obligations, +completion time, and the author's explanation of the generated result. + +Our study should first remove derivable repetition, improve error localization +and clarify ownership. Whether grammar, loader and tests should merge is a +later empirical question; a file-count reduction does not establish a benefit. + +* Next research gates + +1. Binding policy review: classify each syntax-to-datum boundary by name role; + specify use-site dependencies and host-context preservation. +2. Observation review: choose immediate versus fragment-flattened public fields, + ordinary versus recovered artifacts, and the type vocabulary. +3. Mathematical checks: exercise capped sums, nullable bodies, recursion without + a base, repeated wrappers and repeated field instances. +4. Native witnesses: compare predicted field events and exact ASTs with existing + Scheme and qualified generated products before publishing metadata. +5. Author tasks: perform the same operator/optional-clause/fragment changes with + the current surface and a small proposed abstraction; preserve negative cases. + +This study advances gates 1–3 through source inspection and a candidate model. +It does not close gate 4, a new schema/API, or a performance claim. + +* Executable host binding evidence + +[[file:scheme-macro-binding-experiments.org][Gerbil macro binding experiments]] now supply six native Gerbil host-expansion +controls for imported/renamed macros, caller parameters and local shadowing. +They explain the contextual choices available to authors of a composable +grammar vocabulary; they are host-expansion controls. diff --git a/docs/research/scheme-list-role-composition.org b/docs/research/scheme-list-role-composition.org new file mode 100644 index 00000000..8dd3ab1f --- /dev/null +++ b/docs/research/scheme-list-role-composition.org @@ -0,0 +1,432 @@ +#+title: Scheme list components: helper identity, POO composition and phases +#+date: 2026-10-06 + +* Question and local status + +The [[file:scheme-rule-context-design.org][rule-context study]] removes repeated self names from single-rule templates. +This study asks what changes when a component contributes multiple rules, and +whether the existing checked POO grammar protocol can compose those contributions +at an explicitly selected compilation phase. + +All code here is a local research fixture against committed baseline +=783381f33fa1b309ac4cc4b0012696f24c852d93=. No public engine or language package +has been migrated, and no general role-import API has been published. + +* Concrete component and tree contract + +[[file:../../t/fixtures/language-pack-research/list-roles.ss][list-roles.ss]] defines a checked POO role factory. The two uses are: + +#+begin_src scheme +(make-nonempty-list-role 'call-arguments 'arguments + '(reference name) '(literal ",") 'argument) + +(make-nonempty-list-role 'array-elements 'elements + '(reference name) '(literal ",") 'element) +#+end_src + +This is the underlying Scheme construction interface, not the final author +syntax. It accepts canonical item/separator expressions and an explicit per-item +field. It delegates role construction to the existing =make-grammar-role=. +The first role contributes these canonical rule values: + +#+begin_src scheme +(arguments + (sequence + (field argument (reference name)) + (reference call-arguments/tail))) +(call-arguments/tail + (repeat + (sequence (literal ",") (field argument (reference name))))) +#+end_src + +The second produces =elements= and =array-elements/tail= with =element= fields. +Both helper identities derive deterministically from the explicit component owner. +They are grammar-domain symbols, not fresh host bindings or automatically private +module identifiers. The caller still chooses the public entry rule name. + +The component contains one required item expression followed by repeated +separator/item pairs. The containers explicitly wrap its entry reference in +=optional= to permit an empty list. There is no list alias and no outer collective +field: =argument= or =element= is attached separately to each item. A comma must +have a following item in this study. Delimiters belong to the container role. + +"Nonempty" here means one required item expression, not a guarantee about +consumed bytes for every possible parameter. This fixture supplies a nonnullable +Name rule and a comma separator. General dependency/nullability constraints need +whole-grammar analysis; the factory's shape check alone cannot establish them. +The agreed default =item= vocabulary remains an author-interface candidate; these +language-specific instances select their fields explicitly. + +* Actual POO composition and selected phase + +A base role supplies tokens, Name, Call, Array, SourceFile and the root rule. +The component roles supply only their two rule contributions. The grammar uses +existing =merge= for the base role and explicit =append= for each list role. + +[[file:../../t/fixtures/language-pack-research/list-stage.ss][list-stage.ss]] imports the provider with =for-syntax=. Its local adapter: + +1. Constructs real checked GrammarRole/Grammar values at expansion phase 1. +2. Calls existing =compile-grammar/receipt= at that phase. +3. Retains the normalized composition IR and receipt as inert inspection values. +4. Reifies the admitted section rows into an existing =deflanguage-grammar= + declaration, whose compiler generates the parser products. + +Runtime parsing then uses the generated parser through existing =parse-source=. +The runtime driver's normalization calls are verification controls, not parser +request work. Nothing here requires POO object dispatch for each token. + +This demonstrates a concrete phase-qualified provider approach. It does not +assume that a module-load phase-0 POO value is available inside a transformer, +and it does not prove that arbitrary roles can be imported at any phase. + +** Important adapter limitation + +The composition receipt is retained separately. Reifying the sections into the +verbose declaration does not propagate =compositionDigest= into the declaration's +newly constructed Grammar IR. Likewise, generated row provenance is not yet a +complete author/component origin map. Section equivalence is checked, but +composition identity and source provenance are not admitted into the final +language artifact by this adapter. + +A production bridge should specify how the already composed canonical IR, +composition receipt and declaration origin are admitted together. Reconstructing +sections alone is insufficient evidence for that contract. The current adapter +is useful to test grammar behavior before choosing this integration design. + +* Native receipts + +[[file:../../t/fixtures/language-pack-research/list-language.ss][list-language.ss]] also defines an inline control language with the same public +structure and no list helper rules. +[[file:../../t/fixtures/language-pack-research/list-runtime.ss][list-runtime.ss]] checks composition and actual parser artifacts: + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" GERBIL_PARSER_LR_TRACE=1 \ + gtimeout 10 gxi t/fixtures/language-pack-research/list-runtime.ss +#+end_src + +The 62-check invocation exited 0 and printed =LIST-RUNTIME-OK: 62 checks passed=. +It covers: + +- Composition operations =merge/append/append= and the three role owners. +- Identical normalized composition when the provider is instantiated at phase 0 + for inspection and phase 1 for generation. +- Deterministic helper identities and equality of nine reified grammar sections. +- Duplicate explicit append rejection with the existing diagnostic. +- Eight accepted inputs: empty, single-item, multi-item and whitespace/newline + variants for calls and arrays. Both parsers accept, the candidate roundtrips, + full event streams equal, and per-item field counts match. +- Eight invalid inputs: leading/trailing separators, adjacent items and missing + delimiters reject in each container family. + +Complete event equality covers node structure, field placement, tokens and +spans. It confirms that the transparent helper rules add no public wrapper nodes +on these inputs. It is finite sample evidence, not general language equivalence +or a performance result. + +** Separate collision receipt and retained timeout + +[[file:../../t/fixtures/language-pack-research/list-collisions.ss][list-collisions.ss]] isolates two composition-only controls without loading parser +runtime/generation modules: + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout 10 gxi t/fixtures/language-pack-research/list-collisions.ss +#+end_src + +It exited 0 and printed =LIST-COLLISIONS-OK: 2 checks passed=. Both diagnostic +message and identity-bearing irritants are asserted: + +| Collision | Expected diagnostic identity | +|-----------+------------------------------| +| Different component owners export the same entry | =rules shared= | +| Different entries use the same component owner/helper | =rules shared-owner/tail= | + +The larger combined attempt emitted all 64 successful checks but exited 124 +under the ten-second bound. It is retained as a timeout, not a passing receipt. +The final drivers separate parser/tree controls from pure composition controls; +the timeout was not widened. The passing evidence is the 62-check driver and +the independent 2-check driver, each with exit 0. + +* Research interpretation + +[[https://www.cs.utexas.edu/~wcook/Drafts/2012/ecoop2012.pdf][Oliveira/Cook, Extensibility for the Masses]] motivates separating construction +interfaces from observations over their products. Our application is to let +Scheme factories construct checked contributions and let existing normalization +and compilation observe their canonical result. A GrammarRole factory is not +therefore the paper's Java object algebra, and its static typing guarantees do +not transfer to this Scheme experiment. + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Culpepper/Felleisen, Fortifying Macros, section 7.2]] treats context-dependent +grammar checks after collecting declarations. Our design inference is that +component shape checks should precede whole-grammar reference/ownership checks, +while errors should retain the author's component and rule vocabulary. The +current collision tests verify existing row identities, not a richer origin map +or Racket syntax-class support in Gerbil. + +This is where Scheme and POO cooperate concretely: macros choose declaration +context and phase; ordinary Scheme builds parameterized grammar contributions; +POO contracts admit and compose them; the canonical compiler generates parser +products. S-expression notation alone does not provide those capabilities. + +* Working recommendation for the next design discussion + +Keep an expression template, a multi-rule component and a complete role as +separate extension scopes. A multi-rule component needs explicit exported entry, +owned helper identities, external rule dependencies, recognition policy and tree +contract. The phase-qualified provider is now a viable study option because it +has actual POO composition and runtime controls. + +Before proposing a public role-import clause, design preservation of composition +identity and declaration origins. Then compare author-facing syntax against a +concrete multi-rule example so the final API can omit mechanical sections without +hiding exports, dependencies, empty/trailing policy or deliberate overrides. + +* Follow-up: existing origin inputs and composition admission + +The [[file:scheme-component-provenance.org][component provenance study]] reuses existing =source-ownership= and =lineage= +inputs. The local adapter now carries rule owners and generated flags into actual +Bound IR; seventeen identity/artifact checks and fourteen language checks pass. +Composition identity is still lost by section reification, and the follow-up +compares admission designs without modifying the public compiler. + +* Scheme design rationale: two required questions + +The following construction is executable through the local checked role factory. + +#+begin_src scheme +(make-nonempty-list-role 'call-arguments 'arguments + '(reference name) '(literal ",") 'argument) + +;; Question 1: Why is this DSL design better? +;; One declaration generates entry and tail rules, sharing the separator/item pattern. +;; An explicit owner makes helper identity traceable; the component emits each field. +;; Gerbil macros supply syntax; checked POO GrammarRole values supply composition. +;; Question 2: Why does this DSL design have a scientific basis? +;; Recognition is item (separator item)*; each item receives its own field. +;; Expansion enters canonical Grammar IR and uses the existing parser engine. +;; Evidence: 62 checks plus 2 collision checks, including selected event parity. +;; Falsifier: admitted trailing separators, changed field shape or undetected collision. +;; Limit: global consumption analysis is needed; the phase adapter loses compositionDigest. +#+end_src + +The improvement is reduced duplicated component authoring under this tree +contract. A smaller declaration alone establishes neither runtime speed nor +better usability for every language author. + +* Follow-up: what does a required item guarantee? + +A list's structural cardinality and its token consumption are different contracts. +The local factory always emits one required item expression followed by a tail. +It cannot establish that a referenced item consumes tokens without knowing that +item's rule. This is a parameter-contract distinction, not a defect in the engine. + +The new [[file:../../t/fixtures/language-pack-research/list-nullability-contract.ss][list-nullability-contract.ss]] obtains the actual checked role's rules +through =grammar-role-ref= and invokes the existing engine functions +=lower-rules=, =compute-nullable= and =compute-first= from +[[file:../../src/compiler/lr.ss][lr.ss]]. The current LR compiler already calls those same analysis functions. +No alternate nullable algorithm, optimizer or parser is introduced. + +** Scheme rationale: the two required questions + +The role construction below is executable through the existing local factory; +the item row is a canonical rule to compose alongside it. + +#+begin_src scheme +(make-nonempty-list-role 'call-arguments 'arguments + '(reference item) '(literal ",") 'argument) + +;; A possible supplied item rule: +'(item (optional (token identifier))) + +;; Question 1: Why is this DSL design better? +;; A shared list idiom removes repeated sequence/repetition structure while +;; retaining explicit item, separator, field and ownership choices. +;; Explaining required expression occurrences separately from token consumption +;; makes its contract usable with reusable references without hidden assumptions. +;; Scheme macros describe intent; POO composes contributions; existing LR analysis +;; resolves whole-grammar properties rather than duplicating them in the macro. +;; Question 2: Why does this DSL design have a scientific basis? +;; The lowered rule predicts nullable(list-entry) = nullable(item), since its tail +;; is a repeat and therefore nullable. This is checked against the engine's fixed point. +;; Evidence: 35 native checks with exit 0 across seven matched item grammars. +;; Falsifier: disagreement with the item/list nullable relation or expected FIRST sets. +;; Limit: nonnullable does not imply productive; nonempty FIRST does not prove +;; successful completion. No scanner-byte, parse-event or conflict-admission claim follows. +#+end_src + +For this list shape, field annotations preserve recognition nullability. The +zero-or-more tail is nullable regardless of the separator. Consequently the +entry's nullable fact equals the item's. With a comma separator, a nullable item +also permits comma in the lowered entry's FIRST set. This is a CFG derivation +fact, not a claim that the full language is admitted under a particular LR +conflict policy or accepts that input at runtime. + +** Native receipt and counterexamples + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout -k 2 10 gxi t/fixtures/language-pack-research/list-nullability-contract.ss +#+end_src + +The complete driver reported =LIST-NULLABILITY-CONTRACT-OK: 35 checks passed=, +exit 0. An earlier six-case version passed 30 checks with exit 0; the seventh +case was added to test the limit of a nonempty FIRST set. + +| Item rule | Item/list entry nullable | Entry FIRST includes identifier | Entry FIRST includes comma | +|-----------+--------------------------+---------------------------------+----------------------------| +| Token identifier | No | Yes | No | +| Optional token | Yes | Yes | Yes | +| Empty expression | Yes | No | Yes | +| Self-reference or token base | No | Yes | No | +| Reference to optional peer | Yes | Yes | Yes | +| Self-reference with no base | No | No | No | +| Token followed by a peer that only references itself | No | Yes | No | + +Every case checks item nullability, tail nullability, entry nullability and both +FIRST memberships. Recursive and indirect cases require the existing fixed-point +analysis; inspecting the immediate surface constructor is insufficient. + +The last two closed grammars have no finite item derivation: one never reaches +a terminal base, while the other has a token prefix but never finishes its tail. +Their results distinguish three obligations: absence of epsilon derivations, +possible starting terminals, and existence of a complete finite derivation. +These facts are not interchangeable. The driver does not add or claim a general +productivity analyzer; the two examples' nontermination of derivation follows +from their explicit recursive rules. + +** Connection to abstract interpretation and author interface + +The [[https://www.di.ens.fr/~cousot/COUSOTpapers/POPL77.shtml][Cousots' author summary of Abstract Interpretation (POPL 1977)]] describes +properties obtained through monotone equations and fixed points, with conclusions +limited by the abstraction. We use that principle to state exactly what the +existing nullable/FIRST facts tell the DSL. This is a code-level application of +the summary, not a formal proof of the parser or a new analysis derived from the +full paper. + +The working interface recommendation is to specify required grammar occurrences +separately from consumption, productivity and output-field contracts. If a future +constructor promises token-consuming items, record that as an obligation on the +resolved item identity and consume the engine's existing analysis after grammar +composition and reference validation. Do not infer it from a reference name or +reimplement nullable analysis within each template. + +A nullable item can be intentional. This study does not propose rejecting all +such components, renaming the public vocabulary immediately or migrating any +language. The author policy and diagnostic behavior for stronger consumption +contracts remain under discussion. AST field cardinality and byte consumption +also need their own controls; token-level CFG facts alone do not settle them. + +* Follow-up: separator contracts and resolved repetition bodies + +The required item is not the only consumption parameter. One iteration is +=separator followed by item=, with the per-item field wrapper preserving its +recognition meaning. For this CFG shape: + +=nullable(iteration-body) = nullable(separator) AND nullable(item)=. + +A nullable separator does not imply a nullable iteration when the item is +nonnullable. A nullable item with a consuming separator also yields a nonnullable +iteration. These conclusions concern epsilon derivations over the lowered token +grammar; they are not byte-consumption, productivity or runtime-termination proofs. + +** Scheme rationale: the two required questions + +The public constructor macro and the role's resolved dependencies are distinct +inputs to this local study: + +#+begin_src scheme +(grammar-expression + (repeat (seq (reference separator) + (field argument (reference item))))) + +;; Canonical dependency rows for a consuming iteration with a nullable separator: +'(separator (empty)) +'(item (token identifier)) + +;; Question 1: Why is this DSL design better? +;; One reusable list idiom can support explicit separators and adjacency without +;; repeating its sequence, field and repetition structure in each language package. +;; Its contract states which combined body must consume instead of blanket-rejecting +;; every nullable separator. That keeps author intent and language policy explicit. +;; Question 2: Why does this DSL design have a scientific basis? +;; Nullable analysis predicts the conjunction above and resolves item/separator references. +;; Evidence: 29 native checks with exit 0 cover all four nullable combinations, +;; local-versus-resolved summaries and the existing explicit-empty rejection. +;; Falsifier: a resolved body disagrees with the conjunction or a local reference +;; summary is incorrectly promoted to a whole-grammar consumption guarantee. +;; Limit: no parser admission, runtime liveness, field-count or scanner-byte claim. +#+end_src + +An adjacency use would still require an intentional tree and lexical contract. +This witness does not recommend silently replacing commas with empty separators +in an existing language. The role factory accepts grammar expressions; whether +a particular separator belongs in a language remains the language author's choice. + +** Local constructor checks versus whole-grammar facts + +[[file:../../src/grammar/algebra.ss][grammar/algebra.ss]] already implements a structural +=grammar-expression-nullable?= helper. The internal repeat constructors use it +to reject visibly nullable bodies. The public =grammar-expression= macro routes +through those constructors. The helper takes no reference environment, and its +fallback for references is =#f=. Consequently that result cannot be read as a +proof that every referenced rule is nonnullable. + +The study deliberately uses the same reference-bearing body in all four cases. +Local construction accepts it in each case. Existing LR fixed-point analysis, +after lowering the actual dependency rows, differentiates their resolved facts. +The all-empty case has a nullable iteration although local inspection of its +references returns =#f=. This is the scope boundary of the local structural +helper, not evidence that reference values should be guessed during construction. + +A separate control uses explicit empty operands through the public macro: + +#+begin_src scheme +(grammar-expression (repeat (seq (empty) (field argument (empty))))) +;; Existing rejection: "grammar repeat operand must consume input" +#+end_src + +No constructor, compiler analysis or rejection policy is changed in this study. +The role factory currently constructs canonical values directly; shape admission, +constructor validation and whole-grammar analysis are distinct stages. The +proposed consumption obligation would need explicit placement after reference +resolution, if adopted; this witness does not implement that production gate. + +** Native receipt + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout -k 2 10 gxi t/fixtures/language-pack-research/list-repetition-contract.ss +#+end_src + +[[file:../../t/fixtures/language-pack-research/list-repetition-contract.ss][list-repetition-contract.ss]] reported =LIST-REPETITION-CONTRACT-OK: 29 checks passed=, +exit 0. Four cases each check item, separator, iteration-body, entry and tail +nullability, the structural reference summary and public repeat construction. +The final check verifies the explicit-empty rejection's message and irritant. +An initial attempt called the unexported =grammar-repeat= helper directly and +exited 70. The corrected driver uses the public =grammar-expression= macro; +no private API was exported to accommodate the experiment. + +| Item | Separator | Resolved iteration body nullable | Entry nullable | +|------+-----------+-----------------------------------+----------------| +| Token | Comma | No | No | +| Empty | Comma | No | Yes | +| Token | Empty | No | No | +| Empty | Empty | Yes | Yes | + +The tail is nullable in every case because repetition allows zero iterations. +Its nullable status therefore cannot answer whether one iteration consumes. +Nor does a nonnullable iteration prove that it can finish; the previous +unproductive-rule controls remain relevant. + +** Design consequence and remaining gate + +Use local structural checks for immediately known constructor constraints, and +existing resolved compiler facts for reference-dependent constraints. A future +constructor contract should retain the repeated body's identity and author origin +so a failed resolved obligation can explain the component declaration. That +association and any resulting diagnostic remain design work. + +This follows the bounded-analysis principle discussed in the +[[https://www.di.ens.fr/~cousot/COUSOTpapers/POPL77.shtml][Cousots' author summary]]: conclusions must match the facts their abstraction +actually tracks. The research does not claim a new abstract interpretation proof +or a need to replace the engine's existing implementation. diff --git a/docs/research/scheme-macro-binding-experiments.org b/docs/research/scheme-macro-binding-experiments.org new file mode 100644 index 00000000..a290a4bd --- /dev/null +++ b/docs/research/scheme-macro-binding-experiments.org @@ -0,0 +1,111 @@ +#+title: Gerbil macro binding experiments for language pack design +#+date: 2026-10-06 + +* Question and evidence boundary + +The [[file:scheme-language-pack-semantics.org][semantic study]] separates grammar identities, public labels, macro bindings +and host expressions. This experiment asks a narrower executable question: +what happens to a macro's private host reference when its syntax is preserved +versus converted to datum and reconstructed in a caller's context? + +Baseline examined: =a99b3646e7ee68c1010ca47985c762d658fef611=. +The experiment executes ordinary Gerbil macros across two source modules. It +does not import =deflanguage=, construct Grammar IR, or execute a parser. +Its results establish host expansion behavior for vocabulary composition. +All executable research fixtures are Scheme. + +* Official and paper basis + +The official [[https://gerbil.scheme.org/tutorials/languages.html][Gerbil custom languages tutorial]] uses contextual expanders, syntax +construction and renamed exports. It demonstrates intentional contextual +construction; =datum->syntax= is a tool whose context must match the intended +binding policy, not an operation to prohibit everywhere. + +Matthew Flatt's [[https://www-old.cs.utah.edu/plt/scope-sets/general-macros.html][Binding as Sets of Scopes, section 3]] discusses syntax objects, +identifier comparisons, procedural macros and module phases. Our experiment +uses Gerbil's own expansion and evaluation. It does not assume that Racket's +scope representation or edge cases are Gerbil's implementation. + +* Reproducible Scheme controls + +Files: + +- [[file:../../t/fixtures/language-pack-research/fragments.ss][fragments.ss]] defines a private =private-value= and exports three macros. +- [[file:../../t/fixtures/language-pack-research/consumer.ss][consumer.ss]] imports one macro under a new name, declares its own same-spelling + value, and runs assertions including local shadowing. +- [[file:../../t/fixtures/language-pack-research/binding.ss][binding.ss]] is the executable entry. + +The preserving control returns the definition-context identifier through an +ordinary =defrules= template. The parameter control returns syntax provided by +the caller. The reconstructing control deliberately performs: + +#+begin_src scheme +(datum->syntax #'anchor (syntax->datum #'private-value)) +#+end_src + +Here =anchor= is the caller's identifier. The source reference and caller +reference have the same spelling, so equality of their datum names cannot +establish equality of their binding. + +From the repository root: + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" gerbil env gxi t/fixtures/language-pack-research/binding.ss +#+end_src + +The local research run used a ten-second external =gtimeout= bound around this +command. It completed with exit status 0 and six =BINDING-CASE-OK= lines followed +by =BINDING-RESEARCH-OK: 6 checks passed=. Each case prints only after its actual +assertion succeeds. The experiment adds no dependency and requires no native +parser build. It is independent of compiled parser cache contents. + +* Observed results + +| Case | Observed value | Interpretation | +|------+----------------+----------------| +| Imported and renamed preserving macro | =definition-site= | Import renaming retains the macro's private reference | +| Reconstructed reference at module use site | =use-site= | Caller context changes resolution of the same spelling | +| Explicit caller parameter | =use-site= | Supplied syntax refers to the caller's value | +| Preserving macro inside local shadow | =definition-site= | Definition context survives local same-spelling bindings | +| Reconstructing macro inside local shadow | =local-shadow= | Reconstruction can select the caller's local binding | +| Parameter macro inside local shadow | =local-shadow= | Preserving a caller argument retains its local binding | + +The last two rows intentionally have the same value but different origins. +One accepts an argument whose meaning belongs to the caller; the other discards +a definition-context reference and adopts a caller context. A useful language +implementation must know which transformation it intended. + +* Applying the result to composable DSL vocabulary + +=src/language/grammar.ss= expands grammar fragments before converting expression +atoms to datum. It constructs backend rows from datum using the language prefix +as the syntax anchor. These are different information classes: + +| Value crossing elaboration | Binding obligation | Candidate treatment | +|----------------------------+--------------------+---------------------| +| Node/field label | Intentional published symbolic identity | Extract datum; validate identity and catalog ordering | +| Rule/token reference | Declared DSL namespace identity | Resolve before losing relevant origin; retain canonical core names | +| Macro constructor reference | Host compile-time binding | Expand while its original syntax context is available | +| Backend digest/procedure expression | Host expression with lexical references | Preserve original expression syntax through assembly | +| Generated public parser binding | Deliberate API name derived from the pack prefix | Construct in the documented export context | +| Generated private helper | Implementation binding | Preserve private definition scope or use host-generated fresh identity | + +The experiment explains why a grammar vocabulary should document the intended +context of each parameter and introduced name. Public labels, caller grammar +references and implementation helpers have distinct roles. This classification +supports predictable composition using the existing Gerbil macro facilities. + +* Follow-up vocabulary studies + +1. Compare an explicit binary production with a parameterized infix template. +2. Compose separated-list and delimiter templates while preserving the intended + node and field shape. +3. Import vocabulary from ordinary Scheme modules and rename it at the use site. +4. Explain expanded productions in terms of their original arguments and source. +5. Admit the expanded grammar through the current engine and compare its products + with the explicit rules before evaluating compiler or runtime cost. + +The primary design synthesis is +[[file:scheme-deflanguage-design-synthesis.org][scientific grammar vocabulary under deflanguage]]. These six host controls are +supporting evidence for that design study. Runtime, schema and target-product +qualification remain distinct experiments. diff --git a/docs/research/scheme-nested-declaration-context.org b/docs/research/scheme-nested-declaration-context.org new file mode 100644 index 00000000..24e4fd14 --- /dev/null +++ b/docs/research/scheme-nested-declaration-context.org @@ -0,0 +1,245 @@ +#+title: Nested Scheme vocabulary: parameter origins and author declaration context +#+date: 2026-10-06 + +* Research question and scope + +Can a language package wrap or rename a component vocabulary without losing +precise blame? Does preserving the parameter location also preserve the outer +syntax that the author actually wrote? + +This follows the [[file:scheme-declaration-diagnostics.org][declaration diagnostics study]]. It adds local Scheme +controls against baseline =783381f33fa1b309ac4cc4b0012696f24c852d93=. +No public DSL, diagnostic renderer or general expansion-history API is introduced. + +* Actual vocabulary and the two required questions + +[[file:../../t/fixtures/language-pack-research/nested-list-vocabulary.ss][nested-list-vocabulary.ss]] exports a one-layer wrapper and a two-layer wrapper. +Both forward matched syntax rather than rebuilding the author parameters. + +#+begin_src scheme +(defrules argument-list () + ((_ binding owner entry item separator label) + (deflocated-list binding owner entry item separator (field label)))) + +(defrules public-argument-list () + ((_ binding owner entry item separator label) + (argument-list binding owner entry item separator label))) + +;; At the consumer boundary: +(import + (rename-in "nested-list-vocabulary" + (public-argument-list imported-arguments))) + +;; Invalid author input retained by the diagnostic collector: +(imported-arguments component call-arguments arguments + (reference name) (literal ",") 63) + +;; Question 1: Why is this DSL design better? +;; A package can provide domain vocabulary and rename imports while forwarding +;; the original parameter syntax. Authors need not manage helper-rule details. +;; Compared with datum reconstruction, syntax forwarding preserves exact blame +;; coordinates through these wrappers using Gerbil's existing macro machinery. +;; Question 2: Why does this DSL design have a scientific basis? +;; The contract independently checks error type, message, term, location and context. +;; Evidence: 24 native checks with exit 0, including a source-losing negative control. +;; Falsifier: a forwarded parameter points to different text or a renamed import +;; changes which component validates the parameter. +;; Limit: the inner context remains deflocated-list; the outer author form is not +;; automatically recovered, and no whole expansion-chain or parsing claim follows. +#+end_src + +The forwarding wrappers do not add a new list algorithm or parser. They preserve +an existing component invocation and its explicit field contract. This is a +syntax-abstraction advantage under a specific task, not evidence that every +wrapper improves author comprehension. + +* Matched negative control + +The same vocabulary module exports =reconstructed-argument-list=. Its transformer +keeps lexical context but converts each matched argument to a datum and rebuilds +the component call with source arguments =#f #f=: + +#+begin_src scheme +;; Inside the negative-control syntax-case branch: +(datum->syntax #'reconstructed-argument-list + (list 'deflocated-list + (syntax->datum #'binding) (syntax->datum #'owner) + (syntax->datum #'entry) (syntax->datum #'item) + (syntax->datum #'separator) (list 'field (syntax->datum #'label))) + #f #f) +#+end_src + +The rebuilt call still reaches the intended component and reports the same typed +field-label failure. Its blame datum is correct, but its parameter location is +absent. Correct binding and correct datum are therefore insufficient evidence of +preserved source attribution. This explicit reconstruction is a control, not a +recommendation for production vocabulary. + +* Native receipt + +[[file:../../t/fixtures/language-pack-research/nested-diagnostics.ss][nested-diagnostics.ss]] receives actual declarations as reader syntax. It uses +the existing =core-expand= to reach validation through the nested macro layers, +catches the resulting SyntaxError and returns inert records. It never evaluates +the invalid generated definitions. + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout 10 gxi t/fixtures/language-pack-research/nested-diagnostics.ss +#+end_src + +The invocation reported =NESTED-DIAGNOSTICS-OK: 24 checks passed= and exited 0. +There are four cases, each with six assertions: + +| Case | Blame datum | Reader position | Context supplied by component | +|------+-------------+-----------------+-------------------------------| +| Direct component | 61 | Points to actual 61 | deflocated-list | +| One forwarding wrapper | 62 | Points to actual 62 | deflocated-list | +| Two wrappers, outer import renamed | 63 | Points to actual 63 | deflocated-list | +| Datum reconstruction | 64 | Explicitly absent | deflocated-list | + +Each case checks exception type, exact message, blame datum, presence/absence of +location, source-token correctness or explicit unknown, and the context head. +The positive coordinate assertions read the actual file at the reported position. +The cases use distinct values to identify the author occurrence. They do not +claim that imported alias spelling must replace the implementation's message. + +* What the paper adds + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Fortifying Macros (Culpepper and Felleisen, ICFP 2010)]], section 7.1, discusses +loop clauses rewritten into an internal CPS protocol. The authors reconstruct +original syntax for checking because errors should describe the user's clause. +This motivates a separate author-context obligation here: retaining a bad +parameter's location alone does not recover the outer declaration vocabulary. +Their Racket mechanism is not assumed to exist unchanged in Gerbil. + +The concrete inference from our controls is that forwarding provides precise +parameter origins, while outer declaration explanation needs an explicit +protocol. The control does not establish whether that protocol should be stored +in syntax, a diagnostic record or component elaboration data. + +* Compare design directions before choosing a public protocol + +| Direction | What it provides | Design cost / remaining check | +|-----------+------------------+-------------------------------| +| Forward syntax, accept inner context | Precise parameter blame in current witness | Outer vocabulary absent from error context | +| Validate outer vocabulary directly | Local label witness preserves author context and alias | Other constraints delegate; validation order is a surface policy | +| Carry original declaration with elaboration | Potentially explain inner failure through outer form | Define phase, ownership and persistence; no public implementation yet | +| Reconstruct everything from datum | Keeps explicit values and chosen lexical context | Loses reader origins in this matched control | + +The working recommendation is to forward matched syntax by default. A vocabulary +that changes the author's declaration shape should explicitly own that shape's +validation or carry its original declaration to the validating stage. Blanket +catch-and-rethrow of every compiler exception is not a substitute: an unrelated +failure must not be presented as an author parameter error. + +For POO integration, distinguish three identities: the Scheme wrapper binding, +the grammar contribution owner, and the public tree node/field. Renaming the +wrapper import does not itself rename the grammar-domain symbols it passes. +This study does not bind a complete nested language or change the earlier +compositionDigest and actual-language timeout boundaries. + +* Next qualification questions + +- Preserve an outer declaration across wrappers that intentionally change shape. +- Specify which layer owns a constraint and how shared validators avoid drift. +- Check simultaneous invalid parameters and deterministic diagnostic ordering. +- Connect selected contribution origins to override/removal without rewriting + historical declarations as if they were effective bindings. +- Measure whether a developer can identify the outer declaration and repair it; + metadata assertions alone do not establish that usability improvement. + +* Follow-up: explicit outer contract and error ordering + +The next local witness implements one direction from the comparison: an outer +vocabulary owns its positional label contract, then forwards valid syntax to the +existing component. It does not intercept or relabel arbitrary inner exceptions. + +[[file:../../t/fixtures/language-pack-research/declaration-contracts.ss][declaration-contracts.ss]] defines an ordinary Scheme function imported at +transformer phase. Both the inner component's four identifier checks and the +outer vocabulary use that function. This shares the predicate and exception +construction; each layer still chooses its parameter name, context and order. + +#+begin_src scheme +;; Ordinary Scheme function, imported with for-syntax by vocabulary modules: +(def (require-declaration-identifier term declaration operation parameter) + (unless (identifier? term) + (raise-syntax-error #f + (string-append operation ": " parameter " must be an identifier") + term declaration))) + +;; Matched branch of the local author-arguments transformer: +(require-declaration-identifier #'label stx "author-arguments" "label") +#'(deflocated-list binding owner entry item separator (field label)) + +;; Question 1: Why is this DSL design better? +;; The outer vocabulary explains its own positional label contract using the +;; declaration the author wrote, including an imported alias in the context. +;; A shared Scheme function avoids separate predicate/exception implementations; +;; syntax forwarding retains the original label while the existing POO role is reused. +;; Question 2: Why does this DSL design have a scientific basis? +;; Failure behavior and successful elaboration have separate observable contracts. +;; Evidence: 28 checks with exit 0 cover five failures and three valid-expansion controls. +;; Falsifier: wrong context or blame, changed target binding/core call, lost label source, +;; or a multi-error result inconsistent with the declared validation order. +;; Limit: delegated failures still have inner context; no universal ordering or +;; complete original-declaration bridge is established by this witness. +#+end_src + +The complete wrapper is [[file:../../t/fixtures/language-pack-research/author-list-vocabulary.ss][author-list-vocabulary.ss]]. It validates label first, +then delegates binding, owner and entry checks. The inner component retains its +existing binding, owner, entry, label order. Keeping its label check is necessary +because direct component callers do not pass through this wrapper. Predicate +sharing does not eliminate these distinct admission boundaries. + +** Native evidence + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout 10 gxi t/fixtures/language-pack-research/author-diagnostics.ss +#+end_src + +[[file:../../t/fixtures/language-pack-research/author-diagnostics.ss][author-diagnostics.ss]] reported =AUTHOR-DIAGNOSTICS-OK: 28 checks passed=, +exit 0. After sharing the validator, reader-diagnostics.ss was also rerun: its +48 checks passed with exit 0, preserving the inner component's original four +parameter diagnostics and source coordinates. Five invalid calls each check +exception type, exact message, blame datum, +actual source-token coordinates and declaration context: + +| Author case | Selected failure | Context head | +|-------------+------------------+--------------| +| Outer label 81 invalid | Outer label contract | author-arguments | +| Renamed import label 82 invalid | Outer label contract | imported-author-arguments | +| Outer owner 83 and label 84 invalid | Label 84 first | author-arguments | +| Outer owner 85 invalid, valid label | Delegated owner contract | deflocated-list | +| Direct owner 86 and label 87 invalid | Owner 86 first | deflocated-list | + +The three successful-elaboration controls compare a valid wrapper's expansion +with an explicit component invocation: the target is the same macro binding, +the complete call datum is equal, and the forwarded label retains the original +reader location. These are expansion controls; they do not establish parser-event +parity or artifact integration for this new wrapper. + +** Design consequence: equivalent valid syntax can have different error order + +The multi-error control intentionally retains the difference between the outer +label-first policy and the direct component's owner-first policy. Valid expansion +equality does not imply diagnostic equivalence for invalid inputs. This is an +observable surface-language decision rather than a grammar-recognition change. + +A shared predicate helps maintain consistent parameter acceptance. A shared +ordered contract would be needed if all wrappers were required to report the +same first failing parameter. That stronger policy has not been selected. The +current witness checks the chosen local order, not every permutation of errors. + +The working recommendation is to declare which constraints belong to a wrapper, +share their implementation as ordinary Scheme functions at the appropriate phase, +and state the order before treating diagnostics as an author API. If full outer +context is required for delegated grammar failures too, this witness is only one +part of the solution; the original-declaration protocol remains open. + +* Follow-up: official syntax parameters + +The [[file:scheme-syntax-parameter-context.org][syntax-parameter context study]] supplies a scoped author-form protocol for +delegated identifier checks. Its positive role construction control also exposes +and repairs an initial declaration-versus-expression scope error. Full expansion +history and artifact integration remain open. diff --git a/docs/research/scheme-parser-performance-framework.org b/docs/research/scheme-parser-performance-framework.org new file mode 100644 index 00000000..bb521b41 --- /dev/null +++ b/docs/research/scheme-parser-performance-framework.org @@ -0,0 +1,1829 @@ +#+TITLE: High-performance parsing framework for the Scheme-owned engine + +Current architecture decision and unified acceptance plan: +[[file:../architecture/0010-persistent-recognition-snapshots.org][RFC 0010: Persistent recognition snapshots and explicit publication]]. +Historical experiments below remain receipts, rather than independent migration priorities. +#+DATE: 2026-09-28 +#+OPTIONS: toc:2 + +* Working decision + +Keep the Scheme-owned compiler as the authority, and evaluate two algorithmic +paths against distinct workloads. For fresh, whole-source parsing and grammar +generation, test conflict-directed LR state partitioning and generated lexer +design. For small edits, Tree-sitter's +contextual lexing and reusable syntax subtrees remain the stronger reference. +Both preserve deterministic LR execution on the ordinary path and admit +generalized parsing only where the grammar requires it. This is a research +direction, not an adoption or performance claim. Neither external runtime +would be embedded as the parser authority. + +The isolated Scheme prototype uses canonical LR(1) as a semantic reference. +It has two compact-table candidates: a canonical-state quotient and a direct +LR(0)-derived forward/backward follow partition. The direct route does not +enumerate canonical DFA states, but uses conservative potential-conflict +lookaheads rather than langcc's exact Definition 3.24. Generation and parse +cost must be measured before any product cutover. The existing LALR route +remains the production default. + +The current engine already has important pieces: =src/compiler/lr-automaton.ss= +builds an interned LR(0) automaton; =src/compiler/lr-lookahead.ss= propagates +LALR lookaheads; =src/runtime/lr-parser.ss= executes deterministic actions and +selective GLR; =src/compiler/machine.ss= generates parser-directed lexical modes; +and =src/runtime/incremental.ss= resumes sparse LR checkpoints and reuses +tokens only under matching parser-directed lexical modes. The LR executor +still feeds the changed suffix and publishes a complete artifact. +These are source observations; measured costs appear below. + +* Evidence from papers and maintained implementations + +| Reference | Relevant mechanism | Fit and limit | +|- +| Wagner and Graham, /Efficient and Flexible Incremental Parsing/ (1998), https://doi.org/10.1145/293677.293678 ; author PDF https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf | Sentential-form parsing, balanced sequences, and optimal node reuse for edits. | The theoretical basis for edit-local subtree reuse. It does not prescribe this repository's source identity or artifact contract. | +| Tree-sitter, https://tree-sitter.github.io/tree-sitter/creating-parsers/3-writing-the-grammar.html and https://github.com/tree-sitter/tree-sitter/blob/master/lib/include/tree_sitter/api.h | On-demand, parse-state-aware lexing; LR with GLR at conflicts; old-tree reuse after an exact edit. | Closest complete architecture to the existing Scheme compiler plus lossless tree product. Its C runtime is a reference, not a dependency. | +| Lezer system guide, https://lezer.codemirror.net/docs/guide/ | Offline LR tables, declared GLR sites, contextual tokens, balanced repeat trees, fragment reuse, and error recovery. | Independent implementation showing the same family can be expressed with a small runtime. Its trees and editor recovery contract differ from ParseArtifact v1. | +| Trofimovich, /RE2C: A lexer generator based on lookahead-TDFA/ (2020), https://doi.org/10.1016/j.simpa.2020.100027 ; implementation https://re2c.org/ | Compile regular lexical rules into deterministic automata ahead of time. | Strong model for a first Scheme-generated lexer experiment. External and nonregular scanners need an explicit fallback. | +| Denny and Malloy, /The IELR(1) algorithm/ (2010), https://doi.org/10.1016/j.scico.2009.08.001 ; Bison manual https://www.gnu.org/software/bison/manual/bison.html | Avoid some LALR inadequacies without canonical LR table explosion. | Candidate when grammar acceptance or conflict counts show LALR is the binding constraint; not itself an edit-local performance framework. | +| Horspool, /Even faster LR parsing/ (1990), https://doi.org/10.1002/spe.4380200602 | Compile LR table entries into directly executed parser code. | Candidate for Scheme AOT generation from the embedded Parser IR; integer-packing the existing interpreted table is not an implementation of this method. | +| Sperber and Thiemann, /The Essence of LR Parsing/ (1995), https://doi.org/10.1145/215465.215579 ; author PDF https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf | Specialize a functional LR interpreter against a fixed grammar; the paper presents direct-style and continuation-passing variants without an explicit parsing stack. | A Scheme-specific code-generation model beyond replacing action lookup alone. Its AST and error-recovery contracts differ from this engine's lossless ParseArtifact and selective GLR, so a generated path must retain the existing interpreter as a semantic fallback. | + +The cited sources describe algorithms and implementation capabilities. None +establishes that this repository will become faster merely by adopting a name +or data structure. + +* Recent research and implementation audit (2022–2026) + +| Work | Primary evidence and implementation | What it establishes | Transfer limit | +|- +| Zimmerman, /Practical LR Parser Generation/ (2022), https://arxiv.org/abs/2209.08383 ; implementation report https://arxiv.org/abs/2209.08385 ; code https://github.com/jzimmerman/langcc | Optimized LR NFA construction, grammar CPS transformation, recursive-descent actions, and generated DFA lexer. The author reports generated parser throughput 1.2x that of Go 1.17.8's standard parser and 4.3x that of CPython 3.9.12 on the paper's workloads. | This is the most directly relevant end-to-end fresh-parse prototype. The generated code also uses a pool allocator, and the paper notes differences from the reference grammars; its ratios cannot isolate an LR algorithm or predict Scheme performance. The Go and Python generators produce roughly 69K/70K lines of code in 61s/48s respectively, so generation cost matters. XLR is discussed theoretically but explicitly *not implemented* in langcc (author slides: https://jzimmerman.io/langcc_stanford_oct_20_2022.pdf). | +| Evans, /APLR(1)/ (June 2026), https://github.com/BranchTaken/Hemlock/blob/main/doc/reports/aplr1/aplr1.md ; Hocc code in the same repository | Adequacy-preserving remerging of canonical LR(1) states, including unresolved ambiguities. The author compares automaton sizes and generation costs with LALR, IELR+, and LR(1). | A technical report and author implementation, not independent peer-reviewed throughput evidence. Some reported grammars have substantial generation-time and memory costs. Treat it as a table-correctness and size candidate, not an automatic parser-speed upgrade. | +| Vo et al., /An Empirical Comparison of General Context-Free Parsers/ (June 2026 preprint), https://arxiv.org/pdf/2606.08465 | Unified Rust implementations compare CYK, Valiant, Earley, GLL, RNGLR, BRNGLR, LL(1), and LR(1) over 22 grammars. GLR is the fastest generalized family; on deterministic grammars it has a 3x median cost relative to LR(1). | Scannerless, generated inputs on one Apple M1; the paper reports no production Scheme or ParseArtifact measurements. Its result supports keeping a deterministic fast path, then testing RNGLR only at genuine conflicts. No public artifact repository was identified in this pass. | +| Scott and Johnstone, /Earley Table Traversing Parsers/ (online 2025, journal 2026), https://doi.org/10.1016/j.scico.2025.103335 | Precomputed Earley tables make the authors' Java and ANSI C examples faster and smaller than their Earley baseline. | A possible general-grammar fallback, but the cited abstract does not establish superiority over deterministic LR or RNGLR for this engine. | + +The 2026 comparison and the langcc results address different questions. The +former compares algorithm families under a shared Rust harness; the latter +measures generated full frontends against language-specific standard parsers. +Neither supplies a matched result for this repository's Scheme grammar, +scanner, lossless artifact, or incremental edit workload. + +* Implementation coverage audit (2026-09-29) + +The status below describes implemented mechanisms and evidence gates; a single +percentage would mix papers with different goals and give false precision. +The audited base is PR #13 at =a4e6ed5601d60852df5cdc6f83ea2bd079cef473=. + +| Research route | Current Scheme implementation | Remaining gate | +|- +| langcc direct LR(1) construction | Experimental k=1 LR(0)-derived backward/forward follow partition and canonical reference; candidate conflicts are conservative, and ordinary generated languages still default to LALR. The explicit =lalr-then-follow= choice falls back only when LALR has an unresolved reject-policy cell. | Exact paper candidate construction, larger conflict-bearing language corpus, generation memory/time and complete ParseArtifact throughput comparison. CPS grammar transformation, per-symbol attributes, recursive-descent actions, and XLR are not implemented here. | +| Canonical remerge / IELR / APLR | Canonical LR(1) and a conflict-preserving quotient are experimental comparison routes. | Neither Bison IELR nor Hocc APLR adequacy-preserving subgraph remerge is implemented; do not label the quotient as either algorithm. | +| re2c / Tree-sitter contextual lexing | Production Scheme lexer has interned LR mode admission, a shared literal trie, and a mode-local subset DFA for six closed forms (=whitespace+=, =horizontal-whitespace+=, =newline+=, =decimal-digit+=, =identifier=, =number=). The numeric path tracks the last accepting offset across incomplete fraction and exponent states. Remaining expressions retain per-mode ASCII first-character dispatch and their existing scanners. | Extend the DFA only where closed expression semantics admit it; compare complete grammar fixtures and batch throughput, especially Unicode and mode-boundary cases. | +| Wagner / Tree-sitter incremental tree reuse | Production Scheme sessions retain sparse LR prefixes and mode-certified tokens; same-action local token/event windows can preserve event topology, including changing trivia counts. | General reusable-subtree records with entering LR state, lexical context, lookahead/fragility bounds, and grammar identity; significant-token insertions/deletions still replay the LR suffix. | +| Deterministic LR with selective GLR | Production Scheme runtime runs deterministic actions and branches at declared conflicts; ordinary LALR and selective-GLR tests run under v0.19. | No RNGLR or XLR implementation or matched production-grammar comparison; keep generalized work on conflict scenes. | +| Scheme to Rust | Native FFI exposes borrowed canonical GPA1 nodes and fields using the handle catalog; the independent AOT backend generates source-range indexes and conformance fixtures. | The experimental follow-partition route, new Scheme lexical DFA, and future subtree certificates have not been admitted as a unified generated contract. | + +The focused direct-route test now also compares LALR and follow-partition LR +roots for all 12 accepted embedded HCL v2.24 specsuite fixtures. Both tables +have 133 states in this language, so this extends real-language equivalence +coverage but does not exercise a necessary LR(1) split. The existing mixed +fixture exercises that split (20 direct states versus 23 canonical states), +while the 16-region family checks scale (211 states on both routes). A local +20-run benchmark completed, but short batches had large warmup/GC variance; +state counts and products are firmer evidence than its throughput ratios. + +The closed-form lexer DFA is now in Scheme. It preserves the +existing parser-directed modes and scanner fallback; =t/ranked-regular-scanner-test.ss= +compares every offset of mixed ASCII/Unicode sources, including complete and +incomplete numeric exponents, with the old individual +scanners; =t/grammar-composition-test.ss= also checks that LR modes distinguish +two regular rules with the same spelling. The pure Scheme +=t/benchmarks/regular-mode/benchmark.ss= compares the old ASCII-prefiltered +mode scan against the combined scanner: a 100,000-call sample improved all +four selected sources, while a shorter 20,000-call Unicode sample reversed +direction. A later 100,000-call number sample also favored the combined +scanner, but these local microbenchmarks do not establish stable Unicode or +whole-parser throughput gains. A matched full ParseArtifact comparison now +uses =t/benchmarks/versioned-languages/matched-batch.ss=, with independent +Gerbil v0.19 package trees, identical pinned dependencies, and the same 35 +accepted HCL, GQL, openCypher, and TLA+ sources (5,818 characters). Each +process warms up three batches, then checks success and exact roundtrip in +60 timed batches, with GC outside each timed region. Against the pre-number-DFA +revision =998daf33=, the two current-revision medians were 16.96 and 21.33 ms; +the two baseline medians were 17.54 and 29.61 ms. Fastest samples in all four +processes were 10.67--11.00 ms. The second pair and p95 values varied widely, +so this corpus does not establish an end-to-end gain from the number DFA. These +numbers measure a complete batch, not process import or compilation; they also +do not establish per-language scalability. Keep the DFA for its lexical +semantics and focused speed evidence, but measure phase costs before adding +more rule forms. The language-pack and generated Rust conformance gates +remain required. Add subtree certificates and edit-local LR actions where edits +justify them. The follow-partition route remains an +explicit construction choice until a real conflict-bearing language corpus +justifies switching a scene from LALR. + +The generated lexer now also uses each LR mode's ASCII first-character table +to skip the regular scanner or shared literal trie when the first transition +is impossible. Unicode retains the existing scanner path; longest match, rank, +and declaration order still choose among actual matches. This is a mode-local +DFA entry dispatch, not a new parser authority. The full v0.19 language suite +passed. On a 3,200-term arithmetic parse, the older baseline and this version +had 30-sample p10 values of 9.38 and 7.71 ms; both minima were about 7.60 ms, +and the medians varied from 56.07 to 29.02 ms under host noise. In the +35-fixture four-language batch, medians were 22.64 ms on the older baseline +and 25.30 ms with this dispatch; minima were 10.71 and 10.58 ms. This does +not establish a stable end-to-end gain. The pure Scheme +=t/benchmarks/arithmetic-scale/matched-stream.ss= makes the long-input +comparison reproducible before widening this dispatch further. + +The local minute-scale wait seen during this slice was qualification time: +the package-only scanner import took 2.9 seconds, while isolated lexical, +composition, and HCL test commands took about 5, 7, and 9 seconds. A full +suite through the interpreted =gerbil env gxi= test entry took about 46 seconds; +the same suite through native =gerbil test= took 25.6 seconds with the same +pre-import 1 GiB heap fence. These wall times include process startup and +module admission, so they must not be used as per-token or per-parse latency. +The root =justfile= exposes a focused =test-lexer= gate (about 4 seconds with +native =gerbil test=) and a separate full suite gate; CI uses the native entry. + +* Scheme prototype and measured boundary + +=src/compiler/lr-lookahead.ss= now exposes an optional canonical LR(1) +construction. =src/compiler/lr-partition.ss= groups states with equal LR(0) +cores only when their union adds no raw parser conflict, then repeatedly +splits groups until transition targets are congruent. It uses Gerbil's +=:std/struct/queue=, =:std/vector/extensible=, =:std/hash/misc=, standard +tables, and the existing indexed-set masks. The partition code does not +rebuild groups with =append=. =src/compiler/lr-follow-automaton.ss= now builds +a second experimental path directly from LR(0): it refines follow blocks +backward, coarsens and refines them forward, determinizes the quotient NFA, +and publishes the ordinary LR table contract. =compile-lr-spec= selects these +routes through an explicit fifth argument. =compile-parser= forwards an +optional construction choice from canonical Grammar IR to that Scheme +compiler; both defaults stay LALR. + +The focused Scheme test includes the classic LR(1)-but-not-LALR(1) grammar: +LALR reports a reduce/reduce conflict, while canonical, canonical-quotient, +and direct follow-partition LR(1) compile and produce identical parse results +across all 125 three-character +strings over the fixture alphabet; exactly four are accepted. On the +shared-lookahead LALR grammar, canonical construction has 11 states and both +compact routes have 8, matching LALR. The direct path also matches canonical +parse products for a precedence grammar. The benchmark itself is pure Scheme +at =t/benchmarks/lr1-partition/benchmark.ss=; compile the compiler modules and +shared fixture into a clean Gerbil path before invoking it with =gxi=. +The focused Grammar IR test constructs complete v1 IR from the LR(1)-only and +precedence fixtures, calls =compile-parser= with the direct and canonical +choices, and compares their LR parse products. Under the pinned v0.19 build, +it also compares both routes on the generated arithmetic language using its +actual lexer and three source expressions. It does not exercise Rust syntax. + +** Bounded canonical trial at medium grammar size + +The direct follow constructor has a bounded canonical trial for large +grammars. For a medium family of independent LR(1) contexts, the previous +512-production entry threshold instead built the LR(0) seed first. At 48 +contexts (290 productions), it then entered follow refinement, although the +canonical automaton has only 627 states and needs no follow compression. +The entry threshold is now 256 productions. The trial still stops at three +times the production count and falls back to direct follow construction when +canonical states exceed that budget. The later LR(0)-conflict lower-bound +check and all table contracts remain in place. + +The pure Scheme =t/benchmarks/lr1-partition/route-threshold.ss= prepares the +grammar once, alternates canonical and follow construction, runs =##gc= +before each sample, and reports CPU time, state count, and follow-block +count. Invoke it as =gerbil env gxi +t/benchmarks/lr1-partition/route-threshold.ss 48 20= under the built package. +In separate 20-pair local runs, the 48-context follow-route CPU median fell +from 12.715 to 3.280 ms; the canonical reference medians were 2.880 and +2.873 ms. At 64 contexts (386 productions), the follow-route median fell +from 5.828 to 5.196 ms; that case already used a late canonical trial, so +the gain is limited to avoiding the LR(0) seed. Both results report zero +follow blocks, and the focused reference test checks parse products for all +accepted and rejected context fixtures at 48 and 64 contexts. This is a +construction-time route choice, not a measured runtime parse gain. + +** Mapping langcc's direct construction to the Scheme compiler + +Construction 3.27 of /Practical LR Parser Generation/ partitions k-follow +strings *per dotted production*. It first computes forward-reachable follow +strings, then uses LR(0) DFA co-reachability to identify lookaheads on which +actions could conflict. Compatibility with those lookaheads defines the initial +backward partition. Backward edge refinement runs to a fixed point, followed +by a forward refinement over the resulting blocks. Only then does it build the +quotient NFA and determinize it. Theorem 3.28 states equivalence of conflict +existence with the canonical LR(k) NFA; it does not state equality of generated +tables or parser throughput. Source: https://arxiv.org/pdf/2209.08383, +Definitions 3.12, 3.19, 3.22--3.25, Construction 3.27, Theorem 3.28. + +| Paper object | Existing Scheme asset | Required new object or check | +|- +| Dotted production and LR(0) co-reachability | =make-core-item= and =build-lr0-automaton= already index dotted items and intern LR(0) states. | Expose a stable item-to-state membership index for potential-conflict pairs. | +| Forward-reachable 1-follow strings | =make-core-lookahead-catalog= already caches FIRST tails and nullability; =propagate-lalr-lookaheads= propagates masks on LR(0) item nodes. | Track follow sets per dotted production/partition, preserving context; a LALR union mask alone loses the distinction the paper refines. | +| Potentially conflicting lookaheads | The LR(0) state contains jointly reachable dotted items. | Calculate raw shift/reduce and reduce/reduce overlap before precedence, associativity, or selective GLR changes the action set. | +| Backward then forward refinement | =make-lookahead-propagation-graph= resolves closure and shift edges. | Refine follow-string blocks across both edge directions until stable; use standard queue, table, vector, and set operations rather than hand-built collection machinery. | +| Quotient NFA then DFA | =compile-lr-spec= already publishes action/goto tables. | Determinize the partitioned NFA and prove its conflicts and parse products against the canonical reference before comparing runtime throughput. | + +The older =lr-partition.ss= operates on *already materialized canonical +LR(1) DFA states*. The direct path instead uses +=src/compiler/lr-conflict-candidates.ss=: a standard-library queue computes +forward-reachable 1-follow masks per production without canonical states, and +raw action overlap on the LR(0) propagation graph produces conservative +potential-conflict cells. Compatibility signatures then seed the initial +backward follow-string partition. + +The forward follow worklist now propagates only newly discovered terminal bits +through nullable tails, while each static FIRST(tail) mask is applied once when +its parent production first becomes reachable. This is the k=1 fixed point of +Definition 3.22; the focused test compares its masks with LR(0) propagation +on recursive, mixed-context, and 16-context grammars. The pure Scheme +benchmark reports this phase separately with a follow-mask sum so its cost is +not conflated with LR(0) construction or backward refinement. The 16-context +phase is below a millisecond locally, but this does not establish a gain for +the complete parser generator. + +The candidate stage uses LALR union masks, +so it may include more lookaheads than Definition 3.24 and split more blocks +than the paper's exact initial partition. The seed now intersects each LR(0) +state candidate mask with the dotted item's possible raw actions: completed +items reduce on reachable follows, terminal-dot items shift on their terminal, +and nonterminal-dot items have no action. An inactive-core witness reduces the +seed from 42 to 41 blocks and from four to three split cores while preserving +full follow masks and canonical conflict coverage. This remains a conservative +approximation of Definition 3.24 because the candidate stage does not compute +exact follow-specific co-reachability. =lr-follow-automaton.ss= refines +those blocks by shift and closure successors until stable, groups the +backward blocks by dotted core, refines by predecessor reachability, then +determinizes the quotient NFA. If there are no raw conflict candidates, it +reuses the already computed LALR states and lookaheads. This implements the +paper's forward/backward framework for k=1 with a conservative candidate +stage; it is not an exact port of langcc's implementation. +The backward fixed point now tracks reverse dependencies between dotted cores. +When a follow block splits, only cores whose shift or closure targets that core +are checked in the next round; each round still rebuilds the global block +index so identifiers remain consistent. This preserves the partition rule of +Definition 3.19 while avoiding successor-signature work for unaffected cores. +The generated 16-context LR(1) family retains the canonical reference's 211 +states and matching accepted and rejected parse products across all regions. +Small local construction timings vary enough that this change does not yet +establish a stable end-to-end speedup. +The focused test compares follow masks with all LR(0) completed-item +lookaheads and checks that every canonical raw conflict projects into a +candidate cell, including a grammar with a genuine reduce/reduce conflict. +Raw candidate detection now records the first action per terminal and marks +only a distinct second action, avoiding production-sized bignum action masks. +The pure Scheme benchmark reports this candidate phase separately, with its +conflict-state count. Focused equivalence tests pass; full construction +timings remain noisy and do not establish an end-to-end speedup. +It also checks that initial blocks cover every reachable follow string and +that the LR(1)-but-not-LALR fixture yields a real initial split. A mixed +grammar exercises the full path: the direct table has 20 states versus +23 canonical states, and parse products agree over all 775 strings of lengths +2--4 from the five-character alphabet. Generated language module qualification +and representative parser throughput measurement remain. Keep each phase's +state/block count and elapsed time separate; +a smaller table is not by itself a faster parser. + +** Stop forward refinement at singleton blocks + +The direct follow constructor's forward partition previously ran one more +global signature pass after every block had become its own group. No further +split is possible once =group-count= equals =block-count=. It now materializes +the result immediately at that fixed point. This follows the same monotone +partition rule as Construction 3.27 and changes neither the partition nor the +published LR table. + +The new =mixed-context-family-rules= fixture joins independent LR(1)-only +regions with a shared recursive component. Unlike the independent family, it +requires direct follow construction and compresses the result: at 32 regions, +canonical has 489 states, while follow has 486 states and 1,033 blocks for +1,161 output items. The reference test compares accepted and rejected parse +products at 4, 16, and 32 regions. The existing pure Scheme +=t/benchmarks/lr1-partition/route-threshold.ss= accepts =mixed= as its third +argument, for example =gerbil env gxi +t/benchmarks/lr1-partition/route-threshold.ss 32 20 mixed=. + +For a controlled 30-pair, same-process A/B at 32 regions, a temporary switch +alternated the old confirmation round and the new termination rule. The full +LR specs were =equal?= for every pair. The old and new CPU medians were 10.516 +and 8.636 ms; the new path won 25 pairs. The 16-region effect was smaller: +4.093 versus 4.039 ms and 18/30 paired wins. Static predecessor grouping and +epsilon-closure caching did not show stable whole-construction gains on this +same direct-route fixture, so they were removed. These are grammar-construction +measurements; parser runtime throughput is unchanged by this experiment. + +** Skip singleton backward follow blocks + +Backward refinement previously recomputed a successor signature and created +a grouping table even for a block containing only one lookahead. Such a block +cannot split under Definition 3.19, regardless of later changes to its +successors. The direct constructor now carries it into the next partition +unchanged. Multi-lookahead blocks retain the existing signature refinement. + +The independent 32-context family takes the direct route and ends with 900 +singleton follow blocks. In a temporary same-process A/B with 30 alternating +pairs, the complete old and new LR specs were =equal?= in every pair. The +construction CPU medians were 7.455 and 6.486 ms; the shortcut won 25 pairs. +At 16 contexts the medians were 3.295 and 2.994 ms, but only 17/30 pairs +favored the shortcut. On the compressing mixed 32-context family the change +was small (9.179 versus 8.953 ms; 17/30 wins), so this experiment does not +claim a gain there. The checked-in =route-threshold.ss= benchmark continues +to report the direct route's state, block, and item counts for both shapes. + +** Refine forward groups only while they can split + +The forward pass used to rebuild predecessor signatures for every block in +each round. A group containing one block cannot split under Construction 3.27, +so the Scheme constructor now counts current group members and computes a +signature only for blocks in groups with at least two members. It still assigns +new group IDs in block traversal order, preserving the published LR spec's +state and transition numbering. At 32 independent contexts, the initial +partition has 772 groups for 900 blocks; the next two rounds reach 836 and +900 groups. The mixed family has 905 initial groups for 1,033 blocks and +reaches 969, then 1,033. + +A temporary switch compared the old and new paths in one process, alternating +order with GC before each construction. Each of 30 pairs compared the entire +LR spec with =equal?=. Median CPU time in ms (old to new, paired wins) was: + +| Contexts | Independent | Mixed | +|----------+-------------+-------| +| 16 | 2.914 to 2.497, 24/30 | 3.794 to 3.225, 22/30 | +| 32 | 6.741 to 5.698, 24/30 | 8.187 to 6.415, 28/30 | + +The temporary switch was removed from the production module after the +comparison. These are in-process Scheme grammar-construction measurements on +one host; they do not imply a fresh-parse throughput change. Lowering either +canonical-trial threshold improved the independent fixture but slowed the +compressing mixed fixture, so those threshold experiments were reverted. + +A later bounded-route experiment tried canonical construction for 128--255 +productions only when the nonterminal dependency graph was acyclic. It cut +the independent 32-context family's median from 5.283 to 2.270 ms in 30/30 +paired runs, while the recursive mixed family stayed on direct follow. The +new =acyclic-mixed-context-family-rules= fixture is the counterexample to +that admission rule: it shares a finite component across 32 regions, has no +recursive dependency, and needs 485 follow states versus 487 canonical +states. The canonical trial therefore falls back, but raises the full +construction median from 7.050 to 9.923 ms (only 7/30 wins). The route change +was reverted. The fixture, reference test, and =route-threshold.ss= +=acyclic-mixed= mode retain this performance and semantic boundary for the +next routing algorithm. + +Several local signature and fixed-point refinements were also rejected after +same-process full-spec comparisons: direct comparison within small forward +groups, propagation only to groups reached from a split, sparse predecessor +signatures, and an extra backward singleton confirmation guard. Canonical +closure queue deduplication, lookahead-delta propagation, and eager closure +edge caching likewise lacked consistent paired wins. None of these temporary +switches remains in the compiler. + +A subsequent phase profile under ASP Scheme =v0.1.2.1= measured the full +direct constructor for 32 recursive shared contexts. Over 10 local CPU +samples, LR(0) construction took 1.321 ms median, raw conflict detection +0.109 ms, initial backward partition 0.540 ms, metadata 0.139 ms, +backward refinement 0.319 ms, and quotient-NFA determinization 3.230 ms. +The latter includes roughly 1--2 ms each for forward refinement and DFA +state expansion. These are compiler-construction costs, not parser runtime. + +Two reversible determinization experiments preserved the complete LR spec +in every paired sample but did not retain a stable gain. Replacing the +standard queue in epsilon closure with a bitset worklist changed the +recursive shared 32-context median from 6.937 to 7.167 ms (12/30 wins). +Indexing symbols once and reusing a vector of transition targets initially +changed that median from 7.451 to 7.101 ms (23/30 wins), but a repeat was +6.769 to 6.936 ms (15/30 wins) and 64 contexts were 22.579 to 22.792 ms +(11/30 wins). Both candidate edits were reverted. The existing benchmark +fixtures and full-spec equivalence checks remain the gate for another +representation change. + +The lower-bound canonical trial now also admits medium grammars (at +least 64 productions and eight raw conflict states) when each nonterminal +occurs in at most two production right-hand sides. This static sharing +measure screens out the recursive and finite shared-component families; +it is a route heuristic, not a correctness premise. The exact LR(0) +state-plus-conflict lower bound remains the final admission test, and a +canonical trial that exceeds it falls back to direct follow partitioning. +The bounded trial leaves small grammars unchanged. On this host, alternating in-process 30-pair CPU runs +measured the independent 16-context family at 2.392 to 1.486 ms (29/30 +candidate wins) and independent 32 at 5.600 to 3.045 ms (28/30 wins). +The recursive mixed 32 family was 7.363 to 7.324 ms (16/30); the finite +shared 32 family, after reversing the scan to reject sharing early, was +7.245 to 6.897 ms (17/30). Those latter differences are not treated as +wins. The direct route still builds their compressed automata. These are +construction measurements, not fresh-parse throughput measurements. + +For grammars with at least 256 productions, a reachable production that +repeats one nonterminal now selects direct follow construction before the +large canonical trial. The quick scan finds repeated symbols, then a +standard queue walks the production graph from the augmented start rule; +=base-symbol= unwraps marked alias edges. This is a route heuristic, not a +new grammar equivalence claim. A repeated rule outside the reachable graph +leaves the canonical trial in place. The permanent +=unreachable-repeated-context-family-rules= fixture and +=route-threshold.ss= mode protect that boundary. In paired full-spec runs, +recursive shared 96 contexts changed from 52.104 to 29.546 ms (30/30 wins), +and finite shared 64 contexts from 21.682 to 16.479 ms (30/30 wins). +Independent 64 contexts were 5.085 to 5.116 ms (21/40 wins), while an +independent family with one unreachable repeated helper was 4.683 to +4.964 ms (13/30 wins); the latter is a bounded screening cost, not a gain. +The constructor still checks full parse products against canonical LR(1) in +the focused tests, and these construction timings imply no runtime-parse +throughput gain. + +A follow-up route probe placed the same recursive component once in each +of two distinct productions per region. At 16 regions, the direct constructor +made 262 states and 585 follow blocks, but canonical also made 262 states; at +64 regions the existing canonical trial produced 1,030 states, equal to the +canonical reference, with zero follow blocks. The new +=split-shared-context-family-rules= fixture checks parse-product equality +and preserves this counterexample in =route-threshold.ss=. Sharing a +nonterminal across productions alone is insufficient evidence that direct +follow partitioning will compress the table. + +Two temporary Construction 3.27 preparation changes were also screened and +removed. Computing =FIRST(tail)= and tail nullability once per occurrence, +instead of once per child production, had a 40-pair forward-follow phase median +of 0.156 ms on both paths for independent 32 (23/40 candidate wins). Computing +all =FIRST= suffix masks for a production in one reverse pass improved the +initial-partition phase at recursive shared 32 from 0.5635 to 0.495 ms +(21/30 wins), and on a synthetic long right-hand side from 0.7615 to +0.576 ms (23/30). The complete LR specifications matched in all 30 paired +samples for both latter families, but whole-construction wins were only +16/30 in each: 7.9345 to 7.4035 ms for recursive shared 32 and 8.244 to +8.224 ms for the long-right-hand-side variant. These phase savings do not +establish a stable construction gain, so neither implementation remains. +A third temporary route screen stopped the global production scan at its first +repeated rule and tested repetition during reachable-graph traversal. Full +LR specifications still matched, but recursive shared 96 construction moved +from 28.838 to 29.317 ms (12/30 wins); the search still visits many sibling +productions. It too was reverted. A determinization output-order trial +replaced per-state core sorting with reversal, relying on core-ordered block +numbering. Complete LR specs matched at recursive shared 32 and 96, but +only 16/30 paired full-construction runs favored reversal at either size; +it was not retained. + +The benchmark reports both total time and mean microseconds per invocation. +In one sequential run with the pinned v0.19 toolchain, 1,000 full compilations +of the LR(1)-but-not-LALR fixture averaged 79.5 us for canonical and 409.6 us +for the direct route. The mixed grammar averaged 166.3 us for canonical and +380.1 us for direct. A 64-rule, 130-state chain averaged 464.8 us for LALR +and 758.3 us for the direct route over 100 full compilations. The chain has no +raw conflict candidate, so the direct route takes its LALR reuse path. These +one-host generator timings do not justify a production switch. The previous +"41 ms" and "57 ms" figures were totals for 1,000 *phase-only* runs, not +single-invocation latencies. +It now also times fresh =lr-parse= with prebuilt tables and token streams in +three batches per fixture, reporting the median and range. The local runs +placed short-input parses in the microsecond range, but repeated batches had +large variance and changed the relative ordering. These small fixtures do not +establish a parser-throughput gain or a production workload result. + +An additive four/16-region LR(1)-but-not-LALR(1) family now checks the direct +route at larger grammar sizes. At 16 regions, canonical and direct both made +211 states and 452 output items; one local 10-run batch averaged 1.24 ms and +8.85 ms per full generation respectively. The direct route remains an +explicit experiment, not a production default or a demonstrated speedup. + +** Lazy epsilon closure for large direct LR quotients + +A TLA+ layout grammar probe (40 source rules, 280 lowered productions) +explicitly forced the experimental direct follow route. The same grammar's +default LALR construction produced 390 states in five CPU samples with a +66.9 ms median. The 12-second result below measures the forced experimental +route; it is not the production table-generation latency. Its LR(0), +conflict-candidate, initial, and +backward/forward partition stages completed quickly in CPU time, but the +quotient NFA had 3,984 blocks and DFA construction dominated. The old +closure walk reached state 300 with 653 states discovered at 7.85 process CPU +seconds; a local memoized candidate reached the same point at 5.58 seconds. +These process CPU readings include module loading and earlier phases. With +lazy singleton-closure caching the complete TLA+ direct LR spec finished +within a 300-second wall-clock cap: 12.925 seconds of construction CPU, +8,149 states, 3,984 blocks, and 407,734 output items. After narrowing the +threshold and completing a default build, the final source repeated those +counts at 12.453 seconds of construction CPU. The old path did not +finish within a 420-second wall-clock cap on this busy host, so there is no +matched complete-run speedup ratio or full old/new TLA spec comparison. + +Epsilon closure distributes over set union. The implementation computes each +singleton block's closure on first use, then unions cached masks for a shift +seed. It retains the original walk below 3,500 blocks. That empirical gate +matters: on the mixed 64-context fixture (2,057 blocks), five sample CPU +medians rose from 17.2 to 19.6 ms; at 96 contexts (3,081 blocks), from 28.8 +to 33.5 ms. At 128 contexts (4,105 blocks), three samples were close: +92.3 ms for the original walk and 89.7 ms for caching. The 128-context +parse-product test now exercises the cached path against canonical LR(1). +These observations support a large-quotient specialization, not a general +claim that closure caching helps every grammar. They measure generator work, +not parser throughput. + +** Rejected quotient-NFA closure and follow-index experiments + +Three subsequent Construction 3.27 subset-construction experiments preserved +the complete =LRSpec= on the compared fixtures but did not produce a repeatable +full-generation gain. First, condensing epsilon-edge strongly connected +components and unioning closures over the resulting DAG took 53.4--64.8 ms +versus 56.0--61.2 ms for the existing lazy singleton cache across five +paired 128-context mixed samples, winning only three. A paired TLA+ run did +not complete its first pair within roughly three minutes under concurrent +host load, so it supplies no TLA+ performance or equivalence claim. Second, +memoizing complete shift-seed closures won 16/30 pairs on mixed 64 contexts +(19.227 versus 19.075 ms CPU medians) and 11/30 on acyclic mixed 64 +(18.892 versus 19.520 ms). Third, storing a single follow block as a compact +mask/id pair instead of a terminal-width vector won 16/30 pairs on both mixed +64 (18.331 versus 17.697 ms) and acyclic mixed 64 (18.246 versus 18.988 ms). +Each temporary switch and source change was removed. These measurements +exclude module import time and do not alter the production route. + +** State-local reduction masks in the initial candidate partition + +The initial backward partition now intersects each completed LR(0) item's +state-local LALR lookahead mask with that state's raw conflict candidates. +Previously it used the production-wide follow mask, which could mark a +reduction on a terminal admitted only at another LR(0) state. The local LALR +mask includes all canonical follows projected to that item, so this narrows +an over-approximation without enumerating canonical LR(1) states. Terminal +shift actions retain their static action mask. This is a narrower seed for +Construction 3.27; it does not establish the exact co-reachability relation +of Definition 3.24. + +The =local-candidate= route-threshold fixture replicates regions with an A/B +reduce conflict on =e= and a C reduction whose local follow is only =d=. +C admits =e= elsewhere, exposing the old production-wide overmarking. At 32 +regions, the direct construction fell from 483 to 451 states and from 932 +to 868 follow blocks. Two same-process alternating measurements gave CPU +medians of 6.558 versus 5.8115 ms (15/20 pairs faster) and 6.360 versus +5.573 ms (24/30 faster), about an 11--12 percent reduction on this fixture. +At 8 regions the timing won only 16/30 pairs; at 64 regions only 11/20, +so the evidence does not support a size-independent speedup. Existing mixed +and acyclic 64-context fixtures retained identical complete =LRSpec= values +and had no stable timing gain. + +The permanent canonical-reference test compares complete accepted products +and rejection results for seven inputs in regions 0, 15, and 31. The =axe= +input has two reduction paths with identical output; runtime completion +deduplication accepts that common product. The test compares it directly +with canonical LR(1), rather than expecting an ambiguity diagnostic. These +measurements concern generator construction, not parser throughput. + +Local validation completed with =GERBIL_BUILD_CORES=12 gerbil build= and +52/52 native test modules passing across the full run and fresh bounded +processes for unfinished modules. The single =just test-all= process did +not complete: after 46 modules passed, its large canonical case hit the +Case duration budget under cumulative memory pressure, then the process +reached the 300-second cap. That case passed in a fresh five-module batch. +Two pre-existing compiled-environment gaps were refreshed locally: the +scanner export used by SANY and the records fixture's grammar, fixtures, +and parser modules. Both affected tests then passed. This does not claim +that the monolithic Just gate passed unchanged. + +** Further k=1 seed and LR(0) kernel experiments + +Three follow-up Scheme generator experiments were screened against complete +=LRSpec= equality. Algebraically constructing the k=1 initial partition +instead of folding singleton follows preserved all twelve fixture seeds. +At 64 mixed contexts, 30 paired full-spec CPU medians were 17.093 versus +17.0795 ms (15/30 wins). The local-candidate 32-context result improved from +4.9705 to 4.660 ms (19/30), but the repeat won only 15/30. A cache of static +epsilon-prediction target masks preserved the same complete specs but changed +mixed 64 from 17.094 to 17.6135 ms (13/30 wins). Both were reverted; neither +provided a repeatable complete-construction gain. + +LR(0) kernel interning was also tested, followed by a conservative kernel-to- +target memo retaining the original closure-key interner. In a phase-only +20-pair experiment reusing already lowered TLA+ grammar input, kernel +interning reduced LR(0) graph CPU medians from 35.3445 to 13.935 ms and +LALR graph plus propagation from 53.8545 to 31.709 ms (20/20 wins each). +These are partial construction measurements. They exclude lowering, +nullable/FIRST preparation, conflict resolution, and table publication. + +The real TLA+ layout grammar has 40 rules, 280 lowered productions and 390 +LALR states. Complete production-shaped calls did not reproduce that gain: +the original construction took 67.650 ms, kernel interning 325.7425 ms, +and the conservative memo 344.564 ms (20-sample medians, 2 GiB heap). +After a full 12-core native package rebuild, the memo still took 369.0205 ms. +The serialized complete specs were byte-identical (590,109 bytes). +Temporary same-process comparisons also showed order-dependent timing, so +these measurements do not identify a causal GC, hash, or compiler problem. +The complete-call gate rejected both kernel edits and all temporary switches +were removed. A second full 12-core native build after restoring the original +automaton passed; the permanent default-route benchmark then returned +69.409 ms across 20 samples, close to the original 67.650 ms baseline. + +=t/benchmarks/lr1-partition/construction.ss= adds the real TLA+ grammar +to the existing benchmark infrastructure. Run =just benchmark-lalr-construction +20= with the built package and its isolated =GERBIL_PATH=. It omits the +construction argument to exercise the production default, warms once, emits +each CPU sample and a median, and checks complete =LRSpec= equality outside +the timed interval. Imports and explicit pre-sample GC are excluded; this +measures generator construction, not parser throughput. The restored native +canonical LR reference and TLA+ layout tests both passed with =MODULE-OK=, +=HARNESS-OK= and final =OK=; =git diff --check= also passed. + +** Grammar-wide prediction and sparse LR graph construction + +The implementation now factors LR(0) closure into a grammar-wide reachability +solution and kernel application. For each referenced nonterminal A, a dense +production-id mask starts with A's own dot-zero productions. A production +whose first symbol is nonterminal B adds a dependency on B's mask. A reverse +edge work queue propagates only new production bits until these masks reach +their finite fixed point, including recursive grammar components. Each goto +kernel unions the completed masks of its after-dot nonterminals and merges +the resulting dot-zero items with the sorted advanced items. It no longer +runs a prediction queue, item hash table, or full item sort for every goto. + +State identity is the advanced-item kernel bitset. Closure adds only dot-zero +items, so distinct noninitial kernels identify distinct closed states. The +augmented dot-zero initial kernel has a disjoint key. The state index is +queried before closure; closure is computed only when a new state is +published. Symbol discovery order and the canonical sorted state items retain +the existing state numbering. Each state's transition row is assembled once. + +LALR propagation nodes occupy sorted contiguous slices in the node-item +vector, located by state offsets. Binary search resolves graph edges without +a packed (state,item) hash index. Symbol IDs resolve shifts through scratch +arrays; each state/nonterminal shares one immutable prediction target list. +Static FIRST(tail) evidence is delivered once when a node becomes reachable; +later queue entries carry only new follow bits through nullable tails. + +Action construction uses the existing terminal catalog's numeric IDs for +scratch cells and shift targets, retaining action installation order and +precedence/conflict policy. Only touched scratch cells are cleared after row +publication. Both graph and action scratch storage are reused across states, +so these arrays do not allocate a state-by-symbol or state-by-terminal matrix. +All indexes belong to one construction call; no mutable grammar cache or +runtime selection flag was added. + +Validation compares complete before/after =LRSpec= snapshots for twelve +grammar families on LALR, canonical LR(1), and direct follow construction +(36 comparisons). TLA+ and the three larger scale fixtures are additionally +byte-compared with the original serialized specs. The canonical reference +test checks that every noninitial state's advanced kernel is nonempty and +unique, and that each goto target has exactly the source items advanced over +its symbol. Recursive, nullable, and shared-context fixtures exercise these +invariants. All 52 native test modules passed in fresh bounded batches, with +=MODULE-OK=, =HARNESS-OK= and final =OK=. The final 12-core package build and +=git diff --check= passed; this does not claim the monolithic Just gate ran. + +The original production pipeline at =8db9386= was fully rebuilt and measured +again before the final candidate was restored and fully rebuilt. Each run +uses 20 complete calls, a warmup and 2 GiB heap, with explicit pre-call GC +and complete-spec comparison outside CPU timing. In-call GC is included. + +| Existing workload | Original CPU median (ms) | Structural CPU median (ms) | +|-------------------+--------------------------+----------------------------| +| TLA+ layout, default LALR, 280 productions / 390 states | 54.7395 | 47.517 | +| Mixed 64, direct follow, 452 productions / 966 states | 13.907 | 13.2985 | +| Local-candidate 32, direct follow, 226 productions / 451 states | 4.8295 | 4.7335 | +| Independent 64, direct follow, 386 productions / 835 states | 4.2635 | 3.7285 | + +The TLA+ complete-call reduction is about 13 percent on this host; the small +mixed/local differences do not establish a general gain. These are Scheme +generator construction measurements, not parser or Rust throughput. +An intermediate dual-namespace alternating trial had 30/30 faster candidate +pairs, but its original copies incurred more GC than the fresh production +baseline, so that timing ratio is not used as the production speedup. + +After rebasing onto PR head =7c1533356585154c8b5d340f64ffefb289d18907=, +the upstream layout closing-boundary action semantics were preserved in the +numeric action index. Layout reduction scratch uses terminal IDs and clears +only touched slots together with the action row. All 53 current native +modules passed in bounded batches, including the added SANY closure owner; +the 36 complete grammar-family specs were regenerated from the new baseline +and remained equal. + +The matched production baseline =144901c= (latest PR plus the prior local +follow-mask and benchmark commits) and the structural source =06f7f47= were +separately cleaned and fully rebuilt with 12 build cores. The same 20-call, +2 GiB, pre-call-GC protocol measured TLA+ default LALR medians of 60.1065 ms +and 44.7655 ms, respectively, about a 25.5 percent complete-call reduction. +Both serialized complete specs were byte-identical, with 280 productions +and 390 states. This newer result includes the layout guards; it is a +separate matched comparison from the earlier table, not a cross-revision +comparison of changed action semantics. CPU timing includes in-call GC and +excludes imports, explicit pre-call GC, and equality checking. + +The next structural audit should attribute allocation and GC to propagation +graph construction and immutable state/transition publication before choosing +another graph representation. Diagnostic tracing still records 7,201 nodes +and 23,305 propagation visits, but observed phase timings vary substantially +between calls. These wall-clock traces include logging and are not an +allocation profile or proof that SCC condensation is the next profitable +algorithm. Candidate experiments are flat adjacency storage and condensation +of nullable propagation cycles; each requires matched complete-call timing +and complete-spec conformance before admission. + +Earlier separate-table prediction admission changed 54.2705 to 56.0445 ms; +a shared-list admission prototype produced only a roughly 5 percent TLA+ +gain and was superseded. Transitive prediction alone reached 50.7895 ms. +A kernel-bitset prototype retaining the old downstream graph took 222.9065 +ms despite exact spec equality. Runtime statistics on a subsequent array +prototype recorded four in-call collections and about 120 ms of GC CPU in +a slow sample. These observations motivated the complete sparse graph and +scratch ownership migration rather than retaining a partial phase result. + +** Shared LR(0) prediction templates + +The next audit used the existing TLA+ default-LALR construction benchmark. +Temporary phase-statistics instrumentation observed about 4--6 ms of +lookahead propagation, with substantial GC time in slower complete calls. +On the 390-state automaton, the sorted state items contain 6,224 dot-zero +prediction occurrences and 977 advanced kernel occurrences. Only 63 distinct +dot-zero item sets occur; their unique templates contain 876 items in total. +This count includes the augmented initial dot-zero item. It identifies +repeated prediction grouping across states as a structural reuse opportunity. + +=src/compiler/lr-automaton.ss= now interns a template by the completed dense +production-id prediction mask. A template owns its sorted predicted item +list, goto groups, and sparse symbol-id group index. Each goto group stores +its earliest item and immutable advanced-kernel mask. A new state's closure +merges its kernel with this shared list, retaining an unchanged template +suffix when the kernel is exhausted. Its transitions group only its own +kernel items, merge overlapping template groups, and combine the two rows +by earliest item. This reproduces sorted-closure symbol discovery order +without regrouping every closed-state item. Only the small kernel group row +is sorted; the shared prediction group row is already ordered. + +Template ownership lasts for one construction call. Numeric symbol IDs and +scratch vectors belong to that same call. Scratch masks are cleared for +kernel symbols only; template groups remain immutable after construction. +Every template is requested by a newly admitted state, so the template count +is bounded by the state count. The automaton still publishes the same item +lists, state IDs, transitions and visit count. Complete LRSpec comparisons +also retain lookahead masks, conflict actions and diagnostic counters. + +The baseline is =9d6d32f=, following the prior structural migration and latest +PR rebase. The same complete construction benchmark measured a 20-call CPU +median of 46.5365 ms before this change. A clean 12-core production rebuild +of the template implementation measured 15.229 ms; an independent 20-call +repeat measured 15.567 ms. All use a 2 GiB heap and explicit pre-call GC, +with imports, pre-call GC and equality checking outside CPU timing; in-call +GC stays inside. The complete serialized before/after TLA+ specs are +byte-identical, at 280 productions and 390 states. This is about a 67 percent +complete-call reduction on this host and workload. + +Separate four-call process-statistics samples observed in-call collection +counts of =(0 1 1 1)= for the baseline, with roughly 25--29 ms GC CPU in the +three collecting calls, and =(0 0 0 0)= for the template implementation. +The existing =construction.ss= benchmark now reports per-call collection +count and GC CPU time plus their medians, using Gambit's statistics slots +3/4 for GC user/system time and slot 6 for collection count. Its 20-call +qualification measured 14.908 ms and median in-call GC count zero. GC is a +material contributor to this speedup; these samples do not establish that +all heaps or workloads avoid collections. + +The final production build passed all 53 native test modules in bounded +batches with =MODULE-OK=, =HARNESS-OK= and final =OK=. All 36 complete specs +for twelve grammar families on three routes remained equal. Independent +goto tests now also check symbol completeness, uniqueness and discovery +order, alongside exact advanced kernels. The larger mixed-64, local-32 and +independent-64 direct-follow fixtures retained byte-identical complete specs; +their candidate CPU medians were 13.4945, 4.9255 and 3.7585 ms. Those scale +runs qualify compatibility and bounded construction; they are not a matched +claim of a common speedup across all grammar families. =git diff --check= +passed. The monolithic Just gate and remote CI are separate gates. + +Algorithm references for the audit are GNU Bison's +https://raw.githubusercontent.com/akimd/bison/master/src/lalr.c +(nonterminal-goto READS/INCLUDES relations and digraph propagation), and +Gambit's https://raw.githubusercontent.com/gambit/gambit/master/lib/_kernel.scm +(process-statistics counter definitions). Relation condensation remains a +candidate for a later graph audit; this accepted change shares prediction +templates while retaining the existing lookahead fixed-point algorithm. + +** Shared literal recognition across lexical modes + +=generated-lexer= now builds one trie for all complete =(literals ...)= rules, +including precedence-wrapped literal catalogs. Every interned LR lexical mode +keeps a rule-admission vector; the trie filters terminal candidates by that +vector while retaining longest match, precedence, and declaration order. +Pure literal rules no longer construct unused per-rule tries at machine +initialization. +Choice and external scanner expressions retain their existing generated +procedures, and a mode miss still uses the full scanner for the LR failure +frontier. This avoids rebuilding the same literal trie for every lexical mode; +the GQL grammar has 492 interned modes on the current v0.19 fixture. + +The pure Scheme =t/benchmarks/literal-mode/benchmark.ss= compares 64 separate +literal scanners with one combined trie after both have been constructed. +In one 10,000-call run, matching the last literal took 304.6 ms separately and +3.9 ms combined; missing all literals took 375.6 ms and 3.3 ms respectively. +This measures literal dispatch only, not complete parse throughput. The +scanner tie/admission test and the affected arithmetic, HCL, GQL, Cypher, +FHIRPath, TLA+, and HL7 parser suites pass under the pinned v0.19 build. + +** Linear input extent for parser-directed LR + +The parser-directed path feeds one significant token at a time. Previously, +each feed scanned every token accepted so far to recover the source end offset, +making an otherwise deterministic parse quadratic in the token count. The +immutable LR checkpoint now carries that offset: initial full-token parsing +computes it once, and directed feeds update it from the new token. Checkpoint +advance, fork, suffix resume, and recovery preserve the corresponding extent. + +The pure Scheme =t/benchmarks/arithmetic-scale/benchmark.ss= measures complete +arithmetic parse and ParseArtifact creation, then checks accepted status and +source roundtrip outside the timer. Before the change, one v0.19 run took +0.63 ms for 100 addition terms and 82.9 ms for 1,600. After the change, +five-sample best times were 0.41 ms and 6.47 ms respectively, and 6,400 terms +took 29.67 ms. The larger samples contain GC outliers; these figures describe +one local fixture, not a submillisecond promise for arbitrary documents. The +grammar-composition and arithmetic, HCL, GQL, Cypher, and TLA+ parser suites +pass under v0.19 after this checkpoint change. + +The same benchmark now measures lexing, significant-token filtering, prepared +LR execution, and ParseArtifact construction separately. For 3,200 terms, +five-sample best times before action dispatch indexing were 1.81, 0.09, +7.00, and 1.64 ms respectively. LR execution was the largest measured phase. +The runtime now projects each LR action row at preparation time into literal, +token-kind, and EOF indexes. Short rows use association lists; larger rows use +Gerbil's standard equal?-table. The executor looks up token lexemes and kinds +directly, preserving literal-first and case-insensitive lookup order while +avoiding a fresh compound terminal key at each action. One subsequent run of +the same fixture measured 5.52 ms for prepared LR and 11.77 ms for the full +parse at 3,200 terms. These are local best-of-five observations with GC noise, +not a cross-language speed guarantee. The arithmetic, HCL, GQL, Cypher, +FHIRPath, TLA+, HL7, and grammar-composition suites pass with the new index. + +The parser-directed feed now enters the LR executor with its new token instead +of constructing an intermediate immutable checkpoint before execution. A +checkpoint is still published at the same shift/fork boundary; this removes +one short-lived checkpoint allocation per significant token without changing +the public continuation or Core Observability calls. The 23-byte arithmetic +fixture currently takes about 0.04 ms per full parse in one five-sample run. +The larger full-parse timings were too noisy to establish a separate feed +speedup, so the retained claim is the allocation reduction and equivalence +shown by the parser, selective-GLR, and incremental tests. A trial that +precompiled production reduction records was slower on this fixture and was +reverted. + +** Persistent LR checkpoints for repeated edits + +=parse-source/incremental= previously rebuilt an LR checkpoint from the first +significant token for every edit, even when most prefix tokens were reused. +The explicit =make-incremental-session= / =parse-incremental-session= path +retains an immutable LR prefix snapshot every 32 significant shifts. Each +snapshot owns the LR stack and recognition prefix but not the complete token +stream, so repeated edits do not retain full historical document suffixes. +An edit selects the last snapshot before the unchanged token prefix, rebinds +it to the new token stream, and creates replacement snapshots while parsing +the suffix. Each saved token carries the lexical-mode identity under which it +was scanned. The parser reuses it only when the new LR state has that same +mode and the shifted source byte position matches. +Each sparse snapshot also records its source-token cursor and byte end. Edit +admission selects a snapshot by that byte boundary and splits the saved token +stream once at the cursor; it no longer builds a temporary significant-token +prefix and walks the same prefix again to find suffix reuse candidates. +The edit boundary converter now scans one standard UTF-8 byte vector instead +of encoding each source character twice. It still rejects byte offsets inside +a multibyte character. +The returned session supports subsequent edits without replaying the original +prefix. A selective-GLR fork keeps the deterministic snapshots and mode +certificates before the fork; its remaining suffix is scanned anew. A rejected +parse or an external scanner takes the fresh fallback path and reports +=freshFallback?=. The ParseArtifact schema and Core Observability ownership +are unchanged. + +The pure Scheme =t/benchmarks/incremental-session/benchmark.ss= separates +session construction from per-edit cost. After mode certification, a local +v0.19 five-sample run on 1,600 arithmetic terms measured 8.5 ms to build a +session, 4.1 ms for a cached edit, 7.2 ms for fresh parsing, and 13.1 ms for +the one-shot API. The latter must reconstruct the old directed session and +is intended for compatibility; repeated edits should retain a session. +These are local best-of-five measurements with host-load noise, not a +cross-language throughput claim. Tests compare complete artifacts after +sequential edits, shifted offsets, rejection and repair, contextual GQL edits, +and a synthetic grammar where identical bytes change token kind by LR mode. + +** Mode-certified token reuse after an edit + +Byte alignment alone cannot establish token identity: in GQL, a keyword can +be an identifier in one LR state and a reserved token in another. The +incremental driver therefore retains the old lexical-mode ID per token and +checks it against the new LR checkpoint before reusing that token. Matching +tokens retain identity for equal-width edits or receive shifted byte offsets +otherwise. A mode mismatch invokes the directed scanner at that position. +Lexical rules containing =external= scanners continue to parse fresh because +they can depend on earlier source, as the HL7 delimiter scanner does. + +The receipt's =relexedByteCount= counts bytes passed through the scanner; +=convergedSuffixTokenCount= counts mode-certified reused tokens. Neither is +a measure of LR execution work: the parser still feeds the suffix, rebuilds +the artifact, and hashes the source. On the local 1,600-term edit, 4 bytes +were scanned and 3,196 suffix tokens were mode-certified. Submillisecond +general edits require a further subtree or event reuse algorithm. +On a local middle-edit scaling run after this cursor change, cached edits took +about 1.1, 2.0, 4.0, and 8.0 ms for 400, 800, 1,600, and 3,200 arithmetic +terms. The near-linear growth confirms that checkpoint selection alone does +not remove suffix LR execution and complete artifact materialization. + +** Certified same-width token event reuse + +For an accepted session with a closed, mode-directed lexer, a zero-byte-delta +edit wholly inside one trivia token can preserve the LR frontier. The driver +rescans that token in its recorded lexical mode and admits reuse only if the +kind and byte extent still match. It replaces that token event while retaining +the other recognition event objects, the significant tokens, lexical-mode +records, and LR checkpoints. Subsequent edits can reuse the same session. +Kind or extent changes, external scanners, and rejected artifacts follow the +ordinary incremental or fresh path. This is a bounded structural-reuse case; +significant-token edits without the certificate still execute suffix LR work. + +The same certificate now admits a significant token when both old and new +lexemes are generic in every interned LR lexical mode. Literal actions take +precedence over token-kind actions, so a lexeme matching any literal must +continue through LR execution. With an unchanged kind and byte extent, two +generic lexemes select identical actions; the new artifact replaces one token +event and shares the remaining recognition events. Snapshots after that token +are removed because their semantic values refer to its old object. Earlier +snapshots remain available for later edits. Trivia edits retain all snapshots. +This still requires a deterministic mode record at the edited token; a GQL +edit after its GLR fork follows the ordinary path. + +The pure Scheme benchmark checks complete artifact equality before timing. +On one local Gerbil v0.19 five-sample run with 1,600 arithmetic terms +(9,597 bytes), it reported 25,592 reused recognition events, 0.85 ms +best-of-five for the cached same-width whitespace edit, and 7.37 ms for +fresh parsing of the edited source. Samples varied substantially with host +load, so these figures establish a local workload result rather than a +general latency guarantee. The executed incremental test suite also covers +repeated edits, lexical-kind rejection, Unicode byte positions, and GQL +contextual trivia. + +For a same-width generic number edit in the same 1,600-term workload, a +separate local v0.19 five-sample run reported 25,592 reused events and +0.86 ms best-of-five, versus 6.29 ms for fresh parsing. In that run a +four-byte replacement, before shifted-event reuse was implemented, took +3.84 ms best-of-five through cached LR resume. These figures are workload and host +specific. Regression cases compare complete artifacts after consecutive +generic edits followed by a width-changing edit, which exercises the retained +prefix checkpoints. + +** Certified one-token width changes + +The same action certificate also admits a one-token byte-width change when +the edited token keeps its kind and the adjacent old tokens rescan unchanged +in the new source. The Scheme artifact builder preserves event topology and +rebases offsets at or after the old token end. The edited token event and +shifted suffix events are rebuilt; unaffected prefix events retain identity. +Source tokens after the edit are relocated, lexical-mode IDs remain attached +to their corresponding tokens, and checkpoints at or after the edited token +are dropped. The driver returns to ordinary LR resume if the edited token +changes kind or either neighbor changes. +The closed-lexer and deterministic-mode requirements still apply. + +On a local Gerbil v0.19 run with 1,600 arithmetic terms, the four-byte +number replacement took 0.98 ms best-of-five through cached event relocation. +The fresh parse in the same run took 10.49 ms best-of-five; host-load variance +was substantial, so this is only a bounded workload receipt. Executed tests +compare full artifacts with fresh parses for number, trivia, and Unicode +edits and sequential edits. + +** Certified local token windows + +The one-token certificate extends to a contiguous edited token window without +replaying the unchanged suffix. The Scheme driver scans each new token using +its corresponding saved LR lexical mode. It admits reuse only when token +count, kind, trivia membership, and LR action choice agree pairwise. Unchanged +literal tokens retain their actions; changed significant lexemes must be +generic in every interned LR mode. The immediate old neighbors are rescanned +in the new source to reject lexical fusion at either boundary. + +The artifact builder maps old token starts and ends to new byte boundaries. +It replaces window token events, remaps node and field boundaries, and shifts +suffix token events by the final byte delta. Event topology, token IDs, and +unaffected event objects remain stable. Checkpoints at or after the window +are discarded because their semantic values or byte cursors may be stale. +For a window containing only trivia, the same certificate also permits a +different token count. Trivia does not advance the LR state, so the Scheme +driver scans the replacement with the saved mode until the mapped window end. +It then inserts or removes trivia events, renumbers suffix token IDs, and +relocates byte boundaries. An event boundary inside the replaced trivia +window rejects reuse. HCL comment insertion, contraction, and deletion have +fresh-equivalent artifacts in the executed regression suite. + +The certificate now aligns significant tokens through existing trivia runs +as well. Each significant token must keep its kind and LR action class; +changed generic lexemes remain admissible. A trivia run can have a different +number of tokens if all saved modes in that run agree with the following +significant token's mode. The event builder assigns each significant token to +its old event slot, assigns a replacement trivia run to the first old trivia +slot, and maps only run endpoints and significant token boundaries. An +unmapped node or field boundary rejects reuse. HCL number edits combined with +comment growth or contraction match fresh artifacts; an operator action +change takes the LR path. +On a local Gerbil v0.19 five-sample HCL run with 100 attributes, adding a +second comment at the midpoint took 0.16 ms best-of-five with the cached +session versus 1.58 ms for fresh parsing; 1,835 recognition event objects +were retained. This only characterizes that workload and input size. +In a later five-sample run after the mixed-window extension, changing the +middle number while adding a comment took 0.23 ms best-of-five cached versus +1.89 ms fresh and retained 1,833 event objects. The pure-trivia case in the +same run measured 0.20 ms cached versus 1.91 ms fresh. + +An edit that changes significant token count or LR actions uses ordinary LR +resume. Selective-GLR suffixes lack deterministic mode records and remain on +that path. + +On a local Gerbil v0.19 five-sample run with 1,600 arithmetic terms, replacing +two numbers around an unchanged plus took 0.92 ms best-of-five through the +certified window and 5.97 ms for fresh parsing. The receipt reported 14,400 +unchanged event objects. This is a local workload result. Executed Scheme +tests compare full artifacts for expansion, contraction, Unicode boundaries, +sequential windows, and a changed literal operator that must fall back. + +** Single-pass session construction + +The original session constructor parsed source once to publish an artifact, +then replayed all significant tokens to construct its sparse LR checkpoints. +The directed Scheme parser now captures those immutable prefix snapshots every +32 shifts during its first pass and hands the exact token objects to the +session. Keeping token identity matters because recognition publication checks +that each recognized token is the same object as the corresponding source +token. On a 1,600-term local v0.19 run, session construction fell from the +earlier 11.31 ms best-of-five to 7.84 ms; fresh parsing in the new run took +7.72 ms. Host-load outliers are present, so this is a local comparison. + +The captured path includes deterministic prefixes before selective-GLR forks. +A representative GQL query reuses 32 LR shifts for an edit near its end, +then enters GLR with the new source. Short GQL queries may have only the +initial zero-shift checkpoint and gain no prefix speedup. Sequential GQL +edits and arithmetic edits compare against fresh artifacts in the executed +test suites. + +* Candidate comparison after the recent survey + +| Route | Scheme generator change | Likely benefit | Main risk | +|- +| Tree-sitter-style contextual DFA + LR/GLR + subtree reuse | Extend Grammar IR lowering, lexer automaton generation, parse-state token masks, and reusable tree metadata. | One coherent batch and incremental architecture; keeps deterministic ordinary cases. | Largest migration; context-sensitive or external scanners and recovery must preserve language-specific behavior. | +| Lezer-style compact LR/GLR + fragments | Encode compact tables and balanced tree fragments in Scheme products. | Lower runtime footprint and explicit ambiguity policy. | Fragment semantics and tree representation diverge from current lossless ParseArtifact/Rust syntax boundary. | +| Bison IELR table generation | Replace or augment LALR state construction. | Better grammar recognition with potentially modest tables. | Does not directly address lexical dispatch, full-source reparsing, or output construction. | +| re2c-style lexer only | Compile closed lexical algebra into DFA or direct code. | An isolated lexical experiment for batch throughput and contextual lexing. | Not a complete parsing framework; must retain maximal-munch, precedence, declaration tie order, UTF-8 offsets, and fallback behavior. | +| langcc-style optimized LR automaton | Generate a lookahead-partitioned LR NFA/DFA from Scheme Grammar IR and compare its states, conflicts, build time, and generated parser throughput with the current LALR path. | Directly targets fresh parsing and grammar expressiveness without replacing Scheme authorship. | The paper's speed results do not isolate each innovation; XLR has no langcc implementation to reuse. | +| Hocc APLR(1) | Build a canonical LR(1) reference and adequacy-preserving remerge in the Scheme compiler. | May avoid LALR-induced conflicts while keeping a compact table. | Canonical construction and remerge can be costly; parser throughput benefit is unproven here. | + +** Explicit construction choice by grammar scene + +=compile-lr-spec= and =compile-parser= now accept +=lalr-then-follow= as an explicit construction choice. It first publishes an +ordinary LALR action table with unresolved cells marked as forks for the +probe only. If there is no unresolved cell, it returns that LALR table +unchanged. Under the =reject= conflict policy, an unresolved cell invokes the +direct follow-partition LR(1) constructor, whose ordinary conflict policy +must still accept the result. A genuinely ambiguous grammar therefore remains +rejected. Under =selective-glr=, the choice keeps the existing LALR/GLR route. +The default remains =lalr= while representative grammar and generation-cost +qualification continues. This is a workload policy, not a new proof of the +paper's partition theorem. + +The 200-run pure Scheme comparison on this host found that a conflict-free +fixture selected LALR with 8 states and about 27--28 us per generation on +both explicit LALR and adaptive routes. The LR(1)-but-not-LALR(1) fixture +selected the follow-partition table with 15 states. Its fallback compiles +two candidate tables, so grammar generation is more expensive than explicitly +requesting the known-good route; repeated small-fixture timings also vary +with GC. This option is for grammars whose LALR adequacy is not known in +advance, not for maximizing generation speed on a known LR(1) grammar. + +For fresh parsing, the Scheme LR runtime is now scanner-driven in one +continuous loop, and semantic reductions fuse operand mapping with sequence +concatenation using the v0.19 standard =foldl=. On the local 3,200-term +arithmetic fixture, the five-sample best full parse changed from about +17.6 ms before the continuous loop to about 10.0 ms after these changes. +The prepared LR stage was about 4.9 ms; lexical scanning and artifact +materialization were about 1.8 and 1.6 ms. These one-host observations do +not imply submillisecond parsing for documents of that size. + +The LR executor now prepares each production's reduction width once and pops +its immutable state and semantic stacks together, yielding source-ordered +values for both deterministic LR and selective GLR. This removes the separate +stack traversals and value reversal from every reduction without changing the +public tables or recognition products. On a local 3,200-term arithmetic input, +the five-sample best prepared LR stage was 6.08 ms before this change and 5.00 +ms in a repeated run afterward; full-parse samples varied more widely between +processes. Parser runtime, grammar composition, incremental, canonical LR(1), +and selective-GLR suites passed. This measures a runtime improvement, not a +new parsing-complexity bound. + +The matched streaming benchmark now records process CPU time beside elapsed +time. On PR head =6a7cab9=, repeated full parses of a 512-line, +1,534-character arithmetic source typically used about 1.3 ms CPU. Some +elapsed samples exceeded 100 ms while +using only a few milliseconds of CPU; elapsed outliers therefore cannot be +attributed to LR computation alone. A separately timed prepared-token path +measured, at five-sample best CPU time, 0.166 ms lexical scanning, 0.020 ms +significant-token filtering, 0.613 ms LR execution, and 0.295 ms artifact +materialization for 512 addition terms. At 3,200 terms these stages measured +1.018, 0.097, 3.679, and 1.633 ms respectively. These are attribution probes, +not a sum of streaming-path costs. A single-character ASCII byte-width shortcut +and a precomputed unit-pass reduction shortcut were both rebuilt and measured; +neither established a consistent CPU improvement and both were reverted. The +remaining runtime experiment should address the LR dispatch and semantic +construction together rather than add another special case to either path. + +An integer-packed action index was then tested on the deterministic executor: +positive integers encoded shifts, negative integers reductions, and zero +acceptance, while GLR retained the original actions. Arithmetic, parser runtime, +and canonical-LR tests passed. The 512-line CPU samples clustered around +1.36--1.41 ms instead of the approximately 1.30 ms baseline, and 3,200-term +samples did not improve. The extra index and decoder were reverted. This +isolates the next AOT boundary: =deflanguage-grammar= already embeds Parser IR +at expansion time, but =defgeneral-parser-machine= still calls =lr-prepare= at +module initialization and the executor interprets action rows. A direct-code +prototype must generate state/terminal dispatch from the embedded IR while +keeping the original tables for GLR, diagnostics, and conformance. A bounded +prototype did exactly that for automata with at most 64 states, retaining the +original indexed path for larger languages and GLR. Gerbil v0.19 build and +arithmetic/runtime/canonical-LR tests passed, including root equality against +the indexed path. A same-process, prepared-token 512-line comparison gave +best CPU times of 0.600 ms for the indexed executor and 0.666 ms for the +generated dispatcher (20 measured iterations each, three warmups). The +512-line complete ParseArtifact samples still clustered near 1.3--1.4 ms CPU; +3,200-term complete samples ranged from 7.5 to 14.3 ms CPU and did not show a +stable advantage over earlier indexed runs. The generated dispatcher still +called a closure per LR action, tested terminal clauses in Scheme, and left +semantic reduction in the interpreter. The prototype was reverted. The next +direct-code experiment needs to fuse state transition and reduction execution, +not merely replace indexed action lookup with generated conditions. + +A separate experiment based on PR head =13f164c= specialized the whole-source +LR driver, removing budget, frontier, +and observation branches from the ordinary action loop while retaining the +indexed action table, immutable stacks, and exact selective-GLR handoff. +Gerbil v0.19 build and focused arithmetic, HCL, GQL, parser-runtime, and +canonical-LR tests passed. In one process on the same prepared 512-line token +stream, best LR CPU time was 0.792 ms through the generic driver and 0.754 ms +through the specialized driver (20 measured iterations after three warmups). +The complete 512-line ParseArtifact benchmark did not improve: best CPU time +was 1.255 ms on the baseline and 1.290 ms with the specialized driver, with +large host outliers in both runs. The duplicate driver was reverted. This +narrows the next vertical slice to grammar-specialized transition, reduction, +and stack behavior together; removing control branches alone is insufficient +for this engine's full parse product. + +On PR head =fbd8c35=, another Scheme experiment prepared a semantic-action +plan per production at =lr-prepare= time, so deterministic LR and selective GLR +would no longer extract each RHS operand's actions on every reduction. The +v0.19 build and focused arithmetic, HCL, GQL, parser-runtime, and canonical-LR +tests passed. In the same pinned package environment, the 20-sample 512-line +complete ParseArtifact best CPU time was 1.240 ms with plans and 1.309 ms on +the rebuilt baseline. The 3,200-term five-sample prepared LR best moved the +other way, from 3.545 ms baseline to 3.662 ms with plans; the complete parse +best likewise moved from 7.218 ms to 7.555 ms. These small, opposite-direction +results do not establish a throughput gain. The plan added a vector lookup and +an extra runtime descriptor while leaving LR transitions and the semantic +action interpreter separate, so it was reverted. The next generated experiment +must fuse transition and semantic work end to end rather than only hoist +production metadata. + +The comparison with Gerbil Djot PR #1509 +(https://git.cons.io/mighty-gerbils/gerbil/pulls/1509) sharpens the output +boundary. Djot's native Scheme implementation parses blocks and inlines into +typed AST nodes, then finalizes document references and sections; its quoted +parse benchmark uses =gxc -O -exe= and disables source locations. Langcc's +implementation report (https://arxiv.org/pdf/2209.08385) describes generated +AST datatypes, a DFA lexer, and generated LR parsing as one frontend. These +are different grammars and products, so their published parse times cannot +rank our lossless =ParseArtifact= executor. The transferable design target is +a Scheme-generated, grammar-specialized transition and semantic constructor, +with the indexed/checkpoint/GLR engine as fallback. This is an algorithmic +candidate, not a measured speedup. + +A smaller preparatory experiment used private vector state/value stacks for +whole-source deterministic parsing and materialized an immutable checkpoint +only at a fork. Six focused parser suites passed. In a rebuilt v0.19 package, +the 512-line, 20-sample complete-artifact best CPU time was 1.253 ms with the +vector path and 1.210 ms on the rebuilt baseline; the 3,200-term seven-sample +bests were 7.116 ms and 7.283 ms respectively. The opposite-direction small +differences and process variance do not justify the additional executor, so +it was reverted. A direct identity reduction for one-operand =pass= rules +also passed those suites, but its 512-line best (1.216 ms) matched the 1.210 ms +baseline and its prepared 3,200-term LR best (3.584 ms) did not improve the +previous 3.545 ms observation; it too was reverted. Neither experiment +generated state-specific semantic code. + +The pure Scheme =t/benchmarks/arithmetic-scale/aot-benchmark.ss= now separates +three native executable workloads on one preloaded source: directed complete +=ParseArtifact=, source-to-recognition tree, and prepared-token recognition. +It preflights recognition against the complete artifact, excludes process +startup and input construction, collects CPU and elapsed time, and permits GC +during each timed batch. With the repository's isolated =GERBIL_PATH= set, +create its output directory with =mkdir -p .gerbil/benchmarks=, then +build it as =gxc -O -exe -o .gerbil/benchmarks/gerbil-parser-aot-benchmark t/benchmarks/arithmetic-scale/aot-benchmark.ss= and run the executable with +=1024 lines artifact 20 10=, =1024 lines source-recognition 20 10=, or +=1024 lines recognition 20 10=. This 1,024-line arithmetic input (3,070 +source characters) is the primary baseline size for subsequent experiments. It +matches Djot's build mode, but does not make the grammars equivalent. +On this host, three independent 20-sample runs of ten parses per sample gave +median CPU ranges of 3.442--3.562 ms per full artifact, 2.914--2.926 ms for +source-to-recognition, and 1.736--1.768 ms for prepared-token recognition. +The 3,200-term shape remains a secondary scale check; an earlier seven-sample +run had medians of 9.868 ms and 7.760 ms for the first two modes. A prior +same-protocol =gxi= run on 512 lines measured 0.838 ms for +source-to-recognition, so native packaging alone is not an optimization claim. +These numbers still use different inputs and AST contracts from Djot; they +mainly establish a reproducible local gate for a generated semantic path. + +A first fused Scheme-code experiment now lives in +=t/benchmarks/arithmetic-scale/generate-fused-lr.ss= and +=fused-lr-benchmark-body.ss=. From the repository root, generate one standalone +source (after =mkdir -p .gerbil/benchmarks=) with =gerbil env gxi t/benchmarks/arithmetic-scale/generate-fused-lr.ss .gerbil/benchmarks/fused-lr-aot.ss=, +compile it with =gxc -O -exe -o .gerbil/benchmarks/fused-lr-aot .gerbil/benchmarks/fused-lr-aot.ss=, +then run =.gerbil/benchmarks/fused-lr-aot 1024 lines prepared 20 10= or substitute =source= +or =artifact= for =prepared=. The generator consumes the compiled arithmetic Parser IR; it +emits state-specific action selection, inlined reduction expressions, and goto +branches as Scheme source. Field/alias constructors remain the shared semantic +helpers. Every timed run preflights the generated recognition tree against the +existing LR result and the complete directed =ParseArtifact= on 24 varied +expressions plus the measured input. Strict mode rejects any fallthrough to the +generic interpreter, so a claimed fast-path result cannot silently reparse. +The executable alternates measurement order between samples, invokes GC before +each batch, and reports CPU and elapsed time with input construction and +process startup excluded. + +On this host, one final-source 20-sample, ten-parse batch measured 1,024-line +prepared LR medians of 2.206 ms for indexed execution and 1.372 ms for the +generated executor; source-to-recognition medians were 3.421 and 2.546 ms. +The 3,200-term secondary scale measured 6.499 versus 3.618 ms prepared and +7.169 versus 4.976 ms from source. Earlier repeated AOT runs of the same +algorithm, including reversed measurement order, also favored generated +execution, though absolute CPU medians varied with host load. + +The same generator now emits a streaming state driver as +=src/compiler/arithmetic-drive.ss=. Its lexical-mode identities come from +the prepared LR catalog; it requests one token under the current state mode, +builds the same recognition values, and records shifts through the existing +source driver's callback. =parser-machine= admits it only when its embedded +grammar digest matches the machine. A missing action, fork, or invalid goto +returns =fallback=, so the ordinary driver restarts with fresh scanner state +and retains its diagnostic and selective-GLR behavior. The arithmetic v1 +production entry uses the generated driver for unobserved fresh parses; +incremental and observed requests keep their existing executor. Run +=gerbil env gxi t/benchmarks/arithmetic-scale/generate-fused-lr.ss check src/compiler/arithmetic-drive.ss= +to compare the checked-in output byte for byte with the current Scheme IR; +native CI runs this check after building the package. + +The experimental streaming benchmark matched the complete directed artifact +for 24 varied expressions, including Unicode identifiers, and both large +input shapes without falling back. Three 20-sample AOT batches gave 1,024-line +complete-artifact CPU medians of 3.412--3.476 ms for the indexed driver and +2.532--2.599 ms for the generated driver; the 3,200-term ranges were +10.257--10.594 ms and 7.363--7.589 ms. After integrating the driver, four +alternating AOT runs of the old and new =parse-arithmetic= executables +measured 3.224--3.260 versus 2.512--2.571 ms at 1,024 lines and +9.377--9.535 versus 6.488--6.558 ms at 3,200 terms. The package's warm +build measured 6.35 s on this host. The corresponding AOT benchmark executable +grew from 22,416,616 to 22,738,832 bytes. These are matched host observations for +one grammar, not a cross-grammar or Djot speed ranking. Generalizing the +full-state emitter beyond arithmetic still requires semantic and build-size +qualification. + +HCL v2.24 gives a second grammar with 133 LR states and 120 productions. +Expanding the whole arithmetic-style state driver for HCL emitted 809 KB of +Scheme and its C compilation was killed after more than three minutes. Moving +one copy of each reduction into a helper reduced the output to 492 KB, but +still exceeded a 30-second compilation limit. Generating only the semantic +expression produced 14 KB and compiled in about 2 seconds; matched 1,024-line +and 3,200-line complete-artifact CPU medians improved by only about 2--3%. +That narrower candidate was reverted. These failed build and runtime gates +rule out unbounded state expansion for HCL on this host. + +The admitted HCL slice instead generates a fused reduction step from the +Scheme Parser IR: each production has static stack pops and semantic actions, +then looks up its exact goto in the existing indexed table. LR action lookup, +parser-directed lexical modes, diagnostics, and selective GLR remain with the +existing runtime. The generated =src/compiler/hcl-reductions.ss= is about +124 KiB; current native compiler receipts are recorded in the closure roadmap. The language module installs it +only when its embedded grammar digest matches the compiled parser machine; +the runtime exposes a benchmark override to run the same scanner and artifact +path with indexed reductions. Run +=gerbil env gxi scripts/generate-reductions.ss check :gerbil-parser/languages/hcl/grammar src/compiler/hcl-reductions.ss= +to verify byte-for-byte regeneration; native CI runs this check. + +The repository-owned =step-benchmark.ss= preflights exact full +=ParseArtifact= equality and alternates AOT timing order, with garbage +collection before each batch and process startup outside the timed region. +At 1,024 Basic lines (11,178 characters), 20 samples of 10 parses had CPU +medians of 11.507 ms indexed versus 10.333 ms fused, with fused faster in 19 +of 20 pairs. At 3,200 lines, 12 samples of 5 parses measured 35.631 versus +32.451 ms, with fused faster in all 12 pairs. The HCL official accepted +corpus and the 1,024-line Basic source also assert exact indexed/generated +artifact equality; the rejected corpus retains typed failure checks. These +are matched observations on this host, not a cross-grammar speed ranking. + +The prepared action row now indexes ASCII single-character literals directly +by character code. Multi-character and Unicode literals retain the existing +list/table index; exact literal lookup still precedes case-folded literal and +generic token-kind lookup. This targets repeated action selection across LR +reductions without changing contextual token admission. On the same local +3,200-term arithmetic input, the five-sample best prepared LR stage was 3.85 +ms in a repeated run after this action-row change, versus 5.00 ms in the +previous run. The ISO GQL 100-query p95 sample was 50.964 ms against its 150 ms +admission ceiling. These timings are host observations, not portable ratios. + +** Forward event construction after fused LR reductions + +The shared =src/runtime/artifact.ss= linearizer now appends events in source +order with a tail cell. Its child dispatch reads the concrete token, node, or +fragment offsets directly, computes a child's end only for a named field, +and admits trivia once before emitting a token. This keeps the existing +lossless event schema and source-order checks. It applies to all grammars; +the arithmetic and HCL generated LR paths need no special artifact code. + +An AOT HCL experiment checked exact =ParseArtifact= equality before timing. +With 20 alternating batches of 10 parses in one executable, the 1,024-line +Basic source (11,178 characters) had median complete-parse CPU time 10.746 +ms for the committed event builder and 10.387 ms for the new builder (17/20 +paired wins). At 3,200 lines (37,290 characters), medians were 32.465 and +30.903 ms (17/20 paired wins). A separate prebuilt recognition-tree +experiment isolated event construction at 1,024 lines: 1.98 to 1.50 ms; +at 3,200 lines: 5.72 to 4.21 ms. These local measurements use =gxc -O -exe=, +exclude process startup and input construction, and include GC during parsing. +The whole suite passed 49 modules with =HARNESS-OK= and final =OK=. The +complete-parser gain is smaller because scanning and LR work remain. + +** Grammar IR recursive actions for HCL + +The larger fresh-parse cost was the sequence of grammar-wrapper reductions, +not event list reversal. On the 1,024-line Basic HCL source, each line drove +about 20 deterministic LR reductions; nullable branches and single-child +aliases accounted for much of that sequence. A generic event-valued LR +prototype built the exact same artifact but took 20.638 ms versus 10.258 ms +for the previous full parser. Indexed actions reduced that candidate to 13.886 +ms, and generated event reductions to 13.672 ms. A guarded reduction-corridor +prototype also kept exact artifacts but measured 11.038 ms versus 11.075 ms +without the corridor in a same-process comparison. None removed enough +semantic work to warrant integration. + +The first =src/compiler/recursive-source.ss= version compiled the +HCL Grammar IR's sequence, choice, optional, repeat, field, alias, literal, +token, and reference forms into Scheme recursive-descent functions. They +constructed the existing recognition nodes and used the standard Scheme +sequence helpers, then published through the lossless =ParseArtifact= builder. +This follows langcc's generated recursive-descent action direction while +retaining this repository's Scheme grammar and artifact contract. The generated +=src/compiler/hcl-source.ss= is checked byte for byte against +the current IR in native CI. It is installed only for the matching grammar +digest and used for fresh unobserved HCL source requests. A non-admitted +source returns to the existing parser-directed LR route, which owns rejection, +diagnostics, checkpoints, and observed execution. +The source generator lives beside the HCL grammar and is excluded from the +runtime library build. + +The integrated AOT benchmark compared the previous fused LR source path and +the public generated entry in one executable, with strict artifact equality +before timing and 20 alternating batches of 10 parses. Median complete-artifact +CPU time was 9.867 to 6.558 ms at 1,024 Basic lines (20/20 paired wins), and +30.114 to 19.823 ms at 3,200 lines (20/20 wins). The generated path matched +all 12 official accepted HCL specimens exactly; the three official rejected +specimens returned to the typed LR failure path. A deterministic local +differential probe compared 301 additional accepted mixed HCL sources against +the prior LR path without an artifact mismatch. These are host and grammar +specific observations; they neither rank Djot's different output contract nor +establish a general recursive-descent compiler for every grammar. + +** Generated HCL event actions + +The next structural step removed the recognition tree from the fresh HCL path. +The same Scheme Grammar IR now generates recursive functions that append token, +field, and node descriptors to a mutable event list. Choice, optional, repeat, +and sequence operations save a list-tail checkpoint and roll back on failure. +Only after a complete accepted parse does =src/runtime/artifact.ss= assign +canonical token/node IDs, interleave trivia, check token order, and build the +unchanged =ParseArtifact=. Non-admitted sources still use the directed LR +failure path. The byte-for-byte generator check remains in native CI. + +The generated event path matched the previous parser on the 1,024- and +3,200-line Basic inputs, the HCL representative and 15 official specimens, +empty input, Unicode source with multibyte offsets, and 500 deterministic +mixed accepted cases. The local compiled-module comparison alternated 20 +batches of 10 complete parses and checked exact artifact equality before +timing. At 1,024 lines the old and new medians were 7.492 and 6.482 ms CPU +(18/20 paired wins); at 3,200 lines they were 24.376 and 20.389 ms CPU +(20/20 wins). These are same-process measurements on this host; a matched +standalone AOT result is a separate verification gate. + +Two narrower experiments were rejected: a mutable recognition-child stack +improved complete parsing by only about 4-5% at both sizes, while FIRST-set +choice guards improved 1,024 lines by about 2% and slightly slowed 3,200 +lines. Applying the recursive generator to HL7 kept exact artifacts on its +standard and dynamic-separator examples but showed no stable gain on a +128-segment message. Those prototypes were not installed. + +** HCL generated control flow at scale + +A subsequent prototype emitted canonical events during speculative recursive +descent, with trivia, token/node IDs, and token order included in each rollback +checkpoint. Its complete artifact matched at 1,024 Basic lines, but the +20-batch CPU median regressed from 7.471 to 8.159 ms (4/20 paired wins), so +it was rejected. Resolving raw events only after acceptance remains cheaper. + +The next generator change replaced the two-value =(position, success?)= +protocol at every grammar expression with a single result: the next token +position on success and =#f= on failure. The existing tail checkpoint and +raw-event finalization remain unchanged. Exact complete artifacts matched +the 1,024- and 3,200-line Basic inputs and the representative HCL specimen. +Two repeated, alternating same-process 20-batch comparisons at 3,200 lines +measured 21.074 to 17.556 ms and 21.935 to 17.930 ms CPU per parse, with +20/20 paired wins in both runs. At 1,024 lines the two runs were near parity: +6.021 to 5.945 ms (13/20 wins) and 6.008 to 6.163 ms (9/20 wins). Thus the +local evidence supports a scale gain, not a 1,024-line gain. The generated +module passes a byte-for-byte freshness check, the complete 49-module native +test suite, and the lossless artifact checks. A matched standalone AOT result +remains unverified for this slice. + +** Rollback flat event tape for the 1,024-line HCL Basic path + +The generated event-action parser previously allocated a pair and a named +descriptor for each speculative raw event before building canonical events. +An intermediate vector of descriptors preserved the named type, but improved +1,024-line complete parsing by only about 1.5% after the concurrent HCL +generator refactor. The admitted path stores =operation=, =name=, and byte +offset in one request-local flat vector. Choice rollback restores an integer +cursor. The single artifact authority in =src/runtime/artifact.ss= materializes +only its accepted prefix. The named descriptor remains available for +inspection. This follows the paper's allocation-oriented direction while +retaining the exact full =ParseArtifact= contract. Initial capacity covers six +raw events per lexed token, with geometric growth for denser expressions. + +Against the clean remote =9ceca64= head, separate-process 20-batch medians +of 10 complete parses at 1,024 Basic lines were 5.607 ms CPU before and +3.369 ms after; a second production run measured 3.084 ms. A same-process +prototype comparison with exact artifact preflight measured 5.558 to 3.149 +ms, with 20/20 paired wins. At 3,200 lines the remote and production medians +were 18.518 and 15.346 ms. These are local compiled-module results; the +separate-process medians are not paired confidence intervals. The complete +49-module native suite passed after the generator move. It covers the official +accepted/rejected specimens, Unicode byte offsets, a 1,024-line indexed +baseline comparison, and compact 128-line input that forces tape growth. +Standalone AOT throughput and peak memory remain separate gates. + +Two predictive-branch prototypes were rejected for this slice. A proven +second-token guard for HCL =body-item= and a disjoint first-token dispatch +for =primary-expression= each improved the 1,024-line input by only about +3% in local paired runs; neither was installed. + +** Generated HCL lexical subset before recursive actions + +An isolated phase measurement on 1,024 Basic lines put the existing ranked +lexer near 1.34 ms CPU out of roughly 3.2 ms for a complete artifact. The +HCL generator now emits a closed ASCII lexical path for identifiers, decimal +integer runs, horizontal whitespace, newlines, and single = signs. It emits +the same token representation and byte offsets directly. Decimal fractions +or exponents, ==, Unicode, comments, strings, and every other unhandled +character cause a return to the unchanged ranked lexer for the whole source. +The generator checks the exact lexical declarations before emitting this +specialization; grammar drift requires review rather than silently changing +lexical precedence. The recursive parser and artifact materializer are the +same on both sides of this comparison. + +The compiled-module lexer alone measured 1.345 to 0.256 ms CPU at 1,024 +Basic lines in 20 alternating batches of ten calls, with 20/20 paired wins. +Using the exact previous PR-head generated parser and the new parser in one +process, complete =ParseArtifact= medians were 3.331 to 2.110 ms CPU at +1,024 lines and 15.611 to 9.108 ms at 3,200 lines; both had 20/20 paired +wins and exact artifact preflight. The preflight artifacts and token lists +were released before timed batches. This is an HCL Basic subset gain, not a +general lexical DFA claim. A complex source that fails the fast path late +can spend time scanning its prefix before restarting with the ranked lexer. +The official HCL corpus and a fixed random token differential test are +semantic gates. The lexical phase can be rerun with =gxc -O +t/benchmarks/hcl-scale/lexical-subset-benchmark.ss= followed by =gerbil env +gxi -e '(import +:gerbil-parser/t/benchmarks/hcl-scale/lexical-subset-benchmark) (main +"1024" "20" "10")'=. Standalone AOT and peak memory are not measured here. + +** Deterministic simple-attribute event corridor + +After the lexical specialization, a 1,024-line Basic profile attributed about +0.25 ms CPU to lexing, 0.18 ms to token/byte/tape setup, 1.12 ms to generated +grammar actions, and 0.50 ms to artifact finalization. The generated grammar +actions attempted 49,159 raw event writes and 54,284 rollbacks to retain +32,770 events. A full FIRST-set guard prototype kept exact artifacts but +improved complete parsing only from 2.005 to 1.950 ms CPU (17/20 wins). +Speculative canonical-event construction removed finalization but regressed +from 2.068 to 5.025 ms; scalar checkpoints still measured 2.013 to 4.986 +ms. Neither prototype was installed. + +The admitted path instead recognizes a closed token sequence: top-level +newlines and attributes containing an identifier, an equals sign, and one number, string, +or identifier value. +It emits the Grammar IR's fixed field/node event shape for each attribute, +then uses the existing shared raw-tape artifact authority. A complete token +preflight runs before any events are emitted, so a complex value near the end +does not discard a large speculative tape. Other inputs use the unchanged +generated recursive parser. The generator pins the grammar digest and fails +closed when that event shape requires review after a grammar edit. + +In the checked-in pure Scheme benchmark, 20 alternating batches of ten +complete =ParseArtifact= calls measured 2.019 to 1.154 ms CPU at 1,024 Basic +lines and 9.159 to 6.236 ms at 3,200 lines, with 20/20 paired wins in both. +Appending one complex tuple after 1,024 simple lines measured 3.454 to +3.533 ms, a small fallback cost. Preflight artifacts are released before +timing. Official HCL fixtures, 5,000 mixed differential inputs, and 1,000 +valid assignments varying strings, Unicode, CRLF, and comments matched the +previous generated parser exactly. Run =gxc -O +t/benchmarks/hcl-scale/simple-attribute-benchmark.ss= and then =gerbil env +gxi -e '(import +:gerbil-parser/t/benchmarks/hcl-scale/simple-attribute-benchmark) (main +"1024" "20" "10")'=. These are same-process compiled-module results for a +bounded HCL subset; standalone AOT and peak memory remain unmeasured. + +** Closed ASCII quoted-string scanner for HCL + +The generated HCL lexer previously restarted the ranked scanner for every +quoted string, even when the source otherwise matched its closed ASCII +subset. The new branch calls the shared =scan-quoted-string= primitive for +escape and doubled-delimiter semantics, then emits the same =string= token and +byte offsets as ranked scanning. It admits only ASCII characters. Unsupported +or malformed strings still use ranked lexing for the entire source. The +parser already needs UTF-8 source bytes for =ParseArtifact=; computing them +before lexing lets it skip the generated scanner altogether when byte length +differs from character length. This prevents a late Unicode character from +forcing a second full lexical pass. The implementation is a closed generated +scanner specialization, not a complete HCL lexical DFA. + +The checked-in pure Scheme benchmark alternates the ranked and generated +lexer in one process, performs strict token and full =ParseArtifact= preflight, +and times 20 batches of ten complete parses with a 1 GiB Gambit heap. For +1,024 =keyN= assignments with ASCII quoted values, ranked and generated +medians were 2.695 and 1.257 ms CPU, with 20/20 paired wins. At 3,200 lines +they were 13.079 and 6.438 ms, also 20/20. With a Unicode value in the last +of 1,024 lines, both paths used ranked fallback; medians were 2.292 versus +2.377 ms CPU, with 8/20 wins for the generated path. The small residual cost +is not a gain. +Focused tests compare escaped, doubled, multiline, and seeded ASCII strings +with ranked tokens and indexed full artifacts, and require Unicode and +unterminated strings to fall back. Run =gxc -O +t/benchmarks/hcl-scale/quoted-string-benchmark.ss=, then =gerbil env gxi -e +'(import :gerbil-parser/t/benchmarks/hcl-scale/quoted-string-benchmark) (main +"1024" "20" "10" "ascii")'=. These are compiled-module measurements, not +standalone AOT throughput or peak-memory results. + +Two other paper-oriented candidates were rejected in this round. A closed +block/attribute event corridor matched complete artifacts but improved +1,024 block-rich lines by only 10-15%, while its 3,200-line results varied +from about 1% to 12% across matched runs. Replacing canonical event-list +construction with a preallocated vector regressed 1,024-line Basic parsing +by 8.6%; replacing only the node-ID cons stack with a vector also regressed +at that size. Neither change remains in the runtime. + +** Closed ASCII comment scanner for HCL + +The generated lexer now recognizes =#=, =//=, and =/* ... */= comments +without restarting the ranked lexer for each comment. It calls the shared +=scan-line-comment= and =scan-block-comment= primitives, preserving their +terminator and newline rules, then checks that the scanned range is ASCII. +Unsupported characters and unterminated block comments use the ranked path. +The parser's existing UTF-8 byte-length guard also avoids a second lexical +pass when Unicode appears late in the source. This is a bounded HCL lexer +specialization; the ranked lexer remains the semantic reference. + +The checked-in pure Scheme benchmark first compares exact tokens and full +=ParseArtifact= values. It then alternates ranked and generated complete +parses in one process for 20 batches of ten parses, with a 1 GiB Gambit heap. +For 1,024 commented assignments, median CPU time in milliseconds was +5.334 to 1.602 for =#= (20/20 paired wins), 5.807 to 1.626 for =//= +(20/20), and 6.006 to 1.477 for =/* ... */= (20/20). For 3,200 =#= +assignments, the medians were 17.557 to 7.692 ms (20/20). A Unicode +character in the final comment of 1,024 lines forces ranked fallback; +medians were 6.073 and 5.843 ms (12/20), which is not evidence of a +generated-path gain. Run =gxc -O +t/benchmarks/hcl-scale/comment-benchmark.ss=, then =gerbil env gxi -e +'(import :gerbil-parser/t/benchmarks/hcl-scale/comment-benchmark) (main +"1024" "20" "10" "hash")'=. Focused tests cover all three comment forms, +CRLF, Unicode and unclosed fallback, 128 seeded ASCII cases, and a 1,024-line +indexed-artifact comparison. These are compiled-module results; standalone +AOT throughput and peak memory remain unmeasured. + +** Closed ASCII fractional and exponent scanner for HCL + +The generated lexer had admitted integer digits but restarted ranked lexing +for decimal fractions and exponents. It now keeps the one-character-at-a-time +integer path and calls the shared =scan-number-literal= primitive only when +the digit run is followed by =.=, =e=, or =E=. An ASCII range check and the +existing UTF-8 byte-length guard preserve the ranked fallback for unsupported +characters. The generated lexer still uses the ranked scanner as its token +and full =ParseArtifact= reference; this adds no second number grammar. + +The checked-in pure Scheme benchmark performs exact token and artifact +preflight, then alternates ranked and generated complete parses with a 1 GiB +heap. At 1,024 assignments, 20 batches of ten parses gave CPU medians of +2.610 to 1.264 ms for decimal values, 2.398 to 1.277 ms for exponents, and +2.731 to 1.466 ms for mixed integers/decimals/exponents; each candidate won +20/20 paired batches. At 3,200 decimal assignments, 20 batches of five parses +gave 16.900 to 4.081 ms, also 20/20. The original 1,024-line integer Basic +benchmark recorded 1.206 ms for the generated corridor before this change +and 1.173 ms afterward in separate runs; it shows no observed regression, +but those two medians are not a matched A/B test. Focused tests cover numeric +boundaries, 128 seeded valid literals, and a 1,024-line indexed artifact. +Run =gxc -O t/benchmarks/hcl-scale/number-benchmark.ss=, then =gerbil env +gxi -e '(import :gerbil-parser/t/benchmarks/hcl-scale/number-benchmark) +(main "1024" "20" "10" "decimal")'=. These are compiled-module results; +standalone AOT throughput and peak memory remain unmeasured. + +** Committed canonical events for the HCL simple-attribute corridor + +The generated simple-attribute route previously wrote raw event triples to a +request-local rollback tape and then read the tape again to allocate canonical +events, node IDs, token IDs, and trivia. Once the closed shape is admitted, +there is no speculative rollback. The generated route now emits the canonical +events directly and asks the shared artifact authority to assemble the final +value. The raw tape is allocated only when the general generated parser runs. +The general path, including late complex syntax and rejection, continues to +use rollback and the existing raw-tape materializer. Strict full +=ParseArtifact= comparison against both raw-tape and indexed routes protects +the specialized event topology. + +The checked-in pure Scheme benchmark alternates the old raw-tape and new +canonical paths in one process after exact artifact preflight. For 1,024 +Basic lines, 20 batches of ten complete parses gave median CPU times 1.201 +and 0.952 ms (17/20 paired wins). For 3,200 lines the medians were 6.521 +and 5.658 ms (20/20). A 1,024-line source ending in a complex array uses +the generic path on both sides; medians were 3.951 and 4.014 ms across ten +batches of five parses (6/10 paired wins for the candidate). Run =gxc -O +t/benchmarks/hcl-scale/canonical-event-benchmark.ss=, then =gerbil env gxi +-e '(import +:gerbil-parser/t/benchmarks/hcl-scale/canonical-event-benchmark) +(main "1024" "20" "10" "basic")'=. Standalone AOT throughput and peak memory +remain unmeasured. + +The parser tests now share the hygienic =check-accepted-parse= syntax from +=src/testing/parser-ast.ss=. It evaluates source and candidate once, checks +accepted artifact validity and exact roundtrip, then compares any independent +parser routes. HCL and arithmetic tests use it; its own test exercises +single evaluation and caller bindings with the same names as macro internals. + +** Closed flat-block event corridor + +The generated HCL module now recognizes an additional closed shape: root-level +blocks with an identifier type, zero or more string or identifier labels, and +a body containing only simple attributes and newlines. The fast lexer covers +the braces in this shape. The generator emits the =HclFile=, =Block=, =Body=, +repeated =label= fields, and attribute fields and nodes directly into the +canonical event stream. Nested blocks, mixed root items, and other expressions +retain the generated recursive fallback. +The grammar digest and lexical declaration check still guard regeneration. +The generated module remains below the 1,000-line native source-policy limit. + +The checked-in pure Scheme benchmark compares complete parses with the same +fast lexer and alternates the generic and specialized order. It first checks +exact ranked tokens and full =ParseArtifact= equality. After a clean default +parallel build, 20 paired batches of ten parses gave median CPU times of +1.240 versus 0.696 ms for 256 four-line blocks (1,024 lines; 19/20 paired +wins), and 3.920 versus 2.213 ms for 800 blocks (3,200 lines; 20/20 wins). +After a clean rebuild, extending this corridor to two labels per block gave +1.412 versus 0.895 ms for 1,024 lines and 7.137 versus 2.945 ms for 3,200 +lines, with 19/20 paired wins at each scale. The original unlabeled 1,024-line +path remained in the same range: 1.256 versus 0.713 ms, 20/20 paired wins. +Run =gxc -O t/benchmarks/hcl-scale/flat-block-benchmark.ss=, then +=gerbil env gxi -e '(import +:gerbil-parser/t/benchmarks/hcl-scale/flat-block-benchmark) +(main "256" "20" "10" "labeled")'=. These are compiled-module, in-process +measurements on one host; standalone AOT and peak memory remain unmeasured. + +** Recursive closed-block generation + +The next candidate extended the block admission scan and canonical event +emitter recursively through =Block → Body → Block=. Each body still accepts +only newlines, simple attributes, and nested blocks; complex expressions and +mixed top-level attributes fall back to the generated recursive parser. The +recursive emitter carries the last child end into the enclosing =Body= close, +preserving nested field ranges, trivia order, and token/node IDs. Exact full +=ParseArtifact= comparison covers varied nesting and a 1,024-line fixture. + +The =nested= shape in the checked-in flat-block benchmark uses two labeled +block levels and one simple attribute in each four-line group. Across 20 +paired batches of ten complete parses, 256 groups (1,024 lines) had generic +and specialized median CPU times of 1.380 and 1.068 ms (19/20 paired wins); +800 groups (3,200 lines) had 6.844 and 3.212 ms (20/20). These are +compiled-module, in-process observations on one host after a clean default +parallel build; standalone AOT and peak memory remain unmeasured. The +unlabeled and labeled 1,024-line paths remained faster than generic parsing +at 1.301 versus 0.725 ms and 1.589 versus 0.931 ms (20/20 paired wins each). + +A separate mixed-root prototype admitted alternating top-level attributes +and blocks. Its exact artifact matched, but its 1,024-line medians were 1.493 +versus 1.508 ms and its 3,200-line medians were 7.104 versus 7.059 ms; +paired wins were 11/20 and 10/20. It had no reproducible throughput gain and +was reverted. The current closed corridor deliberately does not admit mixed +roots. + +** Fixed canonical event template for admitted HCL attributes + +The generated closed-shape attribute path previously called node and field +helpers for every nested event. Each helper checked the token stream for +trivia even though admission fixes the nested topology and trivia can only +precede the name, equals, and value tokens. The generated Scheme path now +assigns the six node IDs in one step and emits the same canonical vectors in +Grammar IR order, flushing trivia at those three token boundaries. Blocks +and nested blocks use the same attribute template; unclosed or complex forms +still enter the existing recursive fallback. + +A temporary same-process A/B switch compared complete =ParseArtifact= values +for 1,024 Basic lines, spaces, comments, CRLF, Unicode, identifier values, +blocks, nested blocks, and a complex-value fallback; all matched exactly. +Alternating 20 batches of ten complete parses on one host gave median CPU +milliseconds of 1.1394 to 0.9069 at 1,024 Basic lines (19/20 candidate +wins), and 5.7036 to 5.1664 at 3,200 lines (20/20 wins). The switch and old +emitter were removed after the comparison. The generated module remains under +the 1,000-line source limit. These results measure full lossless artifacts in +compiled Scheme modules, excluding import and executable build time. A +standalone =gxc -O -exe= link did not complete within a 240-second local cap, +so no standalone AOT throughput is claimed. + +* Proposed proof sequence + +1. Fix one immutable Scheme grammar and a corpus that exercises keywords versus + identifiers, precedence ties, trivia, Unicode byte offsets, errors, and + externally supplied scanner behavior. Record the exact PR head and run + current Scheme parse/roundtrip and compiler phase receipts. The existing + =t/benchmarks/parser-hot-path= and =t/scenarios/performance/lalr-fixed-point= + are starting gates; add a large input and one-edit scenario so small-input + overhead cannot hide scaling behavior. +2. Use the Scheme canonical LR(1) reference and both compact routes to + qualify context-sensitive state splitting on representative grammars. + The direct k=1 route already avoids canonical state enumeration, but its + conservative candidate stage and forward refinement need scale checks. + Compare language acceptance, conflict cells, state count, generation time + and memory, then fresh-parse throughput and lossless artifact equivalence. + Keep the current LALR generator as the reference until these checks pass. + Do not treat XLR as implemented evidence. +3. Prototype a Scheme-produced contextual lexical DFA for the closed, regular + subset of LexicalExpr. Derive admitted token sets from the existing interned + LR lexical modes. Keep the existing scanner path for expressions the DFA + cannot represent. Compare token identities, source ranges, tie decisions, + and rejection points exactly against the current compiler output. + + The first prototype now precomputes the ASCII first transition for each + interned LR lexical mode. It filters closed primitive scanners by their + possible first character, then runs the existing scanners and ranked + longest-match selector; non-ASCII input and expressions without a proven + first-character condition retain the complete candidate set. This is a + contextual prefix dispatch, not yet a full lexical DFA. On the local + 1,600-term arithmetic fixture, five-sample best lexical analysis measured + about 3.31 ms before and 0.78 ms after the change, in separate processes; + process variation limits the precision of that ratio. A clean v0.19 build, + language-surface Unicode byte offsets, parser-runtime, grammar-composition, + incremental, and ISO GQL throughput suites passed. The GQL suite's 100-query + p95 sample was 92.093 ms under its 150 ms admission ceiling. +4. If the batch path improves representative workloads, add reusable subtree + boundaries to the Scheme IR and ParseArtifact path. A reused node must have + unchanged text, grammar identity, lexical context, and entering LR state. + Compare incremental output with a fresh parse for edits before, inside, and + after nested structures, including edits that change downstream lexical + context. Measure re-lexed bytes, LR actions, reused node counts, memory, + and elapsed time; timing alone cannot prove edit locality. +5. Apply the same generated contract to the Rust product only after the + Scheme algorithm and semantic receipts pass. Rust is a downstream consumer + and conformance check, not the source of parser decisions. + +* Current verification limit + +This document records a research decision, experiment design, and bounded +local throughput observations. =gerbil.pkg= directly pins =poo-flow-core= at +=e85fd45303b208b23e5957fd316e7004994109a4= and +=asp-gerbil-scheme= at its recorded revision. Core itself declares +=gerbil-poo= as a transitive dependency. There is no declared =poo-flow= +dependency. Core's declarative Observability testing Case macro runs the +focused canonical/partitioned parser test against the real Core modules. The +local =gxpkg update= built Core and its dependency, then its global =gxtags= +pass included a pre-existing =.gerbil/pkg/poo-flow= symlink and failed. This +is a local package-environment issue, not a parser dependency edge. The +installed Gerbil is the pinned =2591dcd= v0.19 release on Gambit =dcd677c=. +Its standard library provides =:std/encoding/base64=, +=:std/string/utf8=, =:std/encoding/zlib=, and the queue, extensible-vector, +and list-builder modules used by this experiment. It does not provide the +older =:std/text/base64= or =:std/sort= paths. Current Scheme sources use +the v0.19 paths; stale =.gerbil/lib= interfaces from an older toolchain still +import the older paths. A separate temporary =GERBIL_PATH= was populated from +the exact declared revisions, followed by =gerbil clean= and =gerbil build=. +The canonical LR reference, generated arithmetic comparison, and grammar +composition suites all pass there. The original =.gerbil= cache was not +reused. The generated arithmetic module originally imported =deflanguage= +through the broad =language-support= facade, which loads ANTLR4, ISO BNF, +JavaCC, fixture, and runtime adapters. The source-admission implementations +have been moved from =src/language-support/= to the top-level +=language-support/=; versioned language modules import their owning macro +module directly. In repeated warm v0.19 process launches, +the facade itself took 3.0--3.3 s to import versus 2.4--2.5 s for the focused +grammar macro module. Arithmetic module import fell from about 3.9 s to +2.5--3.2 s. A small pre-change AOT executable importing the arithmetic module started +in about 0.01 s after warmup; dynamic =gxi= import costs must not be reported +as LR parse latency. Isolated v0.19 imports measured 1.89 s for +=:core/types=, 1.99 s for =:core/observability/types=, and 2.63 s for +=:core/observability/debug=. The parser continues to pass Core's declared +Observability policy directly; any reduction of this import cost belongs in +=poo-flow-core=, with its contract and tracing semantics preserved. A separate +v0.19 import of =:clan/poo/mop= took 2.11 s, so most of the cold Core import +time is inherited from the POO contract runtime. A Core-only change to +ordinary phase functions cannot remove that mandatory load while preserving +the same POO policy contract. The POO MOP imports its JSON adapter eagerly; +importing v0.19 =:std/encoding/json= alone took 1.72 s. Keep Core's single +declarative Observability entry and accept this dynamic-import cost rather +than adding a second parser/Core facade. In one +process, 1,000 LR parses of a prepared token stream +took 30.5 ms total. These are one-host short-input observations, not a +throughput improvement claim. Run the Rust syntax differential gates before +promoting this optional route. +Do not use =gxpkg uninstall= in this workspace. diff --git a/docs/research/scheme-poo-language-components.org b/docs/research/scheme-poo-language-components.org new file mode 100644 index 00000000..cbd5955a --- /dev/null +++ b/docs/research/scheme-poo-language-components.org @@ -0,0 +1,954 @@ +#+title: Scheme syntax and POO language components: expression and list studies +#+date: 2026-10-06 + +* Working discussion draft + +This local research draft studies how Gerbil syntax expansion, POO grammar +composition and parser compilation can cooperate. Source baseline: +=783381f33fa1b309ac4cc4b0012696f24c852d93=. It makes no public API decision and +implements no new engine behavior. The author discussion should precede selecting +syntax or committing to a component abstraction. + +Latest executable result: the section [[*Complete native language witness: numeric-free tiers][Complete native language witness]] +records 65 runtime checks and a successful bounded native exit for a small +=deflanguage= declaration assembled from numeric-free templates. Earlier sections +record the preceding core-only experiments and their evidence boundaries. + +The primary question is: how can a language author express semantic intent once, +compose reusable pieces, and obtain the existing parser and language services +through explicit, understandable contracts? + +* Important existing foundation: POO already composes grammars + +The current code supplies more than a POO language loader: + +| Owner | Current behavior | Design consequence | +|-------+------------------+--------------------| +| [[file:../../src/modules/parser/syntax.ss][parser syntax]] | =defgrammar-role=, =defgrammar= and =defgrammar-compose= | Reuse existing role and composition concepts | +| [[file:../../src/modules/parser/objects.ss][parser objects]] | Checked GrammarRole/Grammar objects and receiver-owned =.section= protocol | POO is already a grammar representation and extension boundary | +| [[file:../../src/compiler/normalize.ss][grammar normalization]] | Ordered parent traversal and merge/append/override/remove operations | Composition has deliberate semantics and produces a receipt | +| [[file:../../src/language/grammar.ss][language syntax]] | =defgrammar-syntax= templates and =deflanguage= compile-time elaboration | A concise author vocabulary can share canonical grammar semantics | +| [[file:../../src/language/entry.ss][language entry]] | POO loader inheritance, service slots and effective-object admission | Grammar components and service components have related but distinct roles | +| [[file:../../t/grammar-composition-test.ss][composition tests]] | Declared receipts, collision handling and parser qualification cases | Extend established semantic controls instead of inventing another merger | + +The normalization owner traverses parents, then ordinary roles, then explicit +composition operations. =append= requires a new identity; =override= requires an +existing identity and replaces its row; =remove= requires an existing target. +Ordinary =merge= uses the current keyed-row merger. These choices must remain +visible when a higher-level component is elaborated. + +Earlier discussion emphasized POO service organization. Source inspection adds +an important correction: POO grammar roles and explicit grammar composition are +already implemented. The research should connect concise authoring to these +existing capabilities rather than propose them as missing infrastructure. + +* Native POO substrate and parser operation labels + +The four words merge/append/override/remove are not four Gerbil core syntax forms. +They are labels in the parser's existing normalization contract. That statement +must not imply that native POO lacks the corresponding expressive power or that +language authors need a new operation DSL. The preferred author substrate is +upstream POO syntax and ordinary Scheme abstraction. + +The installed pinned implementation in :clan/poo/object and its doc/poo.md +"POO Definition Syntax" define .o, .mix, ordinary slot replacement, inherited +slot computation (=>), nested object override (=>.+), and an explicit inherited +computation binding. POO Flow's :core/extension-graph/interface also supplies +slot-append, slot-prepend, slot-override, slot-remove and node operations. Existing +capabilities should be reused rather than wrapped in another author mini-language. + +| Existing facility | Native meaning | Parser responsibility | +|-------------------+----------------+-----------------------| +| .o / .mix | Compose prototype objects with upstream inheritance semantics | Admit resulting language declarations and grammar identities | +| (slot expression) | Replace that slot computation | Check the effective grammar's references, fields and policy | +| (slot => function arguments ...) | Combine inherited value through a Scheme function | Project accepted grammar data and sources to canonical IR | +| (slot (inherited) expression) | Name the inherited computation inside a native expression | Preserve actual declaration context without inventing coordinates | +| slot-remove in POO Flow extension graph | Remove values from a list-valued slot | Do not equate this automatically with removing a keyed grammar rule | + +Native slot replacement and the old parser merge rule are not identical: +parser merge rejects unequal rows for an existing key. POO Flow slot-remove +removes values from a list slot; parser remove deletes a keyed declaration. +These are domain admission distinctions, not a reason to require authors to use +a second composition language. The existing legacy operation controls remain +independent semantic controls while native authoring is qualified. + +#+begin_src scheme +;; Implemented factory fragment from list-roles.ss: upstream slot combination. +(.o (:: self list-study-base-role) + name: 'native-list-role + (rules => append (grammar-role-ref arguments 'rules) + (grammar-role-ref elements 'rules))) + +;; Implemented inherited-computation fragment: explicit language policy. +(.o (:: self (make-native-list-role)) + name: 'native-single-list-role + (rules (inherited) + (filter-map + (lambda (row) + (case (car row) + ((arguments) '(arguments (field argument (reference name)))) + ((call-arguments/tail) #f) + (else row))) + (inherited)))) + +;; Question 1: Why is this design better? +;; Authors use one existing POO inheritance/computation model with native Scheme +;; functions. The parser projects the resulting checked value, then owns binding, +;; compilation and publication. No new composition-keyword language is needed. +;; Question 2: Why does this design have a scientific basis? +;; An extensible surface can elaborate to one compiler core. We test the native +;; path against independently declared list policies using complete events, +;; accepted/rejected status and roundtrip. This is evidence of the selected +;; implementation's preservation, not a proof of optimal DSL syntax or speed. +#+end_src + +The native factories enter the same canonical compiler, shared machine emission +and checked Loader entries as the earlier composition controls. The runtime +study now contains 119 checks, including 40 native author controls. Native +row-source slot computations retain declarationOwner and generated status; +componentOwner still names the actual normalized receiver. A single resolved +native role has one normalization occurrence: its receipt is not a reconstructed +trace of every upstream prototype operation. Arbitrary reader-coordinate capture +and the production deflanguage integration remain separate obligations. + +The repository-owned [[file:../../test-processes.ss][native process qualification]] reproduces the existing and native author controls +serially, preserving the ten-second child fence and rejecting nonzero exits. +It is also wired into the existing native CI job; current local results do not +establish a remote CI pass or full native machine/runtime qualification. + +* Two stages that must be designed together + +Current grammar roles expand into constructor calls that create checked POO +values when evaluated. The normalization/compiler functions consume those values. +The concise =deflanguage= path uses =for-syntax= compiler dependencies to generate +immutable grammar/parser products during expansion. Its current optional sections +are =node-fields=, =catalog=, =flow= and =backends=; it does not expose a role-import +section merely because POO roles exist elsewhere. + +Therefore a shared core representation does not automatically make all component +values available at the same phase. The study must specify where a component is +created, composed, checked and compiled. A module-load POO object cannot be assumed +to be available to a macro transformer without a phase design. + +Candidate architecture: + +1. Author syntax describes rules, policies and public tree meaning. +2. Gerbil macro expansion elaborates the chosen vocabulary and preserves origins. +3. Existing checked grammar composition applies explicit component operations at + a specified stage, producing the admitted declaration for the compiler. +4. The canonical compiler derives Parser IR and generated products. +5. The POO loader organizes qualified services and effective-object contracts. + +Step 3's integration into concise authoring is a research decision. Possible +approaches include declaration templates expanded before composition, explicit +phase-qualified role modules, or a shared elaborator used by both current entry +paths. Compare author burden, repeated work, determinism and separate compilation +before choosing one. Avoid assuming runtime object construction belongs in the +per-token parsing path. + +* Additional papers for the POO dimension + +** Extensibility for the Masses: Practical Extensibility with Object Algebras + +Bruno C. d. S. Oliveira and William R. Cook, ECOOP 2012, +[[https://www.cs.utexas.edu/~wcook/Drafts/2012/ecoop2012.pdf][author paper]]. Sections 2–3 describe extensibility of variants and operations; +section 7 studies modular combination of independently developed components and +combination of operations. The examples use Java factory interfaces and generics. + +Our interpretation: distinguish extending language constructs from extending the +analyses/products over those constructs. POO grammar roles help the former; +receiver protocols and compiler passes can support the latter. A role is not +therefore an object algebra in the paper's technical sense. Java's static typing +proof obligations do not automatically transfer to our Scheme contracts. + +Study application: compose arithmetic and call syntax, then add a field-summary +operation without requiring every author to rewrite the grammar. We should first +try the existing IR and POO protocols. Whether an additional algebra protocol is +useful depends on a concrete operation and measured complexity. + +** Polymorphic Embedding of DSLs + +Christian Hofer, Klaus Ostermann, Tillmann Rendel and Adriaan Moors, GPCE 2008, +[[https://pure.au.dk/portal/en/publications/polymorphic-embedding-of-dsls/][university publication record and abstract]]. The abstract motivates decoupling +DSL syntax from a single fixed interpretation. This annotation is based on the +record/abstract; the full paper has not been inspected in this study. + +Our interpretation: a declared structure can support more than one compiler +observation, such as grammar construction, shape analysis or explanation. For +this parser, observations should still share the admitted core and recognition +semantics. Alternative analyses are not permission to implement independent +Scheme and Rust parsers with divergent meaning. + +The [[file:scheme-dsl-research-basis.org][six-paper synthesis]] remains the main basis: Scheme syntax abstraction for +vocabulary, hosted DSL elaboration for shared compilation, Fortifying Macros for +declaration structure, scopes for modular composition, PEG for semantic discipline, +and abstract interpretation for derived facts. These two papers add the object +composition and multiple-observation questions to that existing study. + +* Expression component: author decisions and generated obligations + +Study three changes in order: add an operator, add a tier, then combine call syntax +with arithmetic. Use arithmetic, HCL and FHIRPath to expose differing semantics. + +| Concern | Author-owned intent | Possible compiler-owned result | +|---------+---------------------+-------------------------------| +| Operator identity | Spelling and lexical/category meaning | Required literal/catalog entries under declared lexical policy | +| Binding strength | Tier ordering and precedence | Core productions carrying explicit precedence | +| Associativity | Left, right or a deliberately constrained form | Parser policy on each production | +| Tree shape | Public node and operand/operator fields | Consistent catalog and shape facts | +| Operand grammar | Which constructs are allowed in each position | Resolved rule references and dependency information | +| Dialect extension | Add a construct or deliberately replace an existing rule | Explicit composition steps and receipt | + +A template can abstract the repeated binary-node construction. A procedural +=defsyntax= can elaborate an operator table and explain generated alternatives. +POO grammar composition can organize independently declared rule contributions. +Those mechanisms solve different parts of the author task and should cooperate. + +An important existing constraint: =override= replaces a keyed rule row. Adding an +operator alternative to an existing expression rule is not the same operation as +appending a new rule identity. A component vocabulary must state how alternatives +are assembled. Candidate choices are an explicit operator table, a named +extension point whose elaborator constructs one final row, or an explicit full +rule replacement. Implicit object slot inheritance is insufficient to specify +this grammar transformation. + +Case matrix to discuss: + +- Arithmetic: left-recursive binary productions and explicit precedence. +- HCL: binary expression represented by a first operand and repeated suffixes. +- FHIRPath: expression forms following its admitted normative syntax. + +Do not force these into one tree convention for the convenience of a library. +A reusable component should parameterize meaningful differences or remain narrow. +The shared benefit is a contract and predictable expansion, not identical grammar +for languages with different syntax and published trees. + +* List component: recognition, structure and context + +Study argument lists, array elements and declaration sequences. A candidate +parameter set includes item expression, separator, minimum item count, trailing +separator policy, delimiter choice and intended field placement. Before exposing +all parameters, determine which combinations real language packs use. + +| Decision | Recognition question | Tree/product consequence | +|----------+----------------------+--------------------------| +| Minimum count | Is the empty form accepted? | Item field may be absent | +| Separator | Which token separates items? | Separator tokens remain in lossless source | +| Trailing separator | Is it optional, required or forbidden? | Recognition and diagnostic expectations differ | +| Delimiters | Does the list own brackets/parentheses? | Span and recovery context differ | +| Field per item | Does each item contribute a field instance? | Repeated fields | +| Field around the whole list | Is the sequence one labeled value? | Zero/one outer field with possible fragment | +| Named list node | Is the list a public tree boundary? | Catalog identity and field ownership | + +Scheme templates can compose a nonempty list, optionality and delimiters. +=defsyntax= is useful when a structured policy declaration needs validation and +elaboration. A full POO grammar role is useful when the component contributes +named rules, lexical rows or other declarations. An inline list expression alone +need not become a runtime object or a large component declaration. + +Composition cannot choose recovery solely from the separator: a call argument, +an array element and a statement sequence have different enclosing contexts. +Treat recovery as an explicit contract or a caller-provided policy until a +principled shared rule has been demonstrated. + +* Two possible public authoring directions + +A. Template vocabulary over current =deflanguage= expressions, with explicit + POO roles for larger declaration contributions. This minimizes new concepts; + authors may still need to understand the boundary between the two paths. + +B. A unified component declaration that elaborates grammar contributions and + exposes related service obligations together. This may reduce repeated wiring, + but introduces phase, dependency and interface questions. It should be + justified by complete language-pack tasks before a public macro is chosen. + +Working preference: begin with A, using expression and list cases to discover +what B would need. This is an investigation order, not a decision to exclude B. +Do not publish a =defcomponent= form or a new registry before the component's +meaning and consumers have been discussed. + +* Why this develops the Scheme/Gerbil/POO advantage + +S-expression structure makes declarations directly composable. Gerbil templates +make repeated patterns reusable; procedural transformers express richer +elaboration with the same host language. Modules provide ordinary visibility and +phase boundaries. POO supplies checked component values and receiver protocols; +normalization makes ordered composition explicit. The parser compiler turns that +result into shared executable products and analysis inputs. + +The combined advantage is a progression from a local grammar idiom to an admitted +language component, with a common semantic path. It does not rely on parentheses +alone, arbitrary runtime callbacks, or an assumed speed advantage from inheritance. + +* Discussion and evaluation plan + +First discuss the intended author task, then compare explicit rules and candidate +abstractions. For each case record: + +1. Semantic decisions visible to the author. +2. Facts maintained more than once and facts derived by the compiler. +3. Expansion and composition explanation, including rule identity and order. +4. Nodes, fields, spans, accepted/rejected inputs and recovery behavior. +5. Creation/compilation/loading stage and analysis cost. +6. Qualified generated products using the existing semantic controls. + +Current evidence is source inspection and primary-paper reading. Existing test +cases are referenced but have not been rerun by this document change. No new +language component has been implemented or benchmarked. + +* Agreed direction: default tree structure with explicit language customization + +The user agreed to this direction on 2026-10-06. The agreement concerns the design +principle, not the proposed names, exact defaults or implementation stage below. +Components may provide a documented default tree structure. Each language pack +can explicitly choose its node and field conventions. Recognition, precedence, +associativity and recovery remain separately defined language decisions. + +** Representation differences are more than label differences + +| Examined grammar | Recognition organization | Current public representation | Component implication | +|------------------+--------------------------+-------------------------------+-----------------------| +| Arithmetic | Left-recursive alternatives with explicit precedence | Nested Expression nodes with left/operator/right | Binary-node vocabulary fits its existing shape | +| HCL | First operand followed by repeated operator/operand suffixes | BinaryExpression with potentially repeated operator/right fields | A chain vocabulary preserves its shape | +| FHIRPath | Separate precedence-tier rules; repeated suffixes and a recursive implies rule | Expression wrappers and tier-dependent field multiplicity | Parameterize rule dependencies and retain tier structure | + +This is source inspection of the existing grammar declarations, not a runtime +migration receipt. Renaming a default BinaryExpression node to HCL's public name +would not, by itself, preserve HCL's chain representation. Define a structural +strategy before applying node/field name customization. + +** Expression component sketch + +Three useful study strategies are a binary node, a flat chain and a recursive +right-hand tier. These are descriptive research names, not new DSL keywords. + +- Binary-node strategy: one left operand, one operator and one right operand + per node, with the caller's precedence/associativity policy. +- Chain strategy: one initial operand and a sequence of operator/operand pairs; + repeated pairs stay in one public node. +- Recursive-tier strategy: the caller identifies the next-tier and recursive + rule dependencies; wrapper placement stays explicit. + +For a new binary-node component, a candidate default is =BinaryExpression= with +=left/operator/right= fields. A language can explicitly use =Expression= or +another published node, and map the operand fields. The default itself remains +a discussion candidate. Existing languages select their current structure and +labels explicitly during a study so comparison has a stable target. + +Required semantic inputs are operand/rule dependencies, operator expression, +and the intended precedence/associativity organization. A chain's structural +shape does not alone establish how a consumer evaluates its operators; do not +infer semantic associativity merely from repeated suffixes. + +** List component sketch + +Start with a transparent sequence that contributes a field per item; a candidate +default field label is =item=. A named list node or one outer list field is an +explicit structural customization. Delimiters can compose around the sequence +without introducing an additional node automatically. + +A list study must still specify whether it is empty or nonempty and whether a +trailing separator is allowed. The agreed default-tree direction does not settle +those recognition choices. Inline composition of =optional=, a nonempty-list +template and delimiters keeps the choices visible before any larger policy form +is justified. + +For example, a call language can choose repeated =argument= fields while an array +language chooses repeated =element= fields. Choosing an outer =arguments= field +around the full sequence is a different representation with fragment behavior; +it is not a field-name substitution. + +** Customization scope and conflict meaning + +| Customization | Intended effect | Must not be silently changed | +|---------------+-----------------+------------------------------| +| Node label | Published node identity | Nesting or ownership | +| Field label | Name of a semantic position | Cardinality or child boundaries | +| Structural strategy | Binary versus chain/wrapper/list representation | Accepted language and parser policy | +| Grammar dependencies | Referenced operand/item/tier rules | Their resolution and declared semantics | +| Recognition policy | Allowed operators, separators, emptiness or trailing form | Public tree meaning without an explicit structural decision | + +Each customization should have one semantic owner. An unknown setting or two +contradictory structural choices should be explained at declaration time. Public +catalog projection continues to use the existing identity/order authority. +Defaults need documentation and a change policy once they become a public +library; the current study does not publish them. + +** Cooperation between syntax, POO and compilation + +1. A small =defrules= template expresses a common node/list idiom. +2. A =defsyntax= elaborator handles structured choices and produces explicit + grammar declarations plus source explanations when the template is insufficient. +3. Larger contributions use the existing checked GrammarRole representation and + explicit composition semantics at a specified phase. +4. Canonical compilation derives the parser and catalogs from the final grammar. +5. The loader's POO services refer to that admitted descriptor and its products. + +A new tier may introduce a new rule identity. A modification of alternatives +within an existing rule must produce one final rule row or use deliberate +replacement. Treating every extension as =append= would not express these +operations correctly. Keep this choice in the component's elaboration contract. + +** Next concrete comparison + +Prepare a side-by-side author example for Arithmetic's binary structure and +HCL's chain structure using shared vocabulary. For each show the chosen default, +explicit customization, resulting core rules and expected public tree. +Then compare a call argument list and an array element list. + +The next discussion question is whether component selection should make the +structural strategy explicit in the author's form, or choose it through a named +component whose contract fixes the structure. Working preference: named narrow +components such as binary-expression and expression-chain, with node/field +customization. This reduces contradictory option combinations and makes the tree +model apparent at the use site. The public names remain undecided. + +* Concrete expression vocabulary comparison + +** Local executable study, 2026-10-06 + +The study modules are +[[file:../../t/fixtures/language-pack-research/expression-vocabulary.ss][expression-vocabulary.ss]] and +[[file:../../t/fixtures/language-pack-research/expression-comparison.ss][expression-comparison.ss]]. +They use Gerbil =defrules= for structural templates and a small =defsyntax= +bridge to the existing =grammar-expression= macro. Their names are local +research vocabulary, not published parser API. No parser implementation or +language package has been migrated by this experiment. + +The bridge expands these known templates until the outer form is =prec= or +=alias=, then invokes =grammar-expression=. Its bounded expansion loop is a +fixture convenience, not a general extension protocol. An eventual language +elaborator needs a specified extension boundary and declaration diagnostics. + +The native command, run from the isolated checkout, was: + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" gtimeout 10 gerbil env gxi \ + t/fixtures/language-pack-research/expression-comparison.ss +#+end_src + +It exited 0 and reported six checks: Arithmetic core equivalence, HCL core +equivalence, binary defaults, chain defaults, explicit field customization and +distinct binary/chain values. The last check is only a structural sanity check; +it does not establish equivalence or difference of recognized languages. +The other checks compare constructed canonical grammar values, not parser +output, spans, conflict handling, allocation or performance. The run also +reported the existing =std/net/request= deprecation warning. + +** Author-facing forms + +Arithmetic's additive alternative can select its existing published node: + +#+begin_src scheme +(binary-expression left 10 + (reference expression) + (choice (literal "+") (literal "-")) + (node Expression) + (fields left operator right)) +#+end_src + +HCL's existing chain representation can select the chain component: + +#+begin_src scheme +(expression-chain + (reference unary-expression) + (reference binary-operator)) +#+end_src + +The latter uses the proposed defaults =BinaryExpression= and +=left/operator/right=. Both node and fields can be specified explicitly. +Arithmetic supplies associativity and precedence at the use site; neither +component infers an operator precedence table from token spelling. HCL's +component expresses the current chain rule without adding a precedence policy. + +This vocabulary removes repeated structural mechanics while leaving the +language decisions visible: operand dependencies, operator recognition, +precedence where applicable, and any change to the public node or fields. + +** Expansion into existing grammar forms + +The Arithmetic form expands to: + +#+begin_src scheme +(prec left 10 + (alias Expression + (seq (field left (reference expression)) + (field operator (choice (literal "+") (literal "-"))) + (field right (reference expression))))) +#+end_src + +The HCL form expands to: + +#+begin_src scheme +(alias BinaryExpression + (seq (field left (reference unary-expression)) + (repeat + (seq (field operator (reference binary-operator)) + (field right (reference unary-expression)))))) +#+end_src + +The existing =grammar-expression= macro maps these into the canonical +=precedence/alias/sequence/field/repeat= values. Shared compiler analysis can +therefore inspect the explicit result, rather than needing separate knowledge +of each author's shorthand. + +** Expected public structure to verify next + +For =a + b + c=, the following are schematic consequences of the rules. They +omit operand wrappers and lexical details; they are not captured parser output. + +#+begin_example +Arithmetic, additive left-associative binary structure: +Expression + left: Expression + left: a + operator: + + right: b + operator: + + right: c + +HCL, expression-chain structure: +BinaryExpression + left: a + operator: + + right: b + operator: + + right: c +#+end_example + +A node rename cannot turn the second structure into the first. Repeated +=operator/right= fields and nested binary nodes are different public contracts. +A subsequent parser fixture should verify actual child ordering, field +multiplicity, spans and zero-suffix behavior. In particular, the chain permits +zero repeated pairs and still has an alias node, whereas a binary alternative +requires an operator and two operands. + +** Where POO belongs + +A structural template constructs a rule expression. A GrammarRole can own +that expression together with its dependent rules, tokens and declarations. +Composition then operates on the resulting named contributions using the +existing checked grammar protocol. + +Adding a new rule identity may use =append=. Adding operators to an existing +expression rule requires construction of its final alternatives or an explicit +=override= of that rule. The prototype does not alter those meanings and does +not yet bridge runtime GrammarRole construction into =deflanguage='s expansion +phase. That phase bridge remains a design question with its own executable +acceptance criteria. + +POO's contribution is to let related grammar contributions be named, composed +and checked together. The macro system's contribution is to let authors express +those contributions in a vocabulary that expands to explicit grammar forms. +The canonical compiler remains responsible for deriving parser products. + +** Connection to the papers and the design decision + +- Dybvig/Hieb/Bruggeman supplies the distinction between pattern templates and + procedural syntax transformation. These two structural idioms fit =defrules=; + collecting an operator family and its dependencies may require =defsyntax=. +- Ballantyne/Gamburg/Hemann motivates an extensible author vocabulary that + elaborates into a shared core. The checked core-value comparisons are a small + executable step in that direction, not an implementation of their framework. +- Culpepper/Felleisen motivates diagnostics in the author's declaration + vocabulary. The present prototype has only generic macro mismatch errors; + vocabulary-specific diagnostics remain future work. +- Oliveira/Cook motivates separating reusable construction interfaces from + consumers. It informs the POO study, but does not establish that the existing + role protocol is an object algebra with the paper's static guarantees. + +The working recommendation is two named structural components with documented +defaults and explicit customization. The next study should compare call +arguments and array elements, then test a complete language declaration and +its actual tree. Publish names and the POO phase bridge only after discussing +those results; the current drafts remain local. + +* Discussion correction: author intent versus numeric precedence + +The author questioned why the example exposes =left 10= when native AST +structure is independent of indentation. This distinction must be explicit in +our examples, before choosing author-facing syntax. + +** Three different structures + +1. The grammar declaration is Gerbil Scheme source. Parentheses establish its + S-expression structure; ordinary formatting indentation does not establish + precedence or field ownership. +2. The target source may have explicit parentheses, infix operators or meaningful + layout. Its grammar determines which of those distinctions matter. An input + such as =(+ a (* b c))= already makes operator nesting explicit; it does not + require an infix priority table to discover that nesting. +3. The parser produces a public tree under the chosen recognition and reduction + rules. Tree construction preserves the selected structure; it cannot determine + an infix language's intended binding policy merely from AST node names. + +The Arithmetic fixture parses infix input and has recursive alternatives for +=expression operator expression=. In this existing grammar, =left= is the +conflict policy for equal ranks and =10= is a relative priority, not an +indentation level or an AST depth. Its =20= multiplicative tier outranks the +=10= additive tier. The current +[[file:../../src/compiler/lr-compiler.ss][LR conflict resolver]] compares ranks and, at equal ranks, selects reduce for +left associativity or shift for right associativity. This is source inspection, +not a new runtime measurement. + +For the infix input =a - b - c=, associativity distinguishes =(a - b) - c= from +=a - (b - c)=. For =a + b * c=, the relative binding policy distinguishes +=a + (b * c)= from =(a + b) * c=. Whitespace formatting alone does not resolve +these choices in this language. The explicit =extras whitespace= declaration +in [[file:../../languages/arithmetic/grammar.ss][Arithmetic]] is separate from precedence. Layout-sensitive target languages +can use declared layout markers; that does not make Scheme declaration +indentation a priority mechanism. + +** Revised author-facing direction + +The earlier =binary-expression left 10= prototype reproduces existing core +semantics and proves a small macro mapping. It is not the recommended final +author interface. Numeric ranks should remain an available core representation; +author vocabulary can instead name the semantic organization. + +Two alternatives deserve comparison: + +| Author representation | Benefit | Composition question | +|-----------------------+---------+----------------------| +| Named operand tiers, such as sum referring to product | Grammar dependencies express binding; no numeric rank in ordinary author code | Preserve tree wrappers and specify recursive association when extending a tier | +| Named operator groups with relative order | Retains a compact recursive infix rule while hiding arbitrary rank numbers | Independently contributed groups need explicit ordering/equality constraints | + +A local group name should have component-qualified identity. If two groups have +no declared relation, an elaborator must not invent semantic priority from +import order or the lexical spelling of their names. Cyclic strict ordering is +an invalid declaration. Equality of priority needs its own declared grouping or +constraint and a compatible association policy. A total-order prototype would +be insufficient evidence for independently composed partial orders. + +For either alternative, default =BinaryExpression= and per-language node/field +customization remain useful. Recognition organization and public-tree shape +are separate decisions: changing to named tiers must not silently add tier +nodes. Preserve explicit provenance so a conflict can cite the author's operator +group or dependency rather than only a generated integer. + +This follows our interpretation of +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Ballantyne/Gamburg/Hemann, section 3.3]]: an expressive surface can elaborate +into a small compilation core. The paper does not prescribe this operator-group +API. Likewise, +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Culpepper/Felleisen, sections 3.2 and 7.2]] motivates vocabulary-specific +validation and collection of declarations before dependency checks; it does not +supply Gerbil syntax classes or prove our proposed ordering contract. + +Before publishing expression component syntax, compare these alternatives on +Arithmetic and a language with explicit tiers. The design goal is semantic +intent expressed once, with numeric precedence emitted only where the chosen +grammar strategy needs it. + +* Numeric-free expression strategies: concrete comparison + +** What the named-tier prototype establishes + +[[file:../../t/fixtures/language-pack-research/tier-comparison.ss][tier-comparison.ss]] extends the local macro study with six checks. Its native +command was the same bounded Gerbil invocation as the earlier comparison, with +=tier-comparison.ss= as the entry file. It exited 0, printed +=TIER-COMPARISON-OK: 6 checks passed=, and reported the existing +=std/net/request= deprecation warning. + +The checks cover left-binary core structure, right-binary core structure, +FHIRPath's actual =or= tier, FHIRPath's actual =implies= tier, explicit node/field +customization and absence of canonical numeric precedence in the two binary +values. These are canonical-value checks. They do not prove LR admission, +recognition equivalence, actual CST equivalence or performance. + +Two local author forms illustrate numeric-free binary rules: + +#+begin_src scheme +;; Rule named sum: the base operand comes from the product rule. +(left-binary-tier sum (reference product) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right)) + +;; Rule named power: recursive suffix produces right nesting. +(right-binary-tier power (reference atom) (literal "**")) +#+end_src + +Their explicit core rules are respectively: + +#+begin_src scheme +(choice + (alias Expression + (seq (field left (reference sum)) + (field operator (choice (literal "+") (literal "-"))) + (field right (reference product)))) + (reference product)) + +(choice + (alias BinaryExpression + (seq (field left (reference atom)) + (field operator (literal "**")) + (field right (reference power)))) + (reference atom)) +#+end_src + +The first places recursion on the left; the second places it on the right. +Both pass through their base operand without adding a public binary node when +there is no operator. Dependencies encode tier organization, so these examples +have no =prec= annotation. Actual complete grammar admission still needs a parser +fixture, especially when combining prefix, postfix or overlapping alternatives. +The names remain local prototypes, and an eventual rule-declaration form may +avoid repeating the enclosing rule's own identity as an argument. + +** Existing FHIRPath demonstrates a different structural contract + +[[file:../../languages/fhirpath/grammar.ss][FHIRPath]] already expresses relative tiers as references, without numeric +precedence annotations in these rules. The current =or-expression= references +=and-expression= as its operand and repeats operator/right pairs inside an +=Expression= alias. The =implies-expression= references =or-expression= and has +an optional right-recursive pair. Their reusable local forms are: + +#+begin_src scheme +(expression-chain (reference and-expression) + (choice (literal "or") (literal "xor")) + (node Expression) (fields left operator right)) + +(recursive-right-chain implies-expression (reference or-expression) + (literal "implies") + (node Expression) (fields left operator right)) +#+end_src + +The comparison checks these forms against the current explicit core structures. +The chain templates wrap the operand even when there is no operator; binary +tier templates do not. This is a tree contract, not an optimization detail. +A common word such as "tier" must not silently select one of these structures. + +| Template | Recursion / repetition | Zero operators | Multiple operators | +|----------+------------------------+----------------+--------------------| +| Left binary tier | Self reference in left operand | Base operand passes through | Nested binary nodes on the left | +| Right binary tier | Self reference in right operand | Base operand passes through | Nested binary nodes on the right | +| Expression chain | Repeated suffix over one operand rule | Alias remains | Repeated operator/right fields in one node | +| Recursive right chain | Optional suffix with self reference on right | Alias remains | Right-recursive alias structure, including base wrappers | + +These are rule-derived expectations. The prototype checks their declarations; +actual event order, public fields, spans and wrapper counts still need runtime +assertions. Arithmetic migration and FHIRPath reuse are consequently separate +experiments: one restructures recognition rules and must establish equivalence; +the other can first preserve existing expressions exactly. + +** Alternative: named operator groups + +A second vocabulary could keep a recursive infix rule and allow a language to +name groups and their relative binding. The following is design notation only: + +#+begin_src scheme +(operator-groups + (sum (operators "+" "-") (associativity left)) + (product (operators "*" "/") (associativity left))) +(binding-order sum product) ; weaker to stronger, explicitly declared +#+end_src + +Neither form is implemented or proposed as the final spelling. In this sketch, +=associativity= resolves same-group infix chaining; it does not affect Scheme +indentation. Prefix/postfix placement is another declaration dimension and needs +its own contract rather than being inferred from operator spelling. + +Group elaboration could generate the current =prec= core values. Numeric ranks +would be compilation output, while diagnostics retain component-qualified group +names and source declarations. Numeric values are still allowed in low-level +core authoring; the study is about reducing ordinary language-author burden. + +A composed partial order deserves special care. A deterministic topological +sort can assign a total order, but it would also decide undeclared relations. +Likewise, assigning equal numeric ranks to incomparable groups would introduce +a tie policy the author did not select. Neither operation is a sufficient +semantic implementation of partial ordering. + +A defensible design needs to distinguish irrelevant pairs from pairs that +actually participate in a parser conflict. For relevant pairs, it must require +an explicit relation/grouping or retain an unresolved conflict under the existing +admission policy. Ordering constraints, conflicting association declarations, +duplicate operator contributions and cyclic strict order need author-level +explanations. This contract is still under study; there is no ranking algorithm +implemented in the research fixtures. + +** Why these are different Scheme strengths + +Pattern macros can abstract a recurring grammar shape while preserving caller +syntax. Procedural macros can collect richer declarations and check dependencies +before emitting explicit core forms. Module exports can give a language pack its +own reusable vocabulary. These are distinct capabilities beyond parenthesized +notation. + +Our reading of +[[https://www2.ccs.neu.edu/racket/pubs/icfp24-bgh.pdf][Compiled, Extensible, Multi-language DSLs, section 3.3]] supports this surface/core +separation and deliberate extension positions. Its Racket extension classes are +not automatically available in Gerbil, and the present fixtures do not implement +that paper's expander. Our reading of +[[https://www.cs.utexas.edu/~wcook/Drafts/2012/ecoop2012.pdf][Extensibility for the Masses]] motivates considering both contributed constructs +and analyses over their common representation. It does not turn a syntax macro +into a POO grammar object or grant Java's typing guarantees to Scheme. + +For this repository, the sequence is author vocabulary, explicit core expression, +checked named grammar contributions, canonical compilation, and admitted services. +The phase at which those contributions become POO values remains explicit work. +A component cannot assume that template expansion and runtime role construction +are interchangeable. No additional per-token object dispatch follows from this +proposal. + +** Working recommendation and next evidence + +Prefer named rule dependencies as the first numeric-free authoring strategy. +Existing FHIRPath offers a concrete reuse target; the binary-tier prototypes +show a possible Arithmetic direction. Retain operator-group elaboration as a +separate option for grammars where it reduces author effort and has an explicit +composition contract. Do not make all language authors declare a group table. + +The next complete fixture should verify: + +1. Actual left/right nesting and base-operand behavior for a small admitted + language, with =a-b-c= and a right-associative example. +2. Arithmetic precedence and existing public node/field structure for =a+b*c=, + grouped input and prefix interaction, against the current parser. +3. Exact FHIRPath chain/wrapper behavior using the reusable templates. +4. Module import/rename behavior and ownership of any generated helper rule. +5. A minimal POO role composition at a declared phase before integrating a + role-import clause into =deflanguage=. + +Only after those controls should an API spelling be selected or languages be +migrated. This is a local research recommendation for discussion, not a released +implementation or a measured engine performance improvement. + +* Complete native language witness: numeric-free tiers + +** Implementation boundary and result + +The local [[file:../../t/fixtures/language-pack-research/tier-language.ss][tier-language.ss]] imports the template vocabulary and uses the existing +=deflanguage= compiler. Its declaration uses named sum, product, power, prefix +and atom rules. Sum/product are left-recursive binary tiers; power is a +right-recursive binary tier. Prefix binding is explicit: it binds more tightly +than the study's exponentiation rule. None of these rules uses numeric +precedence. The language is a research fixture, not a replacement for Arithmetic. + +The local [[file:../../t/fixtures/language-pack-research/tier-runtime.ss][tier-runtime.ss]] runs the repository's =parse-source= and consumes the +canonical ParseArtifact events. It implements no alternate source parser. +The successful native invocation was: + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" GERBIL_PARSER_LR_TRACE=1 \ + gtimeout 10 gxi t/fixtures/language-pack-research/tier-runtime.ss +#+end_src + +It exited 0 and reported =TIER-RUNTIME-OK: 65 checks passed=. Compiler progress +reported 24 lowered productions and 31 LR states, followed by artifact admission. +These counts describe this fixture, not a speed or allocation comparison. +The driver explicitly exits only after all assertions and its final flushed +summary. No timeout threshold was widened. + +** What is checked + +| Control | Evidence | Boundary | +|---------+----------+----------| +| Twelve shared inputs | Each parser accepts; candidate roundtrips; complete event lists equal | Finite sample parity, not grammar equivalence | +| =1-2-3= | Binary boundaries open at 0, open at 0, close at 3, close at 5 | Left nesting with exact spans | +| =a**b**c= | Binary boundaries open at 0, open at 3, close at 7, close at 7 | Study-only right-associative extension | +| =a-b-c= | No binary node | Current identifier lexical policy permits hyphens | +| =a= | No binary node | Transparent base operand in this binary-tier design | +| Five invalid inputs | Empty, missing right operand, invalid initial operator, adjacent names, unmatched parenthesis reject | Local admission controls | + +The twelve shared inputs are =a=, =42=, =a-b-c=, =1-2-3=, =a - b - c=, +=a+b*c=, =(a+b)*c=, =-a*b=, =a+-b=, =--a=, =a / b - c= and a source with +leading newline/space and trailing tab. The event comparison includes node IDs, +kinds and order, field boundaries, tokens and spans. Grammar digests differ +because the declarations differ; equality here deliberately compares emitted +public events rather than whole artifact identity. + +The right-associative extension is not compared against Arithmetic. Arithmetic's +punctuation inventory includes =**=, but its examined expression alternatives +do not admit exponentiation. Lexical recognition is not proof of grammar support. +The fixture also does not claim any normative exponentiation/prefix policy for +another language. + +** Retained unsuccessful attempts and corrected test meaning + +The first runtime attempt exited 70 on the left-nesting assertion. It used +=a-b-c=, which the current =identifier= scanner recognizes as one token: +[[file:../../src/runtime/scan.ss][identifier-rest?]] explicitly allows a hyphen. Acceptance and event parity alone +therefore did not establish that subtraction had been exercised. The corrected +fixture keeps this identifier case and uses =1-2-3= plus spaced identifiers for +subtraction. The expected tree assertion remains mandatory. + +A later =gerbil env gxi= attempt emitted all 65 successful assertions but exited +124 under the same ten-second bound. That invocation is retained as a timeout, +not a passing receipt. The terminal driver now explicitly exits after its final +summary, and the successful receipt uses the direct =gxi= entry used in the +repository's native commands. This is not evidence isolating the cause of the +wrapper's shutdown behavior. The passing invocation has both the required +summary and exit 0. + +** Design consequences and remaining author burden + +This witness establishes that the current compiler can admit a complete small +language assembled from numeric-free grammar templates. Named rule dependencies +can express binding while keeping public binary nodes, fields and spans aligned +with the current Arithmetic parser on selected inputs. It is stronger evidence +than core-expression equality and supports the named-tier direction. + +It does not establish that every numeric priority can be removed, that grammar +ambiguity disappears, or that every tier strategy produces the same tree. The +FHIRPath chain/wrapper runtime checks, module composition phase bridge and general +operator-group ordering contracts remain open. + +The current prototype still repeats =sum= as both the enclosing rule name and +the template's self-reference parameter. A final vocabulary should study a +rule-declaration elaborator that supplies this context once. The existing +=deflanguage= extension path expands expressions within rule rows; it does not +already provide an arbitrary declaration-row extension protocol. The design must +specify that boundary before proposing context-dependent shorthand. + +Similarly, moving binding into named rules introduces helper-rule identities. +Their visibility and component-qualified ownership need to be settled with POO +composition. Public AST/CST node identity remains separate from helper-rule +identity: transparent helper rules should not become public nodes by accident. + +The next discussion should evaluate author burden across a complete declaration: +explicit dependencies, helper naming, tree customization, lexical policy and +module composition. The successful witness supports simplification of precedence +configuration, while leaving those other decisions visible for research. + +* Follow-up: declare a rule identity once + +The [[file:scheme-rule-context-design.org][rule-context declaration study]] adds a local declaration wrapper and 59 +successful native checks. It removes explicit self-name repetition, while +separately documenting host binding, grammar identity, POO contribution ownership +and the declaration-provenance boundary. Its public syntax remains undecided. + +* Scheme design rationale: two required questions + +This local vocabulary rule row shows structural binding order. The complete +fixture connects sum to product, product to power, and power to its operand. + +#+begin_src scheme +(sum + (left-binary-tier sum (reference product) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right))) + +;; Question 1: Why is this DSL design better? +;; Named dependencies express operands from product without numeric ranks in this example. +;; Macros share tree contracts; POO roles compose contributions; the compiler uses common IR. +;; S-expressions carry structure, defsyntax/defrules abstraction, and POO composition. +;; Question 2: Why does this DSL design have a scientific basis? +;; Tier dependencies and recursion direction specify binding; nodes, fields and spans are observable. +;; Evidence: 65 native checks cover selected Arithmetic events and associativity spans. +;; Falsifier: wrong nesting of 1-2-3 or right-associative inputs, or extra base-operand nodes. +;; Limit: tiers do not suit every grammar; general groups, author tasks and performance remain open. +#+end_src + +The contextual follow-up removes the repeated self identifier shown here. +These successive witnesses measure separate improvements; they are not evidence +that one shorthand solves the entire language-package declaration problem. + +* Latest PR rebase qualification + +The refactor was rebased onto PR #13 head +=f058225a1454732af6144076f5916418bde71153=. The native author fixture +build completed in 7.90 seconds with exit status 0 under the unchanged +240-second total and 5-second idle watchdog. The portable research gate +passed 296 checks across ten processes, each exiting 0. The rebased GQL +fixture checks compare identifiers and sources because Loader copies fixture +records. These local receipts do not establish GitHub CI success or a complete +native runtime qualification. diff --git a/docs/research/scheme-reduction-abstractions.org b/docs/research/scheme-reduction-abstractions.org new file mode 100644 index 00000000..2fafed7d --- /dev/null +++ b/docs/research/scheme-reduction-abstractions.org @@ -0,0 +1,676 @@ +#+title: Scheme abstractions at the semantic reduction boundary + +* One engine boundary, two semantic products + +This study connects the [[file:engine-reduction-generation.org][shared reduction generator]] to the +generic runtime. The canonical operand action chain is shared by recognition +trees and event programs. Its order, empty-chain identity and preparation +contract belong to the engine. Backend-specific field and alias construction +belongs to each semantic product. Language declarations acquire no additional +operation or configuration. + +The earlier [[file:unary-reduction-corridors.org][corridor experiment]] reduced allocation but did not establish +a consistent complete-request CPU improvement. This refactor addresses the +maintainability of the action interpreter without reviving that execution path. + +* Official source evidence + +The installed Gerbil source used for this study is revision =2591dcd=, shipped +with the locally qualified toolchain. Reproduction starts with the toolchain's +source tree rather than assuming that an online development branch matches it. + +- =src/std/text/parser/ll1.ss=: =ll1-bind= expresses parser composition; + =ll1-begin= and =ll1-case= stage control structure; =ll1-char?= uses the + =BufferedReader= interface through =using=. The same file retains named + =let= loops and conditional expressions in concrete execution algorithms. +- =src/std/iter/macros.ss=: =for1= validates declaration shape with + =syntax-case= and generates callbacks or typed bindings; =for1-dispatch= + selects an expansion appropriate to the iterator form. +- =src/std/vector/vector.ss=: typed procedure and vector arguments define + contracts around vector algorithms, while =for= expresses index traversal. + +These examples support separate abstractions for composition, representation +contracts and execution. They do not establish that higher-order calls are +always faster, or that named loops are poor Scheme style. + +* Declarative action admission + +The runtime previously embedded list-length checks and a callback in vector +construction. Preparation now names the two accepted shapes explicitly: + +#+begin_src scheme +(def (operand-action? action) + (match action + (['field _] #t) + (['alias _] #t) + (_ #f))) +#+end_src + +The action-chain boundary still requires a proper list. Names retain the +existing canonical IR contract; this change does not introduce a new name +restriction. Unknown operations, missing payloads, extra payloads and improper +chains fail during preparation. Validation remains outside the request loop. + +* Stage the shared fold + +#+begin_src scheme +(defrule (define-operand-action-fold fold-name action-step) + (def (fold-name value actions offset constructor) + (if (pair? actions) + (fold-name (action-step (car actions) value offset constructor) + (cdr actions) offset constructor) + value))) + +(define-operand-action-fold apply-operand-actions apply-operand-action) +(define-operand-action-fold apply-operand-actions/events apply-operand-action/events) +#+end_src + +The recognition step remains its existing procedure. The event step is a +hygienic rule that binds the selected action once before its dispatch, keeping +the previous inline event evaluation. The shared rule emits closed tail loops; +it does not install a backend callback or capture a request's offset. + +* Question: why is this design preferable? + +The source now states one action-chain traversal contract and two semantic +interpretations. Fixing action order or identity requires changing one fold +definition. The backend operations remain independently readable. Scheme's +advantage here is hygienic staging of an engine algorithm, beyond merely +representing declarations as lists. A runtime combinator remains appropriate +where a caller genuinely supplies a dynamic strategy; these two backend choices +are already known when the module expands. + +This is a local abstraction inside the shared engine. It does not justify +turning every loop into a macro, adding POO objects to private scalar state, or +changing the three language-author interfaces. + +* Question: why is the claim scientifically testable? + +The invariant is =fold(step, seed, actions)= with actions consumed in declaration +order. Empty chains return the original seed. Existing negative preparation +tests exercise malformed action shapes. Decorated chains and widths +=0, 1, 2, 8, 32= compare complete canonical artifacts against the independent +materialized GLR interpreter, for both recognition and event backends. +Unary-corridor controls additionally compare bounded checkpoint action counts. + +Native compilation verifies the macro expansion, while matched GQL, FHIRPath +and Arithmetic requests distinguish maintainability from performance. A +successful source test alone does not establish native request cost. No +end-to-end speedup is claimed from removing duplicated source text. + +* Qualification + +- Source controls: 25 cases, final =OK= and exit 0, 13.873 s. + Receipt: unarchived local execution. +- Official native compilation: exit 0, 18.363 s; candidate artifacts live in + a temporary native overlay. Inspection of the emitted =.scm= + confirms direct recursive calls and inline event operation dispatch. +- Native controls: five cases, final =OK= and exit 0, 15.143 s. + Receipt: unarchived local execution; its + interface trace selects the candidate =lr-parser.ssi= explicitly. + +Two initial harness attempts are excluded: one named nonexistent test files; +the other selected a C-FFI ABI module for interpretation and failed with +=Interpreter does not support ##c-declare=. The final source run uses the +source-owned ABI alignment control; native qualification remains separate. + +** Matched native request costs + +Four sequential runs used 20 samples of 20 requests for each route, with the +same source products, independent GLR admission, toolchain and artifact graph. +Only the candidate runtime overlay changed. All four runs completed with +=LR-REQUEST-STACK-OK= and exit 0; elapsed times were 16.655, 16.384, 18.906 +and 16.335 s. Receipts are +unarchived local execution. + +CPU values below are batch CPU P50 divided by 20, in milliseconds per call. +They are not individual request latency percentiles. Natural collections remain +in timing; admitted allocation samples exclude collection intervals. + +| Route | Baseline 1 | Candidate 2 | Candidate 3 | Baseline 4 | Bytes/call, all runs | +|-------+------------+-------------+-------------+------------+---------------------| +| gql-return 64 prepared-lr | 0.2173 | 0.21595 | 0.2273 | 0.2119 | 230932. | +| gql-return 64 full-source | 0.30345 | 0.30775 | 0.3618 | 0.31085 | 542941.6 | +| fhirpath-addition 64 prepared-lr | 0.2104 | 0.2128 | 0.2397 | 0.2108 | 255540. | +| fhirpath-addition 64 full-source | 0.33965 | 0.3383 | 0.44785 | 0.3427 | 695181.6 | +| arithmetic-addition 64 prepared-lr | 0.09635 | 0.09575 | 0.09765 | 0.094 | 134804. | +| arithmetic-addition 64 full-source | 0.1457 | 0.14455 | 0.15855 | 0.14145 | 462141.6 | +| gql-return 512 prepared-lr | 1.75865 | 1.61955 | 1.89055 | 1.5999 | 1894004. | +| gql-return 512 full-source | 2.64495 | 2.39755 | 2.7127 | 2.4177 | 4330101.6 | +| fhirpath-addition 512 prepared-lr | 1.6594 | 1.6864 | 1.95235 | 1.66655 | 2104980. | +| fhirpath-addition 512 full-source | 3.49855 | 3.0926 | 3.4138 | 3.1203 | 5585421.6 | +| arithmetic-addition 512 prepared-lr | 0.77145 | 0.7646 | 0.76755 | 0.7565 | 1080980. | +| arithmetic-addition 512 full-source | 1.1478 | 1.18005 | 1.2511 | 1.2851 | 3696477.6 | + +All twelve routes retain exactly the same admitted allocation per call. CPU +results vary: the third run is slower across several routes, while the second +is near or below the neighboring baseline for many routes. This does not prove +a speedup or establish a performance equivalence margin. The maintained claim +is shared source policy with qualified semantics and no added request-time +closure; stable CPU improvement remains unproved. The independent Arithmetic +full-source route still uses its generated executor and is a control for costs +outside the changed generic fold. + +* Continue at cohesive engine boundaries + +The remaining review follows the same canonical IR -> prepared runtime -> +execution -> artifact path, rather than a list of surface syntax substitutions. +Preparation should express admitted representations and backend capabilities. +Generated and generic reducers should share action policy while retaining +their independently testable execution strategies. Checkpoint and publication +ownership should be explicit where mutable lifetime actually matters. + +The next performance transformation still requires CPU attribution and matched +complete products. This abstraction refactor does not resolve that evidence +gap, and a larger rewrite is not admitted merely because it uses more macros +or interfaces. + +* Completion records and ranking ownership + +The next boundary is the selective-GLR result retained by configuration memo +entries. Previously it was a six-element positional list. Ranking lived inside +the request executor, traversed token suffixes during comparison and constructed +a provisional result followed by another copy carrying the completion count. + +Official =src/std/text/parser/base.ss= uses final structs for =token= and +=location= and destructures them with =match/with=. The official +=src/gerbil/core/contract.ss= =Using= module implements the lexical class binding +used below. Its syntax admits one typed binding per =using= form; two objects +use nested scopes. + +#+begin_src scheme +(defstruct lr-completion + (root rest score ambiguities winner-reason completion-count remaining-count) + final: #t) + +(def (ranked-candidate winner reason count) + (using (winner : lr-completion) + (make-lr-completion winner.root winner.rest winner.score winner.ambiguities + reason count winner.remaining-count))) +#+end_src + +=src/runtime/lr-completion.ss= owns the private completion representation and +pure comparison policy. Its API exports construction and projections without +mutation. The executor retains chart ownership, exploration budget, diagnostics, +ambiguity rejection and request-local counters. =select-candidate= returns a +representative and an equivalent-merge flag; the executor updates its own merge +counter. No language declaration acquires another interface or policy switch. + +The comparison order remains dynamic score, maximal consumption, then structural +product equivalence. Distinct equal-ranked products retain their ambiguity in a +representative so a surrounding higher-scored completion may still win. Total +completion counts are accumulated even when one candidate outranks the other. +Cached suffix lengths are copied from the selected record; the engine retains +immutable token suffixes throughout a parse. + +** Question: why is this design preferable? + +A named final record exposes the actual result contract and lets Gerbil resolve +typed slot accesses. A standalone policy can be reviewed without the chart, +continuation and observability machinery. One final construction carries both +ranking evidence and completion count. This applies Scheme's type and lexical +abstraction facilities to a private protocol; POO remains appropriate for the +extensible public grammar and parser models. + +** Question: why is the claim scientifically testable? + +Policy controls cover both exploration orders for score and consumption, equal +structural copies, unequal suffixes, accumulated ambiguity/counts, first-result +identity and outer-score resolution. Existing parser integration controls cover +the corresponding complete parse receipts and failures. A benchmark group runs +four complete prepared GLR requests: equivalent merge, dynamic ranking, fragment +interning and typed ambiguity rejection, requiring the same complete receipt. + +Ordinary full-parse LR completions reach EOF, so their suffix length is zero. +Caching its length does not by itself establish a large request-time saving. +The extra record slot and multiple-value result also have a native cost that +must be measured. Deterministic request controls remain necessary because the +shared runtime imports the new module even when it does not enter GLR. + +** Completion qualification receipts + +- 34 source cases: final =OK=, exit 0, 8.222 s, in + unarchived local execution. +- Official native compilation of the producer and consumers: exit 0, 23.260 s, + in unarchived local execution. +- 34 native cases plus the complete GLR scenario and six deterministic source + products: all required markers and exit 0, 33.040 s, in + unarchived local execution. Interface traces + select both candidate =lr-completion.ssi= and =lr-parser.ssi=. + +An initial source run exposed an incorrect multi-binding =using= form. It was +corrected to nested official single-binding scopes before these final receipts; +the failed run is not counted as qualification. + +Four sequential native runs use the same compiled benchmark/scenario modules +for both engines. The baseline is =31c0667=; the candidate changes only the +completion module and its executor integration. Each GLR sample contains 100 complete groups; deterministic +samples contain 20 requests. There are 20 samples per route. Every run requires +the complete product markers and exit 0, with the existing five-second real +output watchdog. Elapsed times are 16.475, 16.159, 15.879 and 16.730 s. +Receipts are +unarchived local execution. + +The table reports batch CPU P50 divided by its iteration count, in ms per group +or request. Natural collection remains in the measured samples. + +| Route | Baseline 1 | Candidate 2 | Candidate 3 | Baseline 4 | Bytes/call | +|-------+------------+-------------+-------------+------------+------------| +| selective-glr-completion | 0.01391 | 0.01165 | 0.01215 | 0.01397 | 60381.6 -> 58173.6 | +| gql-return 64 prepared-lr | 0.2065 | 0.2136 | 0.2127 | 0.2077 | 230932. | +| gql-return 64 full-source | 0.31365 | 0.30495 | 0.2995 | 0.32 | 542941.6 | +| fhirpath-addition 64 prepared-lr | 0.2111 | 0.21335 | 0.2123 | 0.2101 | 255540. | +| fhirpath-addition 64 full-source | 0.3517 | 0.3385 | 0.3384 | 0.34325 | 695181.6 | +| arithmetic-addition 64 prepared-lr | 0.09345 | 0.09365 | 0.0936 | 0.09435 | 134804. | +| arithmetic-addition 64 full-source | 0.1396 | 0.1457 | 0.1522 | 0.1412 | 462141.6 | +| gql-return 512 prepared-lr | 1.6087 | 1.61845 | 1.60085 | 1.6266 | 1894004. | +| gql-return 512 full-source | 2.32895 | 2.3989 | 2.5245 | 2.96145 | 4330101.6 | +| fhirpath-addition 512 prepared-lr | 1.6559 | 1.66305 | 1.6552 | 1.66585 | 2104980. | +| fhirpath-addition 512 full-source | 2.82765 | 3.44985 | 3.3867 | 3.1199 | 5585421.6 | +| arithmetic-addition 512 prepared-lr | 0.74615 | 0.7507 | 0.75525 | 0.7504 | 1080980. | +| arithmetic-addition 512 full-source | 1.12015 | 1.16495 | 1.20145 | 1.1272 | 3696477.6 | + +The four-request GLR group reduces admitted allocation by exactly 2,208 bytes +(3.66%). Its two candidate CPU medians are below both baseline medians; the +mean of the two run medians decreases approximately 14.6%. This is evidence for +this small complete GLR workload, not for all languages or whole-engine speed. +The twelve deterministic routes retain exactly the same admitted allocation. +Their CPU changes are mixed, including higher short GQL prepared and short +Arithmetic complete-request medians. General CPU non-regression is unproved. + +The FHIRPath 512 complete-request batches include nine collection intervals in +every run. Median CPU over the eleven batches without collection is about +2.69-2.70 ms/request across all four runs, while the natural-collection P50 +above varies more. This diagnostic helps separate sample composition from +execution cost; it does not replace the complete-request measurements or blame +Gambit's collector for an engine design problem. + +The existing =t/selective-glr-benchmark-test.ss= gate also passes on the candidate: +40 samples of 100 four-request groups, wall P95 8.363 ms below the 250 ms +maximum, final =OK= and exit 0 in 5.365 s. Receipt: +unarchived local execution. + +After integrating the concurrent scanner change =bb53704=, 37 source cases +pass in 7.369 s. The updated scanner is compiled into the candidate overlay +(9.308 s), and 37 native cases plus all complete benchmark products pass in +12.286 s with exit 0 and every required marker. Receipts: +unarchived local execution. +These establish combined semantic qualification; the preceding ABBA timings +remain measurements of the isolated completion refactor against =31c0667=. + +The declarative optional Rust AOT generator module list also registers +=src/runtime/lr-completion= before its =lr-parser= consumer. Default package +source discovery already admits the new module. No compatibility record or +additional command/script is introduced. +The existing build-product contract's two cases pass, final =OK= and exit 0, +in 2.161 s (unarchived local execution). +This receipt qualifies the product declaration, not a fresh Rust AOT executable. + +Further concurrent Source/Fold commits through =ec4b6b8= are integrated before +publication. The combined source control expands to 44 cases, including the +updated event-fold suite and build-product contract: final =OK=, exit 0, +7.644 s in unarchived local execution. +The 37-case native receipt above covers the LR/scanner graph, whose production +dependencies are unchanged by those later Source/Fold commits. It does not +qualify their new native Source/Fold implementation. Exact-head CI retains +ownership of the complete published package and optional Rust AOT integration. + +* Generic functions as preparation protocols + +The available mechanisms have different owners. Gerbil =defmethod= associates +behavior with a receiver type. Upstream =clan/poo/mop .defgeneric= invokes a +receiver's prototype slot; it is a single-receiver protocol. The pinned +=core/poo-clos/interface= provides actual multiple dispatch, =eql=/class/prototype +specializers, method combinations and generic generations. Treating these as +one interchangeable mechanism would obscure the extension contract. + +Source evidence is =gerbil-poo/mop.ss .defgeneric= and POO Flow Core's +=poo-clos/{syntax,dispatch,objects}.ss=. In particular, +=applicable-ranked-methods= computes applicability from an immutable generic +generation and sorts by =argument-precedence-order=; =compute-effective-method= +admits protocol hooks. The checked runtime is already a dependency. The parser +must not implement a second generic registry or method-combination evaluator. + +The engine uses receiver protocols for grammar/context role access. The initial +audit found no CLOS preparation protocol for semantic backends; the LR migration +below now installs one. The dependency already provided the required mechanism. + +** Separate selection from repeated execution + +The executable control in =t/generic-dispatch-contract-test.ss= uses the actual +Core CLOS declaration syntax and ordinary Scheme values: + +#+begin_src scheme +(.defgeneric test-select-semantic-plan (backend capability)) +(.defmethod (test-select-semantic-plan (backend (eql 'event)) + (capability (eql 'plain))) + (lambda (value) (list 'event value))) + +;; Selection belongs to preparation; the returned kernel is reused. +(let (kernel (test-select-semantic-plan 'event 'plain)) + (kernel 'a) + (kernel 'b)) +#+end_src + +The probe counts method-body selection separately from kernel calls and checks +that repeated execution does not redispatch. It is a design control, not an +installed event parser, an allocation benchmark or an implementation of a new +public grammar operation. Its unsupported-pair rejection is an explicit probe +contract; it does not change the existing optional event backend fallback. + +| Engine boundary | Appropriate generic operation | Prepared result | +|-----------------+-------------------------------+-----------------| +| Admitted recognition dialect and semantic backend | Resolve a supported lowering strategy | Checked closed recipe plus derived executor | +| Result schema and publication target | Resolve encoding/publication strategy | Schema-admitted publication kernel | +| Open POO authoring role and compiler target | Elaborate inherited behavior | Canonical IR with source origins | + +These are independent variation axes. The first migration should move strategy +selection out of nested runtime branches into one preparation owner, retain +the prepared runtime's identity, and keep each kernel's traversal explicit. +Grammar authors continue to use the existing three interfaces. Ordinary values +are sufficient for fixed strategies; POO/class specialization supports genuine +extension. A closed scalar algorithm such as completion score comparison does +not acquire a class hierarchy solely to increase generic usage. + +Prepared products must bind the admitted generic generation or immutable method +snapshot; otherwise changing methods could silently change a retained parser. +Procedure-valued derived executors must not enter canonical, digest-bearing IR. +Reusing generic selection requires preserving both constraints before replacing +the current backend policy. + +** Contextual conflict semantics remain an independent contract + +Two applicable methods can be more specific on different arguments. Ordinary +CLOS chooses using argument precedence. Current contextual grammar selection +uses componentwise dominance: incomparable maximal methods with unequal results +are an error, while equal results retain both declaration origins. + +The executable control tests both behaviors on the same =shell/command= +intersection. It shows why blindly substituting ordinary primary-method +selection could hide a grammar conflict. Core's existing generic-protocol and +method-combination facilities are the appropriate place to express the required +policy if that resolver migrates; parser-specific conflict admission remains +explicit. No new dispatch kernel is justified. + +** Question: why is this design preferable and testable? + +Multiple dispatch states which combinations the engine supports without making +each caller reproduce a decision tree. Hygienic declarations, POO extensions, +normal values and prepared kernels can cooperate: the generic owns selection, +the compiler owns admission, and the executor owns repeated work. + +The controls demonstrate actual two-argument selection, fixed-kernel reuse, +unsupported combinations and the contextual conflict boundary. Production +migration additionally requires generation invalidation controls, complete +artifact/diagnostic parity, preparation cost and complete-request benchmarks. +These executable controls alone do not close that migration or establish its +performance. + +** Generic control qualification + +Eight source-client/contextual cases pass over the preloaded pinned native +Core CLOS dependency: final =OK=, exit 0, 13.951 s, in +unarchived local execution. +Official native compilation of the new suite passes in 13.917 s. Its three +compiled cases then run directly through the native =std/test= harness: final +=OK=, exit 0, 5.101 s, in +unarchived local execution. + +The first fully interpreted dependency load hit the unchanged five-second +no-progress watchdog and is excluded. A native harness attempt incorrectly +addressed the re-exported =TestHarness= as a runtime symbol owned by =base=; +using the module's normal imported binding fixes that driver error. These +negative receipts establish neither a generic-dispatch performance result nor +a defect in Core. Native dependency reuse and source-client qualification are +separate from a fully interpreted dependency run. + +** Production LR preparation: select admitted executors once + +=src/runtime/lr-parser.ss= now separates the grammar's semantic capability from +the request's backend preference. =lr-prepare= validates reduction plans, +classifies the admitted grammar and selects both executors through the existing +Core CLOS implementation: + +#+begin_src scheme +(.defgeneric select-lr-semantic-reducer (backend capability)) +(.defmethod (select-lr-semantic-reducer (backend (eql 'recognition)) capability) + reduce-value) +(.defmethod (select-lr-semantic-reducer (backend (eql 'event)) capability) + #f) +(.defmethod (select-lr-semantic-reducer (backend (eql 'event)) + (capability (eql 'plain))) + reduce-value/events) +#+end_src + +The general event method denotes unavailable optional execution; the more +specific =event/plain= method supplies the executor. Layout, dynamic precedence +and branching grammars retain recognition execution. Unsupported semantic +actions still fail during reduction-plan validation. No public grammar syntax, +POO wrapper or alternate generic registry is introduced. + +The prepared runtime stores the returned procedure or negative admission. +=lr-runtime-for-current-semantic-backend= performs constant-size preference and +identity checks, then obtains or constructs the cached event runtime. It no +longer traverses the LR action and production tables, including repeated +requests against an ineligible grammar. Generated step installation invalidates +future runtime selection while previously retained runtimes keep their executor. + +This first protocol is closed and module-private. Core's lowering constructs a +lexical =poo-clos-generic-binding=; the parser exports neither that binding nor +the generic. Preparation captures the selected executor itself, so later +requests do not consult a generic generation. An open extension protocol still +needs explicit generation admission; this change does not implement one. +Executors remain derived runtime values, absent from canonical digest-bearing +grammar and parser IR. + +*** Question: why is this abstraction preferable? + +Support for a backend/capability combination is a method declaration owned by +preparation. Callers do not inspect grammar tables or reproduce support rules. +Ordinary Scheme values suffice for these fixed strategies; actual open authoring +objects retain their POO interfaces. Multiple dispatch resolves variation while +named pure algorithms perform capability classification and the fixed reduction +kernel performs traversal. This assigns each abstraction a concrete job. + +*** Question: what makes the claim testable? + +=t/lr-semantic-preparation-test.ss= checks preference-independent preparation, +runtime identity and cache reuse, generated-step replacement without changing +retained executors, and recognition fallback for dynamic, branching and layout +grammars. Existing artifact, GLR, empty-reduction and fragment tests supply +semantic controls. Two generic selections and eligibility scanning now belong +to preparation even when event execution is not requested. Preparation overhead, +cold native dependency loading and complete-request performance must therefore +be measured separately; removing request-time table traversal alone proves no +end-to-end speedup. + +*** Qualification and separated cost observations + +The runtime and new contract suite compile natively with final =OK=, exit 0, +42.902 s. The native =machine= consumer is rebuilt in dependency order, 18.726 s. +The three compiled preparation cases pass directly through =std/test= in +14.981 s. Seven source-client suites over the rebuilt native runtime pass +46 cases in 12.627 s, including artifact parity, empty reductions, GLR, +layout and incremental fragment controls. All accepted processes retain the +five-second real-output watchdog and require both successful exit and final +=OK=. Receipts are unarchived local execution. + +The first integration run exposed an old =machine= product whose inlined +=lr-runtime-event-step= accessor used the preceding record layout. It read the +new executor slot and incorrectly rejected backend installation. Recompiling +the consumer resolves the mismatch; this failed run is excluded. Incorrect +compiler entry names, a delimiter error and an invalid verbosity argument in +temporary drivers are also excluded. No compatibility accessor is installed. + +The existing =t/benchmarks/lr-request-stack/benchmark.ss= now exports a separate +preparation/selection observation alongside complete-request controls: + +#+begin_src scheme +(benchmark-lr-semantic-preparation 8 3 1000) +(benchmark-lr-request-stack 8 3) +#+end_src + +Each prepared runtime is warmed before repeated selection. Preparation samples +include =lr-prepare= and initial backend selection, including construction of +the selected runtime where admitted. Eight samples observe three preparations +or 1,000 warmed selections per batch. These observations are not a paired +baseline/candidate comparison. + +| Language | Event admitted | Preparation plus first selection CPU median, ms/call | Warmed selection CPU median, us/call | +|----------+----------------+-----------------------------------------------------+-------------------------------------| +| GQL | no | 75.079 | 0.081 | +| FHIRPath | yes | 1.139 | 0.053 | +| Arithmetic | yes | 0.565 | 0.054 | + +The complete benchmark also validates six cross-language products at 64/512 +units and runs their prepared-recognition/full-source samples. Both benchmark +markers and final =OK= pass, exit 0, 34.454 s, in +unarchived local execution. +Wall observations contain substantial scheduler delay, so no wall-speedup +claim is admitted. The large GQL preparation cost needs attribution among table +indexing, mode interning and other preparation work; these measurements do not +identify its dominant algorithm or attribute it to generic dispatch. + +Publication integrates concurrent =aff9375= before delivery. The owned LR, +machine and benchmark source graph is unchanged by that commit. Rebased native +preparation controls pass all three cases again: final =OK=, exit 0, 21.962 s, +in unarchived local execution. +These reused native products do not qualify the concurrent prefix-family +implementation; exact-head CI owns the complete merged package and Rust AOT. + +* Contextual matcher preparation across modes and source sizes + +The adjacent audit covers contextual scanner preparation, closed region plans, +lexical source capture and source-engine binding. =region-scanner.ss= already +owns explicit nested frames and a prepared opener index. =source-strategy.ss= +already admits an owned contextual program and checks its publication owner. +The concrete duplication is in =contextual-scanner.ss=: equivalent configured +matchers are lowered separately for each mode, and the short/long source indexes +repeat source-independent lowering when a large literal catalog is present. + +Official Gerbil =std/text/parser/ll1.ss= provides relevant abstraction evidence: +=ll1-bind= constructs a reader procedure, =ll1-or/list= separates declared +alternatives from the later reader, and =peeker= interprets specification data +into predicate procedures. These examples support separating declaration and +execution; they do not justify caching arbitrary closures. The parser's matcher +executors take source, offset and contextual state as arguments. Their captured +configuration belongs to a prepared plan. + +** Share executable configuration, preserve rule semantics + +The implementation separates =lower-rule-matcher=, =prepare-rule-matcher= and +=decode-rule=. A private =contextual-matcher-plan= holds an admitted expression +and executor. One preparation-local equal?-table shares region/guard matcher +plans across modes and both source-size indexes. Their declaration keys begin +with opcode symbols; existing literal-trie keys are catalogs of strings. These +key domains are disjoint. Literal catalogs retain their established short-input +linear and long-input trie paths. Primitive opcodes retain the compiler's +existing configuration-free closures directly. + +#+begin_src scheme +;; A hygienic local constructor retains lexical rule metadata without a closure. +(defrule (finish expression executor) + (make-runtime-scan-rule name mode form expression executor rank action)) +(let (plan (prepare-rule-matcher matcher matcher-cache)) + (finish (contextual-matcher-plan-expression plan) + (contextual-matcher-plan-executor plan))) +#+end_src + +The plan takes ownership of closed IR before interning. No process-global cache, +new grammar declaration, user-visible wrapper or runtime generic dispatch is +introduced. The preparation table is discarded after the completed indexes own +their executors. State, pending delimiters and source identity remain request +owned; rule name, form, rank and action remain distinct records. Canonical IR, +its digest and opcode meaning are unchanged. + +** Question: why is this abstraction preferable? + +Matcher configuration and contextual rule policy vary independently. Naming the +preparation product makes their sharing boundary explicit, while S-expression +data supplies structural equality and pattern-based lowering. A rule's parser +position and method origins stay with contextual dispatch. Existing CLOS +preparation protocols select independent backend/capability axes; this closed +opcode lowering uses structural =match=. Each mechanism has an identified owner +and does not impose another configuration interface on a language author. + +** Question: how is the claim tested? + +The native scanner suite adds a shared =<<= matcher under different ranks, +actions and modes, checking independent requests and short/long input plans. +Existing controls cover longest Unicode literals, guards, ambiguity diagnostics, +IR mutation, deferred delimiters, region nesting and source strategies. + +The first all-matcher interning experiment exposed a negative control: primitive +=identifier=/whitespace executors already have compiler-hoisted closures. +Wrapping and caching them adds allocation without providing sharing. A second +experiment bypassed their cache but used a uniform multiple-value return carrier; +it still added 96 bytes per one-mode preparation and 1,536 bytes at sixteen modes. +Literal-catalog CPU results were mixed, including a regressed median in that +second run. The final implementation confines matcher-plan sharing to region +and guard lowering, preserves the established primitive/literal paths and uses +a local =defrule= to stage record construction. This is a measured selection +boundary, rather than an assumption that every Scheme value benefits from +interning or a uniform return protocol. + +Preparation comparison uses exact =f3dff04= scanner source compiled under an +isolated temporary test namespace. The production package contains no baseline +or legacy executor. The existing =matched-scale.ss fixture-ir= now accepts an +optional declared word matcher, enabling the same mode/position fixture to +exercise identifier, literal-catalog and region configurations. Both native +implementations consume the same compiled IR and source. Each measured product +is scanned afterward and checked for equal token kinds, lexemes, byte boundaries +and final offset. Allocation is admitted only for intervals without collection. +Alternating ABBA order controls sample order; preparation timing excludes IR +compilation and scanning. These controls cannot establish whole-parser speedup. + +** Native matched controls and qualification + +Each run uses four ABBA rounds and three preparations per sample. CPU entries +are per-preparation medians in milliseconds; byte entries are medians admitted +only without GC. Both runs validate every resulting scanner product afterward. + +| Configuration | Baseline bytes | Candidate bytes | CPU first run old/new | CPU confirmation old/new | +|---------------+----------------+-----------------+-----------------------+--------------------------| +| identifier, one mode | 64,709 | 64,709 | 0.0557 / 0.0477 | 0.0573 / 0.0577 | +| identifier, sixteen modes | 1,986,165 | 1,986,165 | 3.3897 / 3.9307 | 3.2960 / 3.2733 | +| literal catalog, sixteen modes | 2,647,259 | 2,647,131 | 4.0503 / 3.5953 | 4.2430 / 3.7817 | +| region, sixteen modes | 6,576,181 | 3,611,723 | 11.1537 / 4.4853 | 9.2747 / 4.8883 | + +The confirmation region baseline is 6,576,309 bytes. The two region controls +reduce preparation allocation by approximately 2.96 MB, or 45%. Identifier CPU +results are mixed, including a regression in the first sixteen-mode run. They +support no general CPU improvement claim. The region preparation control is +narrow: it does not measure scanning, parsing, request latency or application +throughput. The literal path is retained, rather than redesigned on these data. + +Final accepted receipts retain exit 0, final =OK= and the five-second idle gate: + +- Native contextual producer compilation: 35.987 s. +- Native scanner suite and paired benchmark consumer compilation: 22.562 s. +- Native scanner suite: all 17 cases, 11.410 s. +- Source suites over the native producer: all 68 cases across contextual, + deferred, region, source-scanner and source-strategy suites, 22.113 s. +- Exact baseline native compilation: 45.282 s. +- Matched comparison and confirmation: 8.290 s and 19.849 s, both with + =MATCHER-COMPARISON-OK= and validated scanned products. + +Two final compilation attempts stopped at the idle gate during GCC function +compilation and are excluded. An earlier combined baseline compilation emitted +=OK= but had =reason=idle-timeout= and is also excluded. The successful producer +compile enables actual GCC integrated-register-allocator diagnostics with +=-fira-verbose=10=, alongside verbose optimization reports. GCC documents that +values of ten or more send those diagnostics to stderr in its +[[https://gcc.gnu.org/onlinedocs/gcc/Developer-Options.html][developer options]]. This changes observability, not optimization or the idle +deadline. No synthetic progress or relaxed timeout is admitted as a receipt. + +The source controls load reused native dependency products. They qualify this +matcher preparation change, not the complete concurrently changing compiler +package. Exact-head CI owns complete package and Rust AOT qualification after +publication. + +Publication rebases this change onto =b5a3141=, preserving the concurrent +hygienic ranked-candidate-loop work. That commit changes the regular lexical +compiler, not this contextual matcher graph. All 17 native contextual cases pass +again after rebase: 9.655 s, exit 0, =HARNESS-OK= and final =OK=, with the +five-second gate. Reused native products do not qualify the concurrent compiler +change; complete merged-head CI remains a separate gate. diff --git a/docs/research/scheme-rule-context-design.org b/docs/research/scheme-rule-context-design.org new file mode 100644 index 00000000..e6fa889e --- /dev/null +++ b/docs/research/scheme-rule-context-design.org @@ -0,0 +1,203 @@ +#+title: Scheme rule context: declare identity once, elaborate explicit grammar +#+date: 2026-10-06 + +* Research question and status + +Can a rule declaration provide its own name to a reusable recursive template, +so the author does not repeat that identity inside the rule? This study extends +[[file:scheme-poo-language-components.org][the syntax and POO component study]], whose native numeric-free tier fixture +already checks actual nesting and selected Arithmetic event parity. + +The current committed baseline is =783381f33fa1b309ac4cc4b0012696f24c852d93=. +All additions described here are local research fixtures. Public =deflanguage= +and the parser engine are unchanged. Names and declaration syntax remain under +discussion; the study does not introduce a production author API. + +* Concrete reduction in author work + +Before: + +#+begin_src scheme +(sum + (left-binary-tier sum (reference product) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right))) +#+end_src + +The local declaration vocabulary instead accepts: + +#+begin_src scheme +(sum + (left-tier (reference product) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right))) +#+end_src + +The enclosing declaration supplies =sum= to the template. Operand dependency, +recognized operators and explicit tree customization stay visible. The rule +identity is written once without numeric precedence or an implicit global self +variable. The same mechanism supports right binary tiers and recursive right +chains; those keep their distinct tree contracts. + +* Macro boundary and implementation + +[[file:../../t/fixtures/language-pack-research/context-vocabulary.ss][context-vocabulary.ss]] exports the local =deflanguage/tiers= macro. It: + +1. Matches the language's rules section at the declaration boundary. +2. Uses =syntax-case= to recognize =left-tier=, =right-tier= and =right-chain= + only as the outer expression of a named rule. +3. Reuses the matched rule identifier as the explicit template argument. +4. Preserves ordinary rows and language sections for the existing =deflanguage=. +5. Lets existing expression templates and the canonical compiler perform their + established validation, compilation and artifact construction. + +This is a small syntax-object transformation, not a replacement DSL parser. It +has no table compiler, runtime merger or source evaluator. It does not traverse +arbitrary nested syntax and replace every occurrence of a spelling like =self=. +Custom forms outside its declared positions remain the existing expression +expander's responsibility. + +The wrapper requires the existing order of the identity/root/lex/rules sections. +It does not implement an unordered clause collector or a general third-party +rule-declaration extension protocol. Those are separate author-interface choices. + +* Native checks and evidence boundary + +[[file:../../t/fixtures/language-pack-research/context-language.ss][context-language.ss]] imports the declaration macro with =rename-in= and +assembles the same study language without the repeated self arguments. It also +defines host values named =sum=, =product= and =power=. +[[file:../../t/fixtures/language-pack-research/context-runtime.ss][context-runtime.ss]] verifies the result using the existing parser and artifacts: + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" GERBIL_PARSER_LR_TRACE=1 \ + gtimeout 10 gxi t/fixtures/language-pack-research/context-runtime.ss +#+end_src + +The final invocation exited 0 and reported =CONTEXT-RUNTIME-OK: 59 checks passed=: + +- Thirteen inputs: both parsers accept, the candidate roundtrips, and complete + public events equal the explicit-self tier language, including exponentiation. +- Five invalid inputs reject. +- The renamed declaration macro works while the three same-spelling host values + retain their expected values. +- Three malformed rows return the wrapper's expected diagnostics: a missing + operand/operator pair, a nonidentifier rule name and multiple expressions in + one row. The assertion compares messages, not merely whether any error occurs. + +The first attempt exited 70 because the inert diagnostic assertion macro passed +a complete form to =datum->syntax=, whose anchor must be an identifier in this +Gerbil implementation. The assertion now uses its own macro identifier as the +anchor. The declaration wrapper itself preserves matched syntax rather than +rebuilding declarations through =syntax->datum=. The final receipt includes both +the summary and exit 0; no timeout was increased. + +These are selected-input runtime controls. They do not prove grammar equivalence, +general macro capture freedom, complete source-location retention or cross-role +rule collision avoidance. The right-chain declaration case is implemented but +has no new full-language runtime control in this study. + +* Host binding and grammar identity are distinct + +| Identity | Current owner | Consequence | +|----------+---------------+-------------| +| Imported declaration macro | Gerbil module/expander binding | Import renaming preserves the macro's expansion behavior in this fixture | +| Host value such as =sum= | Gerbil runtime binding | It does not become the value of =(reference sum)= | +| Rule name such as =sum= | Canonical grammar declarations and references | It still denotes a declared grammar identity | +| Public node such as =Expression= | Grammar tree contract | It is not the helper rule's host or module identity | +| Role contribution identity | Existing checked POO grammar composition | Merge/append/override/remove meanings still apply | + +The current concise authoring expander projects grammar identifiers to declared +symbols. Hygienically importing a macro therefore does not automatically qualify +its generated helper-rule symbols for POO composition. Two roles contributing a +rule named =sum= still need the existing deliberate collision/override policy. +This experiment reduces repeated author declarations; it does not solve that +separate identity problem. + +A future private-helper naming policy must be deterministic and reference-aware. +Fresh host identifiers are not automatically stable grammar identities. Blindly +renaming every symbol would also rename fields, nodes or token references with +different meanings. A component-qualified rule identity needs an explicit owner, +export map and rewriting boundary before implementation. + +* What the official API and paper contribute + +[[https://gerbil.scheme.org/reference/gerbil/expander/utilities.html][Gerbil's expander utilities]] documents syntax-object operations and =stx-source=. +The prototype uses matched identifiers, syntax templates and =stx-map= to retain +syntax structure while assembling the lowered declaration. Diagnostic errors are +raised with the author row as context. Exact diagnostic source positions have +not been asserted, so the study claims specific messages only. + +[[https://www-old.cs.utah.edu/plt/scope-sets/general-macros.html][Flatt, Binding as Sets of Scopes, sections 3.5–3.6]] distinguishes binding aliases, +module scopes and phases. Our design inference is to preserve syntax through the +declaration transformation and explicitly describe its phase. The paper's model +is about Racket; it does not prove that Gerbil implements the same internals or +that grammar-domain symbols inherit its scope semantics. + +The useful Scheme advantage is the cooperation of declaration context, pattern +expansion and modules. POO then operates on checked grammar contributions at its +specified phase. This prototype compiles through the existing =deflanguage= +expansion path and does not consume a runtime GrammarRole from that phase. + +* Remaining design questions exposed by the witness + +** Declaration provenance + +The current =deflanguage= records expression macro heads in =macro-lineage=. +The outer =deflanguage/tiers= wrapper is consumed before that expression walk, +so the wrapper itself is not added by that mechanism. The language compiler can +record the lowered expression template while missing the author's declaration +vocabulary. This is a source-inspected design boundary, not a checked provenance +receipt in the current fixture. + +A future contract should relate author declaration, lowered rule, component owner +and core expression. That information can explain generated helper rules and +parser conflicts without affecting recognition or exposing implementation details +in ordinary application flows. Prefer an origin mapping alongside existing +compilation data; its schema and lifetime remain undecided. + +** Reusable declaration protocols + +A general extension boundary must state whether a component contributes one +expression, one named rule, multiple helper rules or an entire role. Each needs +different contextual information and ownership checks. The present wrapper only +studies one named rule at a time; it is insufficient reason to publish a broad +component protocol. + +Working recommendation: keep expression templates small, provide rule context +at the declaration boundary, and study provenance plus helper ownership before +merging this vocabulary into public =deflanguage=. Next compare a component that +really introduces multiple helper rules and compose it through the existing POO +admission path at a declared phase. + +* Follow-up: multi-rule components + +The [[file:scheme-list-role-composition.org][list-role composition study]] now supplies a concrete multi-rule provider, +62 native composition/tree checks and two separate identity-collision checks. +It also records the phase adapter's unclosed composition-identity and provenance +contracts, before proposing a public role-import interface. + +* Scheme design rationale: two required questions + +This is a rule row accepted by the local =deflanguage/tiers= declaration +wrapper, not a standalone Scheme expression or a published author API. + +#+begin_src scheme +(sum + (left-tier (reference product) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right))) + +;; Question 1: Why is this DSL design better? +;; Declare sum once; syntax-case derives the recursive target from its declaration. +;; Keep the product dependency, operators and tree fields explicit; renaming needs +;; one fewer coordinated edit. Scheme contributes contextual syntax transformation. +;; Question 2: Why does this DSL design have a scientific basis? +;; Expansion must preserve the explicit left-binary-tier rule's meaning. +;; The 59 native checks cover event parity for 13 inputs, host bindings and diagnostics. +;; Falsifier: changed nodes, fields or spans, or incorrectly admitted declarations. +;; Limit: no author-task timing, runtime speedup or ambiguity-elimination claim. +#+end_src + +Syntactic abstraction motivates preserving syntax context; the local controls +establish this wrapper's bounded behavior. The paper does not prove this exact +DSL optimal. See the paper discussion above and the executable fixture links. diff --git a/docs/research/scheme-source-edit-sessions.org b/docs/research/scheme-source-edit-sessions.org new file mode 100644 index 00000000..e8c61749 --- /dev/null +++ b/docs/research/scheme-source-edit-sessions.org @@ -0,0 +1,209 @@ +#+title: Source Edit Sessions: Scheme Authority and Rust Execution + +* Scope and executable path + +This module continues the [[file:dsl-engine-contracts.org][engine contract audit]] without adding author DSL +clauses. The fixed-position Source path now has owned scanning histories in both +Scheme and Rust: + +#+begin_example +parser declaration -> prepared Source engine -> owned scan session + -> full command recognition -> canonical events +#+end_example + +Scheme's =parse-source-language/session= owns a copied source, scanner states and +tokens, then publishes the existing validated parse artifact. Rust's +=PreparedCommandSource::scan_session= owns a String and private token/context +history. =parse_session= borrows that owned source and executes the existing +command recognizer and canonical event publisher. Rust returns a scanner +Diagnostic when lexical scanning fails; Scheme retains its established +rejected-artifact publication path. A failed lexical scan cannot supply reuse +history on either backend. + +This implements lexical suffix reuse. Source command/CST subtree reuse and +dynamic LR-position sessions remain separate work. The native FFI integration +is described below. + +* Why this design is preferable + +Question 1: Why does this design improve the author interface? + +#+begin_src scheme +;; The parser declaration owns the prepared language product. Its grammar +;; remains syntax and rules; editing adds no scanner cache clauses to it. +(def initial + (parse-source-language/session bash-source-language + "cat < (parse-source-language bash-source-language + "echo猫 < 7) +(check (source-language-session-reused-token-count changed) => 4) +#+end_src + +* Paper relationship and cost boundary + +Wagner and Graham's [[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] separates parser +state matching, sentential-form reuse and optimal node reuse. Its sequence +balancing assumptions underpin its incremental complexity results. Our scanner +continuation argument is an engineering derivation for this engine's lexical +transition system; it is not an implementation of that paper's optimal parser. + +A session retains one token record and one context record per token boundary. +Common suffix discovery reads suffix bytes. Admission binary-searches old +boundaries; pending FIFO comparison visits marker values when necessary. Reusing +a suffix copies token ranges and context records while sharing persistent marker +storage. Full command recognition and event publication still run. Consequently, +reused-token counts measure avoided scanner transitions, not total work or an +end-to-end latency improvement. No logarithmic edit-cost claim follows. + +* Qualification controls + +The existing Scheme source-scanner tests and Rust command-source tests use the +same UTF-8 prefix and quoted-marker edits: 1 scanned token for the prefix edit, +and 7 scanned / 4 reused tokens for the quote edit. Rust additionally compares +multi-edit token streams and published events with fresh execution, exercises +two pending heredoc markers, checks incomplete obligations and foreign plan +rejection, and verifies that a failed edit leaves the previous session usable. +Both generated POO products also run every Scheme acceptance/rejection control +through successive sessions; edit controls compare canonical event IDs, fields, +symbols and byte spans with fresh recognition. + +The Rust source is owned by the session; parse results borrow that allocation. +Token access is read-only. The ordinary full-scan API remains the independent +semantic control. Runtime suite and strict Clippy qualification are required; +exact-head CI is recorded separately from local receipts. + +* Native FFI ownership and commit boundary + +The language ABI now consumes Scheme-owned Source history behind +each registered language handle. Callers continue to use the same parse entry. +Canonical LR and contextual LR products retain their existing execution paths; +only Source descriptors enter the Source session engine. + +The engine separates =source-language-history= (descriptor and private scanner +snapshot) from =source-language-session= (history plus this call's artifact). +=source-language-session-history= extracts a history value with no artifact +field. The handle retains that value after publication, so an old AST and its +canonical event list do not remain alive merely to authorize the next scan. +Source text, tokens and scanner contexts are retained where lexical reuse needs +them. Retention is still proportional to the latest scan; this is not a +constant-memory incremental algorithm. + +Question 1: Why does this preserve the Scheme and POO author boundary? + +#+begin_src scheme +;; The language product still supplies syntax and construction. Native +;; ownership uses the admitted descriptor; editing adds no author clauses. +(def handle (register-language! bash-source-language)) +(language-handle-parse handle (string->utf8 "cat <utf8 "echo猫 < 1) + (check (> reused 0) => #t)) +(release-language! handle) +#+end_src + +Specification admission, context equivalence and byte relocation remain in the +Source engine. The handle registry owns lifetime and the C callback owns result +allocation. A second handle has independent history even when both handles share +the same immutable language product. Descriptor queries leave histories intact. +Releasing a handle removes its history with the registry entry. + +Question 2: Why is the publication transaction justified? + +#+begin_example +admit owner and input -> scan candidate -> recognize complete source + -> encode GPA1 -> allocate and copy C result + -> commit candidate history +#+end_example + +The handle's =language-handle-parse/publish= invokes its result owner +before replacing history. The C callback reports successful allocation and +copying through that publisher. Invalid input, encoding exceptions, a publisher +exception or allocation failure therefore cannot advance the stored history. +Tests simulate both a false publisher result and a throwing publisher, then +verify that the next edit still uses the previous successful scan boundary. +They do not inject a real process-wide allocator failure. + +Syntax rejection remains a successful GPA1 transport. A rejection with a valid +lexical scan may retain that scan. An incomplete lexical obligation publishes +the established diagnostic artifact with no reusable snapshot, and the next +input must scan fully. Canonical grammar/source digests and all publication +records are compared with fresh execution. + +The native Rust fixture transfers two distinct admitted Source handles to Rust +on the VM owner thread. Rust parses successive UTF-8 and heredoc edits through +the actual C callbacks, compares complete payloads with an independent handle +whose token history is cleared by an empty parse, and keeps an old result alive +through those calls. That retained result must keep its bytes and source view. +Dropping Rust owners must release both transferred handles. This complements +the Scheme scan-count/isolation controls and the existing ABI lifecycle tests. + +The host's =NativeSession= still owns VM admission; it is distinct from this +per-language lexical history. Full Source recognition and publication still +execute on every call. The existing ABI shape is unchanged, and this work does +not establish Source subtree identity reuse or an end-to-end latency gain. + +** Local qualification + +The focused Source scanner, native language and test-style suites completed +28 cases with final =OK= and =HARNESS-RETURN status=0= in 10.44 seconds. The +linked Rust/C/Scheme probe completed its nine Source edit controls, retained +result checks, existing 100-call control and handle-release check in 5.37 seconds +with =RUST-NATIVE-SOURCE-EDITS-OK= and =RUST-NATIVE-OK=. Rust formatting, strict +fixture Clippy, the four native ownership/thread compile-fail doctests and the +ASP Rust source policy gate passed. + +The local dependency cache contained stale interfaces and missing native +products. An official package build stopped at its 5-second GCC output-inactivity +fence after 76.74 seconds. That attempt is a failed build receipt. The required +current dependency closure was then completed with the native compiler's real +GCC diagnostics under unchanged fences before the focused tests ran. These +receipts qualify the changed call path; they do not substitute for clean, +exact-head Linux/macOS CI or establish a performance improvement. diff --git a/docs/research/scheme-source-origin-contract.org b/docs/research/scheme-source-origin-contract.org new file mode 100644 index 00000000..c2217a67 --- /dev/null +++ b/docs/research/scheme-source-origin-contract.org @@ -0,0 +1,144 @@ +#+title: Scheme component origins: definition, invocation and generated rules +#+date: 2026-10-06 + +* Research question and evidence boundary + +The [[file:bound-declaration-publication.org][ordered declaration publication study]] implements the shared normalization, +syntax-origin and Bound IR construction path, with source-selection controls and +complete sidecar measurements. It preserves the historical origin experiment +below and its separate qualification limits. + +Can a concise multi-rule component retain both its author invocation and its +macro definition origin, through the existing POO and Bound IR owners? +This follows the [[file:scheme-component-provenance.org][component provenance]] and +[[file:scheme-composition-history.org][composition history]] studies. + +The local study runs against committed baseline +=783381f33fa1b309ac4cc4b0012696f24c852d93=. Its syntax is experimental; +no public language API has been selected. All location values come from reader +syntax, not a manually maintained line table. + +* Official mechanism and a useful negative result + +The [[https://gerbil.scheme.org/reference/gerbil/expander/utilities.html][official Gerbil expander utilities]] document =stx-source= returning a locat +or =#f=. Container and position accessors expose file paths and coordinates; +a location may be a point without an end position. The fixture converts line +and column to one-based values and preserves a missing end as =#f=. + +The initial attempt queried =stx-source= on a literal identifier created inside +the transformer with =#'template-origin-anchor=. In this installed Gerbil, +the result was =#f=. The reader location of the author's matched =owner= +identifier survived. A retry reached that check and exited 70 when the driver +incorrectly treated the absent template origin as an alist. This is a limitation +of the attempted collection strategy, not evidence of a parser defect. + +The corrected strategy captures the real =defsyntax= declaration while an outer +macro still receives its reader syntax. It binds the resulting inert origin at +phase one, where the component transformer can read it. The invocation origin +is independently read from the component owner's matched identifier. + +* Actual Scheme declarations and the two required questions + +The local [[file:../../t/fixtures/language-pack-research/reader-origin.ss][reader-origin.ss]] wrapper records the source of =definition= and +expands to a phase-one origin binding plus the unchanged definition. The shape +below abbreviates the transformer body; it is an explanatory excerpt, not a +standalone module. + +#+begin_src scheme +(with-reader-origin template-definition-origin + (defsyntax (deflocated-list stx) + ;; syntax-case matches owner from the author's invocation. + ;; authorOrigin <- stx-source of the matched owner + ;; templateOrigin <- template-definition-origin + ;; Generated role + sources enter existing POO/Bound IR construction. + ...)) + +;; Actual author invocation from located-list-roles.ss: +(deflocated-list arguments-component call-arguments arguments + (reference name) (literal ",") (field argument)) + +;; Question 1: Why is this DSL design better? +;; Authors state list intent once, without repeated entry/tail patterns or manual source line tables. +;; Invocation origins identify component declarations; definition origins identify generating templates. +;; Scheme syntax objects and phase-one bindings retain origins; POO roles carry multi-rule contributions. +;; Question 2: Why does this DSL design have a scientific basis? +;; Coordinates must point to actual owner and defsyntax text, not merely populate a source field. +;; Evidence: source-origins.ss passes 12 checks with exit 0 through existing Bound IR construction. +;; Falsifier: wrong token coordinates, inconsistent entry/helper origins or invented missing locations. +;; Limit: actual-language loading times out; nested chains and rendered diagnostics remain open. +#+end_src + +The complete [[file:../../t/fixtures/language-pack-research/located-component.ss][located-component.ss]] constructs the checked role and source record; +[[file:../../t/fixtures/language-pack-research/located-list-roles.ss][located-list-roles.ss]] composes it with the base role and associates the +record with both contributed rules. It currently uses the unique-append source +mapper. The override/removal history study explains why that mapper cannot be +promoted unchanged to general composition. + +* Native checks and retained failures + +The focused driver [[file:../../t/fixtures/language-pack-research/source-origins.ss][source-origins.ss]] constructs and normalizes the POO grammar, +then invokes the existing =bind-grammar-ir= with the captured source map. +For each of two components it checks owner, generated status, author token, +template token, distinct source files, and equal entry/helper source records. +The token checks open the actual files and slice the reported line/column. +The template token is =(defsyntax=; the author token is its owner name. + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout 10 gxi t/fixtures/language-pack-research/source-origins.ss +#+end_src + +The corrected definition-location version reported +=SOURCE-ORIGINS-OK: 12 checks passed= and exited 0. + +| Invocation | Observed result | Admission | +|------------+-----------------+-----------| +| Initial combined source-locations driver | Artifact generation progress, exit 124 | Timeout | +| Combined retry with literal template anchor | Author token succeeded; absent template source, exit 70 | Failed strategy | +| Focused corrected source-origins | 12 checks, exit 0 | Passing construction/Bound IR receipt | +| Combined language/event driver with wrapper-binding origin | 21 checks and summary, exit 124 | Timeout, not a passing parity receipt | +| Actual language driver with defsyntax-definition origin | 12 checks and summary, exit 124 | Timeout, not a passing language-loading receipt | + +The last driver is [[file:../../t/fixtures/language-pack-research/source-bound-locations.ss][source-bound-locations.ss]]. It loads the actual declaration's +artifact-backed Bound IR. Its assertions completed, but its exit status remains +a failed bounded invocation. The ten-second limit was not increased. The timing +cause has not been isolated; neither the parser nor GC is identified as its cause. +No successful end-to-end parsing or performance claim is inferred here. + +* Paper interpretation: coordinates do not complete diagnostics + +[[https://www2.ccs.neu.edu/racket/pubs/icfp10-cf.pdf][Culpepper and Felleisen, Fortifying Macros (ICFP 2010)]], section 2, explains +that tracking source locations does not suffice when errors expose an expansion's +implementation instead of the author's declaration. Their guard examples also +show that rejecting syntax alone does not explain the reason. This supports +separate obligations for origin capture and author-level error explanation. +It does not establish that Gerbil provides Racket's syntax classes. + +Our design consequence is an explicit diagnostic contract: report the component +operation and its author declaration first; retain generated rule identity and +template definition as supporting detail. That presentation remains a proposal; +this fixture checks metadata, not diagnostic rendering or editor navigation. + +* Working contract for discussion + +| Origin facet | Meaning | Current evidence / remaining work | +|--------------+---------+-----------------------------------| +| Invocation | Component instance chosen by author | Actual owner identifier coordinate checked | +| Definition | Template implementation declaration | Actual defsyntax coordinate checked | +| Generated identity | Rule admitted by current compiler | Existing binding plus owner metadata checked | +| Effective contribution | Declaration selected after composition | Override requires operation-aware selection | +| Expansion chain | Intermediates between definition and invocation | Existing flat lineage does not yet prove this chain | +| Missing source | Explicit unknown, never invented coordinate | Source-free expansion checked; artifact integration pending | + +The working recommendation is to collect origins at their actual syntax +boundaries, carry them through existing compilation, and keep these facets +separate. The author should not need provenance bookkeeping in every language +package. Before publishing syntax, qualify missing-source and nested-expansion +cases and complete the bounded actual-language receipt. Author-task experiments +must separately establish whether these origins make diagnostics easier to use. + +* Follow-up: declaration diagnostics and absent source + +The [[file:scheme-declaration-diagnostics.org][declaration diagnostics study]] now supplies passing real-reader and source-free +expansion controls. It distinguishes syntax constructed within a transformer +from actual reader arguments, and leaves rendered diagnostic presentation open. diff --git a/docs/research/scheme-syntax-parameter-context.org b/docs/research/scheme-syntax-parameter-context.org new file mode 100644 index 00000000..40b29299 --- /dev/null +++ b/docs/research/scheme-syntax-parameter-context.org @@ -0,0 +1,202 @@ +#+title: Gerbil syntax parameters for scoped declaration context +#+date: 2026-10-06 + +* Research question and implementation boundary + +Can an outer declaration delegate parameter validation while preserving both +its author syntax and the inner declaration that performs the check? This +extends the [[file:scheme-nested-declaration-context.org][nested declaration-context study]]. + +The local witness uses Gerbil's existing =:std/stxparam=. Its committed baseline +is =783381f33fa1b309ac4cc4b0012696f24c852d93=. It changes research vocabulary +and shared fixture validators, not public parser compilation or runtime APIs. + +* Official mechanism and paper interpretation + +The [[https://gerbil.scheme.org/reference/std/stxparam.html][official syntax-parameter reference]] describes passing syntactic information +from parent to child macros and scoped reparameterization. The installed library +implements =syntax-parameterize= using =let-syntax= over the parameter's internal +key. We inspected that source before selecting the local protocol. + +[[https://www.barzilay.org/misc/stxparam.pdf][Barzilay, Culpepper and Flatt, Keeping it Clean with Syntax Parameters]], +sections 4 and 4.1, explains deliberately adjustable syntactic bindings and their +composition with hygienic macros. Section 6 discusses subtleties when macros +introduce additional parameterization. The design inference here is to use an +explicit syntax parameter for declaration context, while testing the actual +Gerbil scope and declaration behavior. The paper's Racket implementation is not +a proof of our Gerbil fixture or a general source-history facility. + +* Actual Scheme design and the two required questions + +[[file:../../t/fixtures/language-pack-research/author-context.ss][author-context.ss]] defines the local auxiliary syntax parameter. The wrapper +in [[file:../../t/fixtures/language-pack-research/scoped-author-vocabulary.ss][scoped-author-vocabulary.ss]] preserves the author form using =with-syntax=. +This excerpt omits its outer binding check and malformed-shape branch: + +#+begin_src scheme +(import :std/stxparam) +(defsyntax-parameter @@author-declaration #f) + +;; Inside the matched scoped-arguments transformer: +(with-syntax ((author-form stx)) + #'(def binding + (syntax-parameterize + ((@@author-declaration (quote-syntax author-form))) + (deflocated-list component-value owner entry item separator (field label)) + component-value))) + +;; At the validating component's transformer phase: +(def author-declaration (syntax-parameter-value #'@@author-declaration)) +(require-declaration-identifier #'owner stx + "deflocated-list" "owner" author-declaration) + +;; Question 1: Why is this DSL design better? +;; Compared with parameter forwarding alone, delegated checks retain the outer +;; author declaration, imported alias, exact bad parameter and inner check context. +;; The official syntax parameter carries context within its scope; ordinary Scheme +;; functions still implement contracts, and existing POO construction supplies the role. +;; Question 2: Why does this DSL design have a scientific basis? +;; Context propagation, scope isolation and valid component construction are separate contracts. +;; Evidence: 9 scope checks, 28 delegated diagnostic checks and 12 valid-role checks, +;; each with exit 0. Existing direct-reader diagnostics also passed 48 regression checks. +;; Falsifier: wrong outer context, lost parameter location, leakage into a direct call, +;; changed POO sections or capture of a same-named author variable. +;; Limit: no general expansion chain, concurrent expansion, artifact integration or speed claim. +#+end_src + +The component's existing shared validator accepts an optional author form. +On failure with that form it records =term, author, inner-declaration= as syntax +error irritants. Without it the existing =term, inner-declaration= shape remains. +The constraint and its message are unchanged. This is explicit metadata at the +contract boundary; it does not catch and relabel unrelated compiler exceptions. + +* A retained declaration-versus-expression failure + +The initial wrapper placed =deflocated-list= directly inside =syntax-parameterize=. +Invalid inputs reached the expected diagnostics, but a valid declaration failed +with =Bad syntax; empty body=, exit 70. The scoped =let-syntax= body contained a +local definition without a result expression. Thus diagnostic success was not +sufficient evidence that a declaration wrapper was usable. + +The corrected wrapper defines the public binding outside the parameterized +expression. Inside it, a hygienic local =component-value= binding constructs and +returns the component value. A host variable with the same spelling remains +unchanged in the valid control. The wrapper explicitly checks the outer binding, +because the generated inner binding is now an implementation identifier. + +This boundary must be specified in DSL design: a declaration introduces an author +binding, while an expression returns a value. The standard library mechanism +supports contextual expansion; it does not erase that distinction automatically. + +* Native receipts and controls + +All drivers use the same ten-second bound. The kill-after option bounds cleanup, +not permitted execution time: + +#+begin_src sh +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout -k 2 10 gxi t/fixtures/language-pack-research/syntax-parameter-scope.ss +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout -k 2 10 gxi t/fixtures/language-pack-research/scoped-author-diagnostics.ss +GERBIL_LOADPATH=.. GERBIL_PATH="$PWD/.gerbil" \ + gtimeout -k 2 10 gxi t/fixtures/language-pack-research/scoped-valid-components.ss +#+end_src + +| Driver | Evidence | Result | +|--------+----------+--------| +| [[file:../../t/fixtures/language-pack-research/syntax-parameter-scope.ss][syntax-parameter-scope.ss]] | Default, nested replacement/restoration, retention and parameter alias | 9 checks, exit 0 | +| [[file:../../t/fixtures/language-pack-research/scoped-author-diagnostics.ss][scoped-author-diagnostics.ss]] | Owner/entry/label blame, imported alias, source text, outer and inner context, direct call after failures | 28 checks, exit 0 | +| [[file:../../t/fixtures/language-pack-research/scoped-valid-components.ss][scoped-valid-components.ss]] | Checked role, same owner, all nine sections, same-named host variable unaffected | 12 checks, exit 0 | +| [[file:../../t/fixtures/language-pack-research/reader-diagnostics.ss][reader-diagnostics.ss]] | Original direct-call messages, blame and coordinates after optional context integration | 48 checks, exit 0 | + +The first complete diagnostic invocation produced no output and exited 124. +A later bounded invocation with actual expansion-start and caught-error logs +passed; the initial timeout remains a failed receipt. Its cause was not isolated. +The definition-only wrapper's exit-70 failure is separately retained above. +Neither is presented as evidence that the parser or GC caused the failure. + +The corrected valid test executes real component construction and compares +receiver-owned sections through =grammar-role-ref=. It does not compare parser +artifacts or parse events. Earlier actual-language artifact-loading timeouts +remain open and are not closed by these macro/role controls. + +* Working recommendation and unresolved contracts + +Use explicit scoped syntax parameters when a vocabulary needs contextual +information in delegated macros. Continue forwarding original parameter syntax, +and retain inner checking context alongside the author declaration. Keep the +public definition boundary outside a value-producing parameterized body. + +Before adopting this as a general declaration protocol, settle: + +- Whether nested wrappers should select the nearest author form, the outermost + form, or an explicit chain. One preserved form is not a complete expansion history. +- How malformed syntax-parameter values are rejected by the context contract. +- Which forms may introduce public definitions versus only expression values. +- How definition-phase behavior, Bound IR origins and persisted composition history + relate; this fixture carries diagnostic context without serializing a full chain. +- How rendered diagnostics present the retained forms, and whether developers + repair declarations more accurately with this information. + +These are protocol and usability questions. The controls establish a concrete +Gerbil mechanism and its bounded behavior, without selecting the public API. + +* Follow-up: nearest context versus first supplied context + +The expanded [[file:../../t/fixtures/language-pack-research/syntax-parameter-scope.ss][syntax-parameter-scope.ss]] now reports +=SYNTAX-PARAMETER-SCOPE-OK: 9 checks passed=, exit 0, under the same command. +The original three checks remain included; six additional checks cover nesting, +retention and renaming. This is a macro scope study, not a new parser benchmark. + +The ordinary nested parameterization selects the inner supplied form while it +is in scope, then restores the outer form. An experimental wrapper can instead +retain an already supplied form and use its candidate only when none is present: + +#+begin_src scheme +(defsyntax (retain-author-context stx) + (syntax-case stx () + ((_ candidate body ...) + (with-syntax ((selected + (or (syntax-parameter-value #'@@author-declaration) + #'candidate))) + #'(syntax-parameterize + ((@@author-declaration (quote-syntax selected))) + body ...))))) + +;; Question 1: Why is this DSL design better? +;; Relative to silently replacing context at every layer, this explicit policy +;; lets vocabulary retain an author entry form across cooperating wrappers. +;; It reuses the existing syntax parameter and forwards syntax instead of +;; manufacturing caller bindings or storing a process-wide mutable context. +;; Question 2: Why does this DSL design have a scientific basis? +;; Nearest-context and retain-existing policies predict different observable values. +;; Nine scope checks verify those predictions, restoration and renamed parameter binding. +;; Falsifier: the retained root is replaced, the outer context fails to restore, +;; or a renamed parameter ceases to adjust the original binding. +;; Limit: first supplied context is not necessarily the outermost source declaration; +;; these controls do not establish diagnostic rendering or a complete expansion chain. +#+end_src + +| Policy | What the check observes | Suitable design question | +|--------+-------------------------+--------------------------| +| Replace in each scope | outer, inner, outer | Which participating layer currently owns this contract? | +| Retain an existing value | root survives a candidate inner form | Which supplied entry should cooperating wrappers keep? | +| Supply only when absent | candidate becomes current form | Where does an entry-context region begin? | +| Explicit chain | Not implemented by a single-value parameter | Which intermediate transformations led here? | + +The renamed-parameter control imports =@@author-declaration= as +=@@renamed-author=. Parameterizing that alias changes the value observed through +the original identifier. This tests parameter binding identity directly; earlier +controls renamed the outer vocabulary macro rather than the auxiliary parameter. + +Retaining an existing value is cooperative. A wrapper that deliberately +reparameterizes can still replace it; a preexisting value may also designate +something other than a source-level outermost declaration. Consequently, the +working proposal is to distinguish an entry form from a current validating form +if both views are needed. That two-facet protocol remains a proposal, not an +implemented public schema or an automatically collected history. + +Before choosing a default, compare actual developer tasks: locating the call to +repair versus understanding the component contract that rejected it. Preserve +both views when qualified, rather than assuming a single selected form explains +all diagnostic needs. POO contribution ownership and persisted Bound IR origins +remain separate from this lexical expansion context. diff --git a/docs/research/scheme-test-process-ownership.org b/docs/research/scheme-test-process-ownership.org new file mode 100644 index 00000000..72652333 --- /dev/null +++ b/docs/research/scheme-test-process-ownership.org @@ -0,0 +1,82 @@ +#+TITLE: Native ownership of process qualification + +* Responsibility + +Language suites own assertions, examples, and benchmark algorithms. The compiled +=t/conformance-main.ss= entry owns their shared runtime and ten-second group +fences. The source Suite runner owns worker scheduling, affinity, ownership +checks, and its five-second silence fence. + +=test-processes.ss= owns the observable contract at a process boundary. Its +three profiles declare commands, expected normal exits, required diagnostics, +and deadlines. Expected negative controls are data in that declaration; they +are not interpreted by a shell pipeline. Every case writes its combined output +and an accepted exit to the profile's log directory. A signal termination can +never count as an expected normal nonzero exit. + +=build-conformance.ss= enters the existing Scheme preparation owner. It enables +real compiler import and GC diagnostics before loading that owner. Preparation +input admission, locking, compilation, and product publication remain in the +existing build implementation. + +* Why use this declaration? + +#+begin_src scheme +(process-case "duplicate" + '("gxi" "scripts/tests/conformance-runner-test.ss" "duplicate") + 70 "duplicate conformance Suite" 10 5) +#+end_src + +The record keeps the expected failure beside the command that must produce it. +Its executor preserves actual bytes and waits for the actual child exit. This +makes the accepted result inspectable without pipeline exit conventions, GNU +=timeout= selection, shell arrays, or a separate grep-based success protocol. +The S-expression is an ordinary value with a native record contract, rather +than an additional parser DSL. Suite declarations and execution retain their +existing Scheme owners. + +The architectural improvement is centralized ownership, not the filename +extension. Four independent Bash entrypoints are removed; one Scheme process +qualification declaration replaces their test coordination. The native build +entry exposes the existing preparation implementation directly. + +* Bounds and evidence + +Research semantic execution retains its 240-second total and five-second +real-output budgets. Its five research children retain ten-second deadlines. +Conformance controls have ten-second total and five-second output budgets. +Worker controls retain the source runner's silence fence; the qualification +parent adds a thirty-second total bound. It does not race a second silence +watchdog against the intentional silence control. + +The total timer covers output reads and the final process-status wait, including +a child that closes its output while continuing to run. Partial output bytes +count as real progress. Parent status messages do not count as child progress. + +These test entries own Scheme threads inside a direct child process. The native +qualification executor kills and reaps that child on timeout. It is not a +compiler process-tree supervisor. CI's existing host build watchdog still owns +compiler descendants, host process cleanup, and the build's 240/5-second fence; +that remaining Python infrastructure is a separate migration boundary. + +=scripts/tests/process-qualification-test.ss= exercises the production executor +with real children: exact exit status, partial stdout/stderr output, rejected +wrong exits, rejected missing diagnostics, silence timeout, continuous-output +total timeout, and timeout after closed output. The profiles additionally run +the original four conformance controls, seven worker controls, all compiled +semantic groups, and all five research entries. + +* Post-link inventory admission + +The first delivered head, =7419c1d=, passed Linux and Rust CI. macOS linked the +native conformance executable, then failed the unchanged five-second output +fence during post-link input collection. This result is retained as a failure, +not a successful preparation receipt. + +Preparation now expands the invariant module-product prefixes once per catalog +and reuses the matcher across inventory files. Its sorted depth-first walk +uses one accumulator and reports actual discovered files before digesting starts. +A source-level A/B checked identical ordered lists for six complete roots and +identical admission for 9,633 library files. Native controls additionally check +ordering, hidden paths, missing roots, and single-file roots. Cache identity, +input mutation rejection, digest admission, and deadlines remain unchanged. diff --git a/docs/research/selector-input-model-first.org b/docs/research/selector-input-model-first.org new file mode 100644 index 00000000..45d80771 --- /dev/null +++ b/docs/research/selector-input-model-first.org @@ -0,0 +1,236 @@ +#+title: Token input, captured owned stacks and prepared LR transition selection + +* Baseline and completion criterion + +This study starts at =7c184acd77b9fcdfec0537be11129b3a8df6616b=, after the +previous sparse-source capture improvement. Its complete 64-record reference +already contains that earlier improvement. The additional CPU result below is +relative to this starting point. Earlier =54746a3= results remain separate. + +The workload is 64 ASCII =a=1= records, 256 source tokens, 256 significant +shifts and 322 captured reductions. Every measured request creates a fresh +capture-enabled incremental session, completes scanning and LR execution, +publishes the artifact, and checks complete artifact equality and source validity. +The independently replayed recognition projection is checked before timing. +A later recorded-query replay corrected the earlier descriptive count of 192 +significant shifts: each of the 64 optional newline tokens is present and shifted, +so the count is 256. The 322 reductions, artifact/frontier fingerprints, timed +requests and recorded CPU observations are unchanged by this count correction. +No language grammar or Rust/C event format is specialized for these records. + +The unchanged gate runs five groups of 32 alternating legs, 256 requests per +leg: 122,880 timed requests, with 256 startup requests in each worker. Before +and after reference/reference controls must remain in 0.90 through 1.10; +production P90 must be below 0.95. The fences remain 90 seconds total and five +seconds idle. CPU is worker user plus system time, including natural GC. There +is no forced GC, timed process startup, runtime binding replacement, adjusted +threshold or added warmup after an unsuccessful observation. + +* Papers, upstream functions and existing implementation + +Sperber and Thiemann's [[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]] motivates deliberate +representation and binding-time choices, constructor/selector elimination and +preserving ordered computations. Wagner and Graham's +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] motivates retaining sufficient +context for reuse and accounting for its maintenance. These sources motivate +the study; their performance results do not establish this implementation's CPU. + +LeanPoo is pinned to its current open PR #13 revision +=8561a3ef8120101f956855d128d44ff9507c6703= in both Lake inputs. The review used +[[https://github.com/tao3k/lean-poo/blob/8561a3ef8120101f956855d128d44ff9507c6703/Examples/PrototypeFunctions.lean][PrototypeFunctions]], +[[https://github.com/tao3k/lean-poo/blob/8561a3ef8120101f956855d128d44ff9507c6703/docs/contracts/generalized-prototypes.org][the generalized Raw/Cooked protocol]], and +[[https://github.com/tao3k/lean-poo/blob/8561a3ef8120101f956855d128d44ff9507c6703/LeanPoo/Prototype/DictionaryBridge.lean][DictionaryBridge]]. Functions can be first-class prototype results, while +inverse-bearing representation conversion preserves their typed consumers. +Our model now reuses =DictionaryBridge= for token/EOF versus empty/singleton +input and proves state-specific selector transport. The runtime keeps its +existing Scheme POO preparation boundaries and retained functions; LeanPoo +supplies checked model contracts. No POO lookup is added per token or reduction. +The Lean toolchain remains 4.34.0, and model/library versions are unchanged. + +The latest upstream revision also adds +[[https://github.com/tao3k/lean-poo/blob/8561a3ef8120101f956855d128d44ff9507c6703/docs/contracts/closure-lifecycle.org][closure lifecycle costs]] and +[[https://github.com/tao3k/lean-poo/blob/8561a3ef8120101f956855d128d44ff9507c6703/Examples/LifecycleAccess.lean][LifecycleAccess]]. These distinguish cold construction, optional selector +preparation, retained metadata and repeated solves. Minimum declared operation +ceilings do not establish minimum CPU time, and future reuse must not be assumed +to repay preparation. Here the goto plan has an explicit allocation ceiling; +its preparation belongs to runtime construction, while the measured complete +request includes scanning, LR execution, artifact publication and verification. +The CPU result below is therefore a prepared-runtime request result, not a +cold-construction or lifecycle amortization claim. Grammar replacement must +construct a new admitted plan; this change does not introduce cross-source reuse. +The latest library revision rebuilt all 56 model jobs successfully. + +The code audit identified three boundaries: + +- The selector consumed a transport list although it observes only token/EOF. +- Captured execution forced immutable state/value stack spines, despite existing + private owned-buffer algorithms and sparse immutable snapshot publication. +- Each ordinary reduction read its production LHS and searched a goto row even + though production identity and grammar transitions are prepared constants. + +* Four-layer contract + +| Layer | Responsibility | Evidence boundary | +|-------+----------------+-------------------| +| Papers and code | Identify the actual representation, paid work and retained observations | No inferred speedup from an operation count | +| [[file:../../formal/ParserEngine/Lookahead.lean][Lean input]] | Pending token/mode preservation, representation inverses and selector transport | Uses upstream =DictionaryBridge= and actual producer premises | +| [[file:../../formal/ParserEngine/OwnedStack.lean][Lean stack]] | Freeze/pop equivalence, post-pop entry state and capacity arithmetic | Scheme must establish active-prefix fit and implement copying correctly | +| [[file:../../formal/ParserEngine/Preparation.lean][Lean preparation]] | Goto column indexing, source-cell provenance and preallocation budget | Stable admitted grammar tables are required throughout runtime lifetime | +| [[file:../../formal/quint/SparseCapture.qnt][Quint lifecycle]] | Ordered admission/consumption, pending modes, sparse frames and alias refusal | Bounded histories; semantic LR tables remain outside this protocol model | +| Scheme controls and CPU | Establish actual dispatch, ownership, complete outputs and measured benefit | Native compiler and runtime behavior require executable checks | + +The buffer model proves that freezing the remaining active prefix after a pop +matches dropping the same width from the immutable frontier. Its entry-state +observation therefore remains exact. The new Quint counterexample deliberately +rewrites a retained snapshot as if it shared later-mutated buffer storage; +=SparseSafety= rejects that implementation. Model input identity binds the +real stack helper, source owner, selector, batch executor and their controls. + +* Implemented engine changes + +** Token/EOF selector protocol + +The ordinary and layout selectors now take =(state, token-or-false)=. Literal +priority, Unicode case folding, first duplicates, EOF, rejected layout shifts +and original action-pair identity are retained. Batch/GLR callers project the +head of their borrowed suffix; streaming execution holds the token directly. +Capture reduction receives token/EOF and the already known post-pop entry-state. +It continues reading lookahead metadata at each existing capture operation. + +One LR transition macro has two compile-time axes: stack storage and input +storage. Each of the four generated executors has a direct static call site. +Generated reduction steps retain their borrowed list-tail contract. The input +policy checks this capability once at entry so those steps can reuse one tail +across reductions, instead of constructing a list for every query. +Arbitrary five-argument shift observers retain immutable list frontiers. +Language declarations acquire no new worker, scanner or input-policy option. + +** Captured owned state/value buffers + +Capture-enabled execution can use the existing request-private state/value +buffers. Its recognition stack remains persistent. Only selected snapshot +boundaries copy the active buffer prefixes into immutable checkpoint lists. +Fork/failure/public checkpoint boundaries also freeze their state and values. +Owned arrays never become snapshot fields or observer arguments. + +The capture owner receives the post-pop entry-state directly; it does not walk +a second state-list prefix. The physical-frontier control verifies that frozen +state spines are distinct after all later buffer writes, while their values, +complete grammar tree, modes, callbacks and artifacts equal the persistent +reference. This checks actual admission into the owned path. + +** Bounded goto preparation + +Preparation transposes each distinct LHS into a state-indexed target column. +Productions sharing an LHS share that column. Reduction then indexes by +production and remaining state. The original sparse row primitive constructs +each cell, preserving first-key and missing-target behavior. + +Before allocating columns, preparation checks =states * distinct-LHS <= 65536=. +Above that budget it retains the sparse algorithm. The bound covers column +cells; production references and other existing grammar data remain separate. +The original goto index is built once and remains available to generated steps. +Runtime clones and semantic-backend preparation retain the same admitted plan. + +* Generated-code finding and rejected measurements + +The first procedure-valued dispatcher computed an =execute= binding. On the +selected SDK/GCC build, inspection of its emitted Scheme showed an unused +selector followed by a direct persistent-executor call. A physical state-spine +control then failed even though value comparisons passed. The generated-code +observation is scoped to this build; compiler-version coverage remains open. + +Explicit static branches now emit all four intended executor calls. The physical +spine control passes. This is an executable admission condition, not an inference +from the presence of source branches or a successful artifact comparison. + +All observations used the same =7c184ac= baseline and gate: + +| Experiment | Production P50 | Production P90 | Admission | +|------------+----------------+----------------+-----------| +| Input protocol with procedure-valued dispatcher | 0.9938555499 | 1.0103229144 | Rejected; new executor not admitted | +| Flat private selector-row plan | 0.9819386307 | 1.0288044933 | Rejected; representation removed | +| Goto columns before dispatch correction | 0.9727722440 | 1.0375286922 | Rejected as a CPU result | +| Owned capture before dispatch correction | 0.9600650761 | 1.0429698349 | Rejected; physical admission failed | +| Owned capture with direct static calls | 0.9201990201 | 0.9609368122 | Rejected by unchanged P90 gate | +| Static input/owned capture plus bounded goto | 0.8910579899 | 0.9095538302 | Passed before final generated-step boundary control | + +The last observation exited 0 in 54.396 seconds with the success marker and +no supervisor reason or missing receipt. It is retained separately from final +qualification after the generated-step policy refinement. + +The native stage diagnostic used 256 complete requests. Inclusive checkpoint +execution was 55.695 ms out of 72.501 ms, while benchmark-only output verification +was 8.870 ms. Instrumentation contributes overhead and inclusive stages overlap; +these values guided investigation rather than establishing a speedup. +Verification remains inside the matched CPU interval. Reference/candidate request +ages differ after the before phase; the existing warmup and controls are retained, +and these observations do not establish a platform-independent latency bound. + +Early local setup failures are retained: a native admission terminated with +SIGSEGV before successful rebuilding of affected consumers; malformed Scheme +bindings and a typed-let form rejected by the SDK were corrected before their +controls executed; the diagnostic initially referenced an unavailable sorting +predicate; the alias model initially used set mapping operators before switching to the +list =foldl= API documented by the pinned Quint package. +A printed success or a model-only result was never admitted as process success. + +* Reproduction and current ownership + +The reference builder freezes all eight protocol owners, including runtime state, +layout, prepare, recognition, batch, checkpoint, transfer and source parser. +Each worker must load those native files from its requested image. Shared stack +storage helpers, incremental-session code and the records grammar must match. +The package lock, SDK and GCC are shared. Startup and IPC remain outside worker +CPU; complete artifact checks and natural GC remain inside. + +#+begin_src sh +just prepare-capture-reference 7c184acd77b9fcdfec0537be11129b3a8df6616b +just benchmark-capture-prefix +just diagnose-capture-input +#+end_src + +The manual Linux CPU matrix has one =selector-input= lane for this full protocol. +Older partial-image recipes require their historical source revision. Current +CI prepares the complete reference protocol and uses the same CPU fence and gate. + +* Final qualification + +After the generated-step input policy was refined, the final implementation +passed the same complete-request CPU gate against =7c184ac=: + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before reference/reference | 0.9671396763 | 0.9935065896 | 1.0422630225 | +| Production/reference | 0.8641645833 | 0.9100194177 | 0.9267669752 | +| After reference/reference | 0.9560683972 | 0.9802815477 | 1.0059877730 | + +=CAPTURE-COMPLETE-CPU-PROOF-OK= was present; exit 0, no supervisor reason, +no missing receipt, elapsed 48.776 seconds. The additional complete-request +P90 CPU reduction is 7.32% relative to the already optimized starting point. +The result qualifies the combined input/owned-capture/goto implementation on +this SDK and host. Component-isolated percentages, Rust/Tokio performance and +other language/workload CPU results remain separate qualification work. + +Both images retained artifact digest +=sha256:c42a4b2281ad7f1921dcd426606db0a2ccb5cbd9925078e67cd479aea27bf24c= +and frontier/mode fingerprint +=sha256:da526271ffa70150686180d18eab518fcb9e346d4faafc43d3b247cf6b0eb63c=. +The final semantic batch passed 94 cases in 11 Suite owners, including layout, +contextual/deferred scanning, GLR, incremental snapshots, generated steps, +physical ownership and the bounded goto plan. It exited 0 in 21.903 seconds. +The independent capture control passed 198 combinations, exit 0 in 10.989 +seconds. Lean completed 56 jobs; Quint passed 111 directed histories and 11 +sampled invariant runs. The source-input invalidation control also passed. + +A preceding final-image CPU attempt is retained as a total-timeout result at +90.349 seconds with no success marker. It completed 292 batches including +startup, and four production groups were 0.8743 through 0.9124; the after +control was incomplete, so the attempt was rejected. Read-only host inspection +identified a concurrent lean-poo =tools/check_atoms.py --jobs 14= run consuming +about five CPU cores. After that independent check ended, the unchanged +90-second budget and unchanged benchmark admitted the final result above. +No other task was interrupted and the timeout was not extended. + +The baseline parser CI for =7c184ac= completed successfully on Ubuntu and macOS +before this submission. A new pushed revision requires its own CI observation. diff --git a/docs/research/semantic-reduction-cost.org b/docs/research/semantic-reduction-cost.org new file mode 100644 index 00000000..e6eb1d91 --- /dev/null +++ b/docs/research/semantic-reduction-cost.org @@ -0,0 +1,506 @@ +#+title: Actual LR reductions and AST construction cost + +* Constructor shape and endpoint projection + +The next implementation targets the shared recognition constructor boundary. +Selected-derivation reexecution validates every production result before +reporting action shapes. These are work counts, not CPU percentages: + +| Request | Scale | Fields | Aliases | Materializations | Singleton bounded actions | Widest input | +|---------+-------+--------+---------+------------------+---------------------------+--------------| +| GQL RETURN | 64 | 0 | 464 | 2 | 462 | 127 | +| GQL RETURN | 512 | 0 | 3600 | 2 | 3598 | 1023 | +| FHIRPath addition | 64 | 456 | 330 | 1 | 329 | 127 | +| FHIRPath addition | 512 | 3592 | 2570 | 1 | 2569 | 1023 | +| Arithmetic addition | 512 | 2046 | 1024 | 511 | 513 | 3 | +| FHIRPath property/invocation | 1 | 55 | 49 | 3 | 46 | 7 | + +The GQL and FHIRPath list requests materialize only one or two wide sequences. +Thousands of bounded singleton actions instead repeat input-shape selection +and recognition-value classification. This makes singleton construction the +first common-path target. GQL's zero field count applies to these requests and +does not describe other grammars or an omitted measurement. + +=recognition-value-bounds= returns both endpoints after one checked variant +selection. Tokens, nodes, fragments, event values and relocation views retain +the existing admission and displacement behavior. In =reduce.ss=, hygienic +constructor rules select empty, singleton or general input once. Empty aliases +use the current invocation offset; singleton constructors call the combined +endpoint reader. General sequences read their boundaries before action +normalization, avoiding the subsequent list-tail scan. Raw sequence boundary +walks still have a cost; this is not a constant-time-bound claim for every tree. +Fields retain their capped-cardinality policy and supplied fragment constructor. +The borrowed and action-normalized constructor families share this one rule. +The prepared backend/action factory catalog remains the existing Core CLOS +multiple-dispatch owner, selected outside request execution. + +** Question: why does this exploit Scheme's syntax and rule facilities? + +The constructor prefix, child policy and bounded projection are fixed in +hygienic syntax. Multiple values express the two endpoints without a new list +or result record. The installed Gerbil compiler's =with-primitive-bind+args= +and the standard =with-list-builder= supply the organization pattern: bind an +ordered operation once and expose its result to the enclosing construction. +Language grammar and parser declarations acquire no additional switches. + +#+begin_src scheme +(define-recognition-alias recognition-action-alias recognition-sequence-for-action) + +;; One checked classification for a bounded singleton. +(let-values (((start end) (recognition-value-bounds value))) + (make-recognition-node kind start end children)) +#+end_src + +** Question: how is the benefit scientifically tested? + +=t/benchmarks/lr-request-stack/constructors.ss= preserves the construction +algorithm from =4f50dd3= as a qualification reference. Each side receives its +own cloned runtime, with the original selector and shared immutable grammar +tables. Private record bindings are resolved before timing. Both sides run +the current scanner, LR driver and complete artifact publisher. Complete +source admission, artifact equality and executor witnesses precede timing. +Arithmetic's cloned prepared executor is explicitly a representation control; +its public generated drive has separate full-matrix observations. + +The historical measurements below use twenty groups with sixteen alternating +legs, natural GC and the 1ms CPU precision gate. The first normalization-order-only probe exceeded its +90-second combined budget after completing the small controls; its partial +result is retained as a rejection, not a matrix receipt. Small-batch ratios +also had large GC-dependent dispersion. Subsequent complete cases keep the +90-second process budget and use 16,384 GQL or 32,768 FHIRPath small requests, +and the =calls= argument is 256 at scale 64 and 64 at scale 512. Each group +executes four times =calls= per variant. The later shared sampling correction +in =1e5a099= retains that request budget in eight full-length legs rather than +sixteen half-length legs; the current comparison owner uses this shared plan. +No collection interval is removed from the CPU result. + +On Gerbil =2591dcd= / Gambit =dcd677c= with both engine and reference compiled +with =-O=, the frozen original-module comparisons give these local ratios: + +| Request | Scale | CPU ratio P10 | P50 | P90 | +|---------+-------+---------------+-----+-----| +| GQL RETURN | 1 | 0.863 | 0.921 | 0.981 | +| GQL RETURN | 64 | 0.865 | 0.899 | 0.987 | +| GQL RETURN | 512 | 0.874 | 0.917 | 0.945 | +| FHIRPath addition | 1 | 0.870 | 0.891 | 0.914 | +| FHIRPath addition | 64 | 0.815 | 0.911 | 1.029 | +| FHIRPath addition | 512 | 0.845 | 0.910 | 0.964 | + +These are candidate/reference complete-request ratios, with roughly 8--11% +lower CPU medians in this local configuration. Percentile ratios are empirical +observations, not confidence intervals or an attribution of all request CPU. +The committed comparison owner independently passes actual MATCH/WHERE GQL +and FHIRPath property/invocation requests: median ratios 0.960 and 0.975, with +P90 ratios 1.326 and 1.122. Their dispersion prevents a stable benefit claim +for those two structured workloads at the measured batch sizes. + +The structured GQL request requires a GLR path whose deterministic derivation +trace is unavailable. The existing six deterministic corpora retain strict +trace admission. This additional request reports the unavailable scope +explicitly and still requires complete source, GLR-product and executor +equality. It contributes full-request/publication observations without a fake +semantic replay or zero action count. The complete matrix now passes with +50 component summaries, eight source workloads and six paired controls inside +the unchanged 90-second budget. Benchmark support uses the existing CI flags. +Source-owned construction, source-span and sequence suites pass with exit zero. +Fresh exact-head platform CI remains a separate qualification boundary. + +** Closed private recognition variants + +The repeated field/alias work above uses private recognition records. Their +subclass protocol was open even though the engine owns their construction and +exports no class descriptor. The installed Gerbil optimizer supplies a concrete +alternative: =compiler/optimize-call.ss= selects =struct-direct-instance?= and +checked =struct-direct-ref= for final classes. Official =std/io/counter.ss= +uses final private records. This is a representation contract, not a new +dispatch facility or an unchecked calling convention. + +The four variants in =src/runtime/recognition.ss= are now final: nodes, +fragments, children and relocation views. Their layout, constructors, child +mutation and source-offset behavior are unchanged. Public parser POO objects +remain extensible. Generated optimized reducers confirm checked +=##direct-structure-ref= instead of inherited =##structure-ref=; constructor +allocation remains. No module-wide unsafe declaration is introduced. + +*** Question: why does this exploit Gerbil's abstraction facilities? + +The engine states its closed representation once and lets Gerbil specialize +all compiled consumers. Language grammars need no type switch or helper DSL. +Existing constructor rules and preparation-time Core CLOS selection keep their +owners. This applies the partial-evaluation principle from the research +synthesis to an invariant the engine can actually guarantee: + +#+begin_src scheme +(defstruct recognition-child (field value) transparent: #t final: #t) + +;; Checked access; the compiler selects the direct final-class instruction. +(recognition-child-value child) +#+end_src + +*** Question: how is the benefit scientifically tested? + +The independent reference is the six-module recognition/funcs/reduce/plan/LR/ +artifact graph at =d8f91c5=, compiled with =-O= under distinct module names. +Each graph owns its recognition classes and publisher; combining old records +with the candidate publisher would invalidate the comparison. The existing +=compare-lr-staged-case= accepts that independent publisher. Its matrix entry +=compare-lr-staged-actions= now forwards the same publisher consistently. +Neither production runtime bindings nor class descriptors are replaced. + +Both sides use the same canonical grammar, public-source tokens and complete +source-admitted artifact oracle. Every timed invocation executes fresh +prepared LR recognition and artifact publication. Twenty groups use the shared +eight alternating full-quanta legs, natural GC and the 1ms CPU precision gate. +Scanning and preparation are excluded from these ratios. These observations +do not establish end-to-end latency or allocate CPU shares by subtracting +component medians. On Gerbil =2591dcd= / Gambit =dcd677c=: + +| Request | Scale | Calls | CPU ratio P10 | P50 | P90 | +|---------+-------+-------+---------------+-----+-----| +| FHIRPath addition | 64 | 256 | 0.817 | 0.882 | 0.985 | +| FHIRPath addition | 512 | 32 | 0.793 | 0.853 | 0.995 | +| GQL RETURN, initial | 512 | 32 | 0.624 | 0.901 | 1.223 | +| GQL RETURN, larger batch | 512 | 64 | 0.824 | 0.899 | 1.003 | +| Arithmetic addition | 64 | 256 | 0.860 | 0.896 | 0.979 | +| Arithmetic single value, initial | 1 | 4096 | 0.331 | 1.640 | 1.714 | +| Arithmetic single value, longer batch | 1 | 65536 | 0.846 | 0.922 | 1.037 | + +All seven cases pass complete-product equality and exit zero inside their +unchanged 90-second process budgets. The CPU medians improve by about 10--15% +in this prepared-request scope. GQL's dispersed initial result is retained; +the larger-batch observation does not support a stable tail-latency gain. +Arithmetic is a small grammar control and shows no median regression here. + +The single-value request tests the shortest reduction path separately. Its +initial paired median regresses despite identical per-batch allocation +(20,382,032 bytes per 4,096 calls) and lower candidate component CPU medians +(6.938ms versus 8.001ms). Retained natural collections take about 26ms, +dominating the 7--8ms collection-free batches; their placement changes which +variant pays that interval. This is an observed sampling imbalance, not a +reason to remove collection time or blame Gambit. At 65,536 calls per leg, +with the same twenty groups and 90-second limit, the paired median is 0.922 +and P90 is 1.037. This resolves the observed median reversal while still +preventing a stable tiny-request tail improvement claim. Both observations +remain evidence; the longer batch does not retroactively turn the first into +a passing speed proof. + +The existing comparison entry keeps this diagnosis reproducible without a +new runner or mandatory CI workload. Supply the independent =d8f91c5= +module graph's preparation, execution and publication procedures: + +#+begin_src scheme +(compare-lr-staged-case 'arithmetic-addition 1 + baseline-prepare baseline-parse 20 65536 baseline-publish) +#+end_src + +Two earlier candidates were withdrawn: synchronous state/value stack popping +gave FHIRPath-64 P10/P50/P90 ratios 0.897/0.969/1.101; a prepared +zero/unary/multiple reduction descriptor gave 0.934/1.007/1.127. Both passed +product checks but failed to establish useful CPU improvement. Neither is +production code. + +The source-recognition Suite rejects subclass construction for all four private +variants and wrong-kind slot access, while checking child mutation and relocated +bounds. It passes with the LR reduction and artifact publication Suites. +The old open reference admits subclass construction, providing a positive +control for the new rejection test. The existing full request matrix passes +50 component summaries, eight source workloads and six paired controls with +exit zero inside 90 seconds. Those controls retain their observed dispersion; +they are not measurements of this representation change. Platform CI remains +a separate gate; +no workflow, threshold or mandatory build entry is added by this change. + +** Publication traversal follow-up + +The recognition publisher classified every child list with =list?= before +=for-each= consumed it. The existing engine sequence consumer already uses +=pair?=/=null?= for this private proper-list-or-record protocol. Publication +now follows the same outer classification; =for-each= still rejects improper +tails. This removes a repeated linear preflight, while complete publication +remains linear in the number of children and events. + +The existing prepared-request comparison accepts an independent original +publisher in addition to original LR preparation/execution. Passing the same +current LR procedures on both sides isolates the publication change; every +timed result must equal the source-admitted public artifact: + +#+begin_src scheme +;; baseline-publish is the independently compiled publisher from 58a069d. +(compare-lr-staged-case 'fhirpath-addition 512 + lr-prepare lr-parse/prepared 20 64 baseline-publish) +#+end_src + +With both publishers compiled at =-O= on the SDK above, the 20-group fresh +prepared-recognition-plus-artifact comparison gave P10/P50/P90 CPU ratios +0.934811/0.982823/1.047521 and exit zero. Scanning and preparation are outside +this timed scope. This small, dispersed result does not establish a stable +CPU benefit or explain the dominant request cost. Natural GC and every sample +remain included. The source-owned publication Suite additionally checks empty, +singleton and 1,024-child field-bearing artifacts, exact event counts and +source admission, and rejects improper tails. It passes together with the +reduction and source-recognition Suites. + +For an optional reproducible comparison, first build the production library +through its existing owner, then use the same production optimization setting: + +#+begin_src sh +gerbil env gxc -O t/benchmarks/lr-request-stack/constructors.ss +gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 \ + --log .cache/constructor-case.log --require LR-CONSTRUCTOR-CASE-OK \ + -- gxi -e '(import :gerbil-parser/t/benchmarks/lr-request-stack/constructors)' \ + -e '(benchmark-lr-constructor-case (quote gql-return) 64) (exit 0)' +#+end_src + +The owner also accepts =fhirpath-addition= and =arithmetic-addition= at scales +1, 64 and 512, and the two structured request names at scale 1. This optional +comparison adds no CI job, production option or mandatory executable. + +* One investigation boundary + +This study follows [[file:request-lookahead-classification.org][the rejected lookahead cache]] and the +[[file:lr-request-cost-boundaries.org][request cost study]]. Action-query counts and selected-derivation counts do +not identify CPU ownership. The next experiment isolates the existing semantic +kernel using operands captured from real deterministic requests. It does not +replace the scanner, LR driver, semantic implementation or published artifact. + +The existing =lr-request-stack= benchmark owns three language controls: GQL +RETURN lists, FHIRPath additions and generated Arithmetic additions. At each +source scale it measures prepared LR execution and complete source parsing. +The new =semantic-replay= and =semantic-replay-control= components use the same +request, production IDs and recognition root. No language author configuration +or CI job is added. + +Artifact publication is also measured with the prepared complete root and the +original source/tokens. This retains its own source encoding, event traversal +and source identity work. Prepared execution, semantic replay, publication and +the public source request remain separate observations. + +The untimed source witness selects prepared checkpoints for GQL and FHIRPath, +but a generated drive for Arithmetic. Arithmetic's manual prepared-LR and +semantic replay rows are representation controls; they do not attribute the +generated source driver's CPU to the prepared semantic kernel. + +* Implementation and responsibility + +=t/benchmarks/lr-request-stack/derivation.ss= captures the deterministic +recognition derivation outside timing. Each private =semantic-invocation= names +its prepared production plan, immutable operand stack, actual lookahead offset +and expected result. Token leaves become the engine's ordinary recognition +children; production children retain the values captured during execution. +The capture validates every invocation against its original result. + +The replay invokes =src/runtime/reduction-plan.ss=, including its real staged +field/alias closures and =src/runtime/reduce.ss= AST constructors. There is no +copied reduction implementation. The traversal control reads the captured +results through the same ordered vector traversal. Both return the complete +root, and the benchmark also compares that root with the prepared parse whose +complete artifact already matches the public parser's source-admitted product. + +=src/runtime/funcs.ss= now owns =recognition-sequence-root-child=: acceptance +requires exactly one unfielded child. Deterministic LR, GLR completion and replay +use that boundary. Returning the child keeps absence separate from its payload; +the refactor preserves the previous acceptance guards and payload behavior. +It does not change production plans, AST constructors or backend preference. + +The corpus retains operands. Replaying each reduction independently does not +rebuild the producer-to-consumer dependency chain of a fresh request. Its CPU +and allocation results therefore describe component work on retained inputs. +They must not be subtracted from request medians or presented as an exact AST +percentage of request time. The control is traversal/result-access work, not +an AST-free parser. Installed noncanonical reduction procedures, GLR fallback +and event-program backends require separate admission; this experiment rejects +missing deterministic derivations and mismatching production results. + +* Gerbil implementation study + +The installed =1cfb032= sources provide two concrete organization patterns: + +- =src/std/list/list-builder.ss= defines =with-list-builder= through hygienic + =defrules=. One lexical builder owns its private list head and tail; the + caller supplies the traversal rather than copying builder mechanics. +- =src/gerbil/compiler/compile.ss= defines =with-primitive-bind+args= through + =defrules=. Syntax binds one ordered traversal and exposes its final + bindings to the surrounding compiler operation. +- =src/std/hash/misc.ss= uses =absent-value= in =hash-ref/default= so a present + false value does not mean absence. The root-child result follows the same + separation of presence from payload. + +The replay uses the standard list builder during untimed capture and a small +=defrule= for its two ordered traversals. The macro stages the fixed operation +into each traversal. Named records express ownership; dense vectors express +numeric addressing. This follows =scheme-coding-style.org= without introducing +a generic dispatch or callback selection inside each measured reduction. + +* Two design questions + +** Why does this use Scheme's syntax and rule facilities well? + +One engine reduction implementation remains authoritative. Hygienic syntax +shares the traversal between the control and actual execution while keeping +their operations explicit. Grammar declarations remain pure syntax; parser +metadata and language tests retain their existing interfaces. This is the +implemented shape, not a new grammar DSL: + +#+begin_src scheme +(define-semantic-replay replay-lr-semantics execute-semantic-invocation) +(define-semantic-replay replay-lr-semantic-control semantic-invocation-expected) + +;; Preparation validates every captured invocation before timing. +(let (replay (prepare-lr-semantic-replay runtime spec significant-tokens)) + (equal? (replay-lr-semantics replay) + (replay-lr-semantic-control replay))) +#+end_src + +** Why is the proposed attribution scientifically testable? + +The experiment can fail at capture, per-production equality, complete-root +equality, source admission or native dependency admission. Nullable aliases, +ordered field/alias composition and different request offsets are exercised +by =t/lr-stack-reduction-test.ss=. Repeated replay checks that the captured +operands remain usable. Shared sampling retains raw CPU and GC observations; +allocation intervals crossing GC remain inadmissible. + +* Qualification and reproducibility + +Use the existing production =build.ss= owner, native benchmark-support build +and optional =benchmark-lr-request-stack= entry. The ordinary CI support step +already compiles =derivation.ss= before =benchmark.ss=; the remaining source +test owner runs the reduction suite. The workflow declaration is unchanged. + +Initial local imports exposed a stale scanner symbol. A rebuilt scanner fixed +that request control, but later compiler controls exposed stale LR producer +objects. An incremental production build exited zero yet native admission +still reported missing =validate-production-semantics= and +=layout-end-action?= symbols, followed by exit 139. Those runs are rejected; +successful compilation alone does not qualify a mixed native graph. + +The clean-output production build subsequently exited zero. Its current +compiler, machine, scanner and LR runtime were rebuilt together. The reduction +suite first caught a replay-only error: an invocation returns a semantic child +sequence, whereas acceptance returns the root payload. Per-invocation equality +had passed; the complete-root control rejected the mismatched boundary. After +correction, the suite passed, including decorated operands, nullable origins, +three request offsets and repeated replay. + +An initial whole matrix hit its unchanged 90-second deadline during the +512-unit replay: 1,000 calls per batch were excessive at that scale. The partial +run is not a complete matrix receipt. The subsequent probe completed all six +workloads and 24 components with complete product controls. Its traversal +controls were below one microsecond per corpus at 64 units, while production +semantic replay was over 50 microseconds for each language. This directs further +investigation toward semantic construction rather than diagnostic traversal. +It does not establish an additive request percentage or an engine speedup. + +The local receipt below uses at least 100 semantic replays and publications per batch, +and 10,000 traversal controls. The observed minimum 64-unit semantic replay median exceeds 50 microseconds, +so even its smallest semantic batch is above one millisecond. The larger +traversal batch independently exceeds the timer resolution. Batch sizes are +published with each component; independent medians must not be subtracted. + +The full existing entry subsequently passed with =LR-REQUEST-STACK-OK= and exit +zero inside its 90-second budget: 42 component summaries and six paired request +controls. The smallest new semantic/control/publication CPU batch was 2.478ms. +The previous small Arithmetic paired control rejected 900 calls because one +leg was below 1ms. Small-request repetition is now 4,096 calls; the threshold +and timeout are unchanged. No CI declaration was changed by this investigation. + +** Final native component receipt + +The core execution and shared-root source are from =71554da=. The later +small-request and publication sampling corrections change batch sizes only. +The concurrent =0e02190= FFI integration does not change this core source closure; +local core measurements do not qualify that integration. Remote exact-head +CI owns combined repository qualification. + +Measurement SDK: Gerbil =2591dcd= on Gambit =dcd677c=, using the existing pinned +toolchain. The =1cfb032= source study above is an implementation reference, not +the measurement SDK. Each component has 20 samples. CPU includes naturally +occurring GC. Values below are CPU P50 milliseconds per corpus: + +| Workload | Units | Prepared LR | Semantic replay | Traversal control | Publication | Full source | +|----------+-------+-------------+-----------------+-------------------+-------------+-------------| +| GQL RETURN | 64 | 0.19090 | 0.10352 | 0.000628 | 0.04425 | 0.27810 | +| FHIRPath addition | 64 | 0.19435 | 0.12413 | 0.000661 | 0.04065 | 0.31910 | +| Arithmetic addition | 64 | 0.08710 | 0.05231 | 0.000253 | 0.02531 | 0.14470 | +| GQL RETURN | 512 | 1.43355 | 1.04256 | 0.004475 | 0.57790 | 3.34060 | +| FHIRPath addition | 512 | 1.51640 | 1.26184 | 0.004784 | 0.60643 | 4.02110 | +| Arithmetic addition | 512 | 0.81045 | 0.51949 | 0.001710 | 0.45123 | 1.24630 | + +At 64 units, each semantic replay admitted 19/20 allocation intervals, with +110,995.36, 133,955.36 and 77,251.36 observed bytes per corpus respectively. +At 512 units, semantic replay admitted 7/20, 5/20 and 12/20 intervals. FHIRPath's +512-unit publication admitted no allocation intervals; its allocation result +is absent, not zero. Changing batch size also changes GC exposure, so the +earlier probe medians are not a before/after performance comparison. + +GQL's 64-unit derivation executes 935 reductions: 661 unary, including 197 +identity unary, and 464 decorated operands. FHIRPath has 985 reductions and +657 decorated operands. These counts and the replay measurements justify +prioritizing field/alias construction and semantic sequence materialization. +They do not prove that one particular constructor, allocation site or rule +dominates fresh requests. The previously rejected unary corridors, identity +reuse and universal deferred sequences remain rejected. + +** Reproduction + +After the existing native production and CI benchmark-support preparation, +the exported entry is sufficient; it does not require a new executable: + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/lr-request-stack/benchmark) +(benchmark-lr-request-stack 20 20) +#+end_src + +Repository documentation records the configuration, results and evidence +boundary. Raw local logs and output-directory paths are not tracked. + +Further performance conclusions require a coherent current native graph and +the complete request controls. No engine speedup follows from adding replay +diagnostics or reducing the number of observed allocations. + +** Cross-host budget rejection and scale-aware repetition + +The macOS matrix at =6d7f5c5= passed the Scheme semantic suite, then exceeded +the unchanged 90-second request-matrix budget with exit 137 and no completion +marker. The six component corpora completed; the first small GQL paired +control was still running. This is a rejected matrix, not an AST mismatch or +a successful cross-host receipt. Large GQL and FHIRPath component corpora took +about 22 and 26 seconds with 100 semantic/publication calls per batch. + +The first correction retained 100 calls for 64-unit semantic/publication corpora, +20 for 512 units and 10,000 for traversal controls. The earlier 512-unit local +per-corpus medians put these 20-call batches above 9ms, providing resolution +without small-corpus repetition at every scale. Twenty samples and all complete +product controls remain. Small GQL and FHIRPath paired requests use 900 calls; +the faster generated Arithmetic control alone retains 4,096 calls to resolve +its original one-millisecond minimum. The paired precision gate and total +deadline are unchanged. Batch changes alter GC exposure, so results must not +be compared as an engine before/after speedup. + +The coherent local native matrix with these batch sizes completed inside the +same 90-second budget with exit zero and =LR-REQUEST-STACK-OK=: 42 component +summaries and all six paired request controls. Across the 360 new +semantic/control/publication samples, the smallest CPU batch was 2.429ms. +This admits the local sampling correction; exact-head remote CI still owns +cross-host qualification. + +The =8cd2da5= macOS matrix also rejected the 90-second deadline. All six +component corpora and four paired controls completed; the fifth paired +control was still running. The remaining dominant sampling cost was the 512-unit +traversal control: CPU P50 per 10,000-call batch was 765.708ms for GQL, +862.501ms for FHIRPath and 270.777ms for Arithmetic. The local support had +been compiled with =gxc -O=; CI's existing support owner uses =gxc= without +that option. These are different support compilation configurations, not an +engine regression. The revised 1,250-call large traversal control reduces +that sampling work by eight while the earlier optimized local medians still +predict more than 2ms for its smallest batch. Qualification rebuilds both +support modules with CI's flags before repeating the complete matrix. +No CI declaration or precision threshold changes. + +After that rebuild, the revised native matrix passed all 42 component +summaries and six paired controls inside 90 seconds, with exit zero and +=LR-REQUEST-STACK-OK=. The smallest CPU batch among the 360 new component +samples was 2.449ms. GQL's 64-unit traversal-control CPU P50 was 86.002ms per +10,000-call batch, consistent with the slower CI support configuration rather +than the earlier optimized support receipt. Remote exact-head qualification +remains a separate gate. diff --git a/docs/research/semantic-sequence-representation.org b/docs/research/semantic-sequence-representation.org new file mode 100644 index 00000000..bbbbf1a4 --- /dev/null +++ b/docs/research/semantic-sequence-representation.org @@ -0,0 +1,133 @@ +#+title: Semantic sequence representation and repeated validation + +* Shared engine question + +After the [[file:request-owned-lr-stacks.org][rejected vector-stack experiment]], this study examines +the semantic sequence boundary used by generic and generated reductions, +field/alias actions, source relocation, and artifact publication. The owner is +=src/runtime/funcs.ss=. Grammar declarations and parser metadata do not change. + +The admitted internal sequence representations are proper lists of recognition +children, concatenation branches, position views, and event-program sequences. +List leaves come from engine constructors and the standard list builder. The +sequence visitor is not an arbitrary Scheme list validator; malformed or cyclic +lists are outside this internal contract. + +* Repeated full-list scans + +=recognition-sequence->list= previously called =list?= to classify an already +materialized leaf. =recognition-sequence-for-each= repeated this check, and +branch materialization classified its leaves again with =list?=. Validating a +proper list traverses its spine, although these sites need only distinguish +the outer representation. Repeated actions over a width-n materialized leaf +therefore paid an extra O(n) scan per classification. + +The engine now recognizes empty/list leaves using the outer =null?=/=pair?= +tag. This makes classification O(1); actual publication still visits every +child. Endpoint lookup, field/alias construction, and event-program behavior +retain their original algorithms. No claim is made that all reduction work +becomes constant time or that this eliminates every quadratic construction. + +#+begin_src scheme +;; Inputs are admitted engine sequences, not arbitrary external values. +(def (recognition-sequence->list sequence) + (if (or (pair? sequence) (null? sequence)) + sequence + ;; The existing branch/view/event materializer follows here. + (materialize-sequence sequence))) +#+end_src + +Why is this design preferable? Grammar authors declare rules and closed +field/alias actions. The engine establishes and preserves the semantic +representation invariant. Scheme's disjoint pair/record representations allow +the engine to exploit that invariant without introducing author-facing storage +annotations or backend-specific syntax. + +Why is it scientifically testable? Returning an admitted list must preserve +its identity. Branch order, nested relocation contexts, source bounds, visitor +exceptions, and complete artifacts must remain identical. The existing sequence +controls cover those observations. Matched native full requests are required +for performance claims; removing a scan alone is a structural complexity result. + +* Official source correspondence + +The installed Gerbil =std/list/list.ss= uses =pair?= in algorithms whose inputs +are admitted lists, including =duplicates= and =length=?=. Its +=std/list/list-builder.ss= builds proper lists using one fresh cons per emitted +element and returns the head's cdr. Our materializer already uses this official +builder. These local source observations support responsibility for the input +invariant; they do not prove an end-to-end parser speedup. + +* Qualification + +Twelve source cases passed for sequence publication, generic reduction stack +controls, and accepted/rejected products in the all-language ABI fixture. +The child completed in 14.633 seconds, exit zero, with final =OK= and no +supervision reason. Receipt: +unarchived local execution. + +The changed module and shared request benchmark compiled using official +Gerbil =-O= in 15.947 seconds, exit zero. Receipt: +unarchived local execution. +Neither receipt is evidence that the remote PR CI has completed. + +The same twelve controls also passed with the candidate native sequence module, +exit zero in 14.917 seconds, final =OK= and no supervision reason. Receipt: +unarchived local execution. +After synchronizing the parallel Fold publication commit =3ba88c7=, sixteen +source cases passed for these controls and native-source generation contracts. +The child completed in 22.830 seconds, exit zero, final =OK=, with no +supervision reason. Receipt: +unarchived local execution. +The measurements below retain their original =7c32c81= baseline; they are not +whole-build native qualification of the parallel Fold implementation. + +* Native complete-request measurements + +The baseline was =7c32c81=. The only candidate runtime change was the three +representation tests in =funcs.ss=. The existing shared benchmark measured +20 samples of 20 calls per route, in baseline/candidate/candidate/baseline +order. The table reports CPU batch P50 divided by 20, in milliseconds/call; +it does not report individual-request latency percentiles. + +| Workload and route | A1 | B1 | B2 | A2 | +|--------------------+----+----+----+----| +| gql-return 64 prepared-lr | 0.23095 | 0.22130 | 0.23300 | 0.21840 | +| gql-return 64 full-source | 0.34290 | 0.44750 | 0.33270 | 0.33880 | +| fhirpath-addition 64 prepared-lr | 0.30215 | 0.26175 | 0.21390 | 0.21910 | +| fhirpath-addition 64 full-source | 0.43790 | 0.42325 | 0.36330 | 0.35995 | +| arithmetic-addition 64 prepared-lr | 0.09955 | 0.09685 | 0.09475 | 0.09905 | +| arithmetic-addition 64 full-source | 0.15145 | 0.19260 | 0.14175 | 0.14785 | +| gql-return 512 prepared-lr | 2.07810 | 2.20255 | 1.79940 | 1.84415 | +| gql-return 512 full-source | 3.04955 | 3.35515 | 2.64850 | 2.89045 | +| fhirpath-addition 512 prepared-lr | 2.28725 | 2.18380 | 1.71585 | 1.79290 | +| fhirpath-addition 512 full-source | 4.41905 | 4.23735 | 3.22615 | 3.55255 | +| arithmetic-addition 512 prepared-lr | 1.06330 | 0.91855 | 0.77220 | 0.81970 | +| arithmetic-addition 512 full-source | 1.87475 | 1.71810 | 1.14750 | 1.49135 | + +Allocation bytes per call were identical between baseline and candidate in +all twelve routes. CPU/wall observations varied substantially, including the +generated Arithmetic full-source control. The results do not establish a +stable complete-request speedup, a P95 improvement, or the largest remaining +CPU hotspot. The admitted improvement is elimination of redundant linear +representation scans under the engine input invariant. + +All four admitted native children completed with the required +=LR-REQUEST-STACK-OK= marker and exit zero in 40.686, 43.752, 18.372, and +17.082 seconds. Their receipts are +unarchived local execution. +These are local evidence files, not durable CI artifacts. + +An earlier candidate cold import failed the five-second silence gate after +14.326 seconds, before measurement. It is excluded. The admitted trials used +the existing compiled dependency trace for both variants, outside timing; +the timeout and silence gates were unchanged. + +* Continued cost attribution + +The generic LR action lookup also performs case folding after an exact literal +miss in case-insensitive grammars. Several reductions may inspect the same +lookahead before its shift. Before changing that ownership, count actual case +folds and distinguish literal lookup, generic-token fallback, layout, and +contextual lexical modes. This is a research hypothesis, not an implemented +normalization cache or a measured dominant cost. diff --git a/docs/research/shared-parser-cost-sampling.org b/docs/research/shared-parser-cost-sampling.org new file mode 100644 index 00000000..ff75e08d --- /dev/null +++ b/docs/research/shared-parser-cost-sampling.org @@ -0,0 +1,154 @@ +#+title: Shared parser cost sampling and optimization admission +#+startup: overview + +Implementation baseline: PR #13 =ecebab9= plus this slice. This module closes +the ownership gap between the [[file:artifact-publication-cost-boundaries.org][publication storage experiment]], +[[file:engine-reduction-generation.org][shared reduction implementation]] and existing cross-language LR benchmarks. +It is part of the same recognition -> reduction -> canonical publication study, +not another language-specific optimization proposal. + +* Research obligation and implementation gap + +The independent event-container control closely tracks event-publication +allocation in the measured families. It does not reveal a large removable +traversal allocation budget. Recognition/reduction experiments therefore need +consistent complete-product sampling and admission across languages. + +The repository already has cross-language selected-derivation counts, prepared +LR controls, independent GLR publication equality and matched complete-request +comparisons. Those mechanisms are retained. The gap was ownership: the generic +batch sampler, GC snapshots, percentile selection and allocation summaries lived +in =t/benchmarks/gql/runtime/matched-stages.ss=. The cross-language LR benchmark +imported that GQL module and renamed =measure-gql-component=. A shared algorithm +therefore acquired an unrelated language dependency and language-tagged receipts. + +* Implemented owner and consumer contract + +=t/benchmarks/parser-stage-cost/benchmark.ss= now owns the shared operations: + +| Operation | Contract | +|-----------+----------| +| =measure-parser-batch= | Integral, nontruncated calls; no implicit warmup, forced GC or logging; complete expected-result equality outside the interval | +| =measure-parser-component= | Validate sample/batch dimensions before warmup; one warmup and initial GC; preserve naturally occurring GC in timed observations | +| =sample-at-percentile= | Select the actual ranked observation, retaining its own CPU/GC counters; sample identity resolves ties | +| =sample-gc-snapshot= | A new collection exposes only the latest whole-VM snapshot; a no-GC interval cannot inherit a stale collection | +| =component-allocation-summary= | Single-caller intervals admit only nonnegative, no-GC deltas; thread-switching process observations do not acquire request allocation ownership | + +GQL workload selection, GQL actors and the cross-language LR benchmark import +the shared operations directly. The old GQL sampling definitions, names and +rename adapter are removed. GQL stage selection and directed-token reconstruction +remain language-owned. Shared receipts use =PARSER-GC-BASELINE=, +=PARSER-COMPONENT-SAMPLE= and =PARSER-COMPONENT-SUMMARY=; final workload receipts +still identify the workload. This is a direct migration, without compatibility +aliases or a versioned namespace. + +The native local diagnostic plan compiles the shared dependency before GQL +consumers and checks its compiled product in prepared mode. Its required +receipt names now match the shared sampler. The existing conformance build +catalog already owns the shared module and its focused test suite. + +* Scheme witness and design questions + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark + :gerbil-parser/languages/arithmetic/parser + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid-for-source?)) + +(let* ((source "1 + 2 + 3") + (reference (parse-arithmetic source))) + (unless (parse-artifact-valid-for-source? reference source) + (error "reference failed source admission")) + ;; A shared sampler receives a complete result; no GQL import is needed. + (measure-parser-component 'arithmetic-complete 20 20 + (lambda () (parse-arithmetic source)) reference) + ;; The inclusive observed stages serve attribution, not a latency comparison. + (measure-parser-stages parse-arithmetic source 20)) +#+end_src + +** Question: why does this improve the DSL and Scheme abstraction boundary? + +Language declarations continue expressing syntax/rules and parser metadata. +Sampling, VM counter admission and batch validation are shared algorithms, +implemented as named procedures over complete results. A language selects a +workload; it does not implement or rename a different sampling policy. The +canonical product remains the comparison boundary. No new =deflanguage= slot, +grammar-level profiling declaration or public semantic storage protocol is added. + +POO dispatch remains appropriate for extensible parser capabilities and prepared +representation selection. A homogeneous batch's arithmetic, ordering and +counter validation require no extensible type decision. Adding generic methods +here would not improve the scientific contract or remove grammar complexity. + +** Question: why is this scientifically testable? + +Existing controls preserve the wall-ranked observation's own CPU/GC values, +reject stale GC attribution, retain unavailable thread ownership and reject +truncated or changed products. New language-independent controls require invalid +component dimensions to fail before any warmup call, count every admitted batch +call and preserve an original thrown condition. The diagnostic plan explicitly +owns the shared dependency. The cross-language caller still validates prepared +and complete canonical products independently and reports actual source routes. + +Neither copying a sampler nor introducing a new owner proves a runtime speedup. +Prepared LR and full-source costs are different experiments, not additive pieces +of one request. Observer capture and derivation replay remain untimed controls; +they must not be silently included in a production latency claim. Earlier +rejected stack, unary-corridor and deferred-consumption candidates remain rejected. + +* Qualification and next decision + +Qualification must cover the shared native producer, all migrated consumers, +source contract controls and cross-language complete requests. Exact-head Linux, +macOS and Rust AOT CI remains a separate gate. The next optimization needs a +measured recognition/reduction construction boundary and complete-request +admission, rather than another GQL-owned measurement primitive. + +** Native producer, consumer and control receipts + +The shared sampler, GQL stage selector, GQL actor caller, cross-language LR +caller, local qualification plan and focused test compile natively in =76.380s=. +The first source control stops before tests because the plan's existing +=qualification/process= native object is missing (=35.601s=, exit 1, +=error-output=, no final =OK=). Its existing process/environment/products +dependencies compile in =20.367s=. The next control reaches GQL execution +assertions, then rejects an old native =execution-counts= product missing the +current source's case-fold fields (=29.645s=, exit 137, =error-output=). Neither +failed control is qualification success. Current execution/reduction diagnostic +sources compile in =24.823s=; the expected assertions are retained. + +The final four-suite source run over the native producers passes all =30= cases +in =42.303s=, exit zero, four =MODULE-OK= receipts, =HARNESS-OK= and final =OK=. +It covers publication/sampling contracts, GQL stage and execution assertions, +native dependency plans and language source contracts. GQL test input comes from +the staged content with relative imports resolved to the same package modules; +the pre-existing uncommitted deferred-materialization test is not included in +this count or staged for this migration. + +Receipts: unarchived local execution. + +** Cross-language and actor consumers + +The migrated LR caller completes twelve prepared/full-source paired summaries +across GQL, FHIRPath and Arithmetic, two source scales and two preferences, in +=44.542s=. Each comparison retains twenty ABBA/BAAB groups and two calls per +batch. Independent GLR equality, source admission, actual source-route witnesses +and complete timed-batch products remain enforced. Exit is zero with +=LR-REQUEST-BACKENDS-OK= and final =OK=. The identical-executor control is stable; +complete-request benefit admission is =#f=. Ten comparisons are =not-admitted=; +the 64-operand FHIRPath and Arithmetic prepared-only comparisons report +=observed-regression=. These short-batch observations do not qualify a new +default or a portable performance result. No engine representation is changed. + +The native actor consumer completes configurations =(1 client, 1 worker)=, +=(2 clients, 1 worker)= and =(2 clients, 2 workers)= in =21.921s=. Each uses two +samples of four requests plus its warmup. Canonical client results equal the +public reference. All three summaries retain =process-with-thread-switches=, +zero admitted allocation samples and unavailable per-request allocation. +Exit is zero, three =SHARED-ACTOR-SAMPLING-OK= receipts, =GQL-ACTORS-OK= and final +=OK=. The shared owner has not silently converted actor process counters into +single-request allocation observations. + +Receipts: unarchived local execution. All local commands +retain the five-second real-progress idle gate. At verification, the preceding +=ecebab9= exact-head CI has passed Rust AOT; Linux Scheme remains in progress +and macOS Scheme is queued. The migrated source needs its own exact-head CI. diff --git a/docs/research/shared-ranked-scanner-execution.org b/docs/research/shared-ranked-scanner-execution.org new file mode 100644 index 00000000..39df7051 --- /dev/null +++ b/docs/research/shared-ranked-scanner-execution.org @@ -0,0 +1,103 @@ +#+title: Ranked scanner and parser execution study +#+startup: overview + +Part of [[file:shared-scanner-staging-and-fusion.org][the shared scanner study]]. +Historical revisions, failed experiments and receipt limits retain their original scope. + +* One argument from declaration to complete product + +This is the integration entry for lexical specialization, prepared execution and +performance evidence. Its four responsibility modules preserve the original +experiments; they are parts of one implementation sequence, not separate author +interfaces or optimization roadmaps. + +The sequence is =grammar.ss= -> macro expansion and POO composition -> canonical +IR -> compiler publication -> prepared machine -> =parser.ss= request -> canonical +artifact -> =parser-test.ss= qualification. Static decisions belong to preparation; +request values and mutable execution state belong to the request. A cheaper +component is not sufficient evidence for a cheaper complete product. + +* Responsibility and implementation map + +| Engine obligation | Detailed study | Existing code and executable consumer | +|-------------------+----------------+---------------------------------------| +| Specialize only states admitted by the declaration | [[file:ranked-scanner-specialization.org][Lexical specialization]] | [[file:../../src/runtime/scan.ss][Shared scanner]]; [[file:../../t/ranked-literal-scanner-test.ss][literal controls]]; [[file:../../t/benchmarks/regular-mode/request-cost.ss][regular request control]] | +| Bind semantic representation and reduction ownership before execution | [[file:prepared-parser-representation.org][Prepared execution]] | [[file:../../src/compiler/machine.ss][machine admission]]; [[file:../../src/runtime/lr-parser.ss][LR execution]]; [[file:../../t/language-surface-test.ss][installed entry controls]] | +| Share immutable literal vocabulary while preserving state admission | [[file:casefold-request-qualification.org][Casefold complete requests]] | [[file:../../src/runtime/lr-action-index.ss][action index]]; [[file:../../t/lr-index-test.ss][independent lookup oracle]]; [[file:../../t/benchmarks/lr-request-stack/casefold-evidence.ss][request comparison]] | +| Admit observations before claiming benefit or delivery | [[file:parser-execution-evidence.org][Execution evidence]] | [[file:../../t/benchmarks/parser-stage-cost/receipt.ss][shared receipt owner]]; [[file:../../t/benchmarks/ranked-literal/evidence.ss][ranked comparison]]; [[file:../audit/engine-design-realization.org][design realization audit]] | + +The canonical production and table contracts are established by +[[file:prepared-lr-action-plans.org][production/operand/reference admission]] and [[file:lr-table-admission.org][table/target admission]]. +Their semantic controls qualify preparation and publication. They do not upgrade +historical performance observations or substitute for exact-head platform CI. + +* Retained mechanisms and rejected candidates + +| Mechanism or candidate | Decision | Evidence limit | +|------------------------+----------+----------------| +| Closed-start monotone regular DFA path | Retained | Independent 64-subset controls and historical native request qualification; no complete-parser CPU claim | +| First-character ranked callback replacement | Withdrawn | Complete cross-language audit rejects the component-only inference | +| Engine-owned installed-entry representation | Retained contract | Actual branch and complete-product controls; arbitrary procedure bodies are not proved correct by a representation query | +| Prepared identity elimination | Withdrawn | Repeatable prototype allocation savings, unstable CPU controls | +| Shared literal vocabulary/casefold lookup | Retained | Complete-request allocation evidence; CPU benefit is rejected | +| ASCII state-initial mask | Withdrawn | Complete-request decision rejects the additional dispatch refinement | +| Delayed ranked result publication | Withdrawn | Compiled component allocation evidence; unchanged CPU controls drift and no complete-parser benefit is established | +| Shared receipt admission | Retained test infrastructure | Detects inconsistent/forged observations; cannot certify unseen execution or manufacture CPU stability | + +The detailed modules retain revision identities and numerical observations. +This table is a decision index, not a new experiment or a portable speed claim. + +* Scheme witness + +#+begin_src scheme +;; Authors declare a grammatical value; preparation derives immutable execution. +;; This is an illustrative ownership boundary, not another public DSL slot. +(let* ((declaration '(literal "λ")) + (execute (prepare-matcher declaration))) + (values (execute "λx" 0 #f) ; => 1 + (execute "xλ" 0 #f) ; => #f + (execute "xλ" 1 #f))) ; => 2 + +;; Scientific admission consumes the complete observation and its controls. +;; A component allocation result cannot imply complete-request CPU improvement. +(equal? candidate-artifact reference-artifact) +#+end_src + +** Question: why does this exploit Scheme beyond S-expression notation? + +The inspectable declaration gives the engine a static value to admit and lower. +Hygienic syntax retains caller bindings; lexical closures capture prepared data; +POO composes extensible declarations and policies; structural matching handles +closed IR alternatives. Their responsibilities are explicit. Authors need no +matcher cache, dispatch loop, opcode, storage visitor or measurement reader in +=grammar.ss=. The detailed studies show where these mechanisms helped and where +additional abstract calls or traversal transfers were rejected by execution. + +** Question: how is the design scientifically justified? + +The [[file:shared-scanner-staging-and-fusion.org][paper-to-engine argument]] states the assumptions of staging and fusion. +Independent token/field/span/artifact controls test semantics, while paired +native observations with unchanged controls decide performance admission. A +changed result, invalid source ownership, unstable control, incomplete receipt +or failed process gate falsifies the corresponding claim. Papers motivate the +mechanism; their proofs do not certify this LR/GLR implementation. Rejected +candidates remain visible and the current defaults follow the retained decisions. + +* Integration and delivery gates + +Continue through the shared engine sequence above. Every optimization must name +its producer, semantic consumer, independent falsifier and complete-product +boundary before changing a default. Preparation, execution, publication, +allocation, CPU and exact-head CI remain distinct evidence categories. + +The modules preserve all 1,296 original body lines from the 1,302-line study at +=c12dcb0=. A Scheme reconstruction check compares the full historical text +exactly, including order and whitespace. Incoming audit heading links move to +their actual module; all owned local file and heading links are qualified. +The first temporary restructuring attempt failed that equality check due to an +incorrect output-port call and is not admitted as successful publication. + +At =c12dcb0=, focused canonical-reference qualification passed 117 cases and +Rust platform CI passed. Linux Scheme was running and macOS Scheme queued. +Those historical receipts do not close platform CI for this documentation +revision. Fresh exact-head results remain the delivery gate. diff --git a/docs/research/shared-region-execution.org b/docs/research/shared-region-execution.org new file mode 100644 index 00000000..326d0986 --- /dev/null +++ b/docs/research/shared-region-execution.org @@ -0,0 +1,446 @@ +#+title: Region execution and sequence ownership +#+startup: overview + +Part of [[file:shared-scanner-staging-and-fusion.org][the shared scanner study]]. +Historical revisions, failed experiments and receipt limits retain their original scope. + +* Shared region opener indexing and Gerbil source review + +** Fixed SDK implementation evidence + +This review used the source shipped with Gerbil 0.19, revision =2591dcd=, +patch =f5cedd8168cb=. Paths below are relative to that SDK's source tree: + +| Source | Mechanism | Consequence for this engine | +|--------+-----------+----------------------------| +| =std/text/parser/ll1.ss=, =ll1-case= | Prepares peek predicates outside the reader lambda | Prepare declaration indexes once; keep input inspection request-local | +| =std/list/list-builder.ss=, =with-list-builder= | Expands collector calls directly into owned list mutation | Use the existing macro when producing lists; do not add a second collector framework | +| =std/text/parser/ebnf/deflexer.ss= and =defparser.ss= | Validate syntax and generate bindings with source information | Keep grammar admission and generated execution separate; preserve hygienic identifiers and error locations | +| =gerbil/compiler/optimize-call.ss= | Specializes calls using known procedure types and checked primitive information | Do not replace proven call specialization with blanket unsafe declarations | +| =gerbil/runtime/util.ss=, =foldl1= | Implements a generic callback-driven tail loop | The engine's repeated callback construction and full declaration walk are the problem, not the generic fold itself | +| =gerbil/runtime/hash.ss= | Provides specialized tables and checked method boundaries | Reuse an eqv table for Unicode codepoints rather than invent another hash implementation | + +The LL1 implementation is a source-quality reference, not a replacement for +contextual or LR grammar semantics. Its list peeker still prepares nested +predicates during probing; using a standard library abstraction alone does not +prove allocation or latency improvements. The list builder's mutable chain +belongs to one execution; moving it into shared language state would violate +request ownership. + +The binding-time argument in [[https://www.cl.cam.ac.uk/~jdy22/papers/a-typed-algebraic-approach-to-parsing.pdf][A Typed, Algebraic Approach to Parsing]] +(PLDI 2019, Section 6.1) distinguishes static grammar facts from a dynamic +stream. That supports the preparation boundary here. Its benchmark computes +counts instead of constructing AST nodes, so its speed ratios cannot be +transferred to this engine's complete lossless artifact contract. + +** Implemented representation and preserved semantics + +The shared =RegionPlan= now prepares opener buckets by first codepoint: a +128-entry vector for ASCII and an eqv hash table for other codepoints. Buckets +refer to the same owned declaration rows used by scoped caches, and preserve +declaration order. The public specification view remains a deep copy. Every +probe retains scope filtering and longest-prefix selection. No language +declaration, loader schema, C ABI or compiler selection changes. + +Previously, each input position built a capturing callback and walked all P +opener declarations. The generated Scheme for the frozen reference contains +that lambda passed to =foldl__0=. The new named loop walks only its selected +bucket. For T inactive input positions, candidate lookup no longer grows with +P. This is not a trie: a bucket with P colliding first codepoints still requires +a P-row prefix walk, plus the existing allowed-prefix membership checks. + +** Native control on base f471694 + +An initial decomposition of the large fixture found 2,550 words and 6,950 raw +tokens. Word-result construction allocated 15,266,128 bytes at CPU P50 +7.809 ms (10 allocation samples of 11); source-frame preparation allocated +532,912 bytes at 0.205 ms (11 of 11). These isolated subproducts are not +additive full-request costs. They directed attention toward word recognition +rather than a source-frame ownership redesign. + +The region control kept the input fixed at 512 inactive characters and varied +opener declarations. Preparation and endpoint checks were outside timing. +Each row reports 11 CPU samples and 11 no-GC allocation samples: + +| Openers | Reference CPU P50 ms, A / B | Indexed CPU P50 ms, A / B | Reference allocation bytes | Indexed allocation bytes | +|---------+----------------------------+--------------------------+----------------------------+--------------------------| +| 1 | 0.041 / 0.040 | 0.032 / 0.032 | 41168 | 208 | +| 8 | 0.074 / 0.171 | 0.031 / 0.032 | 41168 | 208 | +| 64 | 0.315 / 0.316 | 0.032 / 0.032 | 41168 | 208 | +| 256 | 1.992 / 1.157 | 0.031 / 0.032 | 41168 | 208 | + +The complete recognition control used the same current prepared scanner on +both sides and frozen versus indexed region/word producers. Source-bound +artifact validation was inside both measured requests. Consequently this +control isolates recognition lowering; a scanner-side benefit common to both +variants is not attributed to the change in this table. + +| ABBA variant | Complete CPU P50 ms | Allocation P50 bytes | Allocation samples / CPU samples | +|--------------+---------------------+----------------------+---------------------------------| +| Reference A | 26.147 | 36814992 | 8 / 11 | +| Indexed A | 23.315 | 35939792 | 9 / 11 | +| Indexed B | 24.941 | 35939792 | 9 / 11 | +| Reference B | 40.933 | 36814992 | 9 / 11 | + +The stable allocation reduction is 875,200 bytes, approximately 2.38%. +Reference CPU varies substantially across blocks; this does not establish a +stable complete-entry speedup or close the Tree-sitter performance gap. +Short, here-document, compound, large, malformed, Unicode and empty inputs +passed complete-product equality. The native control exited 0 in 27.548 s +with =REGION-PREFIX-CONTROL-OK=. Seven focused modules passed 92 cases in +14.080 s. New cases cover overlapping opener order, restricted and empty +scopes, Unicode indexes, ownership and invalid offsets; existing cases cover +deep nesting and separate quote/pair cache identities. + +** Rejected experiments and limits + +An ASCII class-table experiment passed semantic tests but did not reduce +complete allocation; CPU controls were mixed. It was reverted. Its controller +also failed the idle guard after returning from main, so that run is excluded +from successful native qualification. The retained region controller exits +explicitly only after completing its assertions. + +One exploratory GCC compile exceeded the idle guard during optimization +diagnostics. The unchanged source and SDK compiled on a retry. This does not +identify the precise optimizer stall or justify changing the compiler. All +experiments retained the 120-second total and 5-second idle guards. Temporary +frozen-reference modules and experiment controllers are not production code. + +Remaining algorithm work is active-bucket prefix discrimination and reducing +word-result intermediate construction. Both require matched complete-product +controls, including negative results; the inactive-input scaling result does +not by itself justify either transformation. + +** Qualification after integrating the latest PR changes + +The final integration base was =f750b961e6db7fda910edbc2e2d191b82ce75e72=. +The shared runtime build completed in 22.930 s, and the updated stage +diagnostic compiled in 10.586 s, both with exit 0. Eight focused modules +passed 157 cases in 11.801 s with =TEST-SUITE-OK modules=8=, =OK= and exit 0. +This includes the newly integrated EventFold and P95 contracts; it is a local +qualification receipt, not a remote CI result or a new performance table. + +An earlier compile attempted the fold runtime before rebuilding the newly +exported =fold-source-lines= interface and failed with an unbound identifier. +An earlier integration test loaded the old stage diagnostic and failed at +=cdr= of the missing =cpu-p95-ms= association. Rebuilding the changed modules +in dependency order resolved both stale-artifact failures. Those failed +rounds are excluded; neither production contracts nor watchdogs were relaxed. + +* Rejected dense region-prefix lowering + +** Algorithm and source review + +This experiment used base =9c4b7635238eeb9f1fc8c8236a345b055589ddea=, the +fixed Gerbil/GCC SDK and unchanged 120-second total / 5-second idle guards. +It investigated the remaining linear walk within a first-codepoint bucket. +The engine's =make-bounded-literal-end-scanner= already follows a character +trie. The experiment adapted that representation to terminals containing the +original owned region rows, preserving cache identity and scope filtering. + +The first candidate used a trie for every nonempty bucket. A later candidate +kept buckets of at most eight declarations as lists and compiled larger ones +into tries. The final candidate additionally sorted short buckets by descending +opener length, allowing the first admitted match to commit immediately. +A unique-prefix admission rule ensures that two different terminals of the +same width cannot both match the same source position. A rejected terminal +cannot stop traversal: a longer descendant might still be admitted. + +Trie preparation uses memory proportional to its distinct prefix nodes. +Lookup follows the source prefix with expected constant-time hash edge access; +scope membership remains a list search at terminals. This improves dense +first-codepoint collision scaling, but adds representation discrimination and +edge lookups to small catalogs. It does not remove word captures, intermediate +recognition nodes or final token publication. + +This distinction agrees with the materialized-token cost discussed in +[[https://arxiv.org/html/2304.05276][flap: A Deterministic Parser with Fused Lexing]], Sections 2.2 and 2.8. +This experiment is static prefix lookup specialization, not lexer/parser +fusion, and does not establish the paper's deterministic grammar assumptions +for this engine. + +** Matched native control + +The control froze the original region/part/word producers and shared the same +current scanner and result plan on both sides. It tested full artifact equality +against the public parser and performed source-bound artifact admission inside +each timed complete request. Preparation and post-measurement equality were +outside timing. Natural GC was retained; allocation deltas were admitted only +for intervals with no GC. No overlapping local build or test was running. + +The collision workload used 512 =a= characters and declarations of the form +=a{=. Each sample performed twenty complete region-end scans. Every +lookup shares the first codepoint but fails its continuation. The final +width-ordered bounded-list/trie candidate produced the following 11-sample +ABBA medians (all eleven allocation samples were admitted): + +| Declarations | Reference CPU P50 ms, A / B | Candidate CPU P50 ms, A / B | Allocation P50 bytes, both | +|--------------+----------------------------+---------------------------+---------------------------| +| 1 | 0.690 / 0.717 | 0.720 / 0.722 | 272 | +| 8 | 1.342 / 1.501 | 1.351 / 1.352 | 272 | +| 64 | 7.625 / 6.331 | 1.087 / 1.073 | 272 | +| 256 | 23.190 / 23.048 | 1.074 / 1.077 | 272 | + +These are repeated region lookup costs, not public parser speed ratios. +The constant allocation includes the measurement call boundary. + +The final complete-request control used 31 CPU samples per block to resolve +variance in earlier eleven-sample controls: + +| Variant | CPU P50 ms | Allocation P50 bytes | Allocation samples / CPU samples | +|---------+------------+----------------------+---------------------------------| +| Reference A | 22.027 | 35939856 | 23 / 31 | +| Candidate A | 21.971 | 35940496 | 23 / 31 | +| Candidate B | 25.720 | 35940496 | 24 / 31 | +| Reference B | 21.842 | 35940496 | 23 / 31 | + +Allocation did not decrease. Candidate B was approximately 17.8% slower than +Reference B; Candidate A was approximately level with Reference A. This +cannot prove a universal regression, but it fails to establish a reliable +complete-request benefit without a regression. The production implementation +was therefore restored to the prepared first-codepoint buckets. Neither the +trie nor width ordering was promoted. The collision speed ratio is not used +to claim progress on the complete Tree-sitter gap. + +The final native controller completed in 13.242 s with exit 0 and +=REGION-TRIE-CONTROL-OK=. Short, here-document, compound, large, malformed, +Unicode and empty inputs passed complete-product parity. The final candidate +semantic run passed seven modules and 96 cases in 12.905 s, with +=TEST-SUITE-OK modules=7=, =OK= and exit 0. Temporary frozen modules and the +private experiment controller were removed after recording these results. + +** Retained contract tests and excluded attempts + +The added tests cover sparse and dense catalogs through the existing public +API, shorter admitted terminals beneath longer forbidden ones, unrestricted +queries after scoped cache publication, truncated openers, and Unicode edges +under both ASCII and Unicode first codepoints. A direct query intentionally +selects its unrestricted longest opener rather than reusing the scoped shorter +row. An initial test expected the scoped endpoint for that direct query; it +was corrected to the actual public contract, not by changing the engine. + +Early private controller builds used an incorrect import operand order, a +keyword instead of an identifier prefix, a nonexistent optional library path +and a malformed optional-argument definition. Gerbil's shipped +=gerbil/core/module-sugar.ss= defines =prefix-in= as a module followed by an +identifier prefix, and validates =rename-in= identifiers. Those failed builds +are excluded. An initial native run printed its final assertion receipt but +hit the idle guard during normal exit cleanup; it is not a successful +qualification. Subsequent controllers used the existing disposable test-child +exit interface only after every assertion and flushed receipt. Guards were +not relaxed and assertion failures were not bypassed. + +The next substantial target remains word-result intermediate construction. +The prior decomposition attributed approximately 15.3 MB to word products +and only 0.53 MB to source-frame preparation on the large fixture. Prefix +lookup is not sufficient to remove that representation cost. Future producer +lowering must preserve the complete artifact, owned source boundaries and +negative-input behavior and demonstrate admitted allocation improvement. + +After restoring the default engine, its native region module rebuilt in +5.547 s. The retained tests passed seven modules and 96 cases in 12.244 s +with =TEST-SUITE-OK modules=7=, =OK= and exit 0. This final receipt qualifies +the restored implementation plus new contracts, not the rejected candidate. + + +* Sequence continuation reuse rejected after latest-PR integration (2026-10-10) + +** Mechanism and source grounding + +The experiment targeted the shared =recognition-sequence-for-each= engine, +not a Bash-specific scanner. Existing immutable right subtrees replaced +per-branch =(vector right delta moved?)= frames. A private scope record saved +parent relocation state only when a position view contained another branch or +view. Leaf-only views visited their list directly. The pending-list tail +identity delimited a scope; restoring it also restored the moved flag, even +when nested deltas cancelled to zero. Shared subtrees remained immutable and +visitor exceptions propagated immediately. + +[[https://www.brics.dk/RS/01/23/BRICS-RS-01-23.pdf][Danvy and Nielsen, Defunctionalization at Work]], Section 2.1, motivates +using the original tree as the representation of deferred traversal work. +This experiment applied that representation idea to offset-aware sequences; +it does not establish a formal refinement theorem. The existing Gerbil +=std/list/list-builder.ss= collector and standard =for-each= remained in use. +Two explicit leaf-loop rewrites produced no additional admitted native +allocation reduction. A local macro expanding the resume logic also produced +no allocation reduction and higher CPU samples; all three were discarded. + +** Artifact admission and experimental controls + +The fixed Gerbil SDK revision was =2591dcd=, with GCC and a 3 GiB heap. +The 120-second total and five-second idle guards were unchanged. SDK +=gerbil/runtime/loader.ss=, =__find-library-module=, selects the last +contiguous numbered object version before falling back to source. Merely +replacing an older object does not select it when later versions remain. +Two initial ambiguous runs therefore loaded the candidate on both sides and +are excluded. Subsequent fresh processes asserted the actual loaded object +basename. Only the owned runtime module artifacts changed between variants. + +The private Scheme controller measured 11 samples for each 10,000-child +synthetic workload and 31 complete-request samples. Allocation admission +excluded GC-crossing intervals through =admit-parser-allocation=. Complete +request timing included public =parse-bash= and an additional +=parse-artifact-valid-for-source?= check. Full serialized product equality +was checked outside timing for four fixtures plus empty, malformed and +Unicode inputs. No forced GC or simultaneous local builds were used during +timed blocks. The controller and artifact snapshots are not repository +interfaces and were removed from the change set. + +** Pre-integration ABBA results + +Reference A, candidate A, candidate B and reference B used the same clients +and control source. Candidate A and B were separately compiled objects of +the same retained plain scope algorithm. CPU values are medians in ms; +allocation values are admitted medians in bytes. + +| Workload | Reference A CPU | Candidate A CPU | Candidate B CPU | Reference B CPU | Reference allocation | Candidate allocation | +|----------+-----------------+-----------------+-----------------+-----------------+----------------------+----------------------| +| Unmoved | 0.305 | 0.298 | 0.301 | 0.307 | 1600288 | 960352 | +| Root view | 0.564 | 0.645 | 0.644 | 0.568 | 2880224 | 2240384 | +| Leaf views | 0.625 | 0.618 | 0.650 | 0.628 | 2880224 | 2240288 | +| Complete request | 32.586 | 27.642 | 24.563 | 23.794 | 40345864 | 39965064 | + +Complete-request admitted counts were 22, 23, 23 and 23 of 31. The root-view +workload admitted 10 of 11 in every block; the other synthetic workloads +admitted all 11. Allocation reductions were reproducible, but whole-request +CPU drift prevents a speedup claim. Root-view CPU regressed in both candidate +blocks. The macro alternative preserved these allocation values but measured +0.832 ms unmoved, 1.312 ms root-view and 34.345 ms complete-request CPU in +its single block, so it was not retained. + +** Latest-PR integration changes the decision + +The PR concurrently added bounded direct materialization in revision +=bb445e33ad1e796140b0c9ef84eb616fd533f746=. The reference was rebuilt with +that implementation, then the scope experiment was rebuilt on top of it. +Both fresh processes asserted their loaded artifacts and passed the same +complete-product proofs. + +| Workload | Latest reference CPU ms | Integrated candidate CPU ms | Latest reference bytes | Integrated candidate bytes | +|----------+-------------------------+-----------------------------+------------------------+----------------------------| +| Unmoved | 0.307 | 0.317 | 1593120 | 957280 | +| Root view | 0.584 | 0.643 | 2880224 | 2240384 | +| Leaf views | 0.642 | 0.649 | 2876128 | 2240288 | +| Complete request | 23.673 | 24.125 | 39607464 | 39608104 | + +Complete-request allocation admitted 23 of 31 samples on each side. The +synthetic admission counts remained 11, 10 and 11. Direct materialization +already bypassed the branch frames on the significant complete-request path. +The integrated experiment added 640 complete-request bytes, and its root-view +CPU increased about 10 percent. Component allocation savings alone therefore +do not qualify this implementation. The scope algorithm was removed; the +latest PR's existing runtime implementation remains the production choice. +These results neither establish a portable CPU regression nor a Tree-sitter +performance comparison. + +** Retained contracts and next cost boundary + +Three public semantic cases remain: independent sibling offsets and zero-net +nested relocation, an independent recursive oracle for shared nested views, +and immediate propagation of a visitor's condition before later branches. +They complement existing deep traversal and alias-preservation coverage. +The candidate passed eight modules and 129 cases before integration; final +qualification below applies to the restored latest-PR implementation. + +The next producer experiment must reduce word-result intermediates that +survive direct materialization, rather than repeat an optimization of bypassed +branch frames. Existing complete-product equality, owned source boundaries, +negative inputs and admitted allocation remain the acceptance boundary. + +Final restored qualification completed in 12.807 seconds with eight modules, +130 cases, =TEST-SUITE-OK modules=8=, =OK= and exit 0. The restored native +runtime rebuilt in 11.264 seconds. Final integration also includes revision +=b9e1e506221d198f1202af7567373de8aab336a5=; this change adds no runtime +optimization on top of that revision. + + +* Request-owned region counters instead of replacement frames + +The pinned Gerbil SDK revision =2591dcd=, =src/std/struct/queue.ss=, updates +queue links and length under an imperative ownership contract. Its +=queue->list= explicitly copies the externally returned spine. The transferable +mechanism is ownership-directed mutation, not blanket unchecked access. The +current parser design documents likewise require explicit source and execution +ownership. This change uses ordinary checked vector operations. + +The region matcher previously rebuilt a pair-frame vector and its top stack +cell whenever a bare delimiter increased or decreased the active pair depth. +These are counter transitions within one region, not new independently +addressable region entries. The matcher now changes only the depth slot in the +active frame. Named nested openers and quote entries still allocate their own +frames. Prepared declaration rows remain immutable. + +All callers create initial frames for one invocation. Frames never escape the +match, and completed boundary caches retain numeric endpoints, not frames. +A failed match abandons its local frames. Independent scans sharing a plan +therefore share no mutable depth. This does not grant concurrent-write authority +to one shared source cache. Regression tests exercise deep counter nesting with +other pair kinds and quotes, repeat calls, independent source caches, four +concurrent scans of the same plan, failure followed by reuse, and existing +5,000-frame cache endpoint checks. + +** Matched native control + +The independent grammar declares =@{ ... }=, with ordinary braces contributing +to its counter. The control has three workloads: 4,096 bare nested braces with +a Unicode payload, a flat region, and 512 explicitly named nested regions. +Plan and source construction occur outside timing. Each of eleven samples +runs ten native matches and checks the summed endpoint. The same interpreted +orchestrator invokes native kernels in every block. + +The GCC-backed SDK, 3 GiB heap, 120-second total guard and five-second silence +guard remain fixed. Processes verify actual loaded artifacts: reference +=region-scanner~0.o6= and candidate =region-scanner~0.o7=. Complete owned +artifact snapshots retain contiguous loader suffixes; other clients remain +unchanged. The sequence is reference A, candidate A, candidate B, reference B. + +| Workload | Reference A CPU ms | Candidate A | Candidate B | Reference B | +|----------+--------------------+-------------+-------------+-------------| +| Counter depth, ten calls | 3.106 | 2.624 | 2.606 | 2.996 | +| Flat, ten calls | 0.005 | 0.005 | 0.003 | 0.003 | +| Distinct frames, ten calls | 0.533 | 0.471 | 0.464 | 0.462 | + +Counter-depth median allocation decreases from 11,799,296 to 2,816 bytes +per ten-call sample; admission counts are 10/11 for each reference and 11/11 +for each candidate. This includes identical orchestration overhead. Flat +allocation remains 2,816 bytes and distinct-frame allocation remains 738,656 +bytes, with 11/11 admitted samples in every block. All four controls return +=REGION-CONTROL-OK= and exit 0. + +An exploratory 100-call control admitted only one of eleven reference +allocation samples because most crossed GC. Its candidate admitted all eleven. +The ten-call protocol improves admission for both versions; the sparse +exploratory allocation receipt is excluded from the table. Guards and allocation +admission rules were not changed, and no forced collection was added. + +The repeatable result is elimination of depth-transition allocation in the +shared matcher. Counter-depth CPU improves modestly in these micro controls. +The flat and distinct-frame timings are controls, not claimed improvements. +There is no complete-request or Tree-sitter comparison in this experiment. + +The repository-owned =t/benchmarks/region-depth/benchmark.ss= reproduces these +three workloads with configurable positive exact sample/iteration counts, +endpoint validation, per-call CPU/allocation output, and existing GC-aware +allocation admission. It is a Scheme benchmark, not a frozen alternate engine. + +** Integration and cache provenance + +The candidate scanner build passed in 14.400 seconds. Integration rebased onto +=c1237c5=, including staged LR action vectors and semantic leaf classification. +Changed native dependencies were rebuilt before qualification. Two existing +local interfaces were stale: the IR interface omitted =event-fold-ir= and the +program interface omitted =snapshot-fold-data=. The first failing invocation's +error-prefix watchdog truncated its diagnostic; a caught compiler invocation +rendered the complete syntax exception as one structured record. Rebuilding +the producer interfaces before the Scheme generator consumer repaired the +local artifact graph. Neither source compatibility nor guard relaxation was +introduced. Failed builds are excluded from successful receipts. + +Six source modules passed 75 cases, and the repository benchmark executed two +samples of two calls per workload, in 18.550 seconds total. The process returned +=TEST-SUITE-OK modules=6=, =OK=, =REGION-DEPTH-OK= and exit 0. Owners were region +scanning (15), parts profiles (16), LR stack reductions (3), EventFold source +generation (4), recognition sequences (8) and Bash parsing (29). Benchmark +smoke output establishes executability, not a second performance estimate. +The ABBA micro table remains a control against the earlier =3ba88c7= clients. + diff --git a/docs/research/shared-scanner-source-materialization.org b/docs/research/shared-scanner-source-materialization.org new file mode 100644 index 00000000..d3b32e0b --- /dev/null +++ b/docs/research/shared-scanner-source-materialization.org @@ -0,0 +1,620 @@ +#+title: Source scanning and materialization +#+startup: overview + +Part of [[file:shared-scanner-staging-and-fusion.org][the shared scanner study]]. +Historical revisions, failed experiments and receipt limits retain their original scope. + +* SDK lexer reading and winner-only materialization + +The SDK binary resolved =:std/text/parser/ll1=, +=:std/text/parser/ebnf/lexer= and =:std/source= through the expander's +=import-module= / =module-context-path=. Source reading then used the matching +installed tree. The lexer module is =ebnf/lexer=; the initially attempted +=:std/text/parser/lexer= is not an installed module. + +#+begin_src scheme +(import :gerbil/expander) +(module-context-path (import-module ':std/text/parser/ebnf/lexer)) +#+end_src + +In =std/text/parser/ebnf/lexer.ss=, =lex1= retains a fallback for the last +accepted extent and invokes the selected reduction only after selection. Its +mutable input stream and declaration-order tie policy are not substitutes for +our immutable checkpoints, rank and ambiguity contracts. The useful lesson is +the lifetime of a result: materialize the public result after deciding which +candidate owns it. + +The shared contextual scanner now carries a borrowed rule and an integer end +through its candidate loop. =prefer-rule?= applies the same length/rank/form/action +comparison and raises the same eager conflict. =scan-match= remains the private +result type, but is constructed once after selection instead of once for every +successful candidate. + +For a completed step with m successful candidates, m >= 1, the source-level +constructor count becomes 1 instead of m. A no-match step still returns false; +a conflict still raises immediately. This is a source construction bound, not a +measurement of heap bytes: compiler scalar replacement may already remove some +allocations, and executor calls and final token construction still occur. + +#+begin_src scheme +;; Engine-private selection: no new grammar callback or author configuration. +(let select ((rules candidates) (current #f) (end #f)) + (if (null? rules) + (and current (make-scan-match current end)) + (let* ((rule (car rules)) + (candidate-end ((runtime-scan-rule-executor rule) source start state))) + (if (and candidate-end (prefer-rule? current end rule candidate-end)) + (select (cdr rules) rule candidate-end) + (select (cdr rules) current end))))) +#+end_src + +This answers the existing design questions with an implementation witness: +authors retain the same declarations, while engine-owned lifetime becomes +simpler. The scientific obligation is exact observable selection and diagnostics, +followed by complete-call measurements; fewer source constructors alone do not +establish a throughput improvement. + +Three new independent Cases cover length before rank in both declaration orders, +Unicode byte endpoints, eager conflicts before a longer later match, and the +identity of the first rule retained by an equivalent tie. Both direct IR and +prepared plans execute these controls. The first conflict fixture was rejected +by the duplicate-declaration contract before execution; distinct overlapping +matchers corrected the fixture. That first run was a failure despite its SDK +process returning zero; final =OK= and absence of Case/Harness failures were +required before admitting the rerun. + +The corrected scanner/deferred/source/parser run passed 55 Cases. Source-backed +recognition and Bash/HCL/HL7/FHIRPath passed 74 more Cases: 129 total, final =OK= +and exit zero for both commands. Existing =matched-scale.ss= completed 16 axes, +8 samples, 256 words, 16 literals and 16 requests per plan batch with +=SCANNER-SCALE-OK=. Existing =matched-parser.ss= completed 16 requests and five +measured samples with =CONTEXTUAL-PARSER-SCALE-OK=, exact prepared/unprepared +artifacts, valid publication and source reconstruction including rejected inputs. +These are source semantic and workload receipts. No old/new native ABBA was run; +no speedup or allocation-byte reduction is admitted. The full-artifact workload +was used as a semantic control, not a performance comparison with the old code. + +The source-span audit also confirmed that =prepare-recognition-source= snapshots +a token and validates its UTF-8 extent, while =prepare-contextual-scanner= owns a +request snapshot. The mutation control proves that simply deleting those copies +is invalid. A shared span design must transfer admitted ownership and character +bounds across both stages before replacing either snapshot; no borrowed mutable +caller string was introduced by this change. + +* Native UTF-8 width queries instead of temporary encoding + +The SDK binary resolved =:std/string/utf8= to its installed interface. Its source +owner is =std/string/utf8.ss=. The checked =string-utf8-length= procedure validates +string/range arguments and delegates to Gambit's =##string->utf8-length=; it does +not create an encoded byte vector merely to obtain its length. + +The [[https://gerbil.scheme.org/reference/std/text/utf8.html][website's UTF-8 reference]] still describes the API under =:std/text/utf8=. +That module was not found in this installed SDK. The implementation imports the +actual =:std/string/utf8= binding established by source inspection; no alias or +legacy fallback was introduced. + +Five length-only conversions now use the SDK procedure: each token's extent in +ParseArtifact validation, rejected-source fallback and complete extent in the +shell executor, and both terminated/final line extents in the Scheme event +strategy. Source bytes used for SHA-256 and actual encoding remain materialized +where consumed. The change neither bypasses validation nor borrows mutable +caller text. It adds no language policy, metadata slot or author option. + +#+begin_src scheme +(import :std/string/utf8) +(let (text (list->string (map integer->char '(97 955 20013 128512)))) + ;; Four scalars, ten bytes. Query width without constructing encoded bytes. + (string-utf8-length text)) ; => 10 +#+end_src + +The existing design questions apply directly. This is better because the engine +expresses the operation it needs, byte extent, through the checked native library +instead of maintaining a custom Unicode loop or an unnecessary encoded product. +It is scientifically testable because byte coverage is observable independently +of the chosen implementation: a token's advertised width must match fixed +Unicode boundary expectations, source reconstruction and the unchanged digest. + +New artifact controls cover U+0000/U+007F, U+0080/U+07FF, U+0800/U+D7FF, +U+E000/U+FFFF, U+10000/U+10FFFF with independently specified widths 1/2/3/4. +Every fixture first admits and roundtrips, then changes both token end and +advertised extent to a false but self-consistent width and must reject. A line +control combines all boundaries with CRLF, CR and a final unterminated line; +expected byte ends are 28, 30 and 31. Surrogate integers were refused by this +SDK's =integer->char= during the initial probe and are not valid fixture scalars. + +A local component observation compared the former encode-and-length expression +with the native procedure in encoded/native/native/encoded order, 1,000 calls +per observation, identical loops, fixed source and explicit pre-run collection. +ASCII contained 1,024 characters; mixed input repeated the four scalars in the +Scheme block 128 times, totaling 512 characters and 1,280 UTF-8 bytes. Each run +checked its summed result against the fixed expected width. + +| Component | Encoded counter delta | Native counter delta | Observed collections | +|-----------+-----------------------+----------------------+----------------------| +| ASCII | 2,209,104 bytes in both observations | 129,104 bytes in both observations | 0 for every observation | +| Mixed | 2,721,104 bytes in both observations | 129,104 bytes in both observations | 0 for every observation | + +These are process-statistics counter differences with no collection, not peak +memory or complete-parser allocation. CPU observations were 1.806/1.847 ms +versus 1.091/1.100 ms for ASCII, and 2.128/3.204 ms versus 0.951/1.628 ms for +mixed input. This small component study does not admit a stable end-to-end speed +ratio. It uses the existing SDK in =gxi=, not two separately qualified native +parser products. + +The first mixed command-line probe failed its expected-width check: the intended +U+1F600 literal arrived in the evaluated string as U+F600. Integer-based scalar +construction corrected the workload and the full rerun ended with +=SDK-UTF8-COMPONENT-OK= and exit zero. This observes the command-line evaluation +path; it does not establish a reader, FFI or parser cause. The failed mixed run +is excluded from the admitted observations. + +The combined source run passed 144 distinct Cases across scanner, deferred +scanner, source/parser, artifact, recognition spans, event strategy and +Bash/HCL/HL7/FHIRPath. Admission required final =HARNESS-OK=, =OK=, no Case or +Harness errors and exit zero. The source controls include structured Rust event +strategy generation; they do not replace native Rust execution or exact-head CI. + +After integrating revision =750b8cd= (test and research changes), the added +projection control Suite passed 13 Cases with final =HARNESS-OK= and =OK= and no +Case or Harness errors. The matched complete-parser workload ended with +=CONTEXTUAL-PARSER-SCALE-OK= and exit zero: prepared and unprepared products +remained equal, validated and round-tripped, including rejection. This is a +semantic integration receipt, not an old/new UTF-8 performance comparison. + +The installed SDK compiler also optimized and expanded all three changed +production modules with =invoke-gsc: #f=, =optimize: #t= and =generate-ssxi: #t=. +Each module ended with =SDK-OPTIMIZE-OK= and the process exited zero. This +confirms compiler expansion and helper binding; it does not qualify native C +compilation, linking or execution. + +* Deferred marker comparison: filter before materialization + +The installed SDK binary resolves =:std/string/misc= through +=module-context-path= and =import-module= to its installed =misc.ssi=. The matching +=src/std/string/misc.ss= implements =string== with a length filter, an identity +fast path and =%string-compare= over explicit string intervals. This is a useful +source-level mechanism to study, rather than a reason to assume every call to +the interval API improves this parser. + +Both =marker-line-end= and =profile-marker-end= previously constructed a +substring for every body line before checking marker equality. Comparing +character counts first rejects impossible candidates without materializing a +comparison string. This is valid after line-ending removal and optional tab +stripping: equality requires the remaining source interval and marker to have +identical character counts. Equal counts still require content comparison. + +#+begin_src scheme +;; Shared private engine operation; no language-facing DSL addition. +(def (marker-slice=? source start end marker) + (and (= (- end start) (string-length marker)) + (string=? (substring source start end) marker))) +#+end_src + +*Why is this preferable?* Length-incompatible body lines avoid their temporary +string copy; equal-length candidates retain Gambit's existing string comparison. +The optimization lives in the shared executor and serves both line-ending +policies without introducing a language callback or new public contract. + +*Why is it scientifically justified?* The length condition is necessary for +string equality, not a heuristic about particular languages. The public scanner +controls include equal-prefix, equal-suffix and repeated-marker body lines, +Unicode markers, stripped tabs, a final marker without a line ending, and both +LF and CR profile separators. Existing controls cover CRLF, empty quoted +markers, checkpoints, source ownership and incomplete bodies. + +The initially proposed direct SDK =string== interval replacement passed semantic +controls, but the component observations did not justify adopting it. For 1,000 +comparisons of ="EOF"=, the copying control observed 0.194/0.196 ms CPU versus +0.609/0.589 ms for the interval API. Long equal Unicode content also favored the +existing comparison. The component fixture normalizes =string=='s truthy result; +Scheme truth is not restricted to =#t=. No API or optimizer defect is inferred +from this result. + +The admitted copying/guarded ABBA component control uses identical source and +marker values, nonzero source offsets and five fixtures. For a 1,024-character +Unicode body versus ="EOF"=, copying observed 0.688/0.341 ms CPU and a 4,529,232 +allocation-counter delta; guarded comparison observed 0.215/0.210 ms and +385,232. All observations had zero collections. Equal-length candidates retain +the same allocation delta and add the length check: short equal markers +observed 0.262/0.274 ms versus 0.194/0.196 ms for copying. This tradeoff is +retained; there is no universal latency improvement claim. The first probe used +incorrect process-statistics slots and its counter results are excluded. The +admitted probe uses cumulative GC-count slot 6 and allocated-byte slot 7. + +The reproducible Scheme component fixture is +=t/benchmarks/contextual-scanner/marker-comparison.ss=. It performs 1,000 calls +per observation, checks every observation's expected match count, collects +before each observation and emits =SDK-MARKER-COMPONENT-OK= only after every +fixture passes. The interval alternative has two exploratory observations; +only copying/guarded are ordered ABBA. These are SDK component observations in +=gxi=, not peak live memory or an old/new native parser qualification. + +The final executor passed 89 source Cases across contextual scanning, deferred +scanning and Bash/HCL parser integration. Admission required final =HARNESS-OK= +and =OK= with no Case or Harness errors. The existing deferred-scale workload +passed three 256-delimiter observations with complete token order, lossless +source coverage and empty final delimiter obligations. The SDK compiler also +optimized and expanded =contextual-scanner.ss= with =invoke-gsc: #f= and ended +with =SDK-MARKER-OPTIMIZE-OK=. Expansion and source receipts do not replace +native compilation, execution or exact-head CI. + +* Incremental checkpoint publication: native bounded list operations + +The deferred-marker study concerns individual recognition decisions. The next +pipeline stage publishes an incrementally scanned prefix and certified suffix. +Previously, that stage converted every old checkpoint into a list and discarded +the prefix with =list-tail=. It also joined both reversed token and checkpoint +prefixes using =append= after =reverse=, allocating an intermediate prefix list. + +The installed SDK provides a direct solution. =src/gerbil/builtin.ssxi.ss= +declares one-, two- and three-argument =vector->list= forms; the latter two accept +fixnum interval bounds. Binary evaluation confirmed =(vector->list #(0 1 2) 1 3)= +yields =(1 2)=. The native =append-reverse= binding is exposed through +=gerbil/core/runtime.ss= and used by the official C3 implementation in +=gerbil/runtime/c3.ss=. These existing primitives avoid a custom traversal API. + +#+begin_src scheme +;; Preserve fresh rebinding while materializing only the certified suffix. +(map rebind-checkpoint (vector->list old-states (+ index 1))) +;; Copy a reversed private prefix once; retain the freshly created suffix. +(append-reverse tokens reused) +(list->vector (append-reverse states rebound)) +#+end_src + +*Why is this preferable?* The executor visits the old checkpoint interval it +actually needs. It removes the discarded checkpoint-prefix list and the two +intermediate reversed-prefix lists. Token strings are still copied, suffix +checkpoints still bind the new worker/source, and published values cannot mutate +session history. Language grammar, parser metadata and DSL admission stay out of +this internal representation change. + +*Why is it scientifically justified?* For a proper reversed prefix, native +=append-reverse= yields the same ordered elements as =append= after =reverse=, +without mutating that prefix. The suffix starts after the converged checkpoint; +the scanned prefix already owns that checkpoint. Omitting that boundary would +lose a state; including it twice would break the one-state-per-token boundary +index. Tests compare complete token traces against fresh scans and replay the +result as history after prefix, middle, tail, EOF and empty-source edits. They +also mutate published tokens and require both old and new histories to remain +unchanged. + +An initial SDK =subvector-for-each= plus =with-list-builder= candidate passed +semantic controls, but its callback path increased component costs for long +suffixes. With 4,097 checkpoints and 100 publications, a one-checkpoint prefix +observed about 63.6/65.5 ms CPU and a 133,147,256 allocation-counter delta, +versus 26.9/28.5 ms and 86,027,408 for the original copying control. This +candidate was rejected. The final implementation uses native interval +=vector->list= and native =append-reverse=, with no added library imports. + +The admitted copying/native-bounded ABBA observations are: + +| Prefix / suffix checkpoints | Copying CPU ms | Native-bounded CPU ms | Copying allocation delta | Native-bounded allocation delta | +|-----------------------------+----------------+-----------------------+--------------------------+---------------------------------| +| 1 / 4096 | 28.482, 26.896 | 27.486, 28.134 | 86027408 | 86017808 | +| 2049 / 2048 | 19.886, 20.028 | 8.627, 8.589 | 72493496 | 55744432 | +| 4096 / 1 | 2.969, 2.735 | 1.517, 1.419 | 62314032 | 22992432 | +| 4097 / 0 | 2.685, 2.688 | 1.458, 1.438 | 62307632 | 22976432 | + +These are SDK component observations in =gxi=, not separately compiled native +parser products. The first fixture has little benefit; long retained prefixes +avoid more discarded work. Counters use cumulative slots 6 (collections) and 7 +(allocated bytes); collections occurred in some observations. They are not peak +live memory or a complete-parser latency ratio. The reproducible fixture is +=t/benchmarks/source-edits/checkpoint-publication.ss=; the callback alternative +is retained as an exploratory comparison, not production code. + +The final source executor passed 55 Cases across source scanning, deferred +scanning and Bash parser integration with final =HARNESS-OK= and =OK=, no Case or +Harness errors and exit zero. SDK optimized expansion of =source-scanner.ss= +ended with =SDK-SESSION-OPTIMIZE-OK= using =invoke-gsc: #f=. This confirms expansion, +not native linking or execution. + +The complete Scheme workload =t/benchmarks/source-edits/session-parser.ss= +checks prefix, middle and tail edits plus grammar and lexical rejection against +independent fresh parsing. Every complete artifact must be equal, valid and +round-trip to the input. Its fresh/session ABBA run ended with +=SOURCE-SESSION-PARSER-OK= and exit zero. Existing session reuse is part of this +comparison, so its CPU observations do not isolate this publication change. +An old/new native parser qualification remains a separate performance gate. + +After rebasing onto =650283d=, the combined source controls passed 131 Cases +including the new result-projection and EventFold suites, with final =OK= and +no Case or Harness errors. The complete session workload again ended with +=SOURCE-SESSION-PARSER-OK= and exit zero. These post-rebase receipts cover the +new composition with the projection and request-isolated EventFold changes; +they do not replace exact-head native CI. + +** LR fragment and GLR fork joins use the same native mechanism + +The source-session publication study also exposed four equivalent joins in the +shared LR executor. Fragment reuse joined ordered token/mode segments to reversed +accumulators with =append= after =reverse=. Both streamed and captured directed +GLR forks reconstructed an ordered prefix plus freshly scanned suffix using +=foldl cons=. The installed SDK binary resolved =:gerbil/runtime/c3= to its +installed interface, and the corresponding C3 source uses =append-reverse= for +this same persistent reversed-prefix operation. =builtin.ssxi.ss= declares the +operation's list arguments, list return and allocation effect. + +#+begin_src scheme +;; Ordered fragment into a reversed accumulated token/mode chain. +(append-reverse segment tokens-reversed) +(append-reverse segment-modes modes-reversed) +;; Reversed scanned prefix into an ordered complete fork input. +(append-reverse tokens-reversed suffix) +#+end_src + +*Why is this preferable?* Fragment joins allocate the reversed segment once +instead of constructing and copying it. Fork joins use the native operation +instead of a general fold invoking =cons=. The existing checkpoint and observer +calls still own admission, capture and GLR fallback; this only replaces private +list assembly. No public DSL, source-byte rule or language-specific shortcut is +introduced. + +*Why is it scientifically justified?* For proper input lists, both +=(append (reverse head) tail)= and =(foldl cons tail head)= equal +=(append-reverse head tail)=. The non-destructive operation preserves the input +head and shares the exact retained tail. The existing publication component +fixture now checks those properties at empty-prefix, empty-tail, singleton, +256-element and 1,024-element dimensions, with each control/native pair in ABBA +order and 1,000 joins per observation. + +At 1,024 prefix elements and 16 suffix elements, fragment copying observed +14.223/13.811 ms CPU and a 98,465,440 allocation-counter delta, versus +1.365/1.356 ms and 49,313,232 for native joining. Copying collected once in each +observation; native joining did not. The fold control observed 5.332/5.328 ms +and 49,457,232 versus 1.382/1.327 ms and 49,313,232 for native joining, all with +zero collections. These are interpreted SDK component observations, including +the observed collection differences, not a stable native-parser speed ratio. +The fixture ended with =PARSER-JOINS-OK= and =CHECKPOINT-PUBLICATION-OK=. + +The production executor passed 54 Cases across runtime, recovery/incremental, +incremental-suffix benchmark controls and GQL parsing, with final =HARNESS-OK= +and =OK=, no Case/Harness errors and exit zero. SDK optimized expansion ended +with =SDK-PARSER-JOINS-OPTIMIZE-OK= using =invoke-gsc: #f=; this does not qualify +native C linking or execution. + +The existing =topology-hcl-capture 64= complete workload passed six first/middle/ +last insertion/deletion controls and inverse edits, with five timing samples per +path. The first insertion admitted 64 reused recognition fragments, so fragment +joining was exercised rather than inferred from a semantic no-reuse path. +Fresh artifact median CPU was 1.226 ms versus 8.524 ms for the cached capture +path in that observation. This negative overall comparison remains visible: +those paths include existing capture, certification and incremental bookkeeping +costs and do not isolate the four join replacements. The component result does +not isolate the overall session costs. The follow-up +[[file:source-edit-cost-contract.org::*Matched LR session responsibilities][matched LR session study]] +adds checkpoint-only and fresh captured-session controls: artifact-only fresh +parsing and cached capture have different executor and publication obligations. + +* Dependency order for further implementation + +1. *Shared source spans.* Own one admitted source and its character/byte boundary + mapping. Internal captures carry owner/start/end; decomposition reuses that + owner. Materialize public strings at the artifact boundary. Preserve rejection + of caller mutation, cross-source spans, non-scalar Unicode and stale checkpoints. +2. *Demand-driven scanning.* Reuse immutable step/checkpoint semantics to provide + lookahead without whole-source token/remaining/emitted list duplication. + Rejected sources must still obtain a complete covering fallback artifact; + edit restoration and heredoc obligations remain intact. +3. *Projection and event fusion.* Compile the closed projection plan into event + emission. Keep recognition nodes where consumers require them. Field order, + identities, trivia, diagnostics, offsets and complete artifacts remain equal. +4. *Static AOT specialization.* Generate Scheme and Rust execution from the same + admitted plan. Prepared closures are not yet grammar-specific generated + character branches. Checkpoint equivalence, IR identity and the common ABI + constrain both backends. + +Each step first establishes ownership and semantic controls, then uses the +existing benchmark owners for matched complete-product ABBA measurements under +the same GCC SDK. Hold input, output validation, natural GC, destruction, timing +boundaries and worker count constant. Record complete CPU, allocation per input +byte and GC. Include ordinary text, Unicode, nesting, overlapping delimiters, +dynamic markers, rejection and edit restoration across Bash, HCL, HL7, FHIRPath +and contextual LR. Component timings cannot replace complete-product timing. + +* Prior implementation receipts and retained negative results + +The implementation started at =bc9a56a= and aligned to =021f97a=; intervening +commits did not change the scanner/region/shell/artifact/source-strategy product +paths. Native scanner preparation took 18.966 s and exited zero. + +Contextual scanner 13, deferred scanner 13, Bash 29, HCL 30 and HL7 5 Cases +completed: 90 total, 9.711 s, exit zero. Guard controls compared ASCII/Unicode +prefix exclusions and exceptions, actual tokens and UTF-8 offsets for direct IR +and prepared plans. Complete short/heredoc/compound/large artifacts matched the +native baseline. Preliminary 12 batches of 10 complete parses remained about +37–40 ms/parse. Without symmetric frozen products, complete ABBA and current +allocation data, no speedup is admitted. + +An initial child printed assertions but did not exit; its five-second idle gate +failed. The existing completed-child entry subsequently provided real completion +without changing deadlines. A cold-start attempt also failed; real dependency +initialization diagnostics enabled a later completed run. Both first failures +remain failures. + +After qualification owners moved to =t/support/qualification=, old products did +not prove that the new entry could load. An initial extra contract run failed at +module loading. Preparing its declared POSIX child module resolved that boundary. +On =021f97a=, nine modules then completed: the preceding 90 Cases plus contextual +parser 15, source scanner 11, source contract 2 and ParseArtifact 8, totaling 126. +The controller reported =TEST-SUITE-OK modules=9=, =OK=, exit zero and no missing +receipt in 14.178 s, under 120 s total and five seconds of real-output inactivity. + +* Source-reading receipt at a725a69 + +At =a725a69=, the standard source command below completed the existing 13 +contextual scanner Cases, =HARNESS-OK= and =OK=, exit zero. This verifies the +implemented scanner controls; it does not qualify proposed spans/fusion, record +a new performance comparison or assert exact-head CI closure. + +#+begin_src scheme +;; Ordinary source Suite; no qualification executable is a prerequisite. +;; GERBIL_LOADPATH="$PWD/.." gerbil test -v 5 t/contextual-scanner-test.ss +#+end_src + +* Implemented: exact-size source reconstruction in shared artifact validation + +The shared ParseArtifact validator previously called =display= into a general +string port for every validated token. Its new reconstruction algorithm retains +references to admitted lexemes in reverse event order, then uses the standard +=string-concatenate-reverse= algorithm: sum actual character lengths, allocate +one result string, and copy each fragment into its final position. + +Reconstruction takes O(characters + tokens) work and O(tokens) fragment-list +storage, alongside the owned output string. Allocation follows actual lexeme +lengths; an untrusted =sourceByteLength= does not determine reserved capacity. +Token coverage, UTF-8 byte lengths, node/field balance, terminal IDs, root count, +status, diagnostics, and source digest checks continue to run. The output remains +a fresh owned string. This algorithm applies to every shared ParseArtifact +consumer and requires no language declarations or ABI changes. + +The original validator was frozen under a private test namespace. Both baseline +and candidate artifact modules were compiled symmetrically with the fixed SDK, +GCC16 -O1, gxc optimization, and identical compiler diagnostic flags. A separately +compiled native Scheme control exercised both implementations in one process. +File IO, module loading, preparation, equality preflight, and sample output were +outside timing. One initial GC preceded sampling; natural GC remained timed. + +Four full public products, for short, heredoc, compound, and large Bash inputs, +were accepted and reconstructed identically by both validators. Corrupt source +lengths, source digests, and truncated event streams retained the same admission +results. The complete engine controls also proved full artifact equality with +the public parser before measurement. + +Each block ran one untimed warmup and 11 measured single requests in old/new/new/old +order. Allocation deltas were admitted only without GC. These component and +complete-engine medians must not be added together. + +| Control | CPU P50 ms/request | No-GC allocation P50 bytes | Admitted allocation samples | +|---------+--------------------+---------------------------+-----------------------------| +| validation-old-a | 2.744 | 14434568 | 10 | +| validation-new-a | 1.315 | 5372840 | 11 | +| validation-new-b | 1.451 | 5372840 | 10 | +| validation-old-b | 2.697 | 14434184 | 10 | +| complete-old-a | 29.347 | 48974320 | 7 | +| complete-new-a | 30.258 | 39912208 | 8 | +| complete-new-b | 28.419 | 39912208 | 8 | +| complete-old-b | 30.519 | 48973552 | 7 | + +Validation allocation falls approximately 62.78%; the complete-engine control +falls approximately 18.50%, or about 9.06 MB per request on this corpus. +Complete CPU pair A regresses 3.10%, while pair B improves 6.88%. Consequently, +this establishes an allocation improvement, not a stable complete-parse speedup. +It does not close the measured Tree-sitter gap. + +The complete controls include prepared engine parsing, complete artifact +construction, the selected validator, grammar identity comparison, and source +identity comparison. Both execute the same producer and publication obligations. +They are independently compiled diagnostic controls, not timing samples from +the public parser entry itself. No public-entry latency claim follows. + +Native ABBA qualification completed in 11.064 s with exit status 0 and +=ARTIFACT-COPY-CONTROL-OK=. Seven test modules completed 85 cases in 29.349 s, +including ParseArtifact, recognition spans, source contracts, Bash, HCL, HL7, +and FHIRPath. The controller reported =TEST-SUITE-OK modules=7=, =OK=, exit 0, +and no missing receipts. The added contract case covers returned-string +ownership, empty source, and rejection of a false 4 GiB advertised byte length. + +The first symmetric compiler run failed its five-second idle guard while GCC +was quiet. The rerun enabled actual GCC optimization-decision reports on both +sides, retaining the optimization level and deadlines. It completed in 23.823 s +with exit status 0. The native control compiled in 26.886 s with exit status 0. +The first compiler failure remains excluded. Compiler diagnostics are outside +runtime timing and do not establish that the original silent compile had stalled. + +The comparison modules and native control are temporary experiment owners, +excluded from the implementation commit. Production changes are confined to +shared reconstruction; permanent semantic controls remain in the existing +ParseArtifact suite. + +* Integration with artifact reconstruction + +The winner-only change was rebased onto =e386be0=, which independently changed +validated artifact reconstruction. The rebased product controls passed 83 Cases: +source recognition, Bash/HCL/HL7/FHIRPath and ParseArtifact, including the new +Unicode reconstruction control. Final =HARNESS-OK=, =OK= and process exit zero +were required. The complete existing parser workload was repeated and finished +with =CONTEXTUAL-PARSER-SCALE-OK=. This extends the semantic receipt to the +combined changes; it is still not an old/new native timing comparison or CI +closure. The 55 scanner/deferred/source/parser Cases were then repeated on the +rebased implementation and completed with final =OK= and exit zero. Together +with the 83 product Cases, 138 distinct Cases passed on the combined code. + +* Rejected experiment: prepared closures for every projection operation + +The general projection interpreter was replaced experimentally with prepared +operation closures. Static field and terminal checks stayed in preparation; +request captures, source offsets and supplied child nodes remained dynamically +admitted. A private request frame held the reverse child list and immutable +sequence. The two existing single-operation fast paths stayed intact. + +This does not implement flap's generated, direct-call staged parser. Section 2.8 +and section 5.4 describe specialization that removes intermediate higher-order +calls. Binding an operation to another closure retains an indirect call per +operation and adds request-frame storage. The distinction matters for this DSL: +prepared vocabulary is not evidence of specialized runtime code. + +Both engines were compiled with the same fixed GCC SDK, optimization flags and +native consumers. Four complete artifacts matched each other and the public +parser before timing. Single-request ABBA blocks included natural GC, one untimed +warmup per block and 11 measurements; only no-GC allocation deltas were admitted. + +| Control | CPU P50 ms/request | No-GC allocation P50 bytes | Allocation samples | +|---------+--------------------+---------------------------+--------------------| +| original-a | 32.932 | 39912208 | 8 | +| closures-a | 47.442 | 39821648 | 8 | +| closures-b | 46.698 | 39821008 | 8 | +| original-b | 46.043 | 39912528 | 8 | + +The allocation reduction is approximately 0.23%. Neither paired comparison +supports a complete-engine speed improvement. The original engine also slows +substantially between blocks; concurrent host work prevents attributing the +entire CPU difference to the transformation. This candidate was removed rather +than promoted. Native qualification completed in 34.306 s, with exit 0 and +=PROJECTION-STAGE-CONTROL-OK=. + +The permanent mixed-projection contract test covers optional spans, required +nodes, repeated and empty parts, Unicode source origins, empty projections, +foreign kinds, malformed parts and parent containment. Six modules completed +87 cases in 19.301 s with exit 0 and all completion receipts. That receipt is +for the closure experiment, before rebasing the latest scanner revision. + +The next experiment fuses dynamic child admission with child construction in +the existing general executor. It retains opcode dispatch and does not claim +full lexer/parser fusion. Compile-time admission proves parent, field and +terminal membership; runtime admission must still prove supplied child kinds +and byte containment. Only those redundant checks and traversals can be removed. + +* Rejected experiment: fused child admission in general projections + +A second candidate retained the general opcode executor, eliminated its final +=result-plan-node= traversal and fused supplied-node kind/span admission with +child construction. Static parent, field and terminal checks stayed in plan +compilation. It introduced no request frame or per-operation prepared closure. +This was a shared-engine transformation; language declarations were untouched. + +The sole PR's scanner match-materialization revision =8e98e83= was rebased first. +Both controls then used that same scanner, the same fixed GCC SDK and flags, +complete publication validation and the original projection baseline. Four +complete products again matched both implementations and the public parser. + +| Control | CPU P50 ms/request | No-GC allocation P50 bytes | Allocation samples | +|---------+--------------------+---------------------------+--------------------| +| original-a | 28.021 | 39461328 | 8 | +| fused-a | 28.806 | 39236048 | 8 | +| fused-b | 31.696 | 39235408 | 7 | +| original-b | 28.133 | 39461648 | 8 | + +The allocation reduction is approximately 0.57%. CPU pairs regress approximately +2.80% and 12.67%; this does not support a full-engine optimization. Qualification +completed in 11.417 s with exit 0 and =PROJECTION-STAGE-CONTROL-OK=. The production +candidate was removed. Small traversal changes are insufficient evidence of a +solution to the measured Tree-sitter gap. + +Compilation initially failed the five-second idle guard, including a run of the +unchanged scanner. Compiler diagnostics were buffered and repeatedly ended in +truncated declaration lines. Disabling buffering on the temporary compiler +control's diagnostic ports exposed actual work without synthetic progress. +The same GCC flags and deadlines then completed both modules in 21.766 s with +exit 0. These compiler failures do not establish parser runtime regressions. + diff --git a/docs/research/shared-scanner-staging-and-fusion.org b/docs/research/shared-scanner-staging-and-fusion.org new file mode 100644 index 00000000..a75f765b --- /dev/null +++ b/docs/research/shared-scanner-staging-and-fusion.org @@ -0,0 +1,847 @@ +#+title: Shared parser specialization, source spans and pipeline fusion + +* Scope and architecture + +The [[file:lr-request-cost-boundaries.org][LR request cost boundary study]] complements this Source-engine study +with public GQL evidence, Scheme-native stage hooks and explicit receipt limits. + +This study owns specialization and representation lifetime in the shared engine. +It extends the [[file:scheme-deflanguage-design-synthesis.org][three-interface design]], rather than proposing another +language-author interface. Grammar intent remains in =grammar.ss=; metadata and +entry services remain in =parser.ss=; independent expectations and benchmarks +remain in =parser-test.ss=. Bash is a stress workload, not the implementation +owner of a private scanner or optimization callback. + +* Existing measurements and their limits + +An independent matched Bash report dated 2026-10-09 records 28,440 UTF-8 bytes +and 1,000 lines. Complete-parse median CPU was 43.320–45.093 ms for Scheme and +3.196–3.203 ms for Tree-sitter, approximately 13.6–14.1 times apart. Its owner is +=tools/benchmarks/bash-matched/results-2026-10-09.org= in a separate experimental +workspace. That uncommitted report is not a product of this implementation +checkout and its measurements have not been repeated by this source study. + +Independent diagnostics recorded about 51.930 MB of cumulative allocation for +the complete artifact, 10.915 MB for scanning and 14.464 MB for publication +validation. Complete allocation was roughly 1,826 times the input byte count; +this is allocation traffic, not peak resident memory. Component medians cannot +be added. Allocation across 200/400/1,000-line samples approximately followed +input size; this motivates investigating constant costs but does not establish +linear complexity for arbitrary nesting. + +Scheme used GCC16/Gambit -O1; the initial Tree-sitter grammar used clang -O3. +A clang -O1 grammar control still took about 4.016 ms, but its runtime was not +rebuilt. Products and allocators differ. These observations do not isolate a +language implementation effect. Follow-up comparisons keep the Scheme SDK +fixed and preserve complete validation. + +* Gerbil source reading: mechanisms and transfer boundaries + +The inspected distribution is Gerbil =0.19.2591dcd.patchf5cedd8168cb=. +Paths below are relative to its source tree. This includes distribution patches; +source observations are not assertions that upstream has identical code, nor +proof that a particular parser representation is faster. + +| Source owner | Observed mechanism | Parser implication | Required control | +|--------------+--------------------+--------------------+------------------| +| gerbil/compiler/optimize-base.ss | optimizer-info owns separate identity-keyed hash tables and local type contexts | Separate identities, ownership and lifetimes; do not reinterpret structural grammar values as identity keys | Equal-but-distinct grammar values, stable output order, independent compiler invocations | +| gerbil/runtime/hash.ss | Specialized eq/eqv/symbol/string table types coexist with checked interfaces | Select a native collection by its actual key contract; replacing make-table also requires changing its consumer API | Constructor, lookup, missing-value and iteration behavior; complete-call allocation and CPU | +| gerbil/compiler/optimize-call.ss | Known procedure/signature information enables selected call transformations; unknown types retain ordinary calls | Preserve real bindings and make private callable contracts explicit where supported; inspect generated code before attributing a dispatch cost | Checked admission, invalid values, generated calls and end-to-end comparison | +| gerbil/core/contract.ss | Type identifiers are resolved through syntax bindings | Keep DSL syntax objects and binding context through elaboration; do not replace the macro system with datum dispatch | Import renaming, phases, nested declarations and source diagnostics | + +The [[https://gerbil.scheme.org/reference/gerbil/runtime/hash-tables.html][official hash-table API]] specifies identity/equivalence choices and its own +operations. It does not promise that changing this parser's Gambit tables to +Gerbil tables will improve complete construction. Public Boolean tables returned +by LR analysis currently have consumers using =table-ref=; a private index can +be studied independently, but changing the public representation is a wider +contract change. + +The [[https://gerbil.scheme.org/reference/std/contract][official contract documentation]] distinguishes checked =:= declarations +from =:-= type assertions. Its LRU example checks the external cache and asserts +private nodes whose ownership is already established. The transferable pattern +is checked boundaries plus justified internal facts. Blanket unsafe accessors, +unchecked foreign input and custom substitutes for compiler optimizations are +not implied by this example. + +In our scanner, =runtime-scan-rule.executor= currently has an unannotated private +slot. A callable annotation is a testable hypothesis, not an admitted speedup: +first inspect whether the current compiler already knows the returned procedure, +then compare generated calls and complete parse cost. A closure prepared once +still incurs a call per candidate; preparation alone does not imply complete +inlining or lexer/parser fusion. + +Existing =compiler-index-set-reachable= and its memoized variant already use +indexed immutable edges, local queues and invocation-owned caches. LR suffix +facts are shared across consumers. This study does not duplicate those kernels +or add a global cache whose validity would depend on another grammar. + +* Paper mechanism and its applicability + +[[https://arxiv.org/html/2304.05276][Yallop, Xie and Krishnaswami: flap, PLDI 2023]] discusses intermediate token +cost in section 2.2, fusion in section 4 and preparation-time specialization in +section 5.4. The transferable mechanisms are moving grammar-static work out of +requests and removing representations used only between internal stages. The +paper's speed ratios do not predict this engine's gains. + +Its deterministic grammar, disjoint regex and single-token lookahead assumptions +are not established for every grammar admitted here. Section 8 treats stronger +LR grammars, state and multiple entries as extensions. Longest match, rank, +conflict diagnosis, contextual modes and delayed markers must retain their +current semantics. Candidate rules cannot simply be discarded. + +Lossless token events are public output. Fusion may remove temporary internal +tokens, nodes and lists; it must retain the final artifact and Scheme/C ABI/Rust +semantics. Internal allocation and observable output have different lifetimes. + +* Implemented preparation and current cost boundaries + +The [[file:../../src/runtime/contextual-scanner.ss][contextual scanner]] retains both a declarative matcher and a prepared +executor. =prepare-matcher= binds static operands after closed-IR admission; +=best-match= supplies the request's source, offset and checkpoint. + +- Prefix exclusions and exceptions use prepared literal tries. +- Literal prefix probes compare characters without allocating a substring. +- Unicode, longest match, rank, declaration order and conflicts retain the + selector's existing behavior. +- Markers, pending FIFO, mode and offset belong to request checkpoints. +- Executors capture plan data, not request state. Language IR cannot supply an + arbitrary callback. Scheme closures are not serialized into Rust AOT IR. +- Declarative matchers remain available for first-character candidate indexing. + +The former per-candidate opcode interpretation has been replaced. The following +remaining locations are visible in source; their individual CPU shares have not +been measured in this study. + +| Stage | Current owner | Representation to investigate | +|-------+---------------+-------------------------------| +| Candidate execution | contextual-scanner / best-match | Executor calls and final winning match allocation | +| Whole-source scanning | source-engines / source-scanner-tokens | Accumulated and reversed token lists before recognition | +| Word decomposition | shell-word / prepare-word / recognition-source | Lexeme snapshots, local boundaries and subsequent token slices | +| Projection | result-profile / result-projection-build | Captures, children and tokens across intermediate stages | +| Publication | artifact / recognition-events and validation | Recognition-tree traversal, event construction, reconstruction and encoding validation | + +* Scheme witness and the two design questions + +This block illustrates the existing private literal specialization. It is not +new public syntax or a proposed author-supplied callback. =prepare-matcher= is a +private engine function; the illustration belongs in that module's context. + +#+begin_src scheme +(let* ((declaration '(literal "λ")) + (execute (prepare-matcher declaration))) + ;; Preparation owns the declaration and static width. + ;; Each invocation supplies its own source and character offset. + (values (execute "λx" 0 #f) ; => 1 + (execute "xλ" 0 #f) ; => #f + (execute "xλ" 1 #f))) ; => 2 +#+end_src + +** Question 1: Why is this design better for the author and engine? + +The author declares a literal once. The engine can inspect and validate that +structure, derive candidate indexes and prepare execution without requiring the +author to write trie builders, dispatch loops or state factories. Scheme gives +us more than parentheses: hygienic macros preserve bindings and source context, +POO composes declared policies, and lexical closures bind admitted static data. +These mechanisms cooperate beneath the existing three interfaces. + +** Question 2: What makes the proposal scientifically justified? + +The specialization claim has a concrete invariant: matching results and emitted +artifacts are unchanged while static decoding moves to preparation. Existing +controls independently check Unicode, guards, ambiguity and rejection. The +paper identifies a related staging mechanism under stronger assumptions; it +does not prove our LR engine equivalent or faster. Performance requires matched +complete-call evidence including preparation amortization, output validation, +allocation and natural GC. Those observations can reject the proposal. + +* Implementation modules and current audit + +Read these by engine responsibility. They preserve the complete historical +experiments and negative receipts; they do not define separate migration plans. + +- [[file:shared-scanner-source-materialization.org][Source scanning and materialization]] +- [[file:shared-source-projection-publication.org][Source projection and publication]] +- [[file:shared-region-execution.org][Region execution and sequence ownership]] +- [[file:shared-ranked-scanner-execution.org][Ranked scanner specialization and execution]] +- [[file:scheme-reduction-abstractions.org][Reduction protocols and contextual matcher preparation]] +- [[file:../audit/engine-design-realization.org][Current design-to-code audit and open qualification gates]] + +* Closed ASCII union dispatch: rejected runtime experiment + +The pinned Gerbil SDK's =src/std/text/parser/char-set.ss= separates codepoint +predicates from checked character entrypoints. Its ASCII predicates use finite +ranges, while Unicode predicates retain Scheme's classification. This suggests +partial evaluation of a closed class declaration; it does not establish that +another dispatch representation is faster in this engine. + +The production =prepare-text-class= compiles a union into short-circuit predicate +closures. The experiment evaluates those pure predicates over all 128 ASCII +codepoints during preparation and captures a private byte vector. Runtime ASCII +characters use one indexed Boolean lookup; other characters invoke the original +union predicate. This moves ASCII classification out of scanning without +changing the class IR, language declarations, schema, C ABI or Rust lowering. +Preparation is bounded by the ASCII domain, but each Unicode character gains +an additional codepoint test before its original predicate work. Nested unions +also retain their prepared predicate structure. The candidate is withdrawn. + +** Independent semantics and complete-request controls + +Six union declarations cover the actual FHIRPath, HL7 and Bash class shapes, +including nested unions and duplicate character declarations. An independent +interpreter checks all codepoints from 0 through 255 plus eight Unicode boundary +and category examples: 1,584 comparisons. A separate test mutates the original +character-declaration string after preparation; ASCII and Unicode behavior +must retain the prepared value. These two cases are added to the existing +=text-profile-test= Suite and are retained against production. + +The new =benchmark-lr-text-class-requests= entry reuses the complete-request +owner's independent materialized GLR, canonical artifact and source-route +witnesses. The timed prepared boundary uses existing tokens; the source boundary +includes actual scanning and complete publication. Their costs are not +subtracted or treated as an additive decomposition. Each timed batch's complete +result is checked. No alternate engine or frozen result snapshot is committed. + +The first matrix uses twenty samples of twenty calls over FHIRPath identifier +widths 32/256 and 16/64 expression terms, a short numeric FHIRPath request and +Arithmetic. Four fresh children execute reference/candidate/candidate/reference, +4,800 calls per child, completing in 23.506, 13.606, 14.480 and 14.092 seconds. +All emit =LR-TEXT-CLASS-REQUESTS-OK= and qualified exit zero. The 948 admitted +collection-free intervals retain identical allocation across all twelve +boundaries. However, prepared LR also becomes faster despite unchanged execution +work. The control gate rejects CPU admission. The initial reference runner +precedes the explicit scanner-object assertion used by subsequent children; +this round therefore does not provide the final matched-runner comparison. + +The concrete drift motivates one bounded repeat with the same scanner-object +assertion throughout, twenty samples of one hundred calls and full Unicode +identifier controls. The final six workloads are ASCII identifier widths 32 +and 256 with 64 terms; identifiers containing 256 Arabic-Indic digits with +16/64 terms; 64 numeric FHIRPath terms; and 64 Arithmetic terms. Each has a +prepared and full-source boundary. All commands retain GCC, the 3 GiB heap, +120-second total and five-second silence limits. Each child executes 24,000 +calls. No other build or test overlaps the measured children. + +| Full-source workload | Reference A CPU ms/100 | Candidate A | Candidate B | Reference B | Ratio of summed candidate/reference medians | +|----------------------+-----------------------+-------------+-------------+-------------+--------------------------------------------| +| ASCII width 32, 64 terms | 54.332 | 56.655 | 65.362 | 56.744 | 1.0985 | +| ASCII width 256, 64 terms | 88.536 | 83.890 | 95.151 | 88.556 | 1.0110 | +| Unicode digits, 16 terms | 30.564 | 36.731 | 43.041 | 32.811 | 1.2587 | +| Unicode digits, 64 terms | 138.616 | 152.830 | 167.875 | 123.502 | 1.2235 | + +The children complete in 42.265, 38.862, 72.844 and 32.725 seconds, respectively. +All completion and exit receipts are present. The Scheme evidence verifier +admits 808 allocation intervals and requires exact counter equality across all +four children. Allocation is unchanged everywhere. All prepared boundaries, +Arithmetic source and Unicode requests are CPU controls. Their median stability +gate still fails; no CPU benefit is admitted. Both candidate Unicode medians +are above both corresponding reference medians. These descriptive regressions +and the absent ASCII benefit do not justify retaining the extra Unicode dispatch. +Control drift prevents a portable causal or percentage claim. + +=t/benchmarks/text-class/evidence.ss= uses the shared sampler receipt admission, +not another log parser. Its batch-count argument admits reproducible analysis +of the first twenty-call receipts as well as the default hundred-call matrix. +The original runtime source is restored and its native module and generated +static source are rebuilt. The older static object is invalidated rather than +claimed as qualification of the regenerated source. No runtime optimization +from this experiment is shipped. + +** Build and test qualification boundaries + +The first candidate GCC build reaches optimization but is rejected after +37.504 seconds by the silence guard. Existing limited GCC CFG, optimized, RTL +expansion, CSE, reload and final-pass products then expose actual compiler work; +the same compiler and optimization settings complete the candidate in 46.833 +seconds. These are real changing compiler outputs, not periodic heartbeats. +The initial request benchmark compiles in 39.102 seconds; the revised matrix +and native evidence consumer compile in 51.501 seconds. Restoration from the +original scanner source completes in 17.399 seconds. Native runtime objects +are asserted as =scan~0.o8= for measured references and =scan~0.o9= for measured +candidates. The restored production object is regenerated after those measurements; +its suffix alone must not be used as a historical content identity. + +An untraced source-test bootstrap is rejected in 6.223 seconds while importing +the test/POO framework. The existing expander-import trace records those actual +cold admission boundaries; the same silence deadline then admits nine Suites, +54 cases, final =OK= and exit zero in 18.007 seconds on the candidate. After +restoration, Bash command and structured-profile consumers join the same single +runtime: eleven Suites and 68 cases pass with final =OK= and qualified exit zero +in 43.603 seconds. These are semantic receipts, not parser speedup measurements. + +Local receipt stems are =parser-text-class-reference-a=, +=parser-text-class-candidate-a=, =parser-text-class-candidate-b=, +=parser-text-class-reference-b= and =parser-text-class-evidence= for the first +round; =parser-text-class-long-reference-a=, =parser-text-class-long-candidate-a=, +=parser-text-class-long-candidate-b=, =parser-text-class-long-reference-b= and +=parser-text-class-long-evidence= for the repeat; and +=parser-text-class-tests-traced= and =parser-text-class-restored-tests= for +semantics. At verification, preceding PR head =6f796fe= passes Ubuntu Scheme +and Rust AOT CI. The new test/evidence commit needs its own exact-head checks. + +** Post-rebase source qualification + +Concurrent PR commit =1cfb23a= adds production-relative LR vocabulary admission +and reachable acceptance. The test/evidence change is rebased onto it before +publication. The performance receipts above remain scoped to =6f796fe=; no +measurement is relabeled as performance evidence for the later graph. + +The current =src/compiler/lr-table.ss= native module rebuilds in 7.034 seconds. +The first expanded source gate then detects a stale native Rust generator: +publication negative controls return a path instead of the exceptions already +required by current source. Its apparent =HARNESS-OK= is followed by +=MODULE-BATCH-FAILED= and qualified failure (=17.687s=, exit 143, +=error-output=). This is not an admitted test result. Rebuilding +=src/compiler/rust-runtime.ss= from current source completes in 19.194 seconds; +no expected exception or capacity is relaxed. + +The final source gate covers the updated LR index and stack controls, Rust IR +publication, Bash command profiles, structured profiles, character profiles, +receipt admission and existing LR semantic/execution controls. Twelve Suites +and 82 cases pass with =TEST-SUITE-OK modules=12=, final =OK=, qualified exit zero +and no missing receipt in 17.835 seconds. Its receipt stem is +=parser-text-class-rebased-tests-final=. This is native/source semantic +qualification; exact-head remote CI is a separate gate. + +The subsequent concurrent =c12dcb0= closes canonical rule references before +executable publication. It is retained by another normal rebase. Its LR +producer rebuilds in 10.652 seconds. Adding its fused-reduction source controls +first exposes missing native Arithmetic/HCL parser-test products: the loader +attempts the generated Scheme fallback and rejects illegal syntax (=26.319s=, +exit 143, =error-output=). These two current-source products compile in 20.294 +seconds. The final expanded gate then passes thirteen Suites and 94 cases with +=TEST-SUITE-OK modules=13=, final =OK= and qualified exit zero in 21.676 seconds. +Its receipt stem is =parser-text-class-final-head-tests-complete=. +The following =27f5471= changes Org research/audit documents only; its code graph +is identical to that qualified =c12dcb0= graph. Both concurrent commits are +preserved. None of these rebases extends the earlier performance claim scope. + +* Compact singleton edges in the shared literal trie + +The engine now prepares empty child sets as false and singleton child sets as +one character/child pair. Branching nodes retain the original eqv character +table. This removes generic hash lookup along non-branching paths in bounded +literal scanning, ranked literal scanning and LR casefold traversal. It changes +prepared edge representation rather than language declarations or matching +priority. The terminal payload and duplicate combination remain unchanged. + +Preparation still inserts into mutable tables, then uses an explicit work list +to compact completed nodes. Literal depth does not grow the preparation stack. +The child macro binds both arguments once. The ranked scanner's former direct +access to the edge table now uses the shared child operation; the LR consumer +is rebuilt because it imports that macro. No compatibility representation or +language-specific dispatch is introduced. + +The existing ranked-literal Scheme benchmark exports +=benchmark-literal-edge-cpu=. Its reference expands an equivalent prepared trie +back into all-table edges and executes the original bounded scanning loop. +The production bounded scanner uses compact edges. Both return the complete +longest admitted endpoint and use the same argument guards. This is a reference +primitive comparator, not a frozen alternate parser engine. An independent +linear literal oracle checks all start/limit boundaries over successful, +failing and Unicode sources before timing. + +Twenty ABBA/BAAB CPU groups are bracketed by twenty identical-operation groups +on each side. The existing parser batch sampler and ASP percentile helper own +measurement. Every timed batch resolves at least one millisecond of process +user plus system CPU; controls require P10/P90 within 0.90..1.10 and candidate +P90 below 0.95. Natural GC is recorded, not subtracted. Preparation and oracle +validation remain outside timed scanning. CPU completion and CPU admission +are separate outcomes. + +The first matrix uses one million path characters per batch across widths +1, 8, 32 and 128. It completes in 50.900 seconds, exit zero. Candidate CPU +P50/P90 is 0.6540/0.7078, 0.2805/0.3521, 0.1839/0.2493 and +0.1482/0.2001, respectively. None passes the identical-operation control gate; +these ratios alone do not authorize a performance claim. A separate repeat +keeps thresholds unchanged and doubles batch work to reduce interval noise. + +GCC compiles the scanner and macro-consuming LR index in 21.272 seconds and +the benchmark in 12.506 seconds, both exit zero. Thirteen focused suites pass +96 cases in 22.798 seconds, including 8,192-character preparation, Unicode +boundaries, duplicate/rank/mode oracles, LR action identity, and macro argument +single evaluation. Build/test wall duration is not runtime CPU evidence. + +Receipt stems are =parser-compact-edge-build=, +=parser-compact-edge-benchmark-build=, =parser-compact-edge-tests= and +=parser-compact-edge-cpu-a/b=. No JSON snapshot or absolute host path is stored. + +** Positive CPU admission and executable performance assertion + +The independent two-million-character repeat completes in 106.287 seconds, +exit zero. Widths 1, 8 and 32 retain failed control admission even though their +candidate medians are lower. Width 128 passes the unchanged gates: + +| Observation | Control before | Production/reference | Control after | +|-------------+----------------+----------------------+---------------| +| CPU ratio P10 | 0.9319 | 0.1325 | 0.9227 | +| CPU ratio P50 | 0.9910 | 0.1471 | 1.0096 | +| CPU ratio P90 | 1.0727 | 0.1635 | 1.0680 | + +Both identical-operation controls remain within 0.90..1.10 at P10/P90. +Production P90 is below 0.95. The admitted median reduction is 85.29 percent +for this bounded 128-character literal path on the tested GCC/SDK/host. +Failed controls for other widths remain visible rather than being averaged +into this admitted workload. + +The benchmark now returns per-width observations. The Scheme entry +=qualify-literal-edge-cpu= runs the 128-character workload and throws an error +unless semantic validation, CPU resolution, both controls and the CPU benefit +gate all pass. It emits =LITERAL-EDGE-CPU-PROOF-OK= only after that assertion. +It can be called with the existing module: + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/ranked-literal/benchmark) +(qualify-literal-edge-cpu 20 2000000) +#+end_src + +A third independent child runs this performance test, completing in 19.863 +seconds with exit zero and both completion and proof markers. CPU ratios are: + +| Observation | Control before | Production/reference | Control after | +|-------------+----------------+----------------------+---------------| +| CPU ratio P10 | 0.9190 | 0.1332 | 0.9076 | +| CPU ratio P50 | 0.9946 | 0.1434 | 0.9945 | +| CPU ratio P90 | 1.0797 | 0.1470 | 1.0688 | + +Thus the positive CPU admission repeats, including a median reduction of +85.66 percent. The final performance-test receipt stem is +=parser-compact-edge-cpu-proof=; its GCC build completes in 11.635 seconds. +The 120-second total and five-second actual-progress watchdog are unchanged. +The CPU timing period runs without other builds or tests from this task. + +This proves a production shared-scanner primitive improvement, not whole-parser +CPU throughput or closure of the Bash/tree-sitter gap. The optimized edge IR +is shared by ranked scanning and LR casefold lookup; those consumers retain +semantic tests, but their complete-request CPU gain is not inferred from this +bounded scanning workload. + +** Final source and latest PR closure + +The child macro retains the original evaluation order: it reads the supplied +node's edges before evaluating the supplied character. This avoids exposing an +argument-evaluation change through the new representation. GCC rebuilds the +final scanner and consuming LR index in 17.231 seconds. A stale local LR index +static object is invalidated; generated static source and dynamic native tests +are qualified here, not a new complete local AOT executable. + +The change is rebased onto =060d8a3=, including checkpoint capability and +contextual admission. Its LR/parser owners rebuild in 38.918 seconds. Fifteen +source suites pass 122 cases in 23.455 seconds, including the eleven layout +boundary cases and fifteen contextual parser cases. The final receipt contains +=TEST-SUITE-OK=, final =OK=, exit zero and no missing products. + +The unchanged executable CPU assertion is run again against the final scanner. +It completes in 18.432 seconds, exit zero, with =LITERAL-EDGE-CPU-PROOF-OK=. + +| Observation | Control before | Production/reference | Control after | +|-------------+----------------+----------------------+---------------| +| CPU ratio P10 | 0.9267 | 0.1261 | 0.9460 | +| CPU ratio P50 | 0.9995 | 0.1400 | 0.9996 | +| CPU ratio P90 | 1.0631 | 0.1581 | 1.0728 | + +Both controls and the CPU benefit gate pass again. The median reduction is +86.00 percent for the same 128-character bounded-scanner workload; other +workloads' failed control admission remains unchanged. Final receipt stems +are =parser-compact-edge-final-build=, =parser-compact-edge-rebased-build=, +=parser-compact-edge-rebased-tests= and =parser-compact-edge-cpu-final-proof=. +The following =104679f= rebase contains documentation changes only and leaves +the qualified production source graph unchanged. + +* Shared CPU admission across ranked scanning and complete layout products + +The paired measurement loop is moved into the existing Scheme parser-stage +sampler as =measure-parser-cpu-pairs=. The bounded proof, ranked proof and +layout product comparison share the same CPU counters, alternating ABBA/BAAB +order, actual batch records, one-millisecond resolution floor and ASP percentile +selection. Complete warmup and final batch results must equal the supplied +expected product. Tests do not add a process per sample or change compiler, +heap, total timeout, idle timeout or CPU admission thresholds. + +The existing ranked benchmark gains =benchmark-ranked-edge-cpu= and the +executable assertion =qualify-ranked-edge-cpu=. The comparator retains the old +ranked traversal with all-table edges, original duplicate combination, terminal +selection, mode admission and four-field result publication. The production +scanner uses the retained compact-edge engine. An independent linear catalog +oracle validates every tested start offset over successful, failing and Unicode +sources, including mode filtering. Scalar endpoint agreement is insufficient. + +Initial ranked fixtures include an accepted one-character prefix before the +128-character terminal. This publishes two results per successful call. A +separate final fixture has only the final terminal, representing a long keyword +without an accepted shorter prefix. Earlier prefix-chain benchmarks and semantic +mode/rank/order tests remain available. These are distinct declared workloads; +results from one do not qualify the other. + +| Ranked workload | Calls per batch | CPU ratio P50 | CPU ratio P90 | Admission | +|-----------------+-----------------+---------------+---------------+-----------| +| Accepted prefix and final terminal | 15625 | 0.1776 | 0.2263 | Rejected controls | +| Same prefix workload, larger batch | 31250 | 0.1926 | 0.3182 | Rejected controls | +| All terminals rejected by mode | 31250 | 0.1888 | 0.2143 | Rejected controls | +| Final terminal only | 15625 | 0.1853 | 0.2288 | Rejected controls | + +The prefix children complete in 30.222 and 62.833 seconds, but their performance +assertions fail. The initial runner lets the Scheme error reach the process +watchdog (=error-output=), which rejects it rather than accepting a completion +marker. The all-rejected diagnostic completes in 80.302 seconds with exit zero; +its CPU admission remains false. The final-terminal assertion fails with an +explicit exit one in 35.616 seconds. Smaller observed candidate ratios alone +are not sufficient to pass unchanged control gates. + +** Complete GLR artifact CPU test and rejected layout candidate + +The latest PR includes =8a206b5=, packed request-owned layout columns. The layout +benchmark gains =benchmark-layout-request-cpu= and +=qualify-layout-request-cpu=. Each complete-product operation executes lexical +scanning, significant-token selection, source-column construction, GLR parsing +and success-artifact publication. Both variants use the same parser machine, +source and recognition runtime. The complete source-admitted ParseArtifact must +match, rather than only a token count or root summary. This is a complete GLR +product path, not measurement of an arbitrary language's public entry route. + +The initial complete-product test uses the independent boxed column oracle. +With twenty groups and one hundred products per batch, its 24,000 timed products +complete in 57.692 seconds; product equality passes, but the performance test +exits one. Production/reference CPU P50/P90 is 0.8065/1.3086. Before-control +P10/P90 is 0.7123/1.3946; after-control is 0.6747/1.4197. The initial missing +native fixture load is a separate failed bootstrap; compiling its source owner +fixes that load without weakening any benchmark gate. + +An engine candidate directly publishes ASCII columns and reserves the byte-fill +loop for multibyte characters. Sixteen semantic suites pass 125 cases in 37.510 +seconds, including byte interiors, EOF, tab/CRLF and complete parser controls. +Against the boxed oracle, twenty groups of five hundred constructions produce +CPU P50/P90 0.7153/0.7595 in 75.797 seconds. Controls fail, so the assertion exits +one. A former-packed comparator then retains the same u32 buffer and private +bounded native setter as the original producer; this isolates the algorithm +from the previously admitted storage allocation change. + +For a smaller 128-space source and five thousand constructions per batch, the +same-storage comparison completes in 29.221 seconds. CPU P50/P90 is +0.7745/0.8438, but before-control P10/P90 is 0.2821/1.3219 and after-control is +0.2695/1.6292. The assertion again exits one. The ASCII runtime candidate is +therefore withdrawn and the retained =8a206b5= producer is rebuilt from source. +No additional runtime gain is claimed from these rejected measurements. + +The same-storage comparator is checked against the independent boxed byte +oracle. Added regression tests cover all ASCII bytes and long ASCII runs that +rejoin Unicode, tab and CRLF handling. This test data is source-owned Scheme; +there are no serialized benchmark results or frozen alternate parser engines. + +Receipt stems are =parser-ranked-edge-proof-a/b=, +=parser-ranked-edge-rejected=, =parser-ranked-terminal-proof-a=, +=parser-layout-cpu-before-retry=, =parser-layout-ascii-columns= and +=parser-layout-packed-small=. Every failed gate remains part of the evidence. + +** Isolated Ubuntu CPU gate and final replay interface + +The larger final-terminal ranked test also fails its control gate: CPU +P50/P90 is 0.1992/0.3120, with before-control P10/P90 0.8181/1.0931 and +after-control 0.9078/1.1290. It propagates exit one in 78.315 seconds. Its +receipt stem is =parser-ranked-terminal-proof-b=. + +The original bounded proof is then replayed through the shared sampler. It +observes CPU P50/P90 0.1307/0.1552 but rejects before-control P10/P90 +0.8862/1.1308. After-control is 0.9166/1.0236. The run ends after 100.100 +seconds with a rejected error-output receipt, not a new admitted CPU result. +This leaves the earlier completed positive proofs intact in their recorded +source/environment scope; it does not manufacture a new positive head receipt. +Several unrelated GCC and GXI processes are active on this shared host. They +are not stopped or included in this task's CPU ownership. The expanded tests +expose drift rather than prove its complete causal decomposition. + +The executable source replay entry is +=t/benchmarks/ranked-literal/cpu-proof.ss=. It loads compiled benchmark owners +with actual import tracing, runs one CPU proof in one Scheme VM and propagates +failed assertions as exit one through the existing test exit binding. A +deliberately insufficient one-group request emits =CPU-PROOF-REJECTED= and +qualified exit one in 9.811 seconds, confirming that successful bootstrap or +logging cannot satisfy the performance proof. + +#+begin_src sh +gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/scanner-cpu-proof.log --require SCANNER-CPU-PROOF-OK \ + -- gxi t/benchmarks/ranked-literal/cpu-proof.ss bounded 20 2000000 +#+end_src + +Use =ranked= instead of =bounded= to execute its separate assertion. The +ranked assertion remains diagnostically rejected in the local receipts above. +The layout entry is available directly in Scheme: + +#+begin_src scheme +(import :gerbil-parser/t/benchmarks/layout-columns/benchmark) +(benchmark-layout-request-cpu 20 100 32768) +(qualify-layout-request-cpu 20 100 32768) +#+end_src + +Conformance preparation owns native compilation of both benchmark modules; +there is no benchmark-specific discovery/compiler supervisor. Ubuntu CI now +runs the retained bounded assertion after source compilation completes and +before the test workers start. It keeps the 3 GiB maximum heap, fixed GCC SDK, +120-second total deadline, five-second actual-progress deadline and original +control/benefit thresholds. Missing =SCANNER-CPU-PROOF-OK= or nonzero exit fails +that step. The gate makes CPU improvement an executable requirement in an +isolated runner instead of treating semantic success as performance proof. +Its exact-head outcome must be reported from CI; it is not assumed here. + +The branch is synchronized with =5bc7d2f=, certified artifact-window publication, +before final source qualification. That producer is rebuilt from its current +source; unrelated concurrent commits are preserved. + +Final source qualification rebuilds the latest artifact producer and its +window fixture with GCC in 27.010 seconds, exit zero. The final Scheme replay +passes 17 suites and 129 cases in 38.464 seconds, with final =OK= and qualified +exit zero. This covers certified artifact windows, ranked scanner results, +layout byte boundaries, contextual LR integration and Rust source generation. +It establishes semantic conformance, not CPU improvement. Receipt stems are +=parser-cpu-proof-rebased-build= and =parser-cpu-proof-final-tests=. + + +** Complete ranked lexical competition and request ownership + +The ranked scanner now stages terminal precedence and declaration order once. +Traversal retains a winning prepared payload and allocates the complete +=(name character-end rank ordinal)= result only at publication. Dense ASCII +root fanout uses one shared directory; non-ASCII edges retain the compact trie. + +=make-ranked-lexical-scanner-factory= forms the product of that trie and the +closed monotone regular DFA. A source character advances both states once. +The last accepting state retains maximal munch; rank and declaration ordinal +resolve only equal endpoints. Numeric fraction/exponent pending states retain +the existing rollback scanner. This is shared engine preparation, independent +of language declarations. + +Mode preparation snapshots its literal catalog and admission mask. Modes with +all literals admitted remove the runtime mask operation. A literal-only mode +with one admitted literal performs bounded direct character matching. Other +modes retain the shared product. Neither specialization changes the result +protocol or exposes a language-specific optimization slot. + +=parser-machine-with-ranked-scanner= rebuilds the same checked lexical rules +and mode catalog with a supplied ranked factory. It preserves the grammar IR, +digest, runtime, source plan and owned program. Generated direct source/drive +products cannot bypass that replacement. The CPU reference is the former +compact-trie primitive plus separate regular scanning and original ranked +competition. It is not a copied parser. Both variants execute =parse-source=, +including source traversal, LR/layout, source identity and complete artifact +publication. Preparation occurs before measured requests. + +The semantic controls cover all 64 regular subsets, multiple literal masks, +all source offsets including EOF, Unicode, numeric rollback, longest matching, +rank ties, mutable input/result ownership, ordinary LR and layout requests. +The complete reference, candidate and production artifacts must be equal and +admitted for their source before any CPU interval starts. + +*** Measurement limits retained during the investigation + +Eight-leg observations include both ABBA and BAAB. Four-leg observations +aliased periodic natural collection onto one variant: identical full-parser +controls produced CPU P50/P90 near 0.59/2.08. All natural GC CPU remains in the +measurement. No samples are removed, no per-batch collection is forced and no +control threshold is relaxed. There remain twenty observations in each of the +before-control, production and after-control phases. + +Runtime CPU entries load compiled phase-zero owners without source expander +interfaces. They still trace real dependency initialization. Source gxtest +entries retain interface admission. Both timed variants share this environment; +removing unused compiler contexts is not counted as an engine CPU improvement. + +The initial complete ranked lexical run at 100000 calls per batch observed +production CPU P50/P90 of 0.5205/0.6240 for keyword competition, +0.5677/0.6688 for identifier competition and 0.5650/0.6536 for literal prefixes. +Its identical controls failed, so these are rejected observations rather than +admitted performance claims. Receipt stem: =parser-ranked-lexical-fused=. +Larger individually bounded scenarios retain the same admission rules: + +#+begin_src sh +gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/ranked-lexical-cpu.log --require SCANNER-CPU-PROOF-OK \ + -- gxi t/benchmarks/ranked-literal/cpu-proof.ss lexical 20 250000 keyword +#+end_src + +Replace =keyword= with =identifier= or =literal-prefix= for the other complete +ranked scenarios. The =parser= workload includes =layout=, =core= and +=assumptions= source scenarios. A rejected workload exits nonzero; it cannot +emit the required success marker. + +The first fused layout request run observed CPU P50/P90 0.9759/1.1688 and +failed both control/benefit admission. The ordinary LR diagnostic likewise +failed: P50/P90 1.0078/1.3272. Receipt stems: +=parser-ranked-public-parser-fused= and =parser-ranked-core-proof-rebuilt=. +These are preserved negative results, not complete-parser CPU wins. + +Missing layout cost observations are now present for lexical analysis, +significant-token filtering and LR execution. For the 157-character layout +request, twenty instrumented requests attributed CPU P50 0.1455 ms to LR, +0.0210 ms to lexical analysis and 0.2210 ms to the observed complete request. +LR allocated 122688 bytes; lexical analysis allocated 39440 bytes. The observer +is diagnostic overhead and these samples are not an uninstrumented latency +comparison. They explain why a large primitive scanner ratio need not produce +a large complete-request ratio. Receipt stem: =parser-ranked-stage-diagnostic-v2=. + + +*** Admitted complete ranked keyword result + +The final keyword workload passes the complete Scheme CPU assertion with +20 groups and 250000 calls per batch. CPU ratio P50/P90 is 0.526162/0.572052, +corresponding to a 47.38 percent median CPU decrease for complete ranked +keyword competition. Before-control P10/P50/P90 is +0.941734/1.006953/1.046188; after-control is +0.947828/0.995368/1.056494. Qualified elapsed time is 92.946 seconds, exit zero, +with =SCANNER-CPU-PROOF-OK=. Receipt stem: +=parser-ranked-keyword-calibrated-cpu=. + +The preceding 500000-call run reaches the unchanged 120-second limit during +the after-control and is rejected. Its completed before/production phases use +28.392/21.575 CPU seconds and 62.051/44.274 wall seconds, including +12.173/7.618 GC CPU seconds. The admitted rerun calibrates the workload unit; +it retains every group, both controls, all natural GC CPU and the same gates. +Receipt stem: =parser-ranked-keyword-final-cpu=. + +This is a complete ranked scanner result. It does not establish a complete +parser speedup; that assertion separately includes every public request stage. + + +*** Prepared layout decisions in complete requests + +LR preparation now owns an action-selector procedure. The layout selector +prepares allowed and blocked projections of grammar-owned actions once, +retaining original entry identity when the action is unchanged. The request +keeps only source columns and branch-local frames. Projection never reindexes +filtered rows: the first literal still wins lookup, and a denied literal cannot +fall through to a generic token or a later duplicate. + +The selector snapshots request columns and frames once. Eligible next markers +retain priority over start markers and ordinary actions. Start eligibility and +ordinary shift eligibility share one column relation. After an eligible next +marker has lost, a pure ordinary reduction with no start marker skips that +relation entirely. Pure reductions also skip the prepared projection lookup. +These are shared layout/LR algorithms, with no language-specific hints. + +=parser-machine-with-lr-action-selector= replaces only the checked action +selection primitive and reparents the parse closure to the new runtime. The +CPU reference combines the former compact ranked scanner with the former pure +layout lookup/projection primitive. Both variants execute the complete public +=parse-source= pipeline and must equal the unmodified production artifact. + +The first prepared layout run at 512 calls per batch observed CPU P50/P90 +0.885612/0.974036, but the after-control P10 was 0.874845. It is rejected. +The subsequent frame-staged run at 256 calls observed 0.884177/0.978420; +control P10 values 0.851950/0.879642 also reject it. These observations do not +establish a complete-request CPU win. Receipt stems: +=parser-ranked-layout-prepared-cpu= and =parser-ranked-layout-frame-batch-cpu=. + +A preceding frame-staged run stopped on the unchanged five-second idle bound +between group outputs. The shared sampler now publishes each completed batch's +CPU, wall, GC and allocation observations immediately, outside the measured +interval. No unfinished batch emits progress. The eight-leg group statistics, +minimum resolved CPU, twenty groups and all admission thresholds are unchanged. + +The complete parser CLI additionally accepts a positive clause count: + +#+begin_src sh +gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log .data/ranked-parser-cpu.log --require SCANNER-CPU-PROOF-OK \ + -- gxi t/benchmarks/ranked-literal/cpu-proof.ss parser 20 384 layout 12 +#+end_src + +Changing the final count to 48 measures a longer complete layout request; batch +work must be calibrated without changing controls, sample count or deadlines. + + +The final pure-reduction staging run at 384 calls per batch observed complete +157-character request CPU P50/P90 0.860601/0.957299. Before-control P10/P90 was +0.896855/1.125763 and after-control was 0.911401/1.107213; admission failed. +The 517-character request at 128 calls observed P50/P90 0.806723/0.943927. +Its before-control P10/P90 was 0.860899/1.111094 and after-control was +0.844897/1.118945, so it too is rejected despite passing the production P90 +benefit bound. Qualified elapsed times were 113.113 and 102.889 seconds. +Receipt stems: =parser-ranked-layout-reduce-cpu-v2= and +=parser-ranked-layout-long-cpu=. No control failures are promoted to wins. + +Ubuntu now enforces the same complete-request admission for both source sizes +and the keyword, identifier and literal-prefix complete ranked products. +Each invocation retains the 120-second total and five-second idle bounds, +three-GiB maximum heap, twenty groups and unchanged percentile gates. + + +*** Admitted complete-request and ranked CPU evidence on Ubuntu + +Code revision =c6a6565b910b1a82168c7bfd618b612b1183be87= passes the complete +Scheme CPU gate in [[https://github.com/tao3k/gerbil-parser/actions/runs/38054496900][CI run 38054496900]]. All five assertions emit +=SCANNER-CPU-PROOF-OK= and finish with exit zero. Ratios below are candidate CPU +divided by reference CPU across twenty balanced paired groups, including +natural GC. Preparation and result admission remain outside timed requests. + +| Complete workload | Calls per batch | CPU P50 ratio | CPU P90 ratio | Median CPU decrease | Qualified seconds | +|-------------------+-----------------+---------------+---------------+---------------------+-------------------| +| Public TLA+ layout, 48 clauses, 517 characters | 128 | 0.775231 | 0.919257 | 22.48% | 37.836 | +| Public TLA+ layout, 12 clauses, 157 characters | 384 | 0.868511 | 0.874858 | 13.15% | 32.242 | +| Ranked keyword competition | 250000 | 0.566157 | 0.580265 | 43.38% | 50.490 | +| Ranked identifier competition | 250000 | 0.640662 | 0.657650 | 35.93% | 56.993 | +| Ranked literal-prefix competition | 250000 | 0.552558 | 0.568744 | 44.74% | 29.125 | + +Every identical-code control also passes the original P10 >= 0.90 and +P90 <= 1.10 bounds: + +| Workload | Before P10 | Before P50 | Before P90 | After P10 | After P50 | After P90 | +|----------+------------+------------+------------+-----------+-----------+-----------| +| Layout, 48 clauses | 0.920056 | 0.995391 | 1.087544 | 0.918577 | 0.998814 | 1.083092 | +| Layout, 12 clauses | 0.990513 | 0.998758 | 1.019033 | 0.930102 | 1.000409 | 1.091494 | +| Keyword | 0.952565 | 0.990232 | 1.034907 | 0.957547 | 0.996479 | 1.021163 | +| Identifier | 0.951134 | 0.991495 | 1.018879 | 0.972458 | 0.999915 | 1.023100 | +| Literal prefix | 0.966161 | 1.001431 | 1.029086 | 0.969034 | 1.001863 | 1.028954 | + +The full-request assertions execute the normal public layout path, including +lexical analysis, significant-token filtering, LR/layout, source identity and +complete artifact publication. The reference and candidate must equal the +unmodified production artifact and pass source admission before timing. The +engine derives all preparation from checked grammar tables and lexical modes; +the language declarations contribute no optimization hints. + +The exact rebased code also passes five focused Scheme suites and 43 cases in +34.518 seconds, covering action projections, full parser requests, layout +boundaries and the concurrently merged canonical identity/publication owners. +The earlier broader qualification passed 21 suites and 146 cases. The new +pure-reduction precedence case is included in the final focused qualification. + +The separate local literal-prefix run passes with CPU P50/P90 +0.508880/0.570142, before P10/P90 0.916130/1.075357 and after P10/P90 +0.911077/1.075920, exit zero in 102.443 seconds. Local identifier runs retain +negative results: 150000 calls fails control P90 1.124001/1.105859, while +250000 and 200000 calls reach the unchanged 120-second total bound. Ubuntu's +isolated run admits the 250000-call workload without changing any threshold. +These local failures remain excluded from the admitted figures above. + +The complete-request evidence covers the TLA+ layout pipeline and these two +source sizes. The matched 28440-byte Bash/Tree-sitter comparison requires its +own complete-source qualification. + + +*** Merged semantic-kernel qualification and GC resolution + +Revision =842767b3dc750666204a04fc36db5a85f48037ae= includes the concurrent +semantic-kernel preparation and fixed factory initialization. The shared +=action-selector= field remains intact, and alternate selector runtimes retain +the new reduction plans. Six focused suites and 54 cases pass in 39.844 seconds +after rebuilding the overlapping owners and their generated consumers. + +In [[https://github.com/tao3k/gerbil-parser/actions/runs/38057436075][CI run 38057436075]], the 517-character complete request again passes: +CPU P50/P90 0.778406/0.926456, before P10/P90 0.916878/1.099238 and after +P10/P90 0.912803/1.094988. It exits zero in 35.842 seconds. +The short complete request observes P50/P90 0.860558/0.881254 but is rejected +because before-control P90 is 1.109956, above the unchanged 1.10 bound. + +That control's group 13 has identical code on both sides. Its left/right +CPU totals are 251.356/278.994 ms, including 70.630/95.819 ms GC CPU and +3/4 collections. The 25.189 ms GC difference explains most of the 27.638 ms +CPU difference. Every collection remains charged to its timed request batch. + +Ubuntu's complete-request batches now contain 256 calls for 48 clauses and +768 calls for 12 clauses. Doubling each group's work averages more natural +collections while retaining twenty groups, eight balanced legs, both controls, +all samples, the original percentile bounds and the 120-second/five-second +qualification limits. The preceding qualified times are 35.842/31.193 seconds; +this calibration fits the existing total budget without changing a timeout. +The original admitted figures remain attributed to their original revision and +batch sizes above; the larger batches require a fresh admission. diff --git a/docs/research/shared-source-projection-publication.org b/docs/research/shared-source-projection-publication.org new file mode 100644 index 00000000..4e412e0a --- /dev/null +++ b/docs/research/shared-source-projection-publication.org @@ -0,0 +1,668 @@ +#+title: Source projection and publication +#+startup: overview + +Part of [[file:shared-scanner-staging-and-fusion.org][the shared scanner study]]. +Historical revisions, failed experiments and receipt limits retain their original scope. + +The [[file:artifact-publication-cost-boundaries.org][artifact publication cost contract]] implements the event/encoding/identity +attribution obligation below, with independent source controls and a native +cross-language receipt. It does not admit a new fused executor or default. + +* Next shared-engine design: direct execution and publication ownership + +The evidence motivates a different experiment, not another generic closure +wrapper. flap's staging removes higher-order dispatch through generated direct +code; its fusion removes internal representations. Neither experiment above +performed these transformations. No speed ratio from the paper transfers here. + +The implementation milestones are: + +1. Lower closed projection IR into generated Scheme constructors with fixed + capture indexes and direct token/node operations. The engine owns generation; + the public POO profile remains the declarative authority. Prove catalog, + capture-order and admission parity before admitting any specialization. +2. Measure projection execution separately from recognition, token lexeme + materialization and artifact publication, while retaining a complete-product + control. Inspect generated Scheme/C to verify that opcode dispatch and + per-operation indirect calls actually disappear. +3. Establish ownership for recognition nodes that exist only until publication. + Prototype an engine-owned event sink that emits the required canonical + artifact once, retaining byte coverage, child order, identities, diagnostics + and rejection semantics. Keep request state private; do not add language + callbacks or a second public artifact protocol. +4. Admit direct emission only after full public products and Scheme/C/Rust + consumers preserve their contracts. Analyze unsupported recursion/context + forms explicitly; deterministic lexer/parser fusion requires properties that + have not been proved for every supported grammar. + +These are design obligations, not implemented AOT or direct-publication claims. +The current engine still uses the original projection executor. Permanent tests +strengthen its admission contract; temporary comparison modules are excluded +from the commit. Future measurements must include the complete publication path +and cannot promote a component allocation saving into an end-to-end speedup. + +The final retained tree, after both candidates were removed and scanner revision +=8e98e83= was incorporated, completed seven modules in 11.544 s with +=TEST-SUITE-OK modules=7=, =OK=, exit 0 and no missing receipts. This includes +result projections, source recognition, contextual scanning, Bash, HCL, HL7 and +FHIRPath. The final receipt validates the retained code and added contract test; +it does not establish a new runtime speed improvement. + +* Implemented compiler milestone: direct Scheme projection generation + +=src/compiler/result-projection.ss= now lowers an admitted ResultPlan to a +standalone Scheme module exporting =bind-generated-result-projections=. Each +projection has a whole-projection executor with fixed capture indexes and direct +operations. Its executable body does not read operation vectors, switch on +opcodes, or call a prepared closure per operation. Variable-length captured +parts still need traversal. Existing single-required-token and single-parts +fast paths retain their constructor and validation behavior. + +=result-plan-bind-projections= checks the complete expected recipe, canonical +projection order, unique IDs and executable entries before returning a separate +executable plan. It preserves the original owner and catalogs. Generated +executors are compiler-owned artifacts, not new callback slots in language +profiles. POO inheritance and override remain declarative: a changed recipe +requires regeneration, and stale generated code is rejected before execution. +No language declaration, metadata version, C ABI or Rust protocol changes. + +The generated general executors deliberately retain =result-plan-node= dynamic +admission and the existing immutable token sequence representation. This compiler +milestone removes opcode interpretation; it does not fuse scanner/parser stages +or eliminate temporary recognition trees and final publication traversals. + +Permanent tests execute the actual generated source for mixed captures and empty +projections, compare complete node/token products, reject invalid spans, missing +required nodes, foreign kinds and escaping child nodes, and reject stale POO +recipes and malformed binding tables. A regenerated inherited profile can bind +its changed field names. Binding returns a different plan and retains the same +recipe. + +A temporary native module was generated from the admitted Bash stress-sample +profile. Both comparison parsers use the same shared scanner and shell parser; +only the selected result plan differs. All four corpus artifacts matched each +other and the public parser before measurement. Both sides include artifact +validation, grammar identity and source identity checks. File IO, preparation, +loading and output are outside timing. Natural GC remains included; ABBA blocks +have one warmup and 11 single-request measurements each. + +| Control | CPU P50 ms/request | No-GC allocation P50 bytes | Allocation samples | +|---------+--------------------+---------------------------+--------------------| +| original-a | 26.666 | 39461648 | 8 | +| direct-a | 28.744 | 39461648 | 8 | +| direct-b | 26.792 | 39461648 | 8 | +| original-b | 26.979 | 39461008 | 8 | + +Pair A regresses approximately 7.79%; pair B improves approximately 0.69%. +Allocation is effectively unchanged. Native qualification completed in 9.006 s +with exit 0 and =DIRECT-PROJECTION-CONTROL-OK=. These controls establish native +execution and semantic parity, not a stable complete-engine speed improvement. +The default parser continues to use the original result plan; this AOT compiler +capability is not automatically selected by language loaders. + +The initial generator/binder native compilation completed in 17.272 s. The final +generator compiled in 7.901 s; generation from its compiled artifact completed +in 10.695 s; the generated executor module and native control compiled in +13.593 s. All successful compiler receipts use the same fixed GCC SDK and the +existing 120-second total and five-second idle limits, with real diagnostics +unbuffered. Temporary stress-sample modules are excluded from the commit. + +The next admission gate is publication ownership: identify which recognition +nodes and token copies exist only to feed canonical event construction, then +measure an engine-owned direct emitter against complete artifact obligations. +The unchanged allocation here is evidence against claiming that opcode removal +alone addresses the current complete-engine allocation cost. + +The ABBA table above was collected on base =750b8cd=, before the concurrent +UTF-8 extent and delimiter comparison revisions. Final integration rebased +those changes through =92fcb1d=. Their affected production modules were rebuilt +with the same GCC configuration. Complete native product parity was rechecked +on the final combined implementation in 9.038 s with exit 0 and all receipts; +that integration rerun is not used to claim an additional speed improvement. + +The retained implementation then completed 134 cases across ten modules in +11.161 s, with =TEST-SUITE-OK modules=10=, =OK=, exit 0 and no missing receipts. +Coverage includes result generation/binding, recognition, immediate and deferred +scanning, artifacts, event strategy, and four language suites. The final POO +regeneration assertion had also passed its focused suite in 7.587 s. No temporary +comparison source is retained in the production or test catalog. + +* Implemented: fuse source publication admission with input identity + +The shared SourceLanguage publisher previously validated the artifact by +reconstructing a string from token lexemes and hashing it, then independently +hashed the actual input to check its identity. The publisher already owns that +input. This duplicates whole-source work and constructs a string used only +between two validation steps. + +=parse-artifact-valid-for-source?= now binds artifact admission to the supplied +input. The existing event walk still checks schema, digest shape, canonical +IDs, byte coverage, UTF-8 token widths, node/field balance, root count, status and +diagnostics. For each admitted token it compares its characters directly with +the next input range, without a substring or encoding. Complete character +coverage is required, then the actual input is hashed once and compared with the +artifact identity. SourceLanguage uses this entry and retains grammar identity +admission. All language declarations and artifact/FFI schemas remain unchanged. + +For lexemes L1 through Ln and input S, the walk proves each Li equals the next +character interval in S and that their combined character count is length(S). +This establishes concat(L1,...,Ln) = S without materializing that concatenation. +Contiguous byte coverage and each lexeme's UTF-8 width remain separately checked. +A forged digest that matches S but disagrees with lexeme content still fails. +Standalone artifact admission and owned roundtrip reconstruction retain their +existing behavior; callers with an actual input use the stronger source-bound +entry. Both paths share the same structural validator. + +Both algorithms are O(events + characters). The improvement removes an internal +representation and a repeated input hash; it does not establish a new asymptotic +bound or complete lexer/parser fusion. Recognition trees and canonical event +allocation remain in the current producer. + +The original artifact validator was frozen under a temporary native namespace. +Both sides used the same fixed GCC SDK and optimization/diagnostic flags. The +baseline publication obligation includes artifact validation, grammar identity +and a second hash of the input. The candidate includes grammar identity and +source-bound artifact validation. Full controls share the same prepared scanner +and parser and require complete artifact equality with the public entry for +short, heredoc, compound and large inputs before timing. + +ABBA blocks have one untimed warmup and 11 measured single requests. Natural GC +is included in CPU measurements; only no-GC allocation deltas are admitted. +Preparation, loading, IO and sample output are outside timing. Component and +complete-engine numbers must not be added together. + +| Control | CPU P50 ms/request | No-GC allocation P50 bytes | Allocation samples | +|---------+--------------------+---------------------------+--------------------| +| validation-old-a | 1.462 | 5054208 | 11 | +| validation-new-a | 1.275 | 4406152 | 11 | +| validation-new-b | 1.171 | 4406152 | 10 | +| validation-old-b | 1.454 | 5054208 | 11 | +| complete-old-a | 29.410 | 39054824 | 8 | +| complete-new-a | 28.697 | 38406768 | 8 | +| complete-new-b | 34.412 | 38406448 | 8 | +| complete-old-b | 37.761 | 39054824 | 8 | + +Publication admission allocation decreases approximately 12.82%. The complete +engine allocates approximately 648 KB less per request, about 1.66% on this +corpus. Complete CPU pairs are approximately 2.42% and 8.87% lower, but the +baseline changes substantially between blocks. These timings do not establish +a stable public-entry speedup or close the Tree-sitter gap. Allocation reduction +and complete-product parity are the admitted outcomes. + +Native qualification completed in 11.108 s with exit 0 and +=SOURCE-BOUND-CONTROL-OK=. Symmetric production compilation completed in +47.756 s and the native control compiled in 13.501 s, both with exit 0. The +existing total/idle guards and fixed GCC configuration were retained. + +Permanent controls cover empty input, accepted and rejected artifacts, wrong +or truncated inputs, non-string input, forged digests and identical-byte-width +Unicode substitutions. The existing UTF-8 width boundary controls also admit +through the source-bound entry. False advertised byte lengths cannot trigger +reconstruction capacity reservation. Eleven modules completed 137 cases in +10.382 s with exit 0 and all receipts before the next concurrent PR integration. +The first focused-suite invocation used an incorrect file path and failed; +only the corrected successful receipts are admitted. + +The ABBA table was collected on base =8fe4a0e=. Subsequent EventFold, checkpoint +and LR reverse-list changes are integrated separately; their costs are not +attributed to source-bound admission. Temporary comparison sources are excluded +from the implementation commit. Future representation experiments should retain +this input-bound publication obligation in both complete-engine controls. + +After rebasing the concurrent engine revisions through =ded00a4=, their changed +production dependency chain compiled in 24.357 s. Complete native product parity +and the matched publication obligations passed again in 12.871 s with exit 0 +and all receipts. These integration samples are not used to attribute additional +runtime improvements to this change. + +The first expanded integration run crashed while importing the existing native +=event-fold-future= interface (exit 139), before suite execution; it is excluded. +Rebuilding that module, its direct compiler helper and the remaining EventFold +IR consumer with the fixed SDK completed in 16.003 s. The isolated EventFold +suite then passed 61 cases in 9.113 s. The exact failing instruction has not been +identified; the loading failure did not recur after that rebuild. + +The full retained implementation subsequently passed 210 cases across thirteen +modules in 16.288 s, with =TEST-SUITE-OK modules=13=, =OK=, exit 0 and no missing +receipts. This includes source contracts, immediate/deferred scanners, +checkpoints, EventFold, result AOT binding, artifacts and four language suites. +The following =5a1074e= integration changes only documentation and a benchmark; +the production code and suites behind this receipt are unchanged. + +* Complete request attribution and the next representation boundary + +The previous complete native controls still allocated about 38 MB per large +request. A shared diagnostic now accepts any public parser procedure and input: +=t/benchmarks/parser-stage-cost/benchmark.ss#measure-parser-stages=. It has no +language imports, scanner replacement, process launcher or alternative parser. +The caller owns workload selection. Each measured request must equal the full +unobserved reference artifact and pass source-bound admission. IO, reference +construction, equality, admission and sample output are outside the request. +The summary includes source and grammar identities, event count, CPU median, +allocation median and the number of admitted allocation observations. + +An audit found an observability gap: session parsing already reported source +preparation, scanning and artifact validation, whereas ordinary SourceLanguage +and receipt entries did not. Shared shell and line providers now report the +preparation and scanning boundaries; public source and receipt entries report +validation. Existing disabled-path macro lowering evaluates the expression +without constructing an observation thunk. These additions measure existing +work; they do not implement a runtime speedup. LR and generated direct engines +are different providers and do not acquire source-provider stage names merely +because they support a language such as HL7 or HCL. + +Natural GC remains inside the measured CPU interval. Allocation deltas are +admitted only with zero collections and a nonnegative counter delta. Observer +callbacks and statistics calls have a cost, and outer measurements include +that cost. Nested observations are inclusive. Medians from different stages +and different admitted subsets must not be added or subtracted to infer an +unobserved stage. These are localization diagnostics, not an uninstrumented +latency comparison with Tree-sitter. + +A scanner exception recovered by the parser can legitimately publish a complete +rejected artifact. The scanner observation must retain =completed=false=; +requiring every stage to complete would misclassify correct rejection. The +benchmark admits a rejected request through full product equality and actual +input admission. Permanent controls cover the public shell entry, its multiple +return-value receipt entry, a POO line provider, Unicode, empty input, lexical +failure and prefix rejection. Dynamic observer scope is restored afterward. + +** Paper interpretation and bounded lowering obligations + +[[https://arxiv.org/html/2304.05276][flap sections 2.5, 2.7 and 2.8]] distinguish three mechanisms: +deterministic single-token branch selection, lexer/parser fusion eliminating an +internal token representation, and staging away statically known dispatch. +Its generated parser still permits allocation introduced by semantic actions. +Our required lossless artifact is such an observable semantic product. Removing +an internal token stream does not authorize removing lexemes, field events, +trivia coverage, diagnostics or public identities. + +A generic lowering must be derived from admitted engine IR. It must establish: + +1. Branch selection and lexical priority preserve the original recognition + language, including maximal-munch behavior and negative lookahead. A FIRST + set alone is insufficient to prove scanner/parser fusion. +2. Skipped characters remain publishable lossless trivia. A recognizer that + discards them does not satisfy the complete artifact contract. +3. Deferred delimiters, source modes and speculative branches establish explicit + commit boundaries. A speculative event cannot escape into canonical output. +4. Source intervals retain byte/character agreement and input ownership through + Unicode and relocation. A borrowed interval cannot silently outlive its + request or become a mutable shared input view. +5. ResultPlan field order, token spelling and failure diagnostics remain equal + to the current public product. The existing event-program and raw-event-tape + publication mechanisms are the targets; a new language callback protocol is + unnecessary. + +The current shell execution chain still creates scanner tokens, recognition +values and projected word token sequences, then flattens those sequences at +component boundaries, copies them into the parser's emitted stream and finally +publishes canonical events. Some of this storage is transient, but canonical +output is required. Word source preparation already uses an ASCII fast path and +lazily prepares region caches; those mechanisms must not be proposed again as +new work. The earlier generated projection experiment removed interpreter +operation dispatch without establishing a complete-entry allocation or speed +win. That negative result argues for changing producer/consumer representation +and ownership, rather than another wrapper around projection execution. + +The next experiment should account for scanner and recognition construction +separately, then lower an admitted deterministic producer/consumer boundary to +existing publication instructions. Request-local source intervals and a single +committed semantic stream are candidate representations, not implemented +capabilities. They require whole-product controls for quoting, substitutions, +here-documents, malformed input, Unicode and another applicable grammar before +any default-path promotion. An implementation should be retained only if it +reduces complete native allocation without a stable runtime regression. + +** Native diagnostic receipt on the integrated engine + +The final diagnostic used the fixed GCC SDK on base =8b8e7e0= plus the public +source observation changes. It ran separately from this turn's test/build jobs. +The large fixture has 28,040 characters and 52,852 canonical events; its source +identity is =sha256:11a684b4a83d51ab0f5565ffad083c209ec07a0af5cdc7756c158022ac042221=. +Its grammar identity is +=sha256:880ba035ae26195eaeae0ce3e5fbef582e2195c46aed31703a7592395f248ce8=. +The earlier source-bound control table has a different base and is not combined +with this stage table to claim a subsequent speedup. + +| Stage | CPU P50 ms | No-GC allocation P50 bytes | Allocation samples / CPU samples | +|-------+------------+---------------------------+---------------------------------| +| source-prepare | 0.010 | 114432 | 11 / 11 | +| source-scan | 8.626 | 7364432 | 10 / 11 | +| command-recognition | 15.111 | 19334192 | 11 / 11 | +| artifact-publication | 1.999 | 7173480 | 9 / 11 | +| artifact-validation | 1.352 | 4407464 | 11 / 11 | +| observed-complete-request | 27.354 | 38421712 | 8 / 11 | + +Recognition has the largest observed allocation and CPU interval. Scanning is +also material. Source preparation is small for this admitted workload. This +supports prioritizing the scanner/recognition producer-consumer boundary over +another preparation cache or event-validator micro-optimization. It does not +prove that all recognition allocation is avoidable, identify the individual +constructors responsible, or establish a portable latency ratio. The next +lowering still requires constructor attribution and an uninstrumented matched +complete-entry experiment. + +Short input and lexical rejection also passed complete-product equality and +source-bound admission. The final native diagnostic completed in 12.526 s with +exit 0 and =STAGE-COMPLETE-PARITY-OK=. Five focused modules passed 51 cases in +12.509 s, with =TEST-SUITE-OK modules=5=, =OK= and exit 0. The final diagnostic +module compiled in 22.556 s; shared source modules compiled in 12.896 s. Total +120-second and idle 5-second guards were unchanged. + +Earlier exploratory runs exposed missing imports, bracket errors, the incorrect +all-stages-completed rejection requirement and a test that assigned source +stage names to an LR language. A subsequent POO fixture needed explicit self +inheritance to pass SourceStrategy admission. Those failed runs are excluded. +One successful intermediate diagnostic overlapped test execution; its CPU +numbers are also excluded from this table. No failed test receipt or concurrent +CPU sample is promoted to the final evidence. + +The diagnostic can be called from another Scheme benchmark after native module +preparation, for example: + +#+begin_src scheme +(import (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark + measure-parser-stages) + (only-in :gerbil-parser/languages/bash/parser parse-bash)) +(measure-parser-stages parse-bash "echo hello\n" 11) +#+end_src + +The procedure argument selects the existing public parser. For an engine with +no cost-stage observations, the complete request row is still reported; absent +stages are not fabricated or inferred from its language name. + +* Owned publication spines in shared Word/result execution (2026-10-10) + +** Producer and consumer ownership + +After direct materialization bypassed the earlier branch-frame experiment, +the remaining Word producer still built a reversed parts spine and copied it +with =reverse= before projection consumed it. General result projection +execution similarly allocated a private reversed recognition-child spine, +then copied it into publication order. The Scheme projection generator emitted +the same redundant reversal. Neither spine is a caller-owned capture list: +=parts= is assembled by this =parse-parts= invocation, and =children= contains +only cons cells created by this projection's emitter. Nested result nodes, +capture spines, immutable token branches and source text are separate values. + +The retained implementation uses =reverse!= on those private spines exactly +at publication. An n-cell temporary list no longer requires another n-cell +copy. Traversal remains linear; this is an allocation and ownership change, +not a new parsing-complexity bound. Public nodes, token values, ordered fields, +source validation and the declared POO projection recipes are preserved. The +single-operation executors are unchanged, since they already construct their +final lists directly. No language grammar, schema version, ABI or language +namespace changes are required. + +This follows shipped Gerbil SDK revision =2591dcd= examples: +=gerbil/runtime/c3.ss=, =c4-compute-class-slots=, reverses its private =r-slots= +at publication, and =gerbil/runtime/util.ss=, =string-split=, uses the same +operation for its private accumulator. The SDK's =std/list/list-builder.ss= +is another owned-list construction mechanism; its dummy head and mutable tail +are unnecessary for an existing private reverse accumulator. + +[[https://arxiv.org/html/2304.05276][flap]], Sections 2.2 and 2.8, separates materialized intermediary costs from +allocation introduced by semantic actions. Here the lossless public token and +node products remain observable and cannot be removed using a token-free +recognizer's benchmark as justification. The eliminated cells are redundant +publication spines, not public token or node values. This implementation is +not flap fusion and makes no claim to its measured speedups or proof. + +** Qualification and native ABBA control + +All three modified modules compiled with the fixed GCC/Gerbil SDK in 29.183 +seconds. The 3 GiB heap, 120-second total and five-second idle guards remained +unchanged. Eight source suites passed 131 cases in 22.691 seconds, with +=TEST-SUITE-OK modules=8=, =OK= and exit 0. The new mixed-projection test builds +from the same 1,024-item capture twice through both interpreted and generated +executors, checking ordered fields, shared node identity and preservation of +the original capture spine. Existing cases cover optional and required +captures, Unicode bounds, invalid fields, missing nodes, source ownership, +nested words and incremental publication. Generated forms are evaluated by +these source tests; this receipt is not qualification of a separately linked +native generated executor. + +The native complete-request control reused the preceding private Scheme +controller and full serialized proofs. Each fresh process reported and, +after the first reference observation, asserted the loaded module basename. +Reference objects were =shell-word~0.o2= and =result-profile~0.o4=; +candidate objects were =shell-word~0.o3= and =result-profile~0.o5=. Swaps +removed all numbered objects for these two owned modules before copying the +selected snapshot, respecting the SDK loader's contiguous-version selection. +All other parser/client artifacts stayed fixed. The candidate was restored +before final tests. No controller or frozen proof is added to the repository. + +The large complete request includes public =parse-bash= plus an additional +source-bound artifact validator. Product equality against the pre-existing +reference was checked outside timing for four fixtures and empty, malformed +and Unicode sources. Every block admitted 23 allocation samples of 31; +GC-crossing allocation intervals were excluded. No forced GC or simultaneous +local builds were used during timed blocks. + +| Block | Complete CPU median ms | Admitted allocation median bytes | +|-------+------------------------+----------------------------------| +| Reference A | 30.341 | 39607464 | +| Candidate A | 25.613 | 39233704 | +| Candidate B | 32.500 | 39233064 | +| Reference B | 28.771 | 39607464 | + +Allocation decreased by 373,760 to 374,400 bytes, approximately 0.94 to 0.95 +percent. The 640-byte candidate spread is retained rather than rounded into +an exact invariant. CPU moves in opposite directions across candidate blocks, +so these measurements establish no CPU speedup. The unchanged sequence-only +control allocation medians were 1,593,120 bytes unmoved, 2,880,224 bytes for a +root view and 2,876,128 bytes for leaf views on all four runs. Their CPU drift +is also retained as evidence against a portable timing claim. + +** Remaining fusion boundary + +This removes one avoidable list copy but leaves the parts list consumed by +projection, the recognition-child list returned by projection, owned lexemes +and lossless nodes/tokens. Eliminating a parts list requires fusing its producer +with the projection's declared field emitter. A future lowering must preserve +validation of node kinds and parent bounds, ordering across optional token, +one-node and many-node operations, and externally supplied capture ownership. +It must also qualify interpreted and generated executors with the same full +products. Moving grammar complexity into a language-specific shortcut or +changing the public capture contract merely to bypass validation would not +address this shared producer/consumer boundary. + + +* Direct parts projection and bounded DSL expansion (2026-10-10) + +** Shared producer boundary + +The default unary =many= projection can consume the final child spine directly. +=result-projection-parts-field= derives its field from the admitted operation; +there is no author-facing mode or new grammar declaration. The Word engine's +existing scan loop uses that field to construct =recognition-child= values in +its private accumulator. =result-projection-build-parts= validates the proper +list, exact field and node type, then delegates catalog and parent-span checks +to =result-plan-node=. It publishes that same spine, removing the intermediate +node list, subsequent =map= and temporary capture pair/vector on this path. +Nested mixed projections and assignments retain their existing capture layout. + +The private projection record marks only engine-owned unary executors as +eligible. =result-plan-bind-projections= clears that eligibility on a cloned +plan: a supplied procedure is authoritative even if its declared recipe equals +the default recipe. For bound executors, the scanner emits ordinary nodes and +the new build boundary supplies the original capture frame. This preserves +POO override behavior; it does not introduce a legacy schema or language mode. +Generated bindings are tested through this authoritative frame path. This +experiment does not claim a streaming native generated executor or Rust AOT +allocation improvement. Public recipe digests and serialized ABI products +remain unchanged. + +[[https://arxiv.org/html/2304.05276][flap]], Section 2.8, explicitly generates code that removes statically known +information and avoids introducing higher-order operations in the parser core. +That motivates specializing the declared field in the producer rather than +assuming a generic callback is free. The retained code removes one semantic +intermediary; it is not DGNF lexer/parser fusion and does not inherit flap's +proof or measured speedups. SDK =2591dcd='s owned-list builders and private +reverse accumulators remain the source model for publication ownership. + +** Rejected callback experiment + +The first candidate invoked a projection emitter for every produced part. It +passed complete product equality but added closure and mutable accumulator +state. On the same complete-request control its allocation median was +39,427,464 bytes and CPU median 34.844 ms, versus reference A's 39,233,704 +bytes and 32.202 ms. All 23 admitted allocation intervals out of 31 agreed +with the increase. This callback candidate was removed, including its proposed +producer API. The replacement constructs children in the existing scan loop; +there is no per-part callback in that path. + +** Actual compiler timeout and DSL correction + +The first build was terminated by the unchanged five-second idle guard after +60.161 seconds. A diagnostic retry wrapped actual compiler pass entry and +completion, without changing ASTs or adding periodic output. It terminated +after 24.176 seconds in =apply-optimize-annotated= immediately after +=Optimizing syntax-case expansion=. Type collection, mutable type collection +and refinement completed in 0.059, 0.054 and 0.074 seconds respectively. +The failure was before GCC and was not resolved by switching compilers or +increasing either timeout. + +SDK =gerbil/compiler/optimize.ss= separates those passes; +=gerbil/compiler/optimize-ann.ss= sends =@syntax-case= annotations to its +specializing optimizer. =defresult-profile= repeated complete combination +patterns for eight legal clause sequences. The retained macro parses each +clause once and checks its tag sequence against those same eight combinations. +Duplicate clauses, reordered clauses, unknown clauses and malformed node +clauses are still rejected. Scanner expressions keep their original syntax +binding; quoted node, token and projection slots retain their existing shape. + +After factoring that syntax, annotation optimization completed in 0.015 +seconds. The direct-construction revision completed the same pass in 0.008 +seconds, and its two-module build completed in 49.225 seconds. Both modules +were compiled in one GXI invocation. The fixed GCC/Gerbil SDK, optimization +options, 3 GiB heap and 120-second total/five-second idle guards were unchanged. +Pass instrumentation was private diagnostic tooling, not a new repository +runner. These receipts establish guarded completion; they are not a portable +whole-build timing comparison under host contention. + +** Complete-request controls and contracts + +Reference A, direct candidate A, direct candidate B and reference B used +fresh processes, unchanged clients and the same existing private controller. +Loaded object identities were asserted: reference =shell-word~0.o3= and +=result-profile~0.o5=; direct =shell-word~0.o5= and =result-profile~0.o7=. +All numbered artifacts for those modules were removed before variant swaps, +and direct artifacts were restored before final tests. Each complete block +admitted 23 allocation intervals out of 31. GC-crossing intervals were +excluded; no forced GC or simultaneous local builds occurred during timing. + +| Block | Complete CPU median ms | Admitted allocation median bytes | +|-------+------------------------+----------------------------------| +| Reference A | 32.202 | 39233704 | +| Direct A | 28.583 | 39103304 | +| Direct B | 24.530 | 39103304 | +| Reference B | 23.199 | 39233064 | + +The reduction is 129,760 to 130,400 bytes, approximately 0.33 percent. CPU +drift again prevents a stable speedup claim. Full product equality and an +additional source-bound validator admitted four fixtures plus empty, malformed +and Unicode inputs. The sequence-only controls retained their reference +allocation values, so the improvement is confined to producer/projection +construction rather than a change in sequence materialization. + +Eight source suites passed 135 cases in 12.888 seconds with +=TEST-SUITE-OK modules=8=, =OK= and exit 0. New cases cover all eight legal +DSL combinations and rejected shapes, empty and 1,024-item parts, comparison +with the original capture API through default and generated bindings, exact +published-spine identity, wrong fields, foreign/escaping nodes, non-unary +recipes and custom executor authority. The initial new fixture omitted the +empty =tokens= clause required between =nodes= and =projections=. The macro +correctly rejected it; the fixture was fixed rather than widening the DSL. +That failed 8.693-second preparation is excluded from successful receipts. + +The complete-request benefit is modest because mixed nested captures and +observable lossless products still dominate allocation. Extending direct +construction to those captures requires an admitted producer plan that +preserves operation ordering and all field/span checks. This result closes +the unary boundary and compiler expansion issue, not the remaining gap to a +different parser or the full semantic-intermediary cost. + +* Literal-run invariant staging and bounded source admission + +The current Scheme design documents already describe checked public boundaries, +known private calls, and compile-time ownership. Reading the pinned Gerbil SDK +revision =2591dcd=, =src/gerbil/runtime/util.ss= (=string-index= and +=__string-index/char=), provides a concrete example: validate the source range +once, retain invariant input and endpoint, and advance only the cursor. This +change transfers that structure without copying unchecked primitives or type +assertions. The emitted Scheme for the previous parts scanner called the full +context matcher on every character; its context index was not hoisted there. + +The shared =part-context-literal-end= now retains the prepared ASCII vector and +Unicode character hash for the entire literal run. Empty initial buckets bypass +rule matching. Candidate initials call the named =match-part-rules= helper also +used by direct matching. Declaration order, prefix bounds, guard rejection, +shorter-prefix overrides, and Unicode keys therefore retain the same matching +algorithm. No extra index, callback, language declaration, compatibility path, +schema version, or FFI surface is introduced. + +Range admission requires exact integer endpoints with +=0 <= start <= end <= string-length=. Empty intervals return =end=. The previous +loop started at =start + 1= and stopped only at equality with =end= or a successful +match. With =start = end=, matching returned false beyond the endpoint and the +cursor kept increasing. This nontermination follows from the source; the old +infinite loop was not executed as a benchmark. Tests cover empty intervals, +negative, reversed, escaping and fractional ranges, independent direct-matcher +comparison at every start position, and truncated Unicode guards. + +** Native reference/candidate/candidate/reference control + +The same GCC-backed SDK, corpus, clients, 3 GiB heap, 120-second total budget +and five-second silence limit were used throughout. Each process verified the +actually admitted parts artifact: reference =part-profile~0.o1=, candidate +=part-profile~0.o3=. Complete owned artifact snapshots were exchanged because +the loader selects the last contiguous object suffix. Other native clients +were unchanged. Temporary control entries and snapshots are qualification +inputs, not repository interfaces. The final candidate build passed in +16.643 seconds. + +Each micro sample executes 100 native literal scans through identical source +orchestration, with 11 samples per block. ASCII and Unicode inputs each contain +10,000 characters; rejected escape guards use a 9,000-character input. Values +below are CPU medians in milliseconds, not complete-parser API latency. + +| Workload | Reference A | Candidate A | Candidate B | Reference B | +|----------+-------------+-------------+-------------+-------------| +| ASCII run | 15.801 | 2.428 | 2.430 | 15.018 | +| Unicode run | 55.701 | 27.921 | 26.373 | 52.412 | +| Guard rejection | 34.709 | 18.437 | 16.529 | 31.119 | +| Complete public request | 31.362 | 29.057 | 23.725 | 24.700 | + +All micro blocks admit 11 allocation samples with the same 8,576-byte median +from shared orchestration. The complete request uses 31 samples per block; +admitted allocation counts are respectively 23, 23, 24 and 24. Its median +allocation remains 39,103,304 bytes in every block. Complete products match +the existing Scheme reference values for four corpus fixtures, empty, malformed +and Unicode sources; source-bound checks also pass. Sequence-only allocation +controls retain their reference values. All four processes return exit 0 and +both literal and complete-product success receipts. + +The scan reduction repeats across both blocks: approximately sixfold for ASCII +and twofold for Unicode, with substantial improvement under rejected guards. +Complete-request CPU drifts across the surrounding references, so this control +does not establish a stable end-to-end speedup or close the Tree-sitter gap. +Allocation is unchanged. Remaining work must measure producer construction and +semantic intermediary costs independently before changing their representation. + +Nine source modules passed 151 cases in 17.334 seconds with +=TEST-SUITE-OK modules=9=, =OK= and exit 0. An initial invocation failed the +five-second silence guard during entry dependency admission, before any suite +ran. The successful invocation used the existing compiled-interface preloader +and real import tracing around the test entry; neither guard was widened. +That failed invocation is excluded from successful receipts. + +After rebasing onto the concurrent shared LR semantic-stack change =d69f015=, +the merged LR runtime was freshly compiled with the same GCC SDK and guards +in 48.724 seconds. Three integration modules (LR stack reductions, parts +profiles and Bash parsing) passed 47 cases in 11.500 seconds, with +=TEST-SUITE-OK modules=3=, =OK= and exit 0. This checks integration with that +revision; the performance table remains a control against the earlier +=0906fa3= client set and is not reclassified as merged-head timing. + diff --git a/docs/research/singleton-scanner-model-first.org b/docs/research/singleton-scanner-model-first.org new file mode 100644 index 00000000..5e76d857 --- /dev/null +++ b/docs/research/singleton-scanner-model-first.org @@ -0,0 +1,240 @@ +#+title: Rejected scanner experiments, capture costs and bounded test supervision + +* Starting point and evidence order + +This follow-up starts at =147716d62367d2b79c4f212ccbcd1d31ffc084f2=. +That commit already includes the selector-input, captured-owned-stack and goto +column improvements in [[file:selector-input-model-first.org][the preceding study]]. Any new CPU claim must compare +with this baseline, including its previous improvements. + +The order is paper and current-code review, explicit model obligations, +implementation, independent semantic controls, then complete-request CPU. +The workload and CPU admission policy are unchanged: 64 ASCII =a=1= records, +fresh capture-enabled sessions, full scanner/LR execution, publication, +artifact equality and source validation inside the request clock. Recognition +projection is independently checked before timing. Five groups of 32 alternating +legs, 256 requests each, cover 122,880 timed requests. Both reference/reference +control phases must stay in =0.90..1.10= and candidate P90 must be below =0.95=. +The CPU fence remains 90 seconds total and five seconds idle. Natural GC is +included; startup, driver IPC and setup remain outside this prepared-request clock. + +* Papers, retained functions and the actual producer + +[[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]] discusses binding-time improvements, duplicating +procedures for distinct contexts and eliminating unnecessary list computation. +[[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] motivates preserving sufficient +context for reuse rather than treating all retained work as free. These motivate +examining actual dataflow; neither paper proves a CPU result for this engine. + +The existing POO preparation protocol resolves backend factories before requests. +The pinned LeanPoo Function/Requirements and representation-bridge work informs +that separation: grammar choices belong to preparation, while invocation inputs +and publication obligations remain request-local. The existing library pin and +Lean toolchain are retained; no model version or public language DSL is changed. + +The current =make-ranked-regular-scanner= already prepares mode-specific rule +winners. Its nonnumeric states are monotone subsets. When the active subset +contains one bit, every accepted next character keeps that same bit. Repeated +state-mask intersection and winner selection are therefore unnecessary for that +extent. Numeric pending/rollback states do not satisfy this premise. + +* Four layers and obligations + +| Layer | Obligation | Limit | +|-------+------------+-------| +| Paper and code | Identify the paid transitions and actual enabled paths | A plausible local improvement is not complete-request benefit | +| [[file:../../formal/ParserEngine/Preparation.lean][Lean preparation]] | Singleton state elimination, exact prepared membership and start-query transport | Immutable classifier/topology premises must hold; live priorities remain query inputs | +| [[file:../../formal/ParserEngine/Recognition.lean][Lean representation]] and [[file:../../formal/ParserEngine/Reduction.lean][ordered kernels]] | Retain the contracts of rejected experiments | These are not shipped runtime optimizations | +| [[file:../../formal/quint/OwnedTerminal.qnt][Quint publication]] | Reject reconstructing a published wrapper after an observer changes it | A bounded protocol model, not native heap refinement | +| Scheme controls and Benchmark | Exact ranking, source offsets, Unicode, fresh results and whole requests | Native correctness and CPU require executable evidence | + +=singleton_state_elimination= proves that a permanently active singleton state's +accepted prefix equals the prepared predicate's prefix. =prepared_class_membership= +requires exact membership equality; a first-character approximation is insufficient. +The rejected prototype admitted only masks =1,2,4,8,16=. An ASCII initial mask with +multiple bits retains the original loop, and numeric states retain rollback. + +* Scanner hypotheses and withdrawal + +The singleton prototype prepared shared ASCII continuation classes and bounded +mode-specific start directories in the engine. Numeric rollback, Unicode and +mixed subsets retained checked paths. The product prototype added prepared +literal/regular starts and skipped an impossible literal probe when the first +trie edge was absent. Candidate identities remained prepared metadata; mutable +ranks and winner payloads remained live query inputs. + +=prepared_start_query_exact= preserves an arbitrary live consumer of prepared +metadata. =absent_literal_probe= states the missing-match premise explicitly. +Neither theorem proves native closure layout, actual producer admission or CPU +benefit. Two retained Scheme controls constrain live winner payloads, fresh result +cells and live literal/regular priority comparison; the original scanner also +passes these contracts. + +Both implementations were withdrawn after the complete-request gate failed. +The production scanner, strict source organization guard, language DSL, schema, +C ABI and Rust representation remain at the starting implementation. The models +record hypotheses and their necessary premises, not deployed optimizations. + +* Rejected experiments and enabled-path audit + +All six experiments used the same starting commit and complete-request +policy. All six runtime implementations were withdrawn. Candidate P90 exceeded +=0.95= in every row; favorable medians are not accepted. The first four also +failed the after-control upper limit. The singleton-only run failed control lower +limits. The final product-start run had stable controls but still failed the +candidate gate; its small median improvement is not admitted. + +| Experiment | Candidate P50 | Candidate P90 | After-control P90 | +|------------+---------------+---------------+-------------------| +| Unary metadata classification | 0.994910 | 1.034958 | 1.128814 | +| Certified losing-fallback pruning | 0.973123 | 1.019841 | 1.119237 | +| Prepared fixed-arity semantic kernels | 0.997544 | 1.044800 | 1.136836 | +| Deferred shifted-terminal wrappers | 1.009932 | 1.043904 | 1.103209 | +| Singleton regular scanner alone | 0.999836 | 1.049317 | 1.035311 | +| Literal/regular prepared product starts | 0.980010 | 1.006539 | 1.084313 | + +The classification direction already had a historical negative receipt; the new +starting representation did not establish a different outcome. Fallback pruning +had an admitted-witness proof and direct probe-elimination controls, but directed +64-record modes already exclude most fallback rules. Source-index construction +is disabled by default and cannot explain this request's CPU. Fixed-arity +unrolling alone did not establish savings. Deferred wrappers passed mixed-slot +and snapshot-observer controls but did not establish complete-request savings. +Captured grammar fragments retain semantic values; avoiding an intermediate +operation does not necessarily remove its retained publication work. + +The rejected wrapper study exposed a necessary publication constraint: once a +wrapper is visible through a checkpoint, its observer changes must survive later +reductions. The retained Quint counterexample rejects a fresh unfielded rebox. +The runtime currently keeps its existing canonical semantic representation. + +* macOS CI root cause and supervision repair + +At this starting commit, run =38718426346= passed Ubuntu semantics, corpus, FFI +and generation. macOS semantics failed at the canonical LR(1) reference owner: +the unchanged 90-second owner budget expired in its first exhaustive Case. +The last successful assertions belonged to a three-node partition comparison. + +The supervisor spawned a new Scheme reader thread for every test output line. +The exhaustive oracle also performed four separately logged checks per graph. +The repair uses one reader and one supervisor per owner. The reader drains the +existing pipe directly; the supervisor joins it under the remaining idle and +total deadlines, consulting timestamps written only after actual output reads. +There are no per-line thread creations or mailbox acknowledgement round trips. The five-second idle fence, 90-second owner budget, +worker concurrency, assertion content and failure propagation are retained. +A graph comparison now carries all four facts in one tuple: partition vector, +group count, unchanged input and examined-work bound. This removes redundant +reporting transactions without removing a graph or semantic assertion. + +Only after all workers join successfully does the existing native child-exit +helper terminate the harness. Earlier successful assertions followed by finalizer +idle timeouts remain rejected receipts. The canonical owner completed all 21 +Cases locally in 39.435633 seconds, exit zero, with no timeout or missing marker. +This is a local repair receipt. The repair was pushed as +=75c5742c11815ac133022f899f55bde22104a086=; exact-head CI is a separate gate. +All seven worker lifecycle controls also passed, covering shared runtime, +affinity, failure propagation, silence, duplicate ownership and both reexports +and suite-object ownership. + +* Recorded-query cost diagnosis + +[[file:../../t/benchmarks/capture-prefix/phase-cost.ss][The Scheme diagnostic]] records actual lexical query modes and both character +and byte positions from the complete request. It checks each token and exclusive +character boundary before timing, then replays the exact lexical query trace and +LR input sequence. Captured and uncaptured LR results must agree. Independent +recognition projection and the complete published artifact must agree as well. + +The actual trace has 256 tokens, all significant, and 322 reductions. The earlier +study's prose incorrectly assumed only 192 significant shifts; that count is +corrected explicitly there. The actual workload, artifact, frontier fingerprints +and previous CPU measurements are unchanged. + +Five samples per scope use 256 calls, after 256 warmups per scope, alternating +scope order. The existing ASP-backed Scheme Benchmark CPU/GC sampling primitive +is reused directly. No forced GC, changed CPU gate, source substitution or +instrumentation in production is introduced. + +| Scope | Median CPU ms per request | +|-------+---------------------------| +| Complete captured request | 0.221035 | +| Recorded lexical queries | 0.042395 | +| Captured LR replay | 0.137559 | +| Uncaptured LR replay | 0.024125 | +| Publication | 0.007949 | +| Source validation | 0.019055 | + +This receipt completed in 10.213509 seconds, exit zero. The replays have different +prepared inputs and retained heaps. These numbers are neither additive exclusive +costs nor matched speedup evidence. In particular, replayed lexical queries omit +frontend bookkeeping, and LR replays reuse already allocated input tokens. + +One captured-LR batch spent 25.928 ms of its 35.981 ms CPU in GC; another spent +25.616 ms of 35.215 ms. No-GC captured batches took 11.833 and 9.532 ms. No-GC +allocation observations were about 257 KB per captured request and 87 KB per +uncaptured replay. The latest whole-VM live-byte snapshot was about 121 MB; this +includes loaded compiler interfaces and is not parser-owned live memory. +Negative allocation deltas across GC are not admitted as allocated bytes. + +The final topology-checked diagnostic also passed in 10.659116 seconds, exit zero. +It found 322 fragments, 256 terminal leaves and eight sparse snapshots, each +with six states and five semantic values. Thus this trace does not support an +explanation based on large snapshot stack depths. Its complete-request and +captured-LR CPU medians were 0.229488 and 0.135379 ms respectively; this is a +second cost observation, not a candidate/reference comparison. + +The next model obligation is allocation and retention ownership across captured +fragments, semantic sequences and published snapshots. CPU cost must include +natural GC and the retained live set. Merely specializing small scanner kernels +has not addressed that cost. The optimized Scheme lowering already uses direct +=##structure= fragment allocation, so replacing a generic constructor call is +not an available explanation for savings here. A runtime-only benchmark loader +could alter the live heap, but that would be a separate harness comparison, +not evidence of a parser algorithm improvement. + +* Reproduction + +#+begin_src sh +just formal +just formal-protocol +just diagnose-capture-cost +just prepare-capture-reference 147716d62367d2b79c4f212ccbcd1d31ffc084f2 .data/capture-study/reference selector-input +just benchmark-capture-prefix .data/capture-study/reference/lib selector-input +#+end_src + +The unchanged reference builder freezes the eight selected LR request owners. +Scanner owners and other dependencies are shared in this restored production +state; comparing a future scanner change requires explicitly extending and +verifying the frozen owner closure first. Native-image checks and full +artifact/frontier fingerprints precede timing. At the restored starting runtime, +reference versus current is a control comparison, not a new candidate proof. + +* Local native-image controls + +The local multi-owner parser batch initially crashed on both a candidate and the +restored baseline. A Darwin crash frame identified the old native +=ranked-publication~0.o2= test helper. A production clean build covered 241 modules +but excluded test helpers. Rebuilding the ranked/action-selection helpers and +record fixture with the current SDK produced a successful same-concurrency batch: +30 Cases in 14.912334 seconds, exit zero. This is evidence for the tested native +closure, not a claim about a Scheme compiler or GC defect. Earlier crashes, +startup timeouts, a malformed new control and finalizer timeouts remain rejected. + +The product-start extension then passed 31 Cases across the same six owners, +including every regular-rule subset, independent literal matching, full TLA+ +requests, lexical modes, Unicode and the strict source organization guard. +That local batch completed in 21.988767 seconds with exit zero and no missing +marker. Test compilation and semantic controls do not establish new CPU benefit. + +The restored scanner then passed its eight Cases, including both new live-payload +controls, in 10.269232 seconds, exit zero. The final Lean build completed all 56 +jobs, the Quint suite passed 114 directed histories and 12 seeded invariant runs, +and the source-binding unit control passed. These qualify their respective +layers; none is a substitute for the rejected complete-request CPU gate. + +* Current conclusion + +The CI repair is published. Lean and Quint obligations, live-payload controls and +the recorded-query diagnostic are executable research progress. No new engine +CPU improvement passed the unchanged 64-record complete-request gate in this +study. The six rejected implementations must not be described as delivered +optimizations or substituted for the earlier baseline's separate CPU receipt. diff --git a/docs/research/source-edit-cost-contract.org b/docs/research/source-edit-cost-contract.org new file mode 100644 index 00000000..5515305c --- /dev/null +++ b/docs/research/source-edit-cost-contract.org @@ -0,0 +1,338 @@ +#+title: Complete Source Edit Cost Contract + +* One producer and one consumer path + +Source editing uses the prepared Scheme descriptor, its private scanner history, +full command recognition, canonical ParseArtifact publication, GPA1 encoding +and the existing language-handle C API. The benchmark invokes the actual C API, +reads the result back into a caller buffer and releases the C allocation. +It does not introduce another language DSL or an alternate parser. + +FFI identities are declared in =src/ffi/schema.ss=. Scheme codec constants and +the pure C ABI-version query derive from that table. Native module namespaces +follow file paths; versions belong to schema values. The build-only Rust source +generator is a distinct service, not a second parsing entry by builtin name. + +* Why the author interface is preferable + +Question 1: Why does this preserve Scheme's syntax/rules and POO advantages? + +#+begin_src scheme +;; grammar.ss still describes syntax; parser.ss owns this prepared descriptor. +(def previous (parse-source-language/session bash-source-language old-text)) +(def next (parse-source-language/session bash-source-language edited-text previous)) +(check (source-language-session-artifact next) + => (parse-source-language bash-source-language edited-text)) +;; Diagnostics belong to the engine and do not add cache clauses to deflanguage. +(parameterize ((current-parser-cost-observer collect-stage)) + (parse-source-language/session bash-source-language edited-text previous)) +#+end_src + +Hygienic declarations and POO composition select the admitted language product. +The engine owns history, admission, allocation and publication. Ordinary values +remain sufficient for callers that do not extend the declaration. Cost hooks +are an optional request-local diagnostic service, not grammar syntax. + +* Why the measurement is justified + +Question 2: What makes this a complete-call comparison rather than a token count? + +#+begin_src scheme +;; The reference does not consume edit history. Both native lanes must publish +;; every byte of that reference, including status, digests, IDs and spans. +(def expected (abi-parse-binary-payload language edited-text)) +(call! fresh-handle empty-bytes empty-output) +(call! session-handle old-bytes old-output) +(def fresh-receipt (measure-call fresh-handle edited-bytes fresh-output)) +(def session-receipt (measure-call session-handle edited-bytes session-output)) +(check fresh-output => expected) +(check session-output => expected) +#+end_src + +Priming and validation are excluded from both timed edit calls. Sample order +alternates. Two warmups precede each case; no forced GC occurs. All measured +calls keep natural GC, and each wall-P95 observation carries its own CPU, +allocation and GC counters. Full payload parity is required on every sample. +These are edits from an established base, not a throughput measure of priming +plus editing or an application actor queue. + +* Phase boundary and memory limits + +A separate instrumented call records UTF-8 decoding, source copy/preparation, +scanning, command recognition, artifact publication/validation, GPA1 encoding +and C input/result copying. The enclosing full-call observation includes C +readback/release and remaining glue. Observations are inclusive and can nest; +they are not summed or subtracted to invent an exclusive decomposition. +Exceptions produce incomplete-stage receipts and preserve their conditions. +Lexical rejection may have no command-recognition stage. + +Uninstrumented latency samples have the observer disabled. The disabled macro +branch does not allocate a measuring thunk. Both lanes still pay the same +parameter checks. Instrumented counters include observer bookkeeping and +Scheme process-statistics overhead. Sub-microsecond C copies may be below clock +resolution. Gambit allocation counters do not measure malloc payload bytes. + +Receipts report actual source byte length, history token-record count and GPA1 +payload byte length. These are retained-data dimensions, not exact retained +heap bytes: context records, token strings, headers and allocator overhead are +not quantified. No forced-GC subtraction is used to attribute VM live memory +to a handle. Lexical failure retains no reusable source/token snapshot. + +* Initial local receipt and next decision + +At 20 samples and a 128-line tail, the UTF-8 prefix control passed complete +GPA1 parity: fresh P50/P95 were 3.066/4.178ms, session 2.553/2.720ms. The +session scanned 1 token and reused 518, retaining 519 token records and 1297 +source bytes; its payload was 93416 bytes. This is one local observation, +not a cross-platform or stable end-to-end performance claim. + +One separate session observation reported scan 0.128ms, recognition 0.632ms, +publication 0.094ms, validation 0.142ms and encoding 1.449ms, with a complete +call of 2.546ms. Encoding is therefore the first candidate for further scale +controls; subtree caching is not preselected. Lexical failure allocated the +same 28104 Scheme bytes in both uninstrumented lanes and supplied no reusable +history. Preserve this negative control. + +All five cases passed: UTF-8 prefix, quoted marker, ordered marker FIFO, +grammar rejection and lexical rejection. The 20-sample benchmark exited zero +with =SOURCE-EDIT-BENCHMARK-OK= in 10.78 seconds. This receipt qualifies the +local call path; clean Linux/macOS qualification belongs to exact-head CI. + +The scanner continuation invariant remains the engineering argument in +[[file:scheme-source-edit-sessions.org][Source edit sessions]]. It does not implement or inherit the optimal node-reuse +bound from Wagner and Graham's incremental parsing work. A later encoding +refactor must preserve the complete GPA1 format, publication ordering and the +same reference controls before claiming a gain. + +A later concurrent 8x32 qualification completed all parity/stage controls and +printed the final marker, but exceeded the 5-second inactivity fence during +implicit native module teardown (22.16 seconds total, exit 124). Preserve that +failed receipt. The disposable benchmark child now uses the existing explicit +child-exit owner, after result release, both handle releases and baseline count +validation. Native VM teardown remains qualified by standalone host tests; +it is not part of an edit's cost. The timeout fence is unchanged. + +After the explicit exit change, the CI-sized 8x32 control is qualified separately +from the initial timing receipt. Focused engine/FFI/cost tests pass 30 cases; +the migrated codec suite passes six further cases. The real handle-based C +probe passes 1000 calls, UTF-8/error/replacement checks and three identical Rust +source generations. The Rust/C/Scheme edit consumer also passes its nine edits, +retained-result checks and final ownership marker. C headers compile with +strict warnings, and the renamed Rust performance scenarios compile as tests. + +* Callback lifetime and encoder allocation closure (2026-10-09) + +PR =5bcc3f6= passed Rust but failed the Ubuntu Source benchmark. Its final +observed lexical-rejection call read an output length of =548840451425= for a +128-byte result. The C fixture retained raw Scheme object references across an +allocating Scheme callback. The corrected caller owns its input and result in C, +then borrows the relocated Scheme output in a new readback call. Dynamic cleanup +releases the result, input copy and private call record on success or failure. +The [[https://gambitscheme.org/4.9.8/manual/][Gambit C-interface manual]] states that pointer-type default release does +not deallocate the pointer: this fixture explicitly owns that cleanup. + +A separate semantic child runs 80 large real parses across all five fixtures. +It requires at least one natural collection during measured calls, every GPA1 +byte equal to fresh execution, and restored handle count. Local qualification +recorded seven collections, all five cases and exit 0 in 5.39 seconds. There is +no forced collection or GC-policy change. Timing children remain separate. + +The complete pinned-SDK package build passed in 69.16 seconds with four compiler +workers and actual import/LR progress. C fixtures built in 7.78 seconds. The +installed newer SDK rejected old dependency binaries; that attempt is retained +as an ABI mismatch, not parser latency. Recursive interface preloading hit the +unchanged silence fence. Importing the compiled benchmark entry with existing +interface diagnostics completed its 20-sample baseline in 3.63 seconds. + +The encoder now counts events once and uses a row counter, removing the allocated +event-sized index list. Before/after use the same corrected C caller, 20 samples, +128 tail lines and natural GC. All ten lane/case pairs pass full payload equality. + +| UTF-8 prefix lane | Before no-GC allocation median | After | Reduction | +|-------------------+--------------------------------+-------+-----------| +| Fresh | 3341640 bytes | 3154968 bytes | 186672 bytes (5.59 percent) | +| Session | 2737592 bytes | 2550920 bytes | 186672 bytes (6.82 percent) | + +A separate observed encoding call drops from 281368 to 94696 bytes; the payload +remains 93416 bytes. This is a local allocation result. It is not a general +latency win: quoted-marker session P95 worsened from 2.782 to 25.475 ms; that +actual observation contained one GC taking 22.930 ms. It remains in the receipt. +No retry selection or GC-free filtering of latency observations is permitted. + +That GC observation also had a negative raw allocation-counter delta. Gambit's +[[https://raw.githubusercontent.com/gambit/gambit/master/lib/_kernel.scm][process statistics implementation]] obtains index 7 from its processor +allocation counter; the [[https://raw.githubusercontent.com/gambit/gambit/master/lib/mem.c][memory implementation]] includes occupied still-object +bytes. This is not a C malloc or retained-heap measurement. Raw signed deltas +remain =allocation-counter-delta=. =allocated-bytes= is admitted only for +nonnegative, zero-GC intervals; other intervals report false. Allocation +summaries state their admitted observation count. Wall/CPU/GC and wall-ranked +P95 observations retain every sample, including the negative-counter case. + +The copied C input, private call record, result allocation and readback/release +are included in complete-call wall/CPU measurements. Scheme counters do not +measure those C allocations. Delivered Linux/macOS CI remains a separate gate. + +Final local qualification includes the three cost-admission cases (2.05 seconds), +twelve independent native language ABI cases (10.24 seconds), all with propagated +exit 0 and final =OK=. The final GC-safe encoder passes the large semantic child +with eight natural collections (3.71 seconds), and the exact CI 8-sample/32-line +benchmark workload (3.06 seconds). Allocation observations are processor counter +windows and CPU observations are process counters; optional request-local hooks +do not isolate unrelated concurrent work. This benchmark uses one owner thread. + +* Matched LR session responsibilities + +The common Source-language C-ABI study above and the LR topology workload have +different execution owners. Their timings must not be combined. The LR workload +=t/benchmarks/incremental-session/benchmark.ss= now reports three fresh controls +alongside the cached edit: an artifact-only parse, a checkpoint session without +recognition capture, and a fresh session with the same capture option as the +cached history. The checkpoint-only reference is reported when capture is +requested; it is =#f= otherwise rather than a duplicate timed control. + +Question 1: Why does this preserve Scheme's engine and authoring boundaries? + +#+begin_src scheme +;; One admitted ParserMachine; capture remains an engine capability. +(def artifact-only (parse changed)) +(def checkpoints-only (make-incremental-session machine changed #f)) +(def fresh-session (make-incremental-session machine changed capture?)) +(check (incremental-session-artifact fresh-session) => artifact-only) +;; Capture additionally publishes a recognition root and lexical checkpoint data. +(when capture? + (check (incremental-session-project-artifact fresh-session) => artifact-only)) +#+end_src + +The benchmark uses the existing Scheme APIs and optional capture capability. +It introduces no grammar clause, callback protocol or language metadata. The +same source, machine and backend settings own each control. The source-index +and recognition projection of the fresh captured session are audited before +timing, just as they are for the cached and inverse-edit products. + +Question 2: What makes the comparison scientifically justified? + +The former artifact-only/cached timing answered different questions. In +=parse-source/with-capture=, the =fresh?= guard requires absent capture, +checkpoint, retained prefix and reuse callbacks before trying =direct-source=. +Fresh session construction passes the checkpoint capture callback and therefore +uses the directed checkpoint path. Recognition capture adds another obligation, +but it does not alone explain the complete path difference. Prefix snapshots +in =lr-checkpoint-prefix-snapshot= retain persistent stacks; source inspection +does not support assuming that every snapshot copies a whole history. + +The updated workload retains the artifact-only numbers and adds +=fresh-checkpoint-cpu-samples-ms= / =fresh-checkpoint-cpu-median-ms= and +=fresh-session-cpu-samples-ms= / =fresh-session-cpu-median-ms=. Every row admits +=fresh-session-artifact-equal?= only after comparing the fresh session artifact +to the independent parse. Existing cached artifact, captured projection and +inverse-edit controls remain required. + +With 64 HCL sibling units, recognition capture enabled, three CPU samples per +path and the default mechanism settings, the source workload observed: + +| Edit | Artifact-only median ms | Checkpoint-only median ms | Fresh captured session median ms | Cached captured edit median ms | +|------+-------------------------+---------------------------+----------------------------------+--------------------------------| +| First insertion | 1.256 | 11.841 | 15.250 | 8.605 | +| First deletion | 1.201 | 11.200 | 14.735 | 8.234 | +| Middle insertion | 1.247 | 11.281 | 15.499 | 6.621 | +| Middle deletion | 1.295 | 11.480 | 14.938 | 5.965 | +| Last insertion | 1.258 | 11.202 | 15.046 | 4.426 | +| Last deletion | 1.207 | 10.781 | 14.675 | 3.750 | + +The cached path was slower than artifact-only parsing and faster than constructing +a new session with matched capture responsibilities in this workload. The +checkpoint-only reference shows a substantial execution-path cost even without +recognition capture. These source observations neither isolate the costs of +individual inner phases nor admit a native speedup, P50/P95 claim or subtraction +of medians as an additive phase profile. + +Five independent source workload invocations also compared the default settings, +fragment reuse disabled, probe reuse disabled, lexical-plan reuse enabled and +source indexing enabled. For first insertion, cached CPU medians were respectively +9.280, 13.600, 8.857, 8.341 and 11.845 ms. Each invocation exercised all six edit +positions/operations and complete/inverse artifact equivalence. Disabling +fragment reuse increased work in this observation. The small probe/plan changes +are not stable performance admissions, and the indexing observation does not +justify globally disabling a capability. The existing defaults are preserved. + +The next optimization gate is an inner-phase profile of the checkpointed driver, +fragment certification and complete publication under matched session settings, +followed by independently compiled native before/after products. It should not +start from the premise that capture or Gambit GC is defective. These controls +first establish which products and executor paths are actually being compared. + +The revised controls completed six edits each for capture enabled, capture +disabled and the combined event-program/sequence-fusion/lexical-plan/source-index +settings, with inverse-edit parity and admitted fresh-session artifacts. The +captured fresh session also passed recognition projection and source-index +preflight; the checkpoint-only reference passed independent artifact preflight. + +* Lazy phase dispatch in the checkpointed driver + +The checkpointed driver invokes =call-with-parser-observed-phase= for lexical +analysis, trivia filtering and LR execution. The former procedure interface +received a zero-argument thunk even when its policy was =#f=. The installed SDK +binary resolves =:std/list/list-builder= to its installed interface; the matching +source demonstrates hygienic =defrules= patterns with direct invocation and a +first-class identifier expansion. The existing optional cost macro in +=parse-cost.ss= similarly places its delayed body inside the enabled branch. +This is an engine control-flow mechanism, not a replacement timer or POO policy. + +Question 1: Why does the Scheme design improve this boundary? + +#+begin_src scheme +;; A zero-argument literal body is lowered at its existing call site. +(call-with-parser-observed-phase policy 'lr-execution + (lambda () (lr-checkpoint-feed checkpoint input-token policy))) +;; The macro evaluates policy and phase once. With #f, it executes the body +;; directly; an enabled policy still delegates its thunk to POO Flow tracing. +;; A computed thunk or first-class use retains ordinary procedure dispatch. +(let (invoke call-with-parser-observed-phase) + (invoke policy 'lr-execution computed-thunk)) +#+end_src + +Hygienic syntax can distinguish the literal delayed-body call from a computed +callback without adding a language declaration or runtime recognition opcode. +The public phase adapter retains first-class procedure use. Enabled execution +continues to use =call-with-poo-flow-debug-trace= and its contracts; multiple +return values and exceptions remain owned by the original body. + +Question 2: What makes the lowering scientifically justified? + +The disabled branch must evaluate the policy, phase and body exactly once and +return the body's values. Its literal closure has no caller-visible identity: +it is immediately invoked by the disabled adapter. The dedicated controls check +effect counts, two return values, original exception irritants, computed thunks +and first-class calls. The full observed openCypher fixture checks enabled phase +records and complete parsing. No procedure-taking call is rewritten as a +language-specific fast path. + +The =phase-dispatch= mode in the existing incremental benchmark compares the +first-class procedure and literal-body forms in one SDK process, ABBA order, +10,000 calls per observation, explicit collection before each observation and +an independently checked sum. Procedure dispatch observed 1.780/1.748 ms CPU +and a 4,641,232 allocation-counter delta; direct expansion observed 1.624/1.612 +ms and 3,361,232. All observations had zero collections, and the run ended with +=PHASE-DISPATCH-OK= and exit zero. This measures the complete component dispatch +path, not an exact heap size for a thunk or independently compiled native parser +performance. Native optimization may already remove some of that overhead. + +The final source change passed 57 Cases covering the phase adapter, parser +runtime, recovery/incremental and GQL parsing, with final =HARNESS-OK= and =OK=, +no Case/Harness errors and exit zero. The matched HCL capture workload again +passed all six complete-artifact and inverse-edit controls. Its CPU observations +were variable: first insertion observed 11.249 ms checkpoint-only, 19.120 ms +fresh captured session and 9.353 ms cached edit. The prior matched source run +was not ordered as a before/after ABBA pair, so these observations do not admit a +stable complete-parser improvement. The remaining checkpoint-driver cost is +still an open phase-profile and native before/after qualification task. + +The SDK compiler optimized and expanded =observability.ss=, =parser.ss= and +=lr-parser.ss= with =invoke-gsc: #f=, each ending with =SDK-PHASE-OPTIMIZE-OK= +and process exit zero. This checks macro binding and optimized module expansion; +native C compilation, linking and exact-head CI remain separate gates. + +After rebasing onto =b4e5a6b= (artifact/source-identity admission fusion), the +combined source run passed 80 Cases with final =OK= and no Case/Harness errors. +The six HCL capture topology controls and complete Source-session workload also +passed; the latter ended with =SOURCE-SESSION-PARSER-OK= and exit zero. diff --git a/docs/research/source-event-fold-ownership.org b/docs/research/source-event-fold-ownership.org new file mode 100644 index 00000000..17cfa1f0 --- /dev/null +++ b/docs/research/source-event-fold-ownership.org @@ -0,0 +1,48 @@ +#+title: Source event Fold ownership and local qualification + +* Engine ownership + +The engine owns bounded byte predicates, source offsets, execution frames, +helper/join scope and event accumulation. Language declarations own markers, +node kinds and semantic composition. Public POO grammar/fragment values remain +the extension surface; named pure functions lower literal patterns into the +existing event algebra rather than introducing another matcher or evaluator. + +=source-pattern-funs.ss= provides exact UTF-8 byte patterns, byte-counted ends, +checked offset advancement and explicitly ASCII-only case folding. One-byte +case-insensitive patterns lower to a predicate, not an invalid unary =and=. +The parser interface and native event support export this shared projection. +SourceReferenceScan also consumes it; its duplicate prefix/offset builders are +removed. Application-specific cursor names and rules stay with the language. + +* Request-local execution + +Private native-table slots replace persistent state-list reconstruction. +Caller declarations are not mutated, helper frames are independent and join +temporaries are removed on exit. Static IR admission retains its declaration +representation; no execution compatibility path is retained. + +Statements thread one persistent reversed event chain. Mutation-only or empty +emissions preserve that chain; loops, helpers and joins do not reverse/copy +each nested result. Forward order is restored at output boundaries. The public +event format and native ABI are unchanged. This follows the official Gerbil +EBNF lexer's local accumulator principle without introducing a shared mutable +list-builder queue across continuation re-entry. + +* Evidence boundary + +The isolated source candidate rebases on PR #13 head =2c38e9f0=. Its complete +184-module SDK and phase/public imports pass. All 56 linked native Fold cases +pass, including UTF-8 spans, single-byte ASCII folding, invalid declaration +rejection, nested event ordering and independent helper/join scope. +The Org consumer's 340-module AOT program =54e30a63= passes source-root checks, +178 cases in 12 native suites and an independent exit-42 failure control. +Both existing maturin/CFFI runtime features pass all 25 Python cases with the +unchanged 5s check. This does not establish normal dependency-pin admission, +large concurrent consumer qualification or cross-platform CI. + +Matched 20-sample dense and entity controls preserve source/event/contract +identity and pass the unchanged 100ms envelope. Dense p95 is 56.839ms versus +56.213ms; entity p95 is 3.439ms in both lanes. This convergence is not promoted +as a latency win. Prior state/event-chain experiments have separate receipts; +one timing sample's allocation delta is not a universal allocation guarantee. diff --git a/docs/research/sparse-capture-model-first.org b/docs/research/sparse-capture-model-first.org new file mode 100644 index 00000000..f885a8ed --- /dev/null +++ b/docs/research/sparse-capture-model-first.org @@ -0,0 +1,185 @@ +#+title: Model-first sparse capture in the complete LR source request + +* Paper and current-code foundation + +Sperber and Thiemann's [[https://www.deinprogramm.de/sperber/papers/lr-essence-pepm.pdf][The Essence of LR Parsing]], section 3.1.3, +examines constructor/selector work and preservation of evaluation order. +Wagner and Graham's [[https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf][Efficient and Flexible Incremental Parsing]] motivates +retaining sufficient context for valid reuse and accounting for its maintenance. +Neither establishes a CPU result for this engine. We derive the following +obligation from current code before selecting an implementation. + +At =54746a3b1617639fac05805f6ac9aae4a1861555=, a fresh capture-enabled source +request enters the LR executor once per significant token. Every successful +shift constructs an immutable checkpoint; the source owner retains only those +whose shift count is a multiple of the requested spacing. Trivia advances the +source position without adding a significant shift. Reductions may occur +between shifts, and the lexical mode follows the current LR state. + +The opportunity is therefore to keep the ordered scanner/LR execution open and +construct checkpoints only at selected post-shift boundaries. It requires exact +retained state, values, recognition stack, action and shift counts, lexical modes, +source token counts and byte offsets. It cannot change scanning, reduction order, +semantic callbacks, fork handling or admission of resumed/reused input. + +The earlier [[file:capture-metadata-model-first.org][unary metadata experiment]] failed the complete CPU gate and was +reverted. Its model and negative observation remain separate from this result. + +* Four layers and their evidence boundaries + +| Layer | Responsibility | Evidence and boundary | +|-------+----------------+-----------------------| +| Papers/current Scheme | Identify paid work and retained semantics | Per-token LR entry and discarded checkpoint construction are concrete code costs | +| [[file:../../formal/ParserEngine/Capture.lean][Lean Capture]] | Pure ordered execution and selected immutable frontiers | =fused_driver_order= preserves ordered transitions; =sparse_frontiers_exact= preserves selected frames; =sparse_frontiers_not_more= bounds retained frame count | +| [[file:../../formal/quint/SparseCapture.qnt][Quint SparseCapture]] | Source/shift lifecycle around trivia, reductions, completion, fork and failure | Ten directed histories and 1,000 sampled 20-step histories with seed 1; =SparseSafety= compares sparse publication with dense selection | +| Scheme implementation/Benchmark | Establish representation equivalence and actual whole-request CPU | Exact dense-control comparisons and matched native-image CPU qualification | + +Lean frames are abstract immutable values; Scheme must establish the complete +post-shift frontier and callback order. The fusion theorem assumes the same +ordered transitions, rather than proving the scanner or LR table correct. +Quint does not replace that pure theorem, implement an LR oracle or establish +unbounded liveness. The evidence collector binds the model observations to the +actual source owners and controls; this binding is traceability, not refinement. + +For 64 ASCII =a=1= records at spacing 32, the workload has 192 significant +shifts, 256 source tokens and 322 reductions. The old driver enters LR 193 times +including final EOF completion, and constructs the initial checkpoint plus 192 +post-shift checkpoints. The new source driver enters LR once and constructs the +initial checkpoint plus six selected checkpoints: 193 becomes 7. These are +logical execution counts, not CPU instructions or an independent speedup claim. + +* Engine implementation + +[[file:../../src/runtime/lr-parser/checkpoint.ss][The shared LR driver]] invokes sparse capture after the shift observer has +published the consumed token, using the same states, values, action count, +shift count and recognition stack as the old returned checkpoint. Persistent +stack ownership retains immutable captured prefixes. Ordinary shifts do not +construct a checkpoint. [[file:../../src/runtime/lr-parser/transfer.ss][The existing drive interface]] validates positive +capture spacing and forwards this capability. + +[[file:../../src/runtime/parser.ss][The source owner]] records trivia immediately and significant tokens on shift; +the capture callback then sees their exact token count and byte end. Scanner +observations keep their original lexical mode and order. Completion publishes +the complete source tokens and modes. Selective GLR keeps the deterministic +prefix and existing suffix completion; resumed, reused, observed and layout +requests retain their existing admission paths. No language grammar, FFI event +encoding or token mutation protocol changes. + +* Semantic qualification + +[[file:../../t/sparse-capture-test.ss][Sparse capture controls]] compare fresh streamed capture with dense resume from +an explicit initial checkpoint. The latter exercises the existing per-token +source driver without rebinding runtime functions or adding a runtime flag. +Comparison includes all four return values, every retained checkpoint field, +the grammar recognition root, lexical modes and the scan callback sequence. + +The four cases cover all 64 records at spacing 1, 2, 32 and 1,024, including +the existing =2.0= integer-valued spacing contract; empty and +trivia-only input; Unicode strings, mixed trivia and CRLF; incomplete EOF and +invalid suffixes. The snapshot, fragment-transfer and runtime-organization +Suites add 32 existing cases, giving 36 passing cases in this focused batch. +Expanded unary metadata controls separately compare 198 combinations across +11 productions, nine piece representations and two lookaheads. + +The first focused process printed all successful assertions and =OK= but did +not terminate on this SDK: the original five-second idle fence rejected it, +with exit 143 after 14.674 seconds. The repeat invoked the existing disposable +child exit helper only after =main= returned successfully, and exited 0 in +11.772 seconds with no supervisor reason or missing receipt. Final native +requalification with four explicit Workers passed in 12.304 seconds. Two local +startup attempts were rejected by the unchanged idle fence before test admission; +actual import start/completion tracing exposed loading progress for the final +run, without a timer heartbeat. An intervening local command used an incorrect +exit-helper module path and was rejected before assertions; its log is retained. +Worker CLI controls initially used an unsuitable outer supervisor that treated +expected errors or child silence as a harness failure. Their final run follows +the existing negative-control contract: child owners retain their original +watchdogs and the parent checks their expected exits. All seven controls passed. A printed =OK= alone was not admitted as process success. Initial local syntax errors in the +new controls were corrected before assertions were executed. + +* Complete native-image CPU qualification + +The reference is the exact =54746a3b1617639fac05805f6ac9aae4a1861555= source +archive with matching package pins. Both workers use the same SDK and GCC. +The selected native image contains checkpoint, transfer and parser modules, +compiled in dependency order with their own interfaces. The first reproduction +helper check exposed a repeated =lr-parser/= directory: Gerbil =path-directory= +retains an already trailing directory separator. Explicit parent expansion +corrected the source root before the helper was admitted. A local check also +incorrectly applied the request idle fence to native compilation and was +rejected. Under the existing reference-build contract (a 90-second total fence, +without a request idle fence), all three frozen modules compiled and exited 0 +in 14.012 seconds. The complete request CPU fence remains unchanged. +The worker checks that +all three load from the requested image. Shared grammar, runtime state, +recognition and incremental session modules must have identical digests. +No runtime binding substitution or process launch occurs inside a timed call. + +Every timed call creates a fresh capture-enabled incremental session over all +64 ASCII records, completes scanning and LR parsing, publishes the artifact and +checks full artifact equality and source validity. An independent recognition +projection is also checked before timing. Startup, IPC and diagnostic inventory +are outside the worker user-plus-system CPU interval. Natural GC remains inside +request CPU; no forced collection is used. + +The existing protocol is unchanged: five groups per phase, 32 alternating legs, +256 requests per leg, 122,880 timed requests overall. Reference/reference phases +bracket the production/reference phase. Their accepted ratios are 0.90 through +1.10, and production P90 must be below 0.95. The total fence stays 90 seconds +and the idle fence stays five seconds. CPU phases run sequentially. + +| Phase | P10 CPU ratio | P50 CPU ratio | P90 CPU ratio | +|-------+---------------+---------------+---------------| +| Before reference/reference | 0.9588554399 | 0.9982012347 | 1.0583280689 | +| Production/reference | 0.8422888301 | 0.8751071600 | 0.8929960704 | +| After reference/reference | 0.9505736618 | 0.9922527778 | 1.0440429463 | + +The complete request gate passed with =CAPTURE-COMPLETE-CPU-PROOF-OK=, +exit 0, no supervisor reason and no missing receipt, in 53.933 seconds. +Production P90 CPU was 10.70% lower than this baseline in the final measurement. +The earlier native measurement also passed in 46.159 seconds with production +ratios 0.8203157773 / 0.8563192248 / 0.8745932190 (P10/P50/P90); its 12.54% +P90 reduction is retained as a distinct observation, not substituted for the +final result. +Both workers matched artifact digest +=sha256:c42a4b2281ad7f1921dcd426606db0a2ccb5cbd9925078e67cd479aea27bf24c= +and frontier/mode fingerprint +=sha256:da526271ffa70150686180d18eab518fcb9e346d4faafc43d3b247cf6b0eb63c=. +This establishes the measured fresh capture-enabled Scheme request benefit; +Tokio/FFI, ordinary capture-disabled requests and edited workloads need their +own matched CPU qualification. + +The following historical reproduction commands apply to revision =9103314=. +Current recipes freeze the full selector protocol; see +[[file:selector-input-model-first.org][the subsequent input protocol study]]. Use the matching SDK/GCC environment +and native qualification products for the chosen revision: + +#+begin_src sh +just prepare-capture-reference 54746a3b1617639fac05805f6ac9aae4a1861555 .data/capture-stream/reference source-driver +just benchmark-capture-prefix .data/capture-stream/reference/lib source-driver +#+end_src + +The manual Linux CPU matrix includes =capture-stream= for this exact baseline. +The prior baseline's CI success does not qualify a later pushed head; new-head +CI remains a separate observation. + +* Requalification after the pending-input model study + +The pending-input model extension does not modify the native engine. The +accepted source-driver algorithm at =91033141244b5e1c1d48dfc3e95d5dc527c483ac= +was independently requalified against the same =54746a3= reference, with the +same complete 64-record request, matching artifacts/frontiers, CPU protocol and +90-second total / five-second idle fences: + +| Phase | P10 | P50 | P90 | +|-------+-----+-----+-----| +| Before reference/reference | 0.9371027388 | 0.9702582529 | 1.0163476848 | +| Production/reference | 0.8216117437 | 0.8643581977 | 0.8703403824 | +| After reference/reference | 0.9839937099 | 0.9990015724 | 1.0429723472 | + +The success marker was present; exit 0, no supervisor reason, no missing +receipt, elapsed 58.205 seconds. P90 CPU was 12.97% lower in this independent +observation. This confirms the existing complete-request engine improvement; +the new scalar-input direction has no implementation or CPU claim yet. +The expanded model gate passes 43 Lean jobs, 110 Quint directed histories and +11 sampled invariant runs, plus the source-input invalidation control. diff --git a/docs/research/unary-reduction-corridors.org b/docs/research/unary-reduction-corridors.org new file mode 100644 index 00000000..bb4dbaa9 --- /dev/null +++ b/docs/research/unary-reduction-corridors.org @@ -0,0 +1,230 @@ +#+title: Decorated unary corridors: rejected execution experiment + +* Decision + +The execution fast path is rejected for production. Three coherent variants +were tested: a named helper returning multiple values, a hygienic macro +publishing the stack heads directly, and that macro carrying its resolved next +action row into the outer loop. Allocation improved, but short-chain CPU +regressions remained and complete requests did not improve consistently. +The engine implementation remains unchanged. The retained code is the shared +derivation diagnostic and long-chain semantic/count regression control. + +* One engine investigation + +This study continues [[file:engine-reduction-generation.org][shared reduction generation]], the +[[file:request-owned-lr-stacks.org][rejected vector-stack representation]], and +[[file:semantic-sequence-representation.org][semantic sequence boundaries]]. It targets the generic LR engine, +not a grammar-specific optimization or a new author declaration. + +Temporary source instrumentation first tested the repeated case-folding +hypothesis. GQL's representative request executed 21 folds over 116 characters; +its 64-item RETURN executed 65 folds over 184 characters. FHIRPath invocation, +FHIRPath addition, and Arithmetic addition executed zero. That does not support +folding once per reduction; no normalization cache was added. The temporary +instrumentation was removed. Receipt: +unarchived local execution, exit zero in 7.447 seconds. +This complements the existing [[file:lr-request-cost-boundaries.org][literal lookup eligibility study]]. + +The earlier census counted only *identity* unary chains. Decorated unary chains +are different: field and alias actions are observable semantic work and must +remain. An untimed selected-derivation census now distinguishes them: + +| Workload | Tokens | Reductions | All unary | Identity unary | Longest all-unary chain | +|----------+--------+------------+-----------+----------------+-------------------------| +| GQL representative | 37 | 288 | 213 | 81 | 11 | +| GQL RETURN 64 | 128 | 935 | 661 | 197 | 9 | +| FHIRPath invocation | 12 | 135 | 68 | 15 | 6 | +| FHIRPath addition 64 | 127 | 985 | 585 | 191 | 4 | +| Arithmetic addition 64 | 127 | 318 | 255 | 127 | 2 | + +These are execution counts for the stated sources, not CPU attribution or +universal grammar bounds. The observer captures original derivations and +therefore disables the optimization. Receipt: +unarchived local execution, exit zero in 6.941 seconds. + +* Representation lifetime rather than rule deletion + +Every unary reduction pops one state and one semantic value, applies its +original action chain, and pushes one state and one value. Consecutive unary +reductions keep the same stack suffix. The changing heads can therefore remain +lexical locals until execution reaches a non-unary action. The implementation +owner was =src/runtime/lr-parser.ss=; that candidate code has been withdrawn. + +#+begin_src scheme +;; Conceptual engine kernel: base-state and suffixes remain unchanged. +(let loop ((state first-target) (value first-value) (actions first-count)) + (let (entry (lookup-original-action state lookahead)) + (if (admitted-unary-reduction? entry) + (loop (goto-original-production base-state entry) + (apply-original-actions value entry offset) + (+ actions 1)) + ;; The caller publishes the final immutable heads once. + (values state value actions)))) +#+end_src + +The candidate helper checked the original indexed LR action and goto at every step. +It executes the same prepared field/alias chain in declaration order, and +increments the action count for every reduction. It does not eliminate rules, +change production IDs, merge AST nodes, alter precedence, or change grammar +digests. A missing goto stops the helper before committing that reduction; +the original executor owns fallback at the reconstructed frontier. + +Eligibility requires an existing non-EOF lookahead, no finite action budget, +no captured derivation stack, and no layout runtime/context. Generated reduction +steps retain their independent protocol. Action-budget checkpoints and captured +derivations preserve every original boundary. EOF remains in the outer executor +so streaming callbacks retain their existing invocation behavior. The helper +uses immutable semantic values and ordinary tail calls; no vector arena, global +cache, unsafe access, or grammar option is introduced. + +Why does this improve the Scheme DSL design? The author describes syntax and +closed actions once. Canonical IR, prepared actions, and Scheme lexical scope +allow the engine to shorten the lifetime of private intermediate stack heads +while retaining the complete author-visible tree. POO composition and macro +expansion retain their current authority; authors need no optimization controls. +This ownership advantage does not prove the rejected runtime implementation +is faster. + +Why is the design scientifically testable? Independent materialized GLR must +produce the same complete artifact for both recognition/event backends. Long +decorated chains must preserve every nested field/alias and the exact action +count observed after the following shift. A one-action checkpoint walk supplies +an independent count control. Complete requests then test whether eliminating +intermediate heads improves actual cost. + +* Paper mechanism and boundary + +[[https://arxiv.org/html/2304.05276][Yallop, Xie and Krishnaswami, flap (PLDI 2023)]] demonstrates removal +of intermediate representations while retaining modular definitions. This +motivates asking which internal values need materialization. Our kernel is a +runtime LR stack optimization, not their DGNF normalization or lexer/parser +fusion algorithm. Their assumptions and speed ratios do not establish ours. + +* Qualification + +Twenty source cases passed, including decorated chain depths 0, 1, 4, and 16, +both semantic backends, exact paused/unbounded action-count equality, +all-language ABI products, fragment transfer, and parser runtime controls. +The child completed in 12.453 seconds, exit zero, final =OK=, with no +supervision reason. Receipt: +unarchived local execution. + +The runtime and dedicated test compiled with official Gerbil =-O= in 16.769 +seconds, exit zero. Receipt: +unarchived local execution. + +Two initial source invocations were excluded: nonexistent test paths and a +fixture that used =ref= where canonical GrammarExpr requires =reference=. +Neither is an admitted pass. The corrected controls above use actual source + +* Matched native results and decision evidence + +The final variant used a hygienic =defrule= to publish directly into the +outer loop, avoiding a per-corridor multiple-value return envelope. It also +carried the already-resolved final action row while the state and lookahead +were unchanged. Shifts and ordinary reductions discarded that private entry. +These changes reduced the first variant's overhead, but did not establish a +general win. + +The baseline LR/sequence implementation was =c1237c5=, reused in an isolated +prepared native graph. Only the candidate LR executor was replaced. Parallel +Source/Fold commits are not measured by these LR comparisons. +The final native order was baseline/candidate/candidate/baseline, 20 samples +of 20 calls per workload and route. The table reports CPU batch P50 divided +by 20, in milliseconds/call. Allocation is admitted only for single-caller +samples without GC; GC-containing timing observations remain in the samples. + +| Workload and route | A1 CPU | B1 CPU | B2 CPU | A2 CPU | A bytes/call | B bytes/call | +|--------------------+--------+--------+--------+--------+--------------+--------------| +| gql-return 64 prepared-lr | 0.21700 | 0.21250 | 0.20755 | 0.20970 | 230932 | 182548 | +| gql-return 64 full-source | 0.32150 | 0.29765 | 0.31020 | 0.30705 | 542941.6 | 494557.6 | +| fhirpath-addition 64 prepared-lr | 0.21380 | 0.21455 | 0.22375 | 0.21260 | 255540 | 225300 | +| fhirpath-addition 64 full-source | 0.36145 | 0.34405 | 0.34995 | 0.35435 | 695181.6 | 664941.6 | +| arithmetic-addition 64 prepared-lr | 0.09440 | 0.09960 | 0.09905 | 0.09400 | 134804 | 128756 | +| arithmetic-addition 64 full-source | 0.15705 | 0.15355 | 0.15455 | 0.14405 | 462141.6 | 462141.6 | +| gql-return 512 prepared-lr | 1.81945 | 1.70225 | 1.59040 | 1.60840 | 1894004 | 1501556 | +| gql-return 512 full-source | 2.80255 | 2.46990 | 2.32485 | 2.66735 | 4330101.6 | 3937653.6 | +| fhirpath-addition 512 prepared-lr | 1.78685 | 1.65430 | 1.77775 | 1.69415 | 2104980 | 1859700 | +| fhirpath-addition 512 full-source | 3.22805 | 3.43450 | 3.19350 | 3.08775 | 5585421.6 | 5340141.6 | +| arithmetic-addition 512 prepared-lr | 0.76130 | 0.92860 | 0.79930 | 0.75930 | 1080980 | 1031924 | +| arithmetic-addition 512 full-source | 1.34725 | 1.24615 | 1.17790 | 1.13235 | 3696477.6 | 3696477.6 | + +GQL 512 full-source allocation fell by 392448 bytes/call, about 9.1%. +FHIRPath 512 full-source allocation fell by 245280 bytes/call, about 4.4%. +Arithmetic's generated full-source route allocated the same bytes in both +variants. Short generic Arithmetic recognition still regressed in both +candidate trials. FHIRPath full-source did not show a consistent improvement. +GQL's larger request benefit therefore does not admit the implementation as +a general shared-engine optimization. No general P95 improvement is claimed. + +The named-helper variant increased generic Arithmetic allocation; its +multiple-value boundary cost exceeded its saved stack cells. The macro +removed that increase, but retained a duplicated final action lookup until +the last variant. The last variant removed that duplicate lookup while +retaining all original semantic actions. CPU regressions persisted. These +are failures of the proposed cost tradeoff, not evidence that Gerbil/Gambit +or the language declarations are inherently slow. + +Receipts are +unarchived local execution. +All four children exited zero with =LR-REQUEST-STACK-OK= and no supervision +reason. Final source controls passed in 12.714 seconds; the final runtime +compiled in 16.264 seconds. Earlier native semantic controls passed 29 cases +in 10.796 seconds for the named-helper variant. They do not substitute for a +performance result or whole-build exact-head CI. + +The rejected final runtime source is preserved locally at +unarchived local execution; it is not a runtime +dependency. These local receipts are not durable CI artifacts. + +* Reusable developer diagnostics + +=t/benchmarks/lr-request-stack/derivation.ss= now exports +=profile-lr-request-reductions=. The shared benchmark prints +=LR-REQUEST-REDUCTIONS= for each workload before timing. It reports total, +empty, unary, multi-operand and identity-unary reductions, decorated operands, +and the longest all-unary chain. The captured derivation is discarded before +measurement. These counts explain execution shape; they do not attribute CPU. + +=t/lr-unary-corridor-test.ss= retains independent GLR artifact comparisons for +both semantic backends at chain depths 0, 1, 4 and 16, plus exact action-count +comparison against one-action checkpoints. It remains useful when changing +semantic construction or publishing a future engine optimization. + +The next cost boundary is semantic node/fragment construction and canonical +publication. Future experiments must retain complete artifacts and quantify +both sides of that boundary, rather than infer speed from reduction counts +or transferred allocation. + +The retained benchmark passed from source in 24.363 seconds, exit zero, with +=LR-REQUEST-STACK-OK=; receipt: +unarchived local execution. +The diagnostic module, benchmark and regression test compiled with official +Gerbil =-O= in 13.830 seconds. Their final native qualification passed five +cases and all six benchmark workload/scale products in 49.098 seconds, exit +zero, with both =OK= and =LR-REQUEST-STACK-OK=. Receipts: +unarchived local execution. +No rejected execution fast path was loaded for that final qualification. +After preserving parallel remote commits through =a65abbd=, twenty-five source +cases passed for the retained long-chain controls, all-language ABI, fragment +transfer, runtime behavior and updated Fold source contracts. The child exited +zero in 12.962 seconds with final =OK= and no supervision reason. Receipt: +unarchived local execution. +This source receipt does not claim whole-build native qualification of the +parallel Source/Fold changes. + +* CPU attribution remains open + +A separate complete-request sampling workload reported its own PID through +the official =std/os/pid= interface and verified the final artifact after every +50 calls. The macOS sampler produced an empty call graph, so it supplies no +usable CPU attribution. The workload completed 3500 calls before its existing +60-second total gate terminated it, without its required completion marker. +It is not admitted as a successful qualification. The gate was not widened. +Receipts: unarchived local execution. + +An empty sampling report does not show that the parser has no hotspots. +Future sampling must first demonstrate actual thread/stack observations and +distinguish native parsing, publication, GC and driver work. Execution counts +alone cannot close this missing attribution. diff --git a/docs/research/upstream-bridge-tokio-admission.org b/docs/research/upstream-bridge-tokio-admission.org new file mode 100644 index 00000000..41389f44 --- /dev/null +++ b/docs/research/upstream-bridge-tokio-admission.org @@ -0,0 +1,325 @@ +#+TITLE: Upstream Bridge Revision and Tokio Admission Boundary +#+DATE: 2026-10-11 + +* Revision and consumed code + +The Gerbil package declaration, Cargo workspace dependency and Cargo lockfile +now pin =gerbil-scheme-rust= PR 18 at +=9cd2604efe2a40d53554ee8ffe38d5a8eff4c79e=, replacing +=345b172765e5607b09049614d10520c2439dcac3=. +The standalone parser fixture lockfiles do not contain this dependency. + +The parser consumes the upstream Rust IR compiler through +=gerbil-parser-build-support=, and the Scheme UTF-8 encoder through +=src/ffi/language-abi-context.ss=. This revision advances the encoder's checked +output using a local cursor. The public encoder and caller-owned capacity +contract remain unchanged. Upstream rooted snapshots and numeric/identifier +bulk conversion APIs are separate capabilities; the parser's GPA1 byte-buffer +transport does not implicitly consume them. + +* Tokio evidence and downstream ownership + +At this exact revision, Tokio is a dev dependency of =gerbil-scheme=. Its Tokio +handoff tests and transport benchmarks qualify requests sent from async callers +to a dedicated Scheme owner thread. The safe production crate does not expose +a Tokio parser executor. The active performance benchmark retains bounded mpsc +and per-request oneshot responses. Receive batching and reused-response-channel +performance candidates were rejected; functional receive-batching tests do not +establish a production scheduling speedup. + +The parser's =NativeSession= and borrowed language/results remain confined to +the initialized VM's original OS thread. Any downstream asynchronous adapter +must create and use them on that owner, bound admission, return owned products, +and qualify cancellation, shutdown and latency. Tokio worker concurrency alone +does not establish concurrent Scheme parsing or lower complete-parser CPU. + +Pinned primary sources: +- [[https://github.com/tao3k/gerbil-scheme-rust/blob/9cd2604efe2a40d53554ee8ffe38d5a8eff4c79e/crates/gerbil-scheme/Cargo.toml][Production and development dependency declarations]]. +- [[https://github.com/tao3k/gerbil-scheme-rust/blob/9cd2604efe2a40d53554ee8ffe38d5a8eff4c79e/docs/bridge-numeric-vectors-20261011.org][Bulk conversion and scheduler evidence boundaries]]. +- [[https://github.com/tao3k/gerbil-scheme-rust/blob/9cd2604efe2a40d53554ee8ffe38d5a8eff4c79e/docs/bridge-utf8-tokio-recv-many-20261011.org][Rejected receive-batching performance candidate]]. +- [[https://github.com/tao3k/gerbil-scheme-rust/blob/9cd2604efe2a40d53554ee8ffe38d5a8eff4c79e/scheme/utf8.ss][Consumed checked Scheme encoder]]. + +* Qualification + +Cargo build-support tests validate the consumed upstream IR revision. Existing +Scheme C ABI tests validate UTF-8 admission and complete artifact parity. The +complete 64-record scanner/LR, event-publication, ranked-request and UTF-8 CPU +proofs remain unchanged in the workflow dispatch. CPU includes GC and the strict +before/after controls remain mandatory. + +A passed scanner/LR comparison at the new pin confirms its complete-request +benefit remains admitted. It is not a comparison of old and new upstream +revisions and must not be relabeled an incremental Tokio or encoder speedup. + +Before publication, upstream advanced from =4d71a12= to =9cd2604=. The added +composite object APIs do not change the consumed =scheme/utf8.ss= source. + +At the preliminary =4d71a12= pin, the local Unicode 64-record complete-request engine proof passed with 20 groups +and 256 calls per batch: production P50 0.825059, P90 0.878090; before-control +P10/P90 0.918958/1.048539 and after-control 0.920178/1.065327. Both sides use the +new upstream encoder. This retains a 17.49% median engine benefit in this local +workload, not an incremental upstream speed claim. Exit status was zero. + +* Rust downstream integration at fbb43ca + +This integration stage used the manifest and lockfile pin +=fbb43ca63d3740cb4e19b26fa2abd1ea3ee4ecdb=. Its additional upstream receipt +rejects an admission fast path; it does not change the production encoder. +The earlier qualification above remains historical evidence at its stated pin. + +The optional =tokio= feature of =gerbil-parser-ffi= now exposes =TokioParser=. +It initializes the standalone bundle and retains a single language handle on +one dedicated OS thread. Bounded mpsc admission and per-request oneshot replies +follow the upstream qualified ownership design. Admission uses normal async +reservation; the rejected upstream fast path is not included. + +The owner copies the completed GPA1 payload into independent Rust storage, +releases its C result before publication, and shares an immutable catalog. +=OwnedParsed= retains the exact source and supports normal artifact views after +VM shutdown. Native handles and GC-managed Scheme values never cross threads. +Queued cancelled requests skip execution. An in-flight cancelled request still +releases its C allocation. Explicit shutdown closes admission and releases the +language before normal SDK cleanup; dropping all clients drains the queue. + +This is an async downstream ownership interface, not evidence of concurrent +Scheme execution or an incremental CPU speedup. The existing =qualify.ss host= +entry now includes a real Tokio consumer in a separate one-shot VM process: +32 concurrent requests, alternating ASCII/Unicode, 64 records per request, +642 events, syntax rejection, closed admission, and views retained after cleanup. +Transport unit tests additionally cover queue bounds, queued cancellation and +shutdown ordering. Rust lint and workspace all-feature gates cover the new API. + +Local GCC qualification completed with =NATIVE-HOST-OK= and exit zero. The +Tokio process completed its 32 full requests, rejection and retained-view checks +with =RUST-TOKIO-HOST-OK= in 0.332 seconds total process wall time. This is a +functional receipt, not a matched CPU performance result. Native SDK linking +required the local OpenSSL library search path; no compiler was replaced. + +* Complete Rust request CPU qualification + +The Rust consumer now exercises the same independent former scanner/LR kernels +as the Scheme engine benchmark. A test-only factory prepares the reference +context, preserving the generated direct reduction step and descriptor identity. +Both handles live on the same Tokio owner; a benchmark-only routing wrapper +selects the handle without modifying the production parser or scheduler. +Both sides use the current upstream UTF-8 encoder and GPA1 row writer. + +Every timed request includes Rust source ownership, bounded Tokio admission, +oneshot completion, C ABI parsing, canonical UTF-8 admission, scanner/LR, +complete semantic publication, a Rust-owned GPA1 snapshot, C result release, +full Rust artifact admission, event-count and root traversal and final Rust result drop. +The source contains 64 assignments: 256 ASCII or 320 Unicode bytes. Every +accepted payload contains 15,488 bytes, 642 events and 64 root children. +Complete byte parity also covers empty, rejected and Unicode quoted inputs. + +CPU is process user plus system time from =getrusage(RUSAGE_SELF)=, including +the owner thread, Tokio caller and natural Scheme GC. VM/library startup and +producer compilation are outside timing. No timed GC is excluded, no sample is +dropped, and no per-request observer changes the runtime schedule. + +The production fixture uses the workspace's release policy: optimization level +3, thin LTO and overflow checks enabled. The current workload has one owner and eight closed-loop async callers on one +Tokio executor, with a bounded queue of four. This measures concurrent admission +and complete requests, not parallel Scheme execution. +Each of the before, production and after phases has 20 groups. A group has eight +alternating ABBA/BAAB legs, 256 requests per leg and four legs per side, matching +the Scheme Benchmark protocol. Each batch must resolve at least one millisecond. +Nearest-rank P10/P50/P90 use every group. Before/after P10 must be at least 0.90 +and P90 at most 1.10; production P90 must be strictly below 0.95. + +** Rejected exploratory receipts + +The initial debug run used 64 calls and only two single-batch controls per group. +Before P10/P90 was 0.716998/1.110200; production P50/P90 0.899910/1.128070; +after P10/P90 0.870028/1.056283. It was rejected. + +Increasing that exploratory debug batch to 512 calls did not admit a benefit: +before P10/P90 0.856625/1.128430, production P50/P90 0.903464/0.998036, +after P10/P90 0.756212/1.067461. It was rejected. + +Release with the same exploratory control layout produced P50/P90 +0.810413/0.937457, but before P10/P90 0.765034/1.164918 and after P10/P90 +0.774124/1.178976 were unstable. It was also rejected. The final protocol fixes +the unequal control sampling to match Scheme Benchmark's four-batch-per-side +sums; it does not relax any acceptance threshold. + +** Reproduction and scope + +Set =GERBIL_PARSER_RUST_CPU_PROOF=1= and run =gerbil env gxi qualify.ss host=. +The existing producer owner builds one Scheme library and one release Rust +consumer, then reuses the same linked host for lifecycle and both CPU processes. +Build/link and execution retain their existing 90-second limits; execution +retains a five-second idle limit with real completed-batch output. +Linux CI applies =taskset= only to the CPU host processes, preserving producer +build parallelism. The full manual Linux workflow enables both CPU proofs; +ordinary lifecycle qualification continues using its debug build. + +This qualifies whether the earlier scanner/LR algorithm benefit reaches complete +Rust Tokio/FFI requests. It must not be reported as an incremental benefit from +the latest upstream revision, or as a Tokio scheduler optimization. + +The first eight-leg release run remained rejected: before P10/P90 +0.914546/1.117402, production P50/P90 0.843691/0.922004, after P10/P90 +0.842646/1.175472. Neither production ratios nor profile changes excuse an +unstable control. A subsequent controlled Darwin experiment explicitly sets +and verifies the same =QOS_CLASS_USER_INTERACTIVE= request on the caller and +owner threads before warmup. This is test-only scheduling setup, not a core +pin or a production API optimization. Linux uses the existing CPU affinity. + +The initial Darwin QoS run also remained rejected: before P10/P90 +0.834763/1.084355, production P50/P90 0.837017/0.907827, after P10/P90 +0.925992/1.082593. The after control passed; the before control did not. +The single-caller warmup experiment used eight balanced legs of 1,024 complete +requests, 4,096 requests per variant, and hit the existing 90-second limit. It includes natural GC and establishes +both independent contexts before the unchanged 20-group timed phases. Warmup +is not selected from timed results and no measured group is discarded. + + +** Concurrent complete-request receipts + +The eight-caller, 256-request protocol also reached the 90-second process limit +before completing its production phase. It was rejected. That bounded +protocol used 128 requests per leg and eight symmetric warmup legs of 256 +requests (1,024 per variant). All 20 groups in each of the three phases remain +mandatory, with unchanged percentile and control thresholds. One progress line +records all eight completed leg costs; no group or GC cycle is removed. + +| Workload | Before P10/P90 | Production P50/P90 | After P10/P90 | Admission | +|----------+----------------+--------------------+---------------+-----------| +| ASCII 64 records | 0.940151 / 1.013562 | 0.844709 / 0.863701 | 0.981808 / 1.114173 | Rejected: after control | +| Unicode 64 records | 0.949809 / 1.031382 | 0.836406 / 0.875211 | 0.920825 / 1.033180 | Passed locally | + +The Unicode receipt proves a local complete-request CPU benefit for this +concurrent workload: production P90 is 12.48% lower than the independent engine +control. The ASCII control failure prevents an ASCII performance claim. These +measurements precede the final dependency synchronization; they are not an +exact-head CI receipt or an incremental upstream-revision performance claim. + +* Dependency synchronization and parallel CI + +The latest inspected bridge PR revision is +=0c18941cc6b6b7ff0ef3943abdb2b5fac8e92ccb=. Scheme and Cargo pins and the Cargo +lock agree. Changes since =ce8d781= concern build-launcher environment and +upstream qualification, not a new parser or Tokio scheduling kernel. The +concurrent parser PR changes at =461a4bf= were retained, including the explicit +Darwin FFI compilation policy for the foreign language context. + +The failing macOS run at =51bbe88= compared the installed bridge against an +obsolete hardcoded =345b172= dependency. The contract now derives the expected +bridge revision from Cargo and verifies the Scheme dependency agrees, while +retaining the other package-ownership assertions. + +CI now has three independent Rust jobs and one preparation producer per Scheme +platform. Each producer builds parser, commands, conformance, test support, +FFI fixtures and the standalone language library once. Its consumers run +semantics, Rust regeneration, FFI lifecycle, and Linux corpus qualification in +parallel. Manual runs add independent diagnostic jobs and six Linux CPU jobs: +scanner, events, engine, ranked parser, UTF-8 and complete Rust Tokio requests. +Ubuntu consumers do not wait for macOS preparation. + +Compiled products are transferred inside a tar archive to retain symlinks, +permissions and mtimes. Admission checks the exact Git head, full SDK digest, +OS, architecture and workspace path before extraction. The native host uses +the admitted standalone library instead of compiling it in every consumer. +The ordinary PR graph has 12 runner jobs; manual qualification adds eight jobs. +Actual concurrency is subject to the account's runner capacity. + +The immutable SDK archive is cached by full SHA-256 and verified on every +restore. Gerbil package and content-admitted conformance cache keys include that +SDK digest and dependency identity. Cargo caches include the Rust toolchain, +platform, lane and manifest/lock identities. Preparation is saved before CPU +qualification, so a rejected CPU proof cannot discard reusable preparation. +Each consumer uploads its own qualification logs, including failed runs. +Existing test gates and bounded child-process deadlines remain in force. + + +Local validation after the final synchronization passed actionlint, original +CI-gate ownership parity, and the producer/consumer dependency graph. Artifact +controls rejected mismatched head, SDK, OS, architecture and workspace and +preserved executable permissions, symlinks and mtimes. The real Scheme product +contract ran both cases with =BUILD-PRODUCT-CONTRACT-OK=. Rust build-support +reported eight passed and two intentionally ignored tests; FFI reported seven +passed tests. Consumer clippy and formatting passed. + +Both ordinary library production and consumption of the already prepared +library completed with =NATIVE-HOST-OK=. The latter reused the shared library +without another AOT build and completed its 32-request Tokio lifecycle probe +with =RUST-TOKIO-HOST-OK=. These local results do not replace remote Linux/macOS +qualification of the committed head. + + +* Single-pass admission and complete ASCII qualification (2026-10-12) + +** Structural change + +The Rust artifact consumer previously decoded every GPA1 event once to validate +it and again to construct its navigation index. Field validation also searched +the open-event stack backwards for its owning node. The production decoder now +builds navigation while validating the same event, retaining the current owner +and restoring it when a node closes. Field-owner lookup is constant time. One +open-event stack supplies both nesting validation and closing-row navigation; +the separate navigation pass and node stack are removed. + +Identity, canonical record, vocabulary, UTF-8, field ownership, nesting and +coverage checks are retained. Invalid artifacts expose no navigation. Initial +index reservation is bounded to 1,024 rows; a malicious declared event count +cannot force a full index allocation before row admission. Larger valid inputs +grow with admitted records. + +The former two-pass decoder is frozen under test support. The optional +=benchmark-controls= feature makes that independent decoder available to the +Rust qualification fixture. Normal consumers use single-pass admission. This +is a test control, not a production compatibility path. + +A semantic test compares all navigation entries and admission errors against +the former implementation for all 20 real native-language fixtures. It also +corrupts every payload byte independently, tests truncation and mismatched +sources, and requires identical rejection reasons and event indices. + +** Matched full-request evidence + +The reference requests use both the independent former scanner/LR kernels and +the frozen two-pass artifact decoder. Production uses the prepared scanner/LR +kernels and fused admission. Both retain the same current upstream UTF-8 +encoder, full row writer, source/catalog identities, bounded Tokio owner and +C allocation ownership. Therefore these results qualify the complete combined +engine/consumer path; they do not attribute the entire gain to decoder fusion. + +The initial fused 128-request ASCII experiment was rejected despite production +P90 0.855386: before P10/P90 was 0.978785/1.050850, after was +0.980403/1.102040. Its failed control remains recorded. The fixed qualification +protocol now uses 256 requests per leg, eight balanced legs per group, 20 +groups per phase and three phases. Warmup remains eight balanced legs of 256 +requests. Thresholds, inclusion of natural GC, all-group admission and the +90-second process limit are unchanged. + +| Complete request | Before P10/P90 | Production P50/P90 | After P10/P90 | Admission | +|------------------+----------------+--------------------+---------------+-----------| +| ASCII 64 records | 0.949719 / 1.072802 | 0.836486 / 0.879088 | 0.922525 / 1.036930 | Passed | +| Unicode 64 records | 0.930487 / 1.045828 | 0.832003 / 0.892620 | 0.963446 / 1.052294 | Passed | + +Both independent complete-request processes returned zero and +=RUST-TOKIO-CPU-PROOF-OK=. ASCII process wall time was 50.021 seconds; Unicode +was 51.478 seconds. ASCII production P90 CPU was 12.09% lower than the frozen +combined control; Unicode was 10.74% lower. These are process user+system CPU +ratios for 64 records, including Tokio/FFI, source allocation, Scheme parse/GC, +full publication, C result release, Rust admission/traversal and result drop. +Every timed request checks the complete 15,488-byte snapshot, 642 events and +64 root children. These are local matched receipts, not remote CI completion. + +** CI preparation dependency repair + +The =8805047= Ubuntu producer failed before library-object qualification: +=qualify.ss= imported the uncompiled qualification process module. Moving the +process-contract tests into a consumer had removed their earlier implicit +build prerequisite. Preparation now explicitly compiles =build-qualification.ss= +before its first =qualify.ss= call. This restores build ownership and leaves the +independent consumer job structure and existing deadlines intact. + + +Final local regression validation passed 171 workspace tests (including the ASP +source gate), workspace and fixture clippy, formatting, the explicit Scheme +qualification-boundary build and independent C/Rust/Tokio lifecycle probes. +The real 32-request Tokio host returned =RUST-TOKIO-HOST-OK= and the complete +host qualification returned =NATIVE-HOST-OK=. Actionlint accepted the repaired +producer dependency graph. The latest committed CI remains a separate gate. diff --git a/docs/research/upstream-ffi-utf8-admission.org b/docs/research/upstream-ffi-utf8-admission.org new file mode 100644 index 00000000..5ad3e876 --- /dev/null +++ b/docs/research/upstream-ffi-utf8-admission.org @@ -0,0 +1,380 @@ +#+title: Upstream caller-owned UTF-8 admission in complete parser requests + +* Ownership and integration + +Both =gerbil.pkg= and the Rust IR dependency pin +=gerbil-scheme-rust= revision =345b172765e5607b09049614d10520c2439dcac3=. +The parser imports the actual upstream =scheme/utf8= module; it does not copy +its encoder or initialize a second Scheme VM. The public Rust and C parse +signatures, GPA1 format and language declarations remain unchanged. + +[[file:../../src/ffi/language-abi-context.ss][Prepared ABI contexts]] own a source-admission procedure. The default proves ASCII and decodes it directly into owned Scheme storage. +For other bytes it decodes and calls upstream +=gerbil-rs-encode-utf8-into= to compare the canonical scalar encoding against +the original bytes. [[file:../../src/ffi/language-artifact-codec.ss][Both fresh and session publication]] +use that admission before parsing. [[file:../../src/ffi/language-handles.ss][Prepared context registration]] gives each +handle its existing independent history; it adds no process or VM isolation. +The private context record is an engine protocol, not a new language DSL slot. + +Each nonempty non-ASCII admission allocates stable C storage with capacity four times +its decoded character count. Upstream writes bounded 256-character chunks, +retaining Scheme polling and strict Unicode scalar checks. The consumer passes +a typed =void*= foreign object, not an integer address. Its no-allocation +comparison leaf borrows the Scheme bytevector body only during that call. +No Scheme body pointer survives a poll, callback or allocation. =dynamic-wind= +frees C storage on success and exceptions; empty input requires no allocation. +Allocation failure and malformed/noncanonical UTF-8 remain errors before result +publication and history commit. This changes temporary encoding storage, not +the original source copy or the owned C result lifetime. + +The upstream encoder and its pointer contract are primary implementation inputs: +[[https://github.com/tao3k/gerbil-scheme-rust/blob/345b172765e5607b09049614d10520c2439dcac3/scheme/utf8.ss][Pinned Scheme encoder source]]. Existing parser transport already uses bulk +=memcpy= for source and GPA1 bytes; reimplementing those copies would not consume +a missing upstream optimization. + +* Complete request experiment + +[[file:../../t/benchmarks/language-abi/utf8-cpu.ss][The Scheme Benchmark]] registers two prepared contextual record parsers. The +reference context independently reproduces the former =utf8->string= plus +=string->utf8= equality admission. The candidate uses the production upstream +caller-buffer admission. Grammar, scanner, parser, catalog, byte source and +expected complete GPA1 payload are shared in meaning. Workloads must be accepted. + +Both timed thunks call the actual length-bearing public C parse ABI through +[[file:../../t/benchmarks/source-edits/abi-call.ss][the existing owned C caller]]. Timing includes input allocation/copy, callback, +strict admission, contextual parsing, GPA1 encoding, C result allocation/copy, +readback and all releases. Complete payload equality is checked by the common +sampler; initialization and context preparation are outside both timed lanes. +This is a Scheme measurement of the public C path also used by Rust callers, +not a measurement of Rust wrapper instruction costs. + +Twenty groups retain complementary paired order and equal request budgets. +Every natural collection remains charged to CPU. Identical reference controls +run before and after the production comparison. Both control P10/P90 bounds +remain 0.90--1.10; positive benefit additionally requires production P90 below +0.95 and every timed sub-batch to resolve at least one millisecond. Each fresh +proof child retains 120 seconds total, five seconds of real-output idle time, +three GiB maximum heap and typed nonzero rejection. No collection is forced +between sub-batches, no sample is dropped and GC CPU is not subtracted. + +* Local executable receipts + +The fixed CI Gerbil 2591dcd/Gambit dcd677c SDK and its GCC toolchain compiled the +final producers in 8.111 seconds. The complete ABI GxTest suite, including +Unicode scalar extremes and 256-character boundaries, returned =OK= and exit +zero in 11.460 seconds. Initial completion followed by an idle kill is retained +as rejected; the standard disposable-child exit owner closes the final process. + +| Complete accepted input | Bytes | Nominal calls | CPU P50 ratio | CPU P90 ratio | Median decrease | Elapsed seconds | Benefit admitted | +|-------------------------+-------+---------------+---------------+---------------+-----------------+-----------------+------------------| +| 4096-character ASCII identifier | 4099 | 1024 | 0.889677 | 0.918142 | 11.03% | 46.890 | yes | +| 4096-character Unicode identifier | 8195 | 1024 | 0.847645 | 0.889088 | 15.24% | 51.331 | yes | +| 64 complete record assignments | 256 | 256 | 1.003263 | 1.035410 | none proved | 71.737 | no | + +ASCII controls have P10/P90 0.957433/1.023836 before and +0.976223/1.020410 after. Unicode controls have 0.969120/1.022878 before and +0.987238/1.018941 after. The many-record controls also pass, but its production +series does not establish positive CPU benefit. A separate no-regression +obligation requires stable controls and production P90 at most 1.05; it cannot +be reported as a speedup. Fresh CI verifies both obligations separately. + +Two independent 256-call regression children retained continuous real-batch +progress but hit the unchanged 120-second bound under host scheduling delays. +Neither is admitted. An equal 128-call experiment completed in 40.993 seconds +but rejected its before-control P10 of 0.894746; its production P90 of 1.024535 +does not qualify the run. The regression obligation retains the original +256-call budget and every statistical/control threshold. + +The final independent 256-call regression child completed in 71.671 seconds +with exit zero and =ABI-UTF8-CPU-REGRESSION-OK=. Its control P10/P90 were +0.940476/1.070487 before and 0.961524/1.039007 after; production P50/P90 +were 0.994419/1.031232. This admits the bounded regression obligation and +still reports =cpu-benefit: false=. The Rust workspace consumer checks against +the synchronized pin passed 167 tests with two ignored. + +Negative experiments are part of the boundary. The ordinary upstream encoder +still allocates/fills a maximum-sized Scheme bytevector. With 128 nominal calls, +ASCII CPU P50/P90 was 0.949521/1.111697 and the controls rejected. With 1024 calls, +controls passed but production 0.959259/1.032450 did not admit benefit. Its +Unicode exploratory series also rejected benefit; it is not positive evidence. +The final caller-owned route removes that temporary Scheme vector rather than +relaxing the admission rule. A wrong integer-pointer integration was rejected by +the upstream typed pointer guard and fixed before final semantic qualification. + +These receipts qualify the specified complete contextual requests. They do not +prove every grammar faster, solve the Bash/Tree-sitter gap, or attribute a gain +to the unlinked upstream Rust runtime. The many-record result makes the limited +CPU contribution of UTF-8 admission visible when parsing/publication dominates. + +* Reproduction and remote closure + +Use the normal package dependency/build owners followed by =build-ffi-tests.ss=. +The compiled entry and its source proof runner are part of that existing owner. + +#+begin_src sh +gerbil env gxi t/benchmarks/language-abi/utf8-proof.ss 20 1024 128 identifier false +gerbil env gxi t/benchmarks/language-abi/utf8-proof.ss 20 1024 128 identifier true +gerbil env gxi t/benchmarks/language-abi/utf8-proof.ss 20 256 64 records false regression +#+end_src + +The existing manual performance workflow executes both positive benefit gates +and the distinct many-record regression obligation on Ubuntu. Ordinary CI keeps +its ABI, source callback/GC, complete conformance and generated-product checks. +Exact-head CI remains a separate gate after integration with concurrent PR work. + +* Ubuntu integration receipt + +The complete request gates on PR revision =30106c5= passed in workflow +[[https://github.com/tao3k/gerbil-parser/actions/runs/38124481599/attempts/2][Ubuntu attempt 2]]. ASCII production CPU P50/P90 were 0.901757/0.945452 +(9.82% median decrease); Unicode were 0.871052/0.883793 (12.89% decrease). +Both before/after controls passed. The 64-record request had production +0.990466/1.022054 and passed the distinct regression obligation; it did not +admit positive CPU benefit. Complete payload equality and natural GC charging +remain unchanged. + +Attempt 1 rejected an existing public-parser benchmark before-control P10 of +0.883024; its candidate P90 of 0.867271 was not enough to admit the run. +Attempt 2 reached the final Scheme suites and caught the existing package +contract's two-dependency expectation. The integration adds a third actual +Scheme dependency, so its exact expected dependency list is corrected. These +attempts do not qualify the full workflow; final closure requires the corrected +revision's complete Ubuntu job to pass. + +* Complete 64-record publication algorithm + +The UTF-8-only 64-record result above remains a negative benefit result. The +next experiment changes the shared GPA1 publication algorithm independently +of source admission. A request-local diagnostic measured 100 accepted ASCII +64-record C ABI calls: complete CPU was 54.732 ms, contextual scan/LR drive +21.860 ms, event validation 2.528 ms and event publication 27.962 ms. The +publication stage was approximately 51% of complete CPU. These inclusive +observer measurements locate costs; they are not a paired speedup proof. + +The fixed Gerbil SDK's =std/vector/u8vector= integer setters dispatch byte +order and check each scalar span before shifting/storing each byte. GPA1 rows +have an invariant 24-byte layout. The shared engine now packs that whole row +through one bounded synchronous leaf instead of five generic scalar writes. +The leaf converts unsigned integers before accessing storage, checks the +bytevector and complete remaining span without offset-addition overflow, and +writes explicit little-endian bytes. It requires neither aligned stores nor +host endianness. The Scheme buffer body is borrowed only during the leaf; +there is no callback, allocation, poll or retained pointer inside it. + +Both deferred contextual publication and already materialized Source artifacts +use the prepared context's row writer. Parsing, two-pass event validation, +source ownership, canonical IDs, descriptor lookups, publication error +classification, exact buffer sizing and history commit ownership are preserved. +No language declarations or scanner rules change. Invalid integer widths and +short output spans fail before writing any part of the row. + +The =events= Benchmark registers two independent public C ABI handles with +identical production UTF-8 admission and parsing. Its reference context owns +the former five-setter row writer; its candidate uses whole-row packing. Both +produce the entire 15,488-byte GPA1 for 64 accepted assignments, including input +copy, parsing, source hash, publication, C result copy, readback and release. +Empty, rejected, quoted, CRLF, Unicode and field-bearing products are compared +byte-for-byte outside timing. The existing 20-group paired CPU admission, +resolution, natural GC and before/after controls remain unchanged. + +The initial ASCII experiment passed with CPU P10/P50/P90 +0.624346/0.642045/0.683865, a 35.80% median decrease. Before-control P10/P90 +were 0.928303/1.037129 and after-control 0.928605/1.047672. The child completed +in 66.762 seconds with exit zero and =ABI-EVENT-CPU-PROOF-OK=. This receipt +predates the extra internal bytevector guard; final receipts are recorded below. +A guard compilation rejected a missing Gambit predicate temporary, and the +premature Unicode child was stopped. Neither is admitted as final evidence. + +Reproduce the independent complete-request proof after the normal builds: + +#+begin_src sh +for unicode in false true; do + gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 \ + --log ".data/ffi-event-records-${unicode}-cpu.log" \ + --require ABI-EVENT-CPU-PROOF-OK \ + -- gxi t/benchmarks/language-abi/utf8-proof.ss \ + 20 256 64 records "$unicode" events +done +#+end_src + +The Ubuntu manual qualification runs these independent proof children in +addition to the existing UTF-8 obligations. A gain here proves the specified +complete multi-record requests faster; it does not establish an arbitrary +language speedup or close the separate Bash/Tree-sitter performance gap. + +The final guarded Unicode 64-record child used 320 source bytes and 15,488 +payload bytes. CPU P10/P50/P90 was 0.620822/0.648361/0.690914 (35.16% median +reduction); controls were 0.946849/1.045681 before and 0.958388/1.046244 after. +It returned =ABI-EVENT-CPU-PROOF-OK= and exit zero in 68.784 seconds. + +The final guarded ASCII child used 256 source bytes and the same 15,488-byte +complete payload. CPU P10/P50/P90 was 0.615059/0.655528/0.687218 (34.45% median +reduction); controls were 0.938971/1.043902 before and 0.936834/1.043005 after. +It returned =ABI-EVENT-CPU-PROOF-OK= and exit zero in 67.669 seconds. The final +producer compiled with the fixed GCC SDK in 7.991 seconds. All these results +charge natural GC and retain every sample. + +The final complete language ABI suite, including row extremes, adjacent-byte +sentinels, unsigned overflow and atomic bounds rejection, returned =OK= and +exit zero in 7.205 seconds. Workflow YAML parsing and patch whitespace checks +also passed locally; remote qualification is a separate gate. + +* ASCII admission proof and first Ubuntu rejection + +The first new-head Ubuntu qualification at commit =18e95e6= rejected the +existing ASCII UTF-8 obligation: controls passed, but complete identifier CPU +P50/P90 was 0.939820/0.979701. Its P90 fails the unchanged 0.95 benefit limit. +The new event proof was not reached. The same head passed macOS Scheme and +Rust AOT qualification. This failed Ubuntu result remains part of the evidence: +[[https://github.com/tao3k/gerbil-parser/actions/runs/38134751349][First new-head qualification]]. + +The shared admission now proves canonical ASCII directly. Every byte below +128 is its unique UTF-8 scalar encoding, including NUL. Bounded byte +classification admits precisely this subset, then direct scalar stores decode +it into owned Scheme string storage. This removes generic decoding, redundant +re-encoding, C scratch allocation and chunked encoder calls. Both ASCII +classification and copying enforce 4,096-byte leaves and return to Scheme +between chunks. Copying checks high bits again; a changed non-ASCII byte fails +admission before result publication. Native string stores follow the fixed +Gambit header representation. Every +other byte source still uses the upstream strict scalar encoder and full +canonical-byte comparison. The proof has no grammar, language, input-length or +benchmark-specific conditions. Capacity and malformed input checks are retained. + +Tests cover classification of all 256 byte values, decoding all 128 ASCII +scalars, empty input, Unicode chunk/scalar boundaries and noncanonical sources. +The complete ABI suite returned =OK= and exit zero in 10.972 seconds. The +candidate was recompiled with GCC in 9.184 seconds. Fresh complete-request +CPU receipts for this final admission follow; prior row receipts remain scoped +to their recorded admission implementation. + +The first ASCII-classification-only local experiment still rejected benefit: +P50/P90 was 0.895506/0.951157 with stable controls, and exit was one after +48.749 seconds. Direct bounded decoding was added instead of changing the +0.95 limit, dropping observations or increasing the timeout. The final producer +compiled with GCC in 9.296 seconds. The 4,095/4,096/4,097 and +8,191/8,192/8,193 boundaries include both pure ASCII and late Unicode fallback. + +Final bounded ASCII decoding passed the complete ABI suite in 9.015 seconds. +The formerly failing complete 4,099-byte identifier obligation now passes: +CPU P10/P50/P90 is 0.764338/0.813574/0.895873, an 18.64% median decrease. +Before control P10/P90 is 0.955807/1.052675 and after is 0.988543/1.086949. +The unchanged 20-group, 1,024-call child returned =ABI-UTF8-CPU-PROOF-OK= +and exit zero in 53.505 seconds, with natural GC charged. + +With final bounded ASCII decoding installed identically in both event lanes, +the 64-record ASCII proof passes: CPU P10/P50/P90 is +0.616443/0.647460/0.686067 (35.25% median decrease). Control P10/P90 is +0.954784/1.056653 before and 0.959466/1.029376 after. The complete 15,488-byte +payload matches; the unchanged 20-group/256-call child returned +=ABI-EVENT-CPU-PROOF-OK= and exit zero in 69.144 seconds. + +Final Unicode 64-record CPU P10/P50/P90 is 0.621700/0.647574/0.688848 +(35.24% median decrease). Control P10/P90 is 0.968489/1.034803 before and +0.956032/1.043374 after. The child returned =ABI-EVENT-CPU-PROOF-OK= and +exit zero in 67.328 seconds. Final full ABI tests, including oversized C leaf +ranges and short destination sentinels, returned =OK= and exit zero in 6.933 +seconds. These final event proofs compare independent writers with the same +final admission and the actual complete public C transport. + +* Matched ASCII batch resolution + +The =4d19730= Ubuntu candidate passed the ASCII benefit limit with production +P50/P90 0.847186/0.930900. Admission nevertheless correctly rejected control +P90 values 1.101847 before and 1.100156 after. No result from this run is +admitted. [[https://github.com/tao3k/gerbil-parser/actions/runs/38136884097][Second new-head qualification]] retains that failure. + +Its 960 measured sub-batches include 678 with zero collections and 282 with +one collection. Collection CPU P90 is 13.644 ms against total sub-batch CPU +P50 53.318 ms. This exposes sensitivity to whole-collection granularity, in +addition to possible runner variation. A matched 2,048-call pilot did not solve the control problem: it admitted +production P50/P90 0.817170/0.880954 but rejected after-control P90 1.111759, +with exit one at 91.282 seconds. The nominal budgets remain unchanged. + +The sampler retains eight full-quanta legs instead of sixteen half-quanta +legs. Both orders balance constant, linear and quadratic time position, and +both variants still execute exactly four nominal quanta per observation, +including odd budgets. This increases each retained timing interval without +adding requests, extending runtime limits or dropping any sample. All twenty +groups, natural GC, 0.90--1.10 controls, 0.95 benefit and 120-second child +limits are retained. The request conservation and position-moment tests cover +both group parities and even, odd and singleton budgets. Fresh CPU receipts +qualify the changed sampler before CI admission. +The +64-record proof now runs first to produce its independent target evidence +before the separate UTF-8 obligations. + +The full-quanta sampler passed its budget/execution and quadratic-position +suite with =OK= and exit zero in 26.903 seconds. Its formerly unstable ASCII +obligation passed at the original 1,024 calls: CPU P10/P50/P90 was +0.782721/0.822706/0.864566. Control P10/P90 was 0.917876/1.045868 before +and 0.936697/1.041502 after. All samples and natural GC remain charged; +=ABI-UTF8-CPU-PROOF-OK= and exit zero arrived in 78.921 seconds. Increased +wall scheduling delay is retained rather than subtracted from the receipt. + +The full-quanta 64-record ASCII experiment retains 20 groups and 256 calls. +CPU P10/P50/P90 is 0.606520/0.646482/0.677537, a 35.35% median decrease. +Control P10/P90 is 0.938066/1.027473 before and 0.929851/1.041150 after. +The complete C ABI/GPA1 proof returned =ABI-EVENT-CPU-PROOF-OK= and exit zero +in 70.174 seconds, with all natural GC charged. + +The full-quanta Unicode 64-record proof also passes at 20 groups/256 calls. +CPU P10/P50/P90 is 0.615277/0.640854/0.686611 (35.91% median decrease). +Control P10/P90 is 0.912586/1.038488 before and 0.953002/1.066427 after. +All complete GPA1 comparisons passed; =ABI-EVENT-CPU-PROOF-OK= and exit zero +arrived in 79.846 seconds. No CPU or collection observation was removed. + +* Linux scheduler control admission + +The =0544706= complete ranked-parser gate rejected a fresh Ubuntu control: +12-clause production P10/P50/P90 was 0.747103/0.883760/0.901997, while +before-control P10 was 0.862753. Two retained groups show one 1,024-request +control leg at approximately 160 ms versus ordinary legs near 95 ms, despite +the same two collections and unchanged 194.8 MiB heap. The collection CPU +also rose from approximately 33 ms to 52 ms. This is CPU cost variation; +the evidence does not identify processor migration as its sole cause. +[[https://github.com/tao3k/gerbil-parser/actions/runs/38140263156][Latest-head rejected control]] retains all samples and exit one. + +Linux CPU proof shells now select the first processor from their actual +allowed processor set and bind themselves with =taskset=. All Scheme children +inherit that affinity. This removes processor migration as one uncontrolled +measurement variable without changing production algorithms, GC accounting, +heap limits, sample admission, request budgets or timeouts. A fresh CI run +must still pass the original controls; affinity itself is not a CPU benefit +receipt. The independent 64-record event proof precedes the ranked-parser +obligations so its target evidence is retained even if another gate rejects. + +* Checked tagged row scalars at the Scheme boundary + +The =92f68d8= Ubuntu affinity experiment stabilized both 64-record ASCII +controls (P10/P90 0.967133/1.030082 before, 0.965834/1.033623 after), but +production P10/P50/P90 0.894996/0.938913/0.966094 failed the unchanged 0.95 +benefit gate. The Linux result is a rejected 6.11% median decrease, not the +approximately 35% admitted Darwin result. Exit one at 86.506 seconds is +retained by [[https://github.com/tao3k/gerbil-parser/actions/runs/38141039420][Pinned Linux rejected benefit]]. + +The shared row writer previously entered seven generic foreign argument +converters on every event. Upstream =scheme/utf8.ss= already demonstrates a +checked =##c-code= leaf with tagged fixnum input and disabled interrupts. +The writer now applies that convention: admit the byte vector, every scalar's +exact fixnum type, unsigned width and complete 24-byte span before borrowing +the buffer. The bounded inline body writes one row and retains no pointer. +Non-fixnum unsigned values, including the full uint64 maximum, retain the +general integer conversion leaf. The public row representation and full +C ABI benchmark control remain unchanged. + +GCC compiled the checked inline producer in 8.789 seconds. The independent +language ABI suite passed with final =OK= and exit zero in 10.779 seconds, +including unaligned unsigned extremes, negative and fractional ids/positions, +late-field overflow without partial writes, natural collection, source history +and complete ABI product parity. + +The checked inline row writer passes the fresh complete 64-record ASCII +experiment: CPU P10/P50/P90 is 0.576581/0.622262/0.682581, a 37.77% median +decrease. Controls are 0.950086/1.050963 before and 0.960248/1.057124 after +(P10/P90). The proof retains 20 groups and 256 nominal calls, complete GPA1 +parity and natural GC, with =ABI-EVENT-CPU-PROOF-OK= and exit zero in 73.616 +seconds. The matching Unicode experiment also passes: production +0.602479/0.646371/0.686731, a 35.36% median decrease; before controls +0.949225/1.061191 and after 0.939566/1.067563. Its marker and exit zero arrive +in 74.295 seconds. These are local Darwin receipts; Linux admission remains +a separate exact-head CI obligation. diff --git a/docs/roadmap/0001-versioned-language-pack-closure.org b/docs/roadmap/0001-versioned-language-pack-closure.org index 10a410de..1a16b433 100644 --- a/docs/roadmap/0001-versioned-language-pack-closure.org +++ b/docs/roadmap/0001-versioned-language-pack-closure.org @@ -1,100 +1,73 @@ -#+title: Versioned language-pack closure plan +#+title: Parser engine convergence plan #+startup: overview -* Outcome - -Land GQL, openCypher, and TLA+ as immutable, source-owned language packs on the -one Gerbil parser compiler and runtime. A language is complete only when its -pinned upstream grammar, admitted corpus, generated artifacts, installed -consumer, negative contracts, and performance receipt are all exercised by the -single native package test entrypoint. - -* P0 Native package closure - -- Keep =build.ss= as a thin =std/build-script= entrypoint over one library - PackageSpec; build the optional =gparse= executable through the sibling - first-party =build-gparse.ss= entrypoint. -- Discover each production module once; never install =*-test.ss= modules. -- Inherit capacity only from =GERBIL_BUILD_CORES= and use the ASP native batch - testing interface without a package-local scheduler. -- CI must clean, build the package and command products, run the native - =gerbil test= interface - once, and exercise installed =gparse=. Build-spec duplication is a failing - contract. - -Status: locally closed. Remote Ubuntu CI is the remaining publication receipt. - -* P1 GQL ISO/IEC 39075:2024 closure - -- Preserve the byte- and digest-pinned OpenGQL 1.9.0 source at its declared - commit. -- Admit all 1,018 grammar rules: 574 parser rules and 444 lexer or fragment - rules, with no handwritten runtime language dispatch. -- Keep all 14 upstream samples, malformed-input failures, CST kinds, lowercase - keyword recognition, roundtrip, and installed sidecar consumption in the - native acceptance suite. - -Status: implementation and local acceptance closed; CI publication is pending. - -* P2 openCypher 2024.1 closure - -- Preserve the official 352-production WG3 BNF source and exact digest. -- Keep the 3,897-query TCK syntax audit reproducible from the pinned corpus; - expected-success and expected-runtime-error queries are syntax acceptance, - while expected compile errors remain an explicit negative set. -- Add the TCK audit to a bounded CI scenario instead of treating a checked-in - receipt as live evidence. - -Status: grammar, representative parser suite, and installed consumer are -closed; reproducible CI regeneration of the full TCK receipt remains. - -* P3 Native TLA+ syntax adaptation - -- Model the native =TLAPlusGrammar= productions from chapter 15 of - /Specifying Systems/ directly in the existing lexical and Grammar IR - algebras. Use the pinned official SANY =tla+.jj= source as the executable - conformance reference, never as a runtime parser dependency. -- Add only generally useful algebra operations to =src/language/= when a TLA+ - production cannot be represented. Every extension must retain canonical IR, - deterministic LALR construction, typed diagnostics, and byte roundtrip. -- Publish source-admission scenarios for unresolved productions, duplicate - identities, source-identity drift, deterministic canonical IR, and corpus - disagreement with SANY. - -Status: the digest-bound 99-production inventory and native-core v1 expression -slice are executable; production-to-GrammarIR closure remains. - -* P4 TLA+ syntax closure - -- Replace =tla-plus.native-core.v1= with a new explicit full-grammar contract; - do not silently widen the existing v1 identity. -- Admit every production and lexical constraint in the pinned native syntax - sources, including expression precedence, proof steps, indentation-sensitive - conjunction/disjunction lists, extramodular text, operator disambiguation, - and PlusCal boundaries. -- Run the official SANY syntax corpus plus normal, Unicode, proof, PlusCal, - rejected-syntax, and crash-regression cases. Preserve source bytes and - compare the required structural CST projection; never downgrade an - unsupported construct to an =unknown= acceptance path. - -Status: four accepted and one rejected native-core fixtures are closed. Full -proof, PlusCal, layout, and official-corpus conformance remain open. - -* P5 Algorithm and performance admission - -- Keep LR(0) construction exactly-once and lookahead propagation delta-only. -- Establish cold expansion, warm artifact reuse, corpus parse, recovery, and - external-scanner scenarios with at least 40 samples where percentile gates - are used. -- Fix an algorithm or data structure when a gate is red; never reduce sample - counts or hard-code a smaller worker count to obtain a pass. - -* P6 Downstream and release closure - -- Link the completed parser into =gql-rust= for development, then pin the - published gerbil-parser PR revision through its real dependency graph. -- Prove the Rust GQL consumer uses the installed Gerbil artifacts and the same - canonical AST/ParseArtifact contract. -- Require clean-source rebuild, single-entry native tests, installed consumer - smoke tests, and green PR receipts before declaring the language packs - released. +* Current baseline and evidence + +Reviewed remote baseline: PR #13 =5bcc3f6=. Rust and macOS CI passed; Ubuntu +failed the Source benchmark because its C fixture borrowed Scheme storage across +an allocating callback. The current unit fixes that lifetime, qualifies natural +GC, removes the encoder's row-index list and admits allocation-counter receipts +conservatively. Local cost and ABI suites pass 15 cases; full delivered-head CI +remains required. Historical receipts below are not current evidence. + +The eight language families use the three author interfaces: =grammar.ss= +(syntax/rules), =parser.ss= (metadata and admitted execution), and +=parser-test.ss= (fixtures and conformance). Ten loader scopes are admitted. +Source lexical edit histories exist in Scheme, standalone Rust and native FFI. +They do not establish Source command/CST subtree reuse: recognition and +canonical publication still execute for every edit. + +* Active sequence and exit gates + +| Order | Complete change unit | Exit gate | +|-------+----------------------+-----------| +| 1 | FFI ownership and schema convergence | One handle-based parse API, no builtin name-dispatch facade, no versioned file/namespace names, Schema-owned ABI identities, migrated real consumers | +| 2 | Source edit cost attribution | Actual C calls, fresh/session full GPA1 parity, uninstrumented matched samples, separate request-local phase observations, retained-data counts and bounded memory claims | +| 3 | Measured engine refactor | Select the dominant complete-call cost; exact semantic controls and matched performance/space evidence; reject unstable or negative candidates | +| 4 | Strategy and language convergence | Audit all ten loaders, remove language-specific engine branches through admitted strategy data where justified, unsupported targets reject explicitly | +| 5 | Downstream and delivery | Reproducible source generation, independent Rust/C/Scheme consumer gates, clean native builds and exact-head CI | + +Steps 1 and 2 are the current implementation unit. Step 3 is selected from their +measurements; additional caches and Source subtree reuse are not preselected. +HCL still has a reviewed fixed lexical/grammar fast path; portable TLA+ proof +policy and general recursive lowering have separate obligations. + +* Cost and research discipline + +Complete-call measurements include UTF-8 admission, scanning, recognition, +publication, encoding, C transfer and the consumer readback/release. History +priming and validation are outside timing. Instrumented stages are inclusive +and may nest; they are never summed as independent end-to-end costs. + +Each engine refactor must document two questions with executable Scheme blocks: +why the author interface is preferable, and why the semantic/algorithmic +contract is justified. Grammar authors use hygienic syntax/rules and POO +composition or ordinary values as needed. Engine mechanics do not become new +cache, allocation or checkpoint clauses in the author DSL. + +Paper claims must identify the implemented invariant and its assumptions. +Scanner suffix reuse does not inherit an optimal incremental-parser complexity +bound. Keep negative measurements and distinguish local receipts, remote CI, +syntax coverage and real downstream admission. + +* Current research modules + +- [[file:../research/scheme-source-edit-sessions.org][Source edit ownership and correctness]] +- [[file:../research/source-edit-cost-contract.org][Complete-call cost contract and qualification]] +- [[file:../research/dsl-engine-contracts.org][DSL and engine contracts]] +- [[file:../research/scheme-native-build-ownership.org][Native build ownership]] +- [[file:../research/source-event-fold-ownership.org][Event Fold execution ownership]] + +* Historical modules + +These preserve prior controls and decisions without appending them to the +active plan. Open obligations must be re-audited against current source. + +- [[file:history/baseline-admission.org][Baseline inventory and admission]] +- [[file:history/source-design.org][Source design]] +- [[file:history/scanner-alignment.org][Scanner alignment]] +- [[file:history/syntax-coverage.org][Syntax coverage milestones]] +- [[file:history/language-interfaces.org][Language interface migrations]] +- [[file:history/generated-strategies.org][Generated strategy migrations]] +- [[file:history/source-composition.org][Source composition]] +- [[file:history/rust-source-execution.org][Rust Source execution]] diff --git a/docs/roadmap/history/baseline-admission.org b/docs/roadmap/history/baseline-admission.org new file mode 100644 index 00000000..00955ea5 --- /dev/null +++ b/docs/roadmap/history/baseline-admission.org @@ -0,0 +1,94 @@ +#+title: Historical baseline admission receipts + +This module preserves prior design and qualification records. Its status notes +are historical; the active execution plan is [[file:../0001-versioned-language-pack-closure.org][the convergence plan]]. + +* Current convergence plan — 2026-10-06 + +Authoring baseline: PR #13 =1b05ee4e0620e42ab062a366c64ee2b0a546780c=. +Rust and Ubuntu native CI passed at that exact head, including shared Bash +scanner generation. The source-backed recognition migration is a subsequent +change and requires its own verification. This is the +single execution plan for language-layer convergence. The coverage milestones +below remain separate syntax obligations; their old publication-status notes +are historical, not current CI state. + +** Current authoring receipt + +The three-interface ownership migration is implemented across all eight packs. +The executable Scheme topology gate constrains the directory and mandatory +Loader/test declarations. Engine source strategies, structured scanner/proof +profiles, record projection and graph syntax expose reusable value contracts. +The old procedural pack paths and compatibility qualification facade are removed. +Generated arithmetic/HCL products and the HCL generator belong to =src/compiler/=; +HCL's recursive provider keeps an explicit reviewed corridor. + +The stage criteria below remain stronger algorithm/portable-IR goals. Source +strategy ownership alone does not satisfy S2/S3 portable shell execution or S4 +universal recursive lowering. No new end-to-end performance gain is claimed. + +** Inventory and ownership + +Eight language families publish ten POO loader objects. All ten already expose +=grammars=, =metadata=, value =fixtures= and named =tests= services through +=deflanguage-parser-loader=. TLA+ has three scopes; the other families have one. +Entry unification does not establish shared scanner execution or Rust admission. +There is no =sky= module in this checkout. Bash scanner modes and deferred +markers now compile from a POO profile to shared contextual Scanner IR. The +former shell scanner module and its alias surface are deleted. + +| Family / scope | Current declarations | Open convergence boundary | Owner | +|---+---+---+---| +| Arithmetic v1 | Concise DSL; one loader | Engine-owned =src/compiler/arithmetic-drive.ss= product | Engine compiler; pack retains expressions and precedence | +| HCL v2.24 | Concise DSL; one loader | =src/compiler/recursive-source.ss= retains a closed HCL fast lexer and source/event actions | Engine strategy admission/emission; pack retains syntax | +| GQL ISO-39075-2024 | Pinned ANTLR4 source; one loader | Qualify portable products and generic parser/actor mechanisms | Engine owns admission/execution; graph queries/projection remain domain code | +| openCypher 2024.1 | Pinned ISO-BNF source; one loader | Qualify portable products and complete corpus coverage | Engine owns normalization/execution; pack retains source/corpus | +| HL7 v2.5.1 | Concise DSL; one loader | No external declarations; header delimiter/data opcodes and segment text profiles are shared; header context now prepares once per request; other state transitions remain S2 | Shared lexical/context IR; pack retains ER7 delimiter roles | +| FHIRPath v2.0.0 | Concise DSL; shared quoted profiles; one loader | Closed text profiles for identifier, number, date, datetime and time; no external lexical declarations | Shared lexical IR; pack retains profiles and projection | +| TLA+ core/layout/SANY | Three concise DSL scopes and loaders | SANY has no external lexical declarations; module framing prepares source indexes and identifier/proof roles use closed text profiles; Scheme proof policy remains; full syntax coverage is separate | Engine owns source state/policy execution; pack retains proof and syntax rules | +| Bash 5.3 | Source descriptor; one loader | POO =ScannerProfile.= compiles command/marker/body transitions to shared Scheme/Rust IR; Word results use shared source spans, nested boundary indexes and persistent token publication; =shell-parser.ss= interprets closed control programs; Word projection IR executes through =ResultProfile.=; standalone Rust Word/Command/Source recognition is implemented; full ParseArtifact identity/diagnostics, native binding and Source stack convergence remain open | Engine owns state/result mechanics; pack retains shell grammar and word structure | + +Total: zero external lexical declarations in grammar packs. +Bash has a source parser rather than external lexical rows and is counted +separately. These counts measure declaration sites, not semantic coverage or +handwritten functions. Imported ANTLR/BNF sources stay authoritative; copying +those grammars into a second handwritten DSL is not a migration requirement. +Domain projections, pinned sources, fixtures and generated artifacts have +legitimate language ownership. File count alone is not a convergence gate. + +** Loader admission is the authoring contract + +| Declaration boundary | Required constraint / engine owner | Current state | +|---+---+---| +| =grammar= / =source= | Descriptor kind, identity and fixed parse dispatch; canonical grammar/catalog checks happen in engine compilation | Admitted; source parser still a separately advertised migration scope | +| Lexical/scanner roles | Versioned closed IR, input/span/progress bounds and exact producer identity; no unregistered worker procedure | Closed lexical roles enforced; header/module source contexts prepare once; Bash scanner is declared IR, while source recognition remains separate | +| =fixtures= | =defsyntax-corpus= data; whole-corpus language/version/contract, unique ids and typed fixtures admitted before any parse | Admitted by the engine fixture service | +| =tests= | Declared service identity plus execution/result protocol bound to the loader; arbitrary return values must not establish conformance | Private typed fixture-test declarations bind the exact descriptor; engine execution owns conformance. Bare procedures, duplicates and foreign descriptors reject | +| =backends= | Closed strategy kind, exact Parser IR digest, procedure signature, duplicate/occupied-slot rejection before installation | Admitted atomically; HCL/Arithmetic strategy generalization remains | +| POO extension slots | Effective inherited object passes base and extension contracts; identity/dispatch cannot be replaced | Admitted; declarations must retain POO extension support | +| Rust / native C targets | Real producer/consumer capability and artifact checks, independent of loader metadata labels | Separate gates; source callbacks and Scheme policies cannot claim standalone Rust | + +The schema =gerbil-parser.language-entry.v2= replaces procedure-valued tests +with =declare-language-fixture-test= declarations bound to the descriptor. +All ten public loaders inherit the engine-derived default; =run-language-test= +checks the registry before fixture conformance runs. New service kinds require +explicit input/result contracts and POO admission. +Additional language assertions continue through native Scheme/ASP test suites. + +** Stages and dependencies + +| Stage | Engine deliverable / affected packs | Exit condition | Dependency | +|---+---+---+---| +| S0 Baseline and worker isolation | Executable inventory over the ten real loaders; loader-admitted worker declarations; checkpoints bound to their creating source worker | Inventory agrees with admitted IR; raw/foreign scanner services reject at loader admission; malformed worker products reject at creation; foreign checkpoints reject before calling language code | Current baseline | +| S1 Closed lexical profiles | Shared character classes, bounded runs, strict numeric/date/time composition and local guards; FHIRPath, TLA+ local lexemes, HL7 segment id/terminator | FHIRPath's scanner module disappears; local external roles disappear; Scheme/Rust endpoint and malformed-suffix behavior agree | S0 | +| S2 Source context and transitions | One prepared source context, named source captures, mode/nesting/deferred obligations; HL7 separators, TLA+ module boundaries, Bash scanner | No header/prefix revalidation per token; closed transitions in both engines; snapshots bind worker/source/artifact; scanner callbacks replaced by data | S1; reuse existing contextual Scanner IR | +| S3 Recognition/result boundary | Source recognition results admitted/lowered by the engine; Bash structured words, nodes, failure and here-document links | Pack emits grammar/recognition values; engine alone owns token bytes, result validation and success/failure artifacts; complete Bash tests remain lossless | S0; S2 for Bash shared-IR execution | +| S4 Generated strategies | General strategy eligibility and emission for Arithmetic/HCL; declarations describe assumptions rather than language-name branches | Reproducible generated products, fresh/optimized semantic parity, no common-case performance regression, no HCL hardcoded lexer in engine | S1; independent of stateful grammar completion | +| S5 Portable policy and downstream products | SANY proof-policy representation; language-independent native C and Rust products through the same descriptor | Each admitted product passes its real producer and consumer gate; unsupported policy/opcodes reject with rule/context; no Scheme callbacks disguised as Rust portability | Relevant S1–S4 parts; complete policy semantics | + +Work proceeds by engine capability, migrating every applicable pack together. +A stage closes only after production callers migrate and obsolete callbacks or +assembly paths are removed. Moving an old callback into =src/= alone does not +close a stage. S4 can proceed alongside S2/S3; it does not wait for Bash full +syntax. SANY portable proof policy does not block ordinary lexical lowering. + diff --git a/docs/roadmap/history/generated-strategies.org b/docs/roadmap/history/generated-strategies.org new file mode 100644 index 00000000..ae35b538 --- /dev/null +++ b/docs/roadmap/history/generated-strategies.org @@ -0,0 +1,276 @@ +#+title: Historical generated strategies receipts + +This module preserves prior design and qualification records. Its status notes +are historical; the active execution plan is [[file:../0001-versioned-language-pack-closure.org][the convergence plan]]. + +* TLA+ canonical tests and engine TLC execution — 2026-10-05 + +Core, layout, SANY candidate and SANY closure now use four +=deflanguage-parser-tests= suites in =languages/tla-plus/parser-test.ss=. +The 57 rows retain all previous source literals and obligations; three raw +scope test files were removed and the aggregate pack suite exports all four. +Model receipt, fixture selection, metadata, native binary status and actual +Rust producer rejection execution are constrained engine-owned operations. + +Production TLC qualification moved from the language directory into +=src/language/tlc.ss=. The admitted descriptor supplies syntax parsing. +The legacy qualification module forwards the canonical parser API; independent +controls require identical function and receipt identities. The closed tool +fixture quotes stdout as data, including quotes and command substitution, and +removes its temporary directory after qualification. + +| Final native gate | Qualified result | Process time | +|---+---+---| +| TLA+ canonical entry | Layout 8, core 17, candidate 22, closure 10; 57 cases | 7.96 s | +| Public DSL controls | 50 malformed declarations rejected; seven runtime controls | 12.60 s | +| Product engine controls | Legacy API identity, inert stdout, exact receipt hashes and cleanup | 10.98 s | +| HCL compatibility | All 29 declaration rows | 10.79 s | + +The three final control runs were concurrent. All gates have explicit =OK=, +=HARNESS-RETURN status=0= and exit 0 under unchanged 90/5 limits; native +compiler phases use SDK/GCC and 90/30 limits. These process times include module +loading and are qualification evidence, not runtime performance improvements. + +The initial migrated nonzero-tool row failed because shell exit 7 is stored as +raw Gerbil wait status 1792. The replacement =(nonzero exit-status)= expresses +the original obligation without changing runtime receipts. A consumer compiler +import also failed before the new TLC native product existed; compiling that +product and importing it in a fresh compiler process resolved the dependency. +Neither failure was admitted, and the native compiler/profile stayed unchanged. + +Synthetic TLC output checks receipt admission, final versus intermediate state +counts and syntax rejection before tool resolution. Actual model checking and +standalone Rust support for the SANY proof policy are not established by these +fixtures; unsupported policy still fails through the actual Rust producer. + +** Current remaining order after this migration + +1. HCL generated product ownership: engine strategy generation and execution + must absorb =generate-direct.ss=, =direct-step.ss= and =direct-recursive.ss= + through general admission rules. +2. TLA+ scanner/proof policy, grammar metadata and corpus organization: converge + remaining language files onto the three public interfaces while preserving + scope semantics and the explicit portable-policy boundary. +3. Bash procedural scanner, word and source recognition: replace remaining + callbacks with engine-owned declarative state and operations. + +This supersedes the earlier core/SANY test migration queue. It closes those +test suites and TLC execution ownership, not every language runtime directory. + +* Generic POO fused reductions and Loader build slots — 2026-10-05 + +The HCL fused reduction producer is now the reusable engine +=FusedReductionStrategy.= prototype, exposed by =language-build-support.ss=. +It derives actions from a descriptor's exact Parser IR and admits the complete +production/action algebra before emission. POO inheritance supplies effective +binding names and metadata; stale digest, invalid names and foreign descriptor +bindings fail admission. No generator branch selects a builtin language. + +HCL and Arithmetic both declare =fused-reductions= in their public Loader +=build-strategies= slot. The Loader contract checks unique declared names and +exact descriptor identity. Empty strategies remain the default for other +languages; the existing Loader schema version is unchanged. All build work is +outside source parser dispatch. + +The HCL generated step moved to =src/compiler/hcl-reductions.ss=, and the old +benchmark generator was deleted. CI uses =scripts/generate-reductions.ss= with +the grammar's module identifier. A second generated native fixture is produced +from Arithmetic's actual IR through the same engine. Existing event tests and +incremental benchmark imports follow the new engine product path. + +Seven independent native controls cover deterministic interleaved generation, +POO names/metadata inheritance, stale and malformed overrides before any output, +Loader duplicate/foreign binding rejection, both languages' full artifact parity +in recognition and event program backends, and malformed-source rejection. +They compare against the indexed runtime without replacing installed routes. + +The initial generator compiler run rejected an unimported list utility; the +name check now compares the three slots directly. Two initial test runs caught +binding-group/closing-parenthesis errors and remain failures. Initial plain +script launchers hit the unchanged five-second idle gate after imports; those +receipts were not admitted. The producer was exercised through the existing +native child harness with explicit main-return/exit propagation, using module +identifiers rather than repeated source imports. No timeout was widened and no +compiler/profile was replaced. + +This closes fused reduction generation and HCL step ownership. The remaining +HCL recursive generator and its fast lexical/source corridor must become a +separately admitted reusable strategy; moving those HCL assumptions into a +nominally generic module would not establish reuse. TLA+ scanner/proof policy +and Bash source/word state remain the following runtime convergence work. + +| Final GCC native gate | Result | Process time | +|---+---+---| +| Generic POO strategy | Seven controls; two grammars and both semantic backends, including protected dependency names | 5.38 s | +| HCL compatibility | All 29 declaration rows | 5.26 s | +| Existing Loader contract | POO inheritance, source/grammar descriptor kinds and extension admission | 4.19 s | +| Public test syntax | 50 malformed expansion forms and seven runtime controls | 5.02 s | +| Existing event program/incremental controls | Complete existing suite | 6.89 s | +| Arithmetic public parser | Complete existing declarative suite | 4.91 s | +| Common producer, HCL | Exact deterministic generated source, explicit return and exit 0 | 3.72 s | +| Common producer, Arithmetic | Same CLI, exact deterministic generated source, explicit return and exit 0 | 5.80 s | + +The native suites require =OK=, =HARNESS-RETURN status=0= and exit 0 with +90/5 limits. GCC compiler phases remain 90/30. Final suites were run in +independent concurrent batches. Generated HCL forms were also compared directly +with the previous artifact: all executable forms are identical except the +language-independent unknown-production error wording. There is no new runtime +speedup claim from this ownership and reuse change. + +The migrated HCL generated module's SDK/GCC native phase completed in 9.30 s +while independent modules compiled concurrently; this is a compiler receipt, +not a runtime comparison. + +* Common POO build protocol across provider kinds — 2026-10-05 + +The previous registry admitted only =FusedReductionStrategyContract= and called +its emitter directly. That restriction is replaced by =BuildStrategyContract= +and =emit-build-strategy=. Identity/digest/metadata/target checks now live in the +common prototype; concrete providers refine admission. The Loader has no +concrete backend import and no language/kind dispatch branch. + +Both HCL and Arithmetic now declare fused Scheme reductions and canonical +Rust output through the same =build-strategies= slot. Rust output is +produced by the existing producer; its source and portable-policy gates remain +unchanged. Public recipe values remain ordinary POO inheritance/override values, +and the Loader schema remains v2. + +Eight controls cover two languages and two real provider kinds, exact canonical +Rust output, renamed downstream declarations and inherited metadata, registered +writer dispatch despite a recipe method override, provider/target/digest forgery +rejection, additional constrained engine recipe/format registration without +changing common dispatch, foreign-descriptor rejection and Scheme/Rust policy +admission differences. The extension control also checks eight actual parser +calls cause zero extra provider admission or emission invocations. + +The first consumer compiler attempted to load the provider's generated macro +SCM before its native phase product existed. The next consumer used a cached +facade export set; after moving the convenience method out of the Loader, that +old interface referred to a removed internal export. The specific generated +facade interface/optimizer files were saved and rebuilt, then consumed in a +fresh compiler process. These failed phases remain failures; SDK/GCC and bounded +native gates were retained. No package uninstall, compiler switch or timeout +extension was used. + +This establishes a common constrained POO capability protocol and two real +implementations. It does not close HCL recursive/source strategies, TLA+ policy +portability, Bash word/source scanning or the three-file language layout. + +| Final GCC native gate | Result | Process time | +|---+---+---| +| Common build protocol | Eight controls; two languages, two real providers, constrained engine extension and eight parses without build admission/emission | 4.67 s | +| Fused reduction compatibility | Seven controls; both languages, deterministic generated forms and both semantic backends | 4.59 s | +| Existing Loader admission | Complete existing contract/extension suite | 7.56 s | + +All three final runs returned =OK=, =HARNESS-RETURN status=0= and exit 0 +under the unchanged 90/5 gates. The first extension fixture hit the five-second +idle gate: its local =provider= value collided with the POO =provider= slot, +causing self-reference while constructing the prototype. Renaming the fixture +binding to =engine-provider= removed that collision. The next run reached the +last case but rejected its symbol policy identity before strategy admission; +the descriptor API requires a string. Correcting that fixture input produced +the complete passing run. Both failed receipts remain failures, not passes. + +These are capability, semantic parity and dispatch-isolation receipts. They do +not claim a runtime speedup or a new standalone Rust execution qualification; +the Rust producer is unchanged and its exact source is compared for both +languages. The new protocol control is discovered by the existing native test +suite's =t= source traversal. + +* Direct build API refactor, without compatibility adapters — 2026-10-05 + +The common protocol is the single public build route. Removed +=declare-language-fused-reductions= from the facade and migrated both actual +language Loaders to =declare-language-build-strategy= with an admitted concrete +strategy value. The fused provider's form builder and raw writer are private +implementation functions; their public exports and =.module= slot were removed. +The generation CLI and semantic controls now use =emit-build-strategy=. There +is no legacy alias, compatibility mode or alternate dispatch retained. + +POO inheritance of supported recipe slots and provider-specific admission remain +part of the new protocol. Semantic controls compare validated artifacts and +canonical generated output; they are correctness checks for the refactor, not +an obligation to preserve the deleted API. All current guide examples use the +new declarations. Earlier dated sections record historical implementation. + +| GCC native gate, concurrent batch | Result | Process time | +|---+---+---| +| Common protocol | Eight controls, including both real providers and build isolation during actual parses | 15.80 s | +| Fused semantic controls | Seven controls, both languages and both semantic backends | 13.27 s | +| HCL production CLI | Exact deterministic generated source; explicit return and exit 0 | 12.32 s | +| Arithmetic production CLI | Same generic producer, exact deterministic source; explicit return and exit 0 | 12.65 s | + +The batch includes process startup/import work and is not a runtime performance +comparison. The unchanged test gates are 90/5; compiler gates remain 90/30. +Both suites returned =OK= and =HARNESS-RETURN status=0=, and both producers +returned =PRODUCER-RETURN status=0=; every process exited 0. + +A consumer compilation first rejected the old HCL generated interface because +it still imported the deleted declaration. The two affected languages' generated +interfaces and optimizer caches were saved then rebuilt from migrated source; +no source adapter was restored. The failed compilation remains a failure. + +This closes the build declaration/output API migration. Scanner and runtime +ownership migration is still open: Bash has source/word execution adapters, +HL7 has external header-derived delimiter callbacks, HCL has recursive source +execution, TLA+ has lexical/proof policies, and GQL retains query-specific APIs. +These need declarative constrained engine capabilities and actual consumers, +not compatibility wrappers or language-specific code relocated without reuse. + +* HL7 scanner removal through shared closed header rules — 2026-10-05 + +Removed =languages/hl7/scanner.ss= and all six language-owned external callback +references. HL7's concise grammar now declares shared =header-delimiter= and +=header-data= operations. The lexical algebra validates their prefix/count/index +or stop data; execution now belongs to =src/runtime/lexical-source.ss= and the +Rust lexical engine. The count is bounded to 1..32 and headers require distinct non-alphabetic, +non-numeric delimiter characters. The engine compares/borrows source characters +without constructing delimiter lists or hash tables. At this scanner-removal +baseline, header validation still occurred per lexical-rule invocation. The +request-owned preparation described below supersedes that execution path; no +measured speedup percentage is claimed. + +The second actual POO Loader is a test language with prefix ="記"=, two dynamic +separators and =";"= stops. It parses accepted/rejected sources with the same +engine operations. This is an independent language declaration used as a reuse +control, not a second production language pack. POO metadata extension preserves +its descriptor-bound identity. No old HL7 entry alias or external fallback is +retained, and Loader/IR schema identifiers are unchanged. + +The common Rust build strategy emits both complete specs from canonical Scheme +IR. Rust executes the real HL7 ADT A08 corpus, a custom-delimiter message, +and the independent record sources; malformed header recipes reject at spec +admission. The two generated source files are executable compiler witnesses, +not benchmark/result snapshots. CI compiles the Scheme declaration, regenerates +both Rust files with required per-product markers, formats them and compares +exactly. The existing Rust workspace gate runs the new native controls. + +| Local bounded gate | Result | Process time | +|---+---+---| +| Shared Scheme header rules | Three controls: endpoints, closed admission, two actual POO Loaders and lossless rejection | 3.97 s | +| HL7 public declarative suite | Five controls including real fixture/projection and alternate delimiters | 6.38 s | +| Existing text profiles | Four controls including HL7 identifiers/terminators and malformed profiles | 6.40 s | +| Rust header controls | Four filtered tests including three new controls; actual two-language AOT parsing, malformed delimiter/data admission and UTF-8 boundaries | 1.58 s | +| Direct generation CLI | Both actual product markers, exit 0, exact formatted source comparison for both products | 6.48 s | + +Scheme suites require =OK=, =HARNESS-RETURN status=0= and exit 0, with unchanged +90/5 gates. SDK/GCC compiler and Rust compilation use 90/30. Independent Scheme +suites and native compiler products were run concurrently. These process times +include startup/import/build work and are not runtime A/B measurements. + +The first metadata extension control replaced bound metadata with an unbound +object and correctly failed the Loader contract. It now extends the existing +metadata value, preserving language/version/contract/digest identity. The first +generation invocation returned 0 but lacked an exported =main= and produced +neither file; that receipt was rejected after checking product existence. +The corrected exported entry requires both product markers, and a separate +actual CLI invocation verifies exact files. The initial Cargo invocation also +failed before execution because Rust tools were absent from PATH; the existing +project native environment resolved the installed paired Cargo/rustc tools. +No failure was turned into a pass by relaxing a gate or restoring an adapter. + +HL7 scanner ownership is closed; its projection module remains outside the +three-interface target. Next runtime migrations remain Bash word/source state, +HCL recursive source strategies and TLA+ scanner/proof policy. Each must become +a constrained declarative engine capability with actual reuse controls. + diff --git a/docs/roadmap/history/language-interfaces.org b/docs/roadmap/history/language-interfaces.org new file mode 100644 index 00000000..f3a5f0fe --- /dev/null +++ b/docs/roadmap/history/language-interfaces.org @@ -0,0 +1,273 @@ +#+title: Historical language interfaces receipts + +This module preserves prior design and qualification records. Its status notes +are historical; the active execution plan is [[file:../0001-versioned-language-pack-closure.org][the convergence plan]]. + +* Three-interface language pack contract + +Every language converges to =grammar.ss=, =parser.ss= and =parser-test.ss=. +=parser.ss= is the only public entry. Grammar identity, upstream digests and +corpus declarations stay with =grammar.ss=; the loader binds their values to +POO metadata and fixture services. No separate =source.ss= or =fixtures.ss= +authoring module is required. Test consumers import the public entry. +Upstream grammar/corpus resources remain source files, not Scheme interfaces. + +Pure declaration consolidation removes seven extra modules across arithmetic, +HCL, openCypher, FHIRPath and GQL. It changes neither language identity nor +loader schema. FHIRPath and openCypher have three Scheme interfaces after this +step. The other languages still contain execution or specialization modules; +they need the shared-engine migrations above before this contract is closed. +File consolidation alone does not qualify portable Rust/C execution. + +** Native verification of declaration consolidation + +The five updated grammar modules and four changed loader entries emitted and +compiled through the native Gerbil SDK with GCC 16. A stale generated parser +interface still imported the deleted fixture owner and caused an import +conflict; refreshing only those generated interfaces resolved it. The failed +compiler receipt remains recorded; no test mode or compiler profile changed. + +The engine now freezes ISO BNF source catalogs at expansion, matching the +existing ANTLR source boundary. Runtime restores records instead of reparsing +BNF when the public loader is loaded. The actual new native openCypher product +contains =iso-bnf-source-from-datum=, with no BNF parse call at that binding. +Full catalog field parity and malformed materialization controls passed. +Source macro identity and expected digest are expansion-time literal values. + +| Native control | Result | +|----------------+--------| +| All ten POO loaders and registered corpora | 4 cases, OK; final run 6.12 s | +| Loader extension/fixture admission | 15 cases, OK | +| FHIRPath native conformance | 5 cases, OK | +| openCypher native conformance | 4 cases, OK | +| ISO BNF catalog and overlays | 6 cases, OK; final run 4.94 s | +| ANTLR catalog and LR lowering | OK; final run 4.89 s | + +Controls ran with the existing 90-second total / 5-second idle test gates and +explicit =OK= plus exit status propagation. These are verification durations, +not a matched end-to-end performance benchmark. Source catalog consolidation +does not qualify new Rust/C capabilities, Bash execution or all pack migrations. + +* Shared closed regions and Bash caller migration — 2026-10-05 + +The first step of the whole Bash migration now has production callers. The +public =defregion-plan= accepts stop values, quote escape/scope values and paired +region values. =languages/bash/grammar.ss= declares the actual Bash plan; +=scanner.ss= and =word.ss= consume the engine matcher. The 103-line procedural +=scan-words.ss= and all its imports are removed. This is generic region +execution, not a language-name dispatch or a copied Bash scanner under =src/=. + +The engine prepares and owns a copied plan and operator trie once. It traverses +with one forward cursor and explicit frames rather than recursively invoking +region scanners. Longest declared openers select arithmetic expansion before +command substitution; process substitutions shield overlapping operator stops; +quotes constrain visible nested regions. Initial-stop consumption is explicit +so transitional marker scanning retains its existing behavior. The closed +=region-word= operation is additive within Scanner IR v1; loader schema stays +=gerbil-parser.language-entry.v2=. + +The existing Scheme producer emits the Rust region tables and nine source +traces. Its final native output is byte-identical, after rustfmt, to the checked-in +Rust fixture. The Rust executor admits the same scalar-depth constraints and +uses explicit frames with UTF-8 byte offsets. No Scheme callback executes in +that consumer. + +| Gate | Evidence | Bounded process time | +|---+---+---| +| Native Bash =parser-test.ss= | All 17 cases, source roundtrip, fields, malformed syntax and here-documents | 6.15 s | +| Native region suite | Six cases; 5000 nested regions, nonzero start, opaque quotes, malformed suffixes, copied-plan isolation and admission | 5.02 s | +| All ten POO loaders | Metadata, registered fixture conformance and independent Bash worker obligations | 6.35 s | +| Existing contextual deferred suite | Nine cases; immutable checkpoints and FIFO controls | 2.98 s | +| POO loader extension suite | Fifteen cases; inherited values and constraints | 5.11 s | +| Native Scheme Rust producer | Nine traces; final generated Rust matches repository bytes | 8.67 s | +| Rust unit suite | 90 passed; actual generated region traces, 5000 levels, UTF-8, malformed scope and admission | 1.07 s | +| Rust clippy | All targets, warnings denied | 1.15 s | +| ASP Rust source policy | Actual workspace dev gate passes without policy changes | 1.05 s | + +Native products were compiled with the configured Gerbil SDK and GCC. The first +control batch failed because the new module's generated aggregate lacked a +native object; compiling that actual aggregate fixed the import. Export-graph +changes also required a fresh compiler process to avoid an old cached grammar +interface. Those failures remain failures, with no interpreter fallback or +compiler/profile change. The timings above are concurrent qualification runs, +not comparative performance evidence. + +Fixtures are now declarative values in Bash =grammar.ss= and the unchanged +regression corpus moved to =parser-test.ss=. The three remaining procedural +modules are =scanner.ss=, =word.ss= and =parser-core.ss=. Command transitions, +embedded word projection and recognition/result assembly are the next shared +engine obligations. This step does not close Bash's three-interface target or +qualify a complete Bash Rust AOT/C ABI product. + +* Constrained parser-test syntax — 2026-10-05 + +=deflanguage-parser-tests= is now the public test declaration through the +=language-test-support.ss= facade. It admits the POO Loader with the existing +=LanguageLoaderContract=, evaluates the Loader once, and uses its effective +=.parse= and named fixture-test service. Syntax rows own source/expected values; +the engine owns evaluation and artifact validity/status/roundtrip/digest checks. +Node/token membership, node counts, absent nodes, diagnostics, exact AST fields +and byte spans, and projection values are typed clauses. Property extensions +have explicit bindings and equality assertions; arbitrary test-case statement +bodies, unknown sections and empty properties are rejected during expansion. +Duplicate labels also reject before publishing a suite. + +| Owner | Current migration | +|---+---| +| Bash | 17 syntax/receipt/fixture rows; repeated parser calls and local event traversal removed | +| Arithmetic | Five rows; direct-driver property, loader identity, fixtures, 1024-line input and exact AST | +| HL7 | Five rows; ADT projection/identity, alternate delimiters, lossless rejection and fixtures | +| FHIRPath | Six rows; official source identity, AU fixtures, normative forms and strict profile admission | +| Independent Records pack | Two rows; public identity and accepted/rejected corpus through installed facades | +| HCL | Existing 27 strategy/corpus cases remain; design shared seeded lexer/route parity test declarations next | +| GQL / openCypher | Existing precedence, normalization and bound-source controls remain; generic AST/IR expectation extensions still required | +| TLA+ scopes | Existing separate scope suites remain to migrate | + +Bash's two source-digest admission and UTF-8 checkpoint tests moved intact to +=t/language-source-contract-test.ss= because they exercise engine contracts. +The regression corpus, ordered marker/body checks, word/node kinds and rejection +inputs were preserved. Tests now additionally check valid lossless artifacts and +source digests for every ordinary syntax row, including rejected source rows. +No parsing implementation, schema, dependency or backend selection changed. + +Native GCC-compiled test support qualified all five migrated consumers. Bash, +Arithmetic, HL7 and FHIRPath bounded runs completed in 5.19, 5.08, 4.55 and 4.11 +seconds respectively. Engine controls cover ten expansion-time negative forms, +single Loader/source/parser evaluation despite multiple expectations, macro +hygiene, and effective inherited fixture override. Records and the retained +source-contract controls also pass. These are qualification timings from +concurrent runs, not performance comparisons. + +The first native support import failed because its newly emitted macro module +had not yet been compiled. The first consumer batch exposed an empty generated +=let= for syntax rows without extra structure expectations and one unmatched +Arithmetic source delimiter. Both source defects were corrected; the exact +failed logs remain separate from passing reruns. No timeout was relaxed and no +interpreter/toolchain fallback was used. Full migration of the remaining +advanced suites is still an open owner obligation; a bare suite wrapper is not +accepted as completion. + +* Advanced graph-query tests and TLA+ layout migration — 2026-10-05 + +GQL and openCypher now use only the public test facade and their =parser.ss= +entry. Their local CST walkers, metadata/IR access helpers, token-driving loops, +source generators and raw test-case bodies have been removed. Existing pinned +identities, Parser/Bound IR counts and ordered references, compatibility and +precedence overlays, digest prefixes, syntax sources, corpus structure and +rejection controls remain declarations. GQL's selected comparison operands +must retain their order; an additional reversed-operand query must fail. +openCypher retains 1,317 states, 321 fork cells, absent nullable-prefix helpers, +initial shift checks and the actual 300-item LR receipt with zero speculative +branches and one completion. + +TLA+ layout's eight controls migrated to =parser-test.ss=; the old +=layout-test.ss= was removed and the native suite owner/build exclusion updated. +Field cardinality, source ordering, CRLF/tab layout, nine concurrent requests, +fresh/incremental artifact equality and disabled recovery rejection all remain. +Parallel scheduling and source-bound result selection belong to the engine; +all workers join before a captured parser exception propagates. Incremental +text selection uses the shared prefix matcher and converts character positions +and deletion lengths to UTF-8 bytes. A Unicode HCL control exercises both a +multibyte prefix and a multibyte replacement. + +| Native gate | Qualified result | Bounded process time | +|---+---+---| +| GQL | Ten cases; source-bound identity, full official fixtures constrained to accepted, syntax and positive/negative precedence queries | 4.58 s | +| openCypher | Eight cases; original Bound IR, overlay, static conflict and actual LR receipt controls | 4.81 s; final current native products | +| TLA+ layout | Eight cases, including nine barrier-coordinated requests, incremental/fresh parity and rejection/recovery | 4.66 s; final canonical parser-test entry | +| Engine DSL controls | Seven cases; 20 expansion-time rejection forms, single evaluation, inherited fixtures, false subtree queries, five parallel sources and Unicode edit | 6.44 s | +| Bash compatibility | All 17 existing declarative rows | 4.62 s | + +The native support was emitted and compiled with the configured Gerbil SDK and +GCC. One emission failed at the imported =string-contains= signature: its +inferred boolean result could not serve as a substring index. The test engine +now uses the existing source-prefix primitive with an explicit forward cursor, +then converts positions to bytes. Failed emission remains a failed receipt; +passing consumers use subsequently emitted and compiled products. These runs +qualify test execution and semantic obligations, not a performance comparison +or a new Rust/C product. No timeout, compiler, profile or schema changed. + +** Remaining migration order + +1. HCL: replace its 27 raw strategy/corpus cases with engine-owned indexed + reference execution, named source/lexer strategy routes and a closed seeded + generator algebra. Preserve the exact LCG seeds, generated source sequences, + fast/fallback coverage, 1,024-line artifacts and complete corpus parity. + Register unsupported routes explicitly; do not hide copied loops in property + expressions or introduce a language-name branch. +2. TLA+ core: move common syntax/corpus rows to the canonical parser-test entry; + model qualification needs constrained temporary-tool fixtures and typed + process/receipt expectations rather than an arbitrary callback body. +3. SANY candidate/closure: constrain source coverage, scanner policy and proof + admission expectations without claiming stronger syntax or portable policy + semantics than the existing corpus. All legacy checks stay active until the + corresponding declarative consumer qualifies. + +The public language interfaces remain =grammar.ss=, =parser.ss= and +=parser-test.ss= as the final target. Partial scope migration does not establish +that TLA+ or HCL has reached that layout, nor does test migration close their +remaining runtime/scanner/source-policy implementations. + + +* HCL strategy and corpus test execution migration — 2026-10-05 + +HCL's 27 raw cases migrated to 29 =deflanguage-parser-tests= declaration rows. +The two added rows split the numeric boundary/generated-source checks and the +Loader identity check; no former obligation was removed. =parser-test.ss= now +imports only the public test facade and =parser.ss=. It has no local function, +LR checkpoint loop, CST walker, random state, thread, scanner execution callback +or free-form test-case body. Its source literals and five seed streams retain +the original input ordering, string/comment terminators, number bounds and +Unicode values. The 1,024-line attribute sources, two 256-block label variants, +256 nested blocks and 128 compact lines keep exact indexed-reference artifacts. + +The engine's =language-strategy.ss= owns the ranked/indexed reference driver, +closed source template interpreter, named generated recognizer routes, lexical +parity and fast/fallback admission counts. The =native-test-profile= Loader slot +is an ordinary extensible POO value. =LanguageLoaderContract= checks its profile, +exact machine digest, source procedure identity and declared lexer procedure. +The profile is optional; arbitrary recognizers do not acquire recursive-source +parameter semantics merely by having a parse entry. This uses the existing +=language-entry.v2= schema and adds no per-call runtime admission. + +Seven engine controls independently compare all 1,024 sources from the original +five port-writing generators, require one lexer call per coverage input, admit +an effective inherited profile and reject a stale digest, foreign recognizer, +unknown profile, absent lexer or profile for another grammar. The public macro +now rejects 35 malformed declaration forms, including unknown/duplicate routes, +invalid bounds, unknown bindings and incomplete coverage obligations. Coverage +uses the candidate lexical result once rather than running the scanner again. + +| Final native gate | Qualified result | Process time | +|---+---+---| +| HCL | All 29 rows, full 15-fixture corpus, exact generated/native/indexed artifact parity | 6.87 s | +| Strategy engine controls | Seven cases; all five full source streams, profile admission and single lexical invocation | 4.26 s | +| Public DSL controls | Seven cases; all 35 expansion rejection forms, hygiene, inherited fixtures, structural queries and Unicode edits | 4.57 s | +| Bash compatibility | All 17 existing declarative cases | 4.68 s | + +The SDK/GCC production path remains unchanged. The first HCL run failed during +macro expansion because a pattern variable collided with the reserved =status= +keyword. A subsequent run caught =cdr= on the empty parser-route list in a +lexical-only row. Both failed receipts remain failures; hygiene and empty-route +handling were fixed in the engine, and final consumers passed with explicit +=OK=, =HARNESS-RETURN status=0= and exit 0 under 90-second total/5-second idle +limits. Native compiler phases retain 90/30 limits. These times are qualification +receipts rather than an end-to-end performance claim. + +** Current remaining order + +1. TLA+ core/model qualification: migrate syntax/corpus rows and constrain the + temporary-tool/model process receipts without replacing existing assertions. +2. TLA+ SANY candidate/closure: declare source coverage, scanner policy and proof + admission checks with their existing evidence boundaries. +3. HCL generated product ownership: move =generate-direct.ss=, =direct-step.ss= + and =direct-recursive.ss= generation/execution responsibility into the engine. + The current test profile is an explicit compatibility adapter to those + generated products; it does not establish closed Scanner IR or portable AOT. +4. Remaining procedural Bash/source backends: finish scanner, word and source + recognition convergence before claiming every language has only three files. + +This closes HCL test execution migration, not every language runtime directory +or Rust/C qualification. All remaining legacy suites stay active until their +replacement declarations qualify. + diff --git a/docs/roadmap/history/rust-source-execution.org b/docs/roadmap/history/rust-source-execution.org new file mode 100644 index 00000000..4b44cdb4 --- /dev/null +++ b/docs/roadmap/history/rust-source-execution.org @@ -0,0 +1,246 @@ +#+title: Historical rust source execution receipts + +This module preserves prior design and qualification records. Its status notes +are historical; the active execution plan is [[file:../0001-versioned-language-pack-closure.org][the convergence plan]]. + +** Loader fixture metadata convergence + +On top of the research revision =b455d402281aa1e0d9616a3acc210ba6c6b52cca=, +fixture status derivation and identity indexing move fully into the Loader. +The engine validates and copies an effective corpus once, then caches ordered +partitions, counts and an ID table in its sealed POO slot. Foreign identity, +duplicate ID and digest failures now reject at admission, before parsing. +Replacing/extending the fixture value list derives a fresh catalog; no user +callback or partition procedure is needed. The language-specific accepted and +rejected aliases in HCL, openCypher and TLA+ are removed with all consumers. +Test strategies and benchmark corpus preparation use the shared Loader APIs. +Benchmark metadata preparation remains outside timed parse calls. This changes +metadata algorithms and authoring constraints; it makes no parser speed claim. + +The macOS failure from the previous commit is a finite generator exit stall: +=STRUCTURED-LEXICAL-RUST-GENERATED= was printed, followed by the unchanged +five-second idle deadline and termination. Seven remaining standalone Scheme +writers now flush and invoke the existing normal child-process exit binding +only after all output ports close. No timeout or compiler selection changes; +generation/formatting must retain byte-exact checked Rust products. + +The first Loader/value native gate passes 22 cases in 20.50 seconds. It includes +POO replacement/extension, sealed prototype behavior, cached slot identity, +ordered Unicode fixture lookup, copied input/publication isolation, foreign and +duplicate identities, corrupt source digests and actual accepted/rejected +semantic conformance. Three Unicode class writers exit normally in 2.26, +2.54 and 2.42 seconds with byte-exact formatted Rust outputs. Structured lexical +regeneration exits zero in 10.28 seconds and matches its checked product. +Local quoted regeneration initially fails because its native test module was +not compiled; that dependency is compiled with GCC before retrying. TLA+ entry +compilation initially sees an old generated =parser.ssi= with removed exports; +only that generated interface is discarded before rebuilding from current +source. These failures remain recorded in the qualification transcript; neither +is counted as a pass. + + +Final native qualification passes 144 Loader/topology/metadata/pack cases in +46.70 seconds, 29 complete HCL cases in 9.33 seconds, and 73 openCypher, +TLA+ and FOLLOW-partition cases in 22.45 seconds. Each process reports final +=OK=, =HARNESS-RETURN status=0= and exit zero under the unchanged 90-second +runtime / five-second silence fences. The initial combined language gate +stops during cold admission at 36.59 seconds without reaching its harness; +GCC compilation of the test entries and separate qualification of their +natural language groups resolves that local gate without changing deadlines. +Both migrated benchmark entries compile successfully with GCC. + +The remaining quoted, text and header writers exit normally in 14.34, +20.68 and 20.46 seconds. All seven migrated finite writers retain byte-exact +formatted Rust products, including both header outputs. This directly +qualifies local output and process termination; fresh CI remains a separate +platform gate. + +Before submission this slice fast-forwards to the concurrent research head +=29dcb79dafcde202fc9818def3ac536cc20f47c4=. That revision adds Scheme macro +binding experiments and Org semantics research, without modifying the engine +or language interfaces qualified above. + +Native language and FFI qualification also passes 15 cases in 23.63 seconds. +These verify independent descriptors, exact binary publications and accepted/ +rejected calls through the public length-delimited C ABI. Small 100-call +controls remain local receipts, not end-to-end performance claims. +HCL and GQL corpus count assertions now use the existing declarative +=fixture-catalog= test clause; no direct indexing call is required in their +public test interface. + +After the declarative count migration, the final HCL/GQL native gate reports +41 cases, final =OK=, =HARNESS-RETURN status=0= and exit zero in 6.93 seconds. + + +** Shared result construction in Rust + +Following =d8ba30d3216fe9d251ead1808a9f0cc0c3d5c35c=, the Rust runtime gains +an engine-owned interpreter for the Scheme ResultProfile construction IR. The +existing compiler emits the table from admitted POO profiles. The execution +boundary preserves typed captures, ordered fields, required/optional values, +UTF-8 byte ranges and source/catalog ownership. Bound instruction entries +avoid repeated name resolution, and nested composition moves subtrees before +one iterative event publication. This is a structural data/algorithm change; +no parser speed improvement is claimed. + +Thirteen controls come from the actual Scheme Word engine, including empty +words, Unicode, all quote forms, nested parameters and subscripts, command/ +arithmetic/process substitutions, assignment and heredoc content. Rust replay +executes the construction table and compares complete events, field order and +Rust syntax roundtrip. This witness deliberately does not establish Rust recognition +of Word or Command syntax. The CI regeneration gate uses unchanged 90-second +total / five-second silence fences and byte-exact formatted Rust comparison. + +Local GCC compilation of the producer succeeds. Scheme ResultProfile and Bash +qualification passes 41 cases in 9.56 seconds with final =OK=, +=HARNESS-RETURN status=0= and exit zero. Thirteen-control regeneration exits zero +in 6.39 seconds. Rust passes all 110 tests and strict Clippy. + +A second category-separated vocabulary executes through the same interpreter +and Rust syntax sink, exercising node/token names with identical spelling. ASP Rust +source policy passes. Its first local link attempt fails to resolve =libiconv= +through the host Nix =cc= wrapper; using the existing platform SDK and platform +linker path resolves the environment failure without a Scheme compiler change. +Final deterministic regeneration matches the checked Rust bytes in 10.43 +seconds, with normal process exit and the unchanged runtime fences. + + +At =02f9ef4cc344946af77a7802080413abba0f7785=, exact-head Rust and Ubuntu CI +both pass. macOS passes the new result projection regeneration gate, then +fails in the existing Bash scanner fixture writer after printing +=BASH-SCANNER-RUST-GENERATED accepted=46 rejected=4=: the child remains alive +until the unchanged five-second silence fence terminates it. This establishes +a completed generation / process-exit stall, not a recognition or build failure. +The four remaining CLI writers in the shared-scanner fixture family now close +their output ports, publish completion and use the existing finite child exit +binding. Local regeneration confirms unchanged Rust, Scheme and C products: +generic scanner 10.83 seconds, Bash scanner 5.83, contextual parser 4.65 and +native pack 0.33; every process exits zero under 90/5-second fences. + + +** Shared region recognition in Rust + +At =60101b7fceaf7906e121ffbb701a162027b613f4=, Rust, Ubuntu and macOS CI +all pass. The next structural boundary extracts the existing RegionWord stack +machine into a shared engine, rather than adding a separate language recognizer. +The same declaration validator and region-field emitter serve scanner AOT and +standalone region AOT. Pinned Scheme whitespace remains authoritative. + +The engine adds once-admitted reusable plans, source-local endpoint caches and +restricted pair scopes. Scanner-only execution leaves cache writes disabled. +Forty source/profile controls are produced by Scheme from production Bash and +an independent Unicode configuration; Rust independently recognizes boundaries, +including rejected requests. A 512-level nested control checks linear outer +visitation and zero character visits for subsequent cached inner requests. +No end-to-end parse speed improvement is inferred from these structural checks. + +The remaining full Source AOT obligations are Part/Binding composition followed +by Command execution and complete ParseArtifact parity. Region execution and +ResultProfile construction are shared engine capabilities; their completion +does not by itself publish an independent Rust Bash Source parser. + +Local qualification: GCC producer compilation succeeds; Scheme region admission +and matching passes ten cases in 6.70 seconds with final =OK=, +=HARNESS-RETURN status=0= and exit zero. Rust passes 114 cases; strict +Clippy and the ASP Rust workspace policy gate pass. Region regeneration exits +zero in 6.68 seconds and matches formatted checked Rust bytes. All four existing +shared-scanner products remain byte-exact (5.44/5.19/4.40/0.37 seconds). The +pinned Scheme whitespace class regenerates byte-exact in 1.34 seconds. Runtime +fences remain 90 seconds total and five seconds of actual output silence. + + +** Shared Word and Binding recognition in Rust + +The region boundary at =619bf1905b9dfc9a4541d30e1daffe9190587254= passes +Rust, Ubuntu and macOS CI. This next boundary adds a generic Scheme-owned +PartProfile producer and independently executing Rust composition engine. +Public language Scheme interfaces remain exactly =grammar.ss=, =parser.ss= +and =parser-test.ss=; the producer and recognition algorithms are engine-owned. + +Bindings and projection signatures are admitted once; context indexes and +projection handles are prepared once. Explicit task stacks construct nested +results, sharing the complete immutable source. Production Bash and inherited +POO Unicode/priority controls compare full events and field order rather than +replaying pre-recognized captures. All 110 controls match, including malformed +regions, optional captures, escaped Unicode and non-assignment outcomes. A +512-level control checks complete lossless output without recursive parser calls. + +The first independent replay exposed an incorrect subscript task order; the +corrected ordering is qualified by nested index/operand event and field parity. +An attempted duplicate rule override was rejected by the existing POO contract; +the extension now replaces the inherited context entry explicitly, preserving +the closed declaration constraint. Neither failure was addressed by a timeout +or compiler change. Full Command interpretation and complete Source artifact +admission/parity remain the next obligations. + +Local qualification: the producer compiles with GCC. The combined Scheme +PartProfile/ResultProfile/Bash gate passes 55 cases in 22.32 seconds with final +=OK=, =HARNESS-RETURN status=0= and exit zero. Rust passes 117 tests in +0.03 seconds; strict Clippy passes, and the ASP workspace source gate passes in +0.90 seconds. A separate 110-control regeneration is byte-exact after rustfmt +and exits zero in 23.74 seconds. The existing 90-second total/five-second silence +runtime fences are unchanged. These are semantic/structural receipts, not an +end-to-end parse-speed claim. + + +** Ordered result construction before Command execution + +The Word/Binding boundary at =e7139c35af91e02f6c50d30d937282fcb8fda92c= +passes Rust, Ubuntu and macOS CI. Command integration needs ordered catalog +construction as well as the existing fixed projection instructions: repeating +fields can interleave with operator tokens and other node fields. + +The generic ResultProfile now binds node constructors once and validates +ordered captures against the same catalog, source ownership and UTF-8 bounds. +Repeated fields retain supplied order; subtrees move without cloning accumulated +events. Four Scheme catalog controls compare complete events and lossless text. +A Rust integration combines independently recognized Word nodes with Command +and Source nodes under one catalog. Negative controls reject foreign fields, +node/token category confusion, wrong source/catalog ownership and out-of-range +captures. This closes the construction prerequisite, not the Command interpreter. +Command programs and complete Source ParseArtifact parity remain open. + +Local qualification: Rust passes 120 tests; strict Clippy and the ASP +workspace policy gate pass. Scheme catalog-control generation exits zero in +9.56 seconds. Independent regeneration exits zero in 9.35 seconds and is +byte-exact after rustfmt. Runtime fences remain 90 seconds total/five seconds +of real inactivity; no Schema, Loader, Source or C ABI version is changed. + + +** Shared Command and Source execution boundary + +The ordered constructor baseline =e1252ed102a1d67f5d2d1fb3ab359f6b7f97aed6= +passes Rust and Ubuntu CI. This change adds the shared closed Command interpreter +and the engine-owned Source cursor; =languages/= retains only declaration +interfaces. One admitted POO source strategy emits a single Rust Source product, +and a prepared Rust engine reuses scanner/Word/Command/result admission. + +Literal/fallback selector preparation follows Scheme precedence. Command +control uses explicit frames/tasks and checks loop progress. Source construction +covers all 14 production forms, arrays, descriptor/redirection spans, pipelines, +and-or lists, deferred bodies and complete trivia publication. Shared call-DAG +admission computes widths once in topological order, with a 32-level double-call +control that does not expand exponentially. Unicode keywords, root projection, +pipeline role and empty-prefix POO overrides use the identical engine. + +The independent Scheme generator publishes 54 full-Source controls, including +rejection outcomes. Rust compares full Rust syntax events, field sequence, FIFO +marker/body links and source roundtrips. The public generated product API and +the token-tape API both run these controls. Invalid tape, wrong catalog and +cyclic/nullable/ambiguous declarations are rejected explicitly. + +This closes standalone Command/Source recognition for the qualified corridor. +Full versioned ParseArtifact identity/hash/diagnostic parity and binding that +Rust Source product through the language-independent native ABI remain open. +The remaining recursive Source command/list bridge has a 256-level rejection +gate; an explicit Source task stack is still needed for unbounded nesting. +The loader v2, Source v1, native metadata v1 and language handle ABI v2 remain +unchanged. Qualification is semantic/structural evidence, not a parse-speed claim. + +Local qualification: GCC compiles the producer and public facade; the SDK's +=gambuild-C= selects =gcc-16=. The native Scheme Command/Bash +gate passes 38 cases in 10.52 seconds with final =OK=, +=HARNESS-RETURN status=0= and exit zero. Rust passes 126 tests, strict +Clippy passes, and the ASP workspace source gate passes. Independent Source +regeneration is compared byte-for-byte after rustfmt. Existing compiler/runtime +fences are retained; no results are admitted by relaxing a timeout. diff --git a/docs/roadmap/history/scanner-alignment.org b/docs/roadmap/history/scanner-alignment.org new file mode 100644 index 00000000..2582d7e8 --- /dev/null +++ b/docs/roadmap/history/scanner-alignment.org @@ -0,0 +1,276 @@ +#+title: Historical scanner alignment receipts + +This module preserves prior design and qualification records. Its status notes +are historical; the active execution plan is [[file:../0001-versioned-language-pack-closure.org][the convergence plan]]. + +** Shared scanner migration — implementation and remaining gates + +The first closed text-profile migration replaces all five FHIRPath lexical +callbacks and removes its scanner module. HL7 segment identifiers and CR/LF +terminators use the same IR; its six source-dependent callbacks have been replaced +by closed header opcodes. TLA+ identifier, proof-start and proof-reference are +closed text profiles. Its module-text context now prepares source-owned indexes. +Bash source state remains S2 work. This does not close all of S2. + +The engine admits character classes, bounded maximal runs, sequences, +rollback optionals, committed next-character branches and non-consuming guards. +Fragments expand during declaration compilation; callbacks and fragment names +never survive in the published IR. Scheme prepares matching closures once; +Rust consumes static AOT data with UTF-8 spans and the Scheme numeric table. +Admission rejects invalid bounds, excessive nesting and nullable roots. +No language-entry schema or upstream source edition was changed. + +Declaration expansion now normalizes the six existing low-level clause shapes +in one compiler function. Grammar identity, bound ownership, parser authority, +immutable artifact caching and hygienic assembly remain engine responsibilities. +Final grammar-composition tests passed natively in 31.70 seconds, with +explicit =OK= and exit status zero. This is semantic evidence, not a runtime +throughput claim. + +Native compilation exposed two distinct local verification problems. A missing +=.o6= in the scanner cache made Gerbil stop at the earlier contiguous object +version; these ignored products were archived and rebuilt from the emitted +Scheme/C. Eager evaluation during a private compiler dependency probe also +executed unfinished targets; that probe now imports interfaces without target +evaluation. Initial idle-timeout receipts remain failures. Exact C backend +phases use the original SDK/GCC optimization settings and the same 90/30 +compiler bounds, with real GCC intermediate-file progress. + +The shared scanner's repaired native product compiled in 77.28 seconds; a +Unicode endpoint smoke check passed in 7.76 seconds. Further qualified gates: + +| Gate | Result | Bounded wall time | +|---+---+---| +| Shared profile Scheme suite, real FHIRPath/HL7 IR | 4 cases; Unicode, rollback/commitment, maximal-run/CRLF and invalid-root admission pass | 45.69 s | +| Original native FHIRPath scanner control | 1,554 endpoints match across suffixes and nonzero offsets | 41.08 s | +| Complete native FHIRPath parser suite | 5 cases, normative source identity, accepted/rejected and lossless fixtures pass | 43.28 s | +| Complete native HL7 parser suite | 3 cases, message-local delimiter and lossless rejection controls pass | 42.37 s | +| Full FHIRPath Scheme-to-Rust producer | Real native descriptor; no language callbacks in generated AOT | 41.22 s | +| Second full AOT generation | Formatted artifact agrees byte for byte, 540,361 bytes | 39.51 s | +| Rust native unit compilation | Original optimized/debug profile | 26.72 s | +| Rust lexical consumer | 16 actual tests, including complete FHIRPath parse and malformed/deep-profile admission | 0.83 s | +| Complete Rust engine unit suite | 88 tests, including existing scanner/contextual/incremental controls | 0.87 s | +| Rust strict Clippy | All crate targets/features, =-D warnings= | 9.54 s | +| Complete POO loader inventory | Ten admitted loaders, external counts, fixture services and Bash worker isolation pass | 33.16 s | + +The initial Rust filter selected zero tests and is not test evidence; the +corrected lexical filter executes the 16 tests above. The complete ten-loader +inventory initially rejected a stale cached TLA loader, then hit its five-second +silence gate while loading missing GQL/Cypher/HCL native products. The current +sources and all their required native products were compiled with the same +SDK/GCC settings; the final inventory suite passed as recorded above. No +measured runtime speedup or new C ABI qualification is claimed for this slice. + +** Stable language modules and metadata + +Language editions are data rather than module identity. The seven edition +subdirectories and the two edition-labelled reference-corpus directories are +flattened into stable family paths. Public language/parser bindings omit edition +suffixes; tests, generators, native FFI callers, CI commands and documentation +use those same paths. Upstream grammar/corpus bytes and pinned authority values +remain unchanged. This is an in-place correction of the unreleased loader +contract, with no v3 schema bump and no compatibility aliases. + +Loader metadata is an inherited POO value augmented by the engine with +=language=, =version=, =contract=, =digest= and =digest-kind=. The effective +identity is checked against the descriptor; language/user metadata fields such +as grammar format and upstream references remain extensible. Grammar digests +bind Parser IR; source-language digests bind declared source identity and must +not be described as hashes of scanner implementation code. + +** Fixture value-slot correction + +The unreleased =gerbil-parser.language-entry.v2= contract exposes =fixtures= +as a data slot, +defaulting to an empty list. Every shipped entry assigns its corpus directly; +no =lambda= wrapper is required or admitted. POO inheritance, replacement and +computed value slots support extension using ordinary values. Loader admission +checks the list/fixture brands; the engine fixture service owns whole-corpus +identity, duplicate-id and parse-result validation. Test execution and scanner +factories remain explicit descriptor-bound services. + +** Stable namespace correction: current qualification boundary + +The loader schema remains v2. The complete =t/language-loader-test.ss= suite +passed all 15 native cases in 69.67 seconds. The three cases in +=t/language-loader-value-test.ss= passed in 49.07 seconds, including an inherited +computed slot reading the final metadata via =self=. Both suites produced +explicit =OK= and propagated gxtest status 0 within the unchanged 90/5 fences. +The independent Rust Records consumer compiled and passed both structural/ +lossless and rejected-input tests after its module/path migration. + +The Scheme reader accepted all 114 changed Scheme modules, and their declared +internal import paths resolve. These reader/path checks do not establish macro +expansion, native compilation or language conformance. The staged GQL, +openCypher and FHIRPath authority sources retain their exact Git blob identity; +the generated Records Rust source is also an exact rename. FHIRPath's =-text= +attribute follows its new path, preserving its pinned CRLF bytes. HL7 external +scanner protocol stamps retain their declared identity. + +The first complete-loader attempt failed at the five-second cold-import fence +before assertions. Preparing the new namespace's native dependencies resolved +that failure; the later complete-suite pass above is a separate receipt. +The fresh Arithmetic generated module and an exact-C GCC backend diagnostic +also reached the 90-second construction fence. The same generated C subsequently +passed the SDK GCC backend in 32.07 seconds with the original optimization +options and without the large internal-tree diagnostic dumps. Its source SHA-256 +remained =2fdea5852d54b29e1b2ea2dbc5889723bd460dca5edf1a0eddb48944cf3c91ae=. +The new Arithmetic grammar/fixture modules compiled in 53.10 seconds. +A Bash dependency batch produced word/word-scanning/scanner products before +hitting the total fence during parser-core dependencies; the follow-up native +parser-core, grammar, Bash parser and Arithmetic parser batch passed in 73.76 +seconds. Earlier failed invocations remain failed receipts. + +No interpreter fallback, compiler-profile switch or timeout relaxation is used +as native evidence. Whole-pack conformance, generated Scheme checks and installed +native consumers still require the new PR head's complete CI qualification. +The earlier e1d0cf2 CI is green on Rust, Ubuntu and macOS, and is not evidence for +this namespace correction. This work does not close the remaining 17 external +scanner declarations or the Bash source-parser strategy stages. + +** Resource-path follow-up + +Head =5bc7a05= failed Ubuntu/macOS native package construction: the fixture +macro still referenced =corpus/opengql-1.9.0/= after moving the corpus to +=corpus/reference/=. HCL had the corresponding local-path omission as well. +The prior import-path audit did not inspect fixture resource declarations. +Both manifests now use the stable reference path; a Scheme reader audit checks +all 54 declared fixture/grammar-source paths across languages and the downstream +Records pack. This fixes the identified missing-file error, and does not turn +the failed native jobs into passes; the follow-up head requires fresh CI. + +** Generated HCL backend identity follow-up + +Head =0a8f1cd= passes Rust but native CI rejects the stale HCL backend digest. +The namespace correction changes the canonical Grammar/Parser IR name from +=hcl-v2-24-grammar= to =hcl-grammar=. A Scheme check of the admitted old Parser +IR and the freshly compiled declaration proves that all other rows are equal. +The fresh digest is +=sha256:657e59afd49381fda2d88ecd2ec0615a8c7e68b1b2f7c5b5074ac27b87a26852=. + +The official step/recursive generators regenerate both products and pass their +reproducibility checks in 69.70 seconds. Generation uses a temporary copy of the +exact declaration without backend installation, resolving the stale-product +bootstrap cycle; public backend admission remains enabled. The generator's +reviewed closed-path digest changes with the identity. Full installed native +qualification remains the next CI gate; the failed native jobs stay failed. + +** Records dependency ordering follow-up + +Head =6a107af= passes both HCL generator checks on Ubuntu and macOS, then fails +while compiling downstream Records test support: =parser.ss= now imports the +fixture value, but CI still compiles the parser before =fixtures.ss=. Both CI +commands now follow grammar → fixtures → parser. No producer or runtime check +is skipped; fresh CI at the follow-up head remains required. + +** Historical S0 implementation and qualification (e1d0cf2) + +The v2 loader migration covers all ten public entries. Test services use private, +descriptor-bound declarations and engine-owned execution; arbitrary procedure +rows and foreign/duplicate declarations reject. The inherited default declares the fixture test without executing fixture parses. Source-worker creation checks source identity, and +checkpoints now bind the creating worker even when two workers share one string. + +Native Scheme verification used the project SDK and GCC 16: + +| Check | Cases | Total seconds | Result | +|---+---+---+---| +| Loader POO admission and service extension | 15 | 50.18 | Passed | +| Source-worker checkpoint ownership and UTF-8 replay | 2 | 18.54 | Passed | +| All ten public loaders, admitted IR inventory and fixture corpora | 4 | 56.64 | Passed | +| Bash structured syntax and deferred obligations | 16 | 56.02 | Passed | + +Each test retains the 90-second total / 5-second silence gate, explicit =OK=, +propagated child status and final zero exit. These are semantic qualification +receipts, not throughput benchmarks. + +The first complete entry compilation timed out at 91.04 seconds while loading +aggregate facade type interfaces. Imports now name the loader, fixture and +policy-descriptor modules directly. A verbose retry still timed out at 91.26 +seconds in type expansion. Disabling detailed compiler type printing retained +all optimization/GCC settings and real interface/artifact observations: all +entry products were emitted, but the aggregate invocation still timed out at +90.61 seconds during final completion. Those invocations remain failed; the +native corpus check does not turn their process statuses into passes. Complete +compile exits passed for the ordinary-entry target (78.95 seconds) and the +independent SANY policy target (45.03 seconds), each with the unchanged 90 / 30 +compiler gate. The final loader core (73.29 seconds) and public facade (71.24 +seconds) also compiled successfully. Direct entry imports avoid the aggregate +facade, but existing grammar/compiler metadata still brings it into some +transitive type closures; that remaining dependency is not declared removed. + +** Design obligations for each stage + +=deflanguage-parser-loader= is the enforcement point, not merely an entry-name +macro. Its descriptor/identity/dispatch slots are sealed; POO extension +contracts validate the effective inherited object. Grammar compiler admission +owns canonical IR, fixture services pre-admit the entire native corpus, and +scanner services require an engine-owned declaration tied to the descriptor. +A row containing an arbitrary procedure is not a scanner declaration. + +The remaining source-parser bridge is explicitly named +=declare-language-source-scan-worker= and is admitted only for source descriptors. +It constrains descriptor ownership, factory result type, source identity and +checkpoint isolation. It does not qualify the callback as portable Scanner IR. +S1/S2 remove the remaining callback implementations; they do not grandfather +undeclared downstream callbacks. Test/fixture extensions must retain their +loader-bound admission and conformance checks, and future IR worker services +must use the same declaration boundary rather than a second registry. + + +- POO remains the extension boundary. =grammars= and metadata are exposed by the + loader; fixture data uses ordinary POO values; test/worker execution remains explicit. Engine identity, descriptor + binding and dispatch remain sealed against inherited overrides. Downstream + users declare options/rules/services rather than maintain scanner drivers. +- Lexical profiles must preserve committed malformed suffix rejection. For + example, FHIRPath's incomplete =-MM= / =:mm= groups must not quietly fall back + to a shorter accepted token. HL7 segment ids reject a maximal four-character + run instead of accepting its first three characters. Preserve Unicode class + semantics through Scheme-generated classes and exact UTF-8 endpoint tests. +- HL7 source captures are validated once per parse: the five MSH delimiters + are distinct and non-alphanumeric. The prepared plan retains no source; + per-source execution owns captures. No global mutable delimiter cache. +- TLA+ module depth is source state rather than repeated scans from offset zero. + Comments, quoted strings, module borders and proof lexemes keep their existing + boundaries. Source-sensitive checks become transitions/guards, not arbitrary + callbacks executed by Rust. +- Bash retains nested Word/WordPart/expansion nodes and stable fields. Generic + queues/checkpoints and marker-to-body links belong to the engine; an opaque + balanced-word token does not qualify structured Bash syntax. Traversal and + child accumulation must avoid repeated copying as input grows. +- Native C ownership and standalone Rust are separate product gates. A common + loader is not a portable-policy receipt; source descriptors are currently + rejected by native artifact serialization and standalone Rust generation. + Runtime creation/FFI handles are reused across calls, not recreated per parse. + +** Qualification and progress tracking + +=t/language-pack-metadata-test.ss= is the executable whole-inventory boundary: +all ten admitted entries, POO metadata/services, native fixture conformance and +explicit external-role counts. Update its reviewed expectations when callbacks +are removed. Native Scheme/ASP remains the semantic authority; no JSON result +snapshots or document-side audit scripts are introduced. + +For each stage: +1. Preserve old endpoint/tree/artifact behavior as a native Scheme differential + witness before removing the old implementation. Include malformed input, + Unicode, adjacency, EOF and source-dependent state cases. +2. Regenerate Rust from the same Scheme-owned declaration. Run its real Cargo + consumer and compare required source spans/CST/field obligations. +3. Preserve identity/catalog ordering; regenerate and review any digest change. + A syntax-coverage change requires an explicit contract decision. +4. Use existing ASP benchmarks for cold/warm execution, FFI handle reuse and + work/allocation growth. Report structural work and paired semantics before + claiming runtime improvement; AOT build time is a separate measurement. +5. Keep native test gates at 90 seconds total / 5 seconds silence and compiler + gates at 90 / 30. Real output and nonzero exits remain required. Do not split + a failing case merely to evade its gate or switch Scheme away from GCC. +6. Commit the qualified owner slice to the existing PR and verify its exact + remote head. Ubuntu native and Rust are the requested publication gate; + inspect macOS failures without treating queued macOS as a reason to stop. + +Current implementation sequence: S0 authoring contracts and S1 closed lexical +roles are implemented. S2 header/module contexts prepare once; Bash source state +still needs portable lowering. S3/S4 use the same descriptor/result/strategy +contracts, and S5 proof policy must become portable data. These remaining stages +must not introduce alternate pack APIs. + diff --git a/docs/roadmap/history/source-composition.org b/docs/roadmap/history/source-composition.org new file mode 100644 index 00000000..b137c9e7 --- /dev/null +++ b/docs/roadmap/history/source-composition.org @@ -0,0 +1,287 @@ +#+title: Historical source composition receipts + +This module preserves prior design and qualification records. Its status notes +are historical; the active execution plan is [[file:../0001-versioned-language-pack-closure.org][the convergence plan]]. + +** Structured local lexemes — closed shared execution + +TLA+ identifier, proof-start and proof-reference no longer publish external +callbacks. =run-containing= checks maximal runs and a required character class +in one traversal; =ends-in= and =ends-not-in= constrain the consumed span without +rescanning its body. These operations are general lexical IR, with Scheme and +Rust admission, preparation and execution. Their spelling and character classes +participate in the grammar digest. The module scanner macro now binds only the +remaining contextual module-text scanner; obsolete local callback exports are +removed, without aliases. + +Rust alphabetic classification is generated from the pinned Scheme +=char-alphabetic?= predicate, just as numeric classification already is. The +portable witness imports the actual TLA+ fragments, emits a standalone parser, +and checks Unicode endpoints at nonzero offsets. This witness establishes local +lexical portability; it does not establish standalone full SANY execution. + +The remaining closure sequence is explicit: prepare HL7 header and TLA+ nesting +context once (S2), lower shell command/word recognition into shared IR (S3), +replace the closed HCL recursive corridor with general lowering (S4), and publish +proof policy as portable data consumed by native C/Rust (S5). No runtime speedup +is asserted without complete-call matched measurements. + +** Source preparation — header captures + +The machine now lowers a distinct header capture plan once. Scheme and Rust +prepare valid and invalid header captures once per parse request. Scheme scope +restoration and parallel requests, Rust borrowed source ownership, and nested +lexical choices have explicit semantic controls. Incremental resume starts a +fresh source scope and retains existing runtime/checkpoint ownership. Record +projection consumes the same canonical header scanner module; the old scan +module forwarding exports are removed. No speedup percentage is claimed from +algorithmic work-count reduction alone. + +** Source preparation — module framing + +All TLA+ lexical roles now use closed IR. The old module scanner profile/binder +and its public macro are removed without aliases. =module-text= contains bounded +framing values; the shared engine prepares header intervals and nesting events +once per request. Long border runs do not expand into per-position records. +Scheme and standalone Rust share lexical admission, source ownership and emitted +values. Pinned Scheme whitespace joins alphabetic/numeric character authority. + +Local controls cover exact endpoints, every offset against the prior framing +algorithm, nested comments/strings, an independent UNIT spelling, numeric extra +name characters, long borders, ordinary parser roundtrip and malformed profiles. +The complete TLA+ pack suite remains a required semantic gate. CI checks exact +regeneration of the standalone framing parser and whitespace table. + +Header capture and module framing preparation are implemented. S2 remains open +for Bash source state; S3 command/word IR, S4 general recursive lowering and S5 +portable proof policy still require implementation and real consumer gates. +No complete-call speedup is claimed from the indexing change alone. + +Local module-framing receipt: 73 Scheme cases passed with explicit =OK= and +zero harness return; 102 Rust syntax tests plus the complete Rust workspace passed. +ASP Rust policy and Clippy passed. The GCC native generator built and its actual +executable emitted the same module parser as the Scheme producer. Exact parser +and whitespace-table regeneration passed. These are local results; exact PR-head +Ubuntu/Rust CI is a separate publication gate. Source-backed Rust tests live in +=tests/unit=, with explicit imports and =#[path]= mounts. + +** Result operation and source native admission qualification + +The GCC native production gate executes 63 cases over result profiles, public +C handles, Bash syntax, source contracts, topology, source spans, native metadata +and parser policies, with +explicit =OK=, =HARNESS-RETURN status=0= and exit zero. A temporary Scheme +control against =46f0c891131b8fd9f732559f0105f9f847077f36= compares all +56 complete ParseArtifacts across scanner controls, compound forms, large +Unicode parts and nested parameters. The former parser modules are removed +from the production checkout after qualification. + +The construction contract adds runtime checks. Paired timings include negative +samples and scheduling/GC variance; they do not establish an overall speedup. +Child type and span admission now use one dispatch with retained bounds rather +than repeated generic span dispatch. The real public C probe records 100 +length-delimited calls for both Bash and the independently declared line +language, and checks UTF-8 rejection, result reuse, release and stale handles. +The latest PR Scheme metadata transport is retained. Regenerated canonical +GPA1 bytes match the Rust fixture exactly; three Rust Bash scanner cases and +six native artifact consumer cases pass. No runtime timeout or compiler +optimization level is relaxed. + +** Part recognition dispatch qualification + +The GCC native gate passes 57 cases, including a second template configuration, +non-ASCII recognition prefixes, rule overrides, owned copies and admission +failures. The gate ends with =OK=, =HARNESS-RETURN status=0= and exit zero +in 32.73 seconds including imports. A temporary control against +=78084bcfa444abc81c7580607aac748488fbc975= matches 57 complete ParseArtifacts, +including lossless unknown-dollar grouping, compound commands, 2,000 Unicode +parts and 512 nested parameters. These former implementations remain outside +the production checkout after qualification. + +The final public C probe records 100 Bash parse calls at 6.344 ms wall/CPU and +100 independent line calls at 0.800 ms wall / 0.799 ms CPU, excluding descriptor +initialization. Alternating complete-parser controls retain negative samples: +flat median 9.296 ms before / 18.604 ms after for three calls, while individual +current samples range from 9.173 to 191.401 ms; plain medians for 10,000 calls +are 498.641 / 425.484 ms and nested medians for three calls are 5.849 / 5.868 ms. +These unstable wall measurements establish neither an overall speedup nor a +zero-regression claim. ASCII dispatch uses a compiled direct index, Unicode +prefixes use a separate index, and the Word executor binds its context outside +the part loop. No timeout, compiler choice or schema version is relaxed. + +** Binding profiles and bounded Rust name qualification + +The GCC native gate passes 75 cases in 34.90 seconds including imports and +retains explicit =OK=, =HARNESS-RETURN status=0= and exit zero. A temporary +Scheme control against =1f0d87f38079d1115148d4d3421cc95416813c94= matches +62 complete ParseArtifacts, adding parameter-name/operator controls, nested +array indexes and literal brackets inside command substitutions. Production +macro cases exercise 512 nested indexes; the region suite checks both query +orders for overlapping openers under distinct scopes. Temporary former +implementations are removed from the checkout after qualification. + +Production Bash name IR generates 135 bounded Unicode endpoint controls. +Two Rust tests check every generated endpoint, nullable-profile rejection and +invalid byte boundaries through =PreparedTextProfile=. Independent regeneration +matches the checked-in Rust artifact byte for byte after formatting. Rust +formatting and crate/all-target/all-feature clippy pass with warnings denied. +CI regenerates the same artifact using the finite native child exit binding. +A first generation printed its completion marker but remained alive during +shutdown and failed the five-second inactivity gate; completion is now accepted +only with process exit zero. No timeout was extended. + +The current complete-parser controls retain negative samples: 10,000 plain +calls have medians 605.387 ms before / 427.741 ms after; three flat calls have +35.738 / 25.175 ms; three nested calls have 6.033 / 6.367 ms. An isolated public +C gate records 100 Bash calls at 7.340 ms wall / 7.153 ms CPU and 100 line calls +at 0.889 ms wall/CPU, excluding initialization. These observations qualify +execution and conformance; they establish no overall speedup or zero regression. +Command grammar, result-projection lowering and complete standalone Rust +Source recognition remain open. + +** Command roles, projections and linear result construction + +=CommandProfile.= now owns token-role indexes, full descriptor TextProfiles +and node projection values. =defcommand-profile= constrains the grammar +interface; Source strategy admission requires the complete effective POO value, +checks scanner/result catalogs and includes the owned recipe in Source identity. +Identity projections are derived inside the engine instead of repeating the +result catalog in the language pack. No schema, loader or ABI version changes. + +The executor binds role matchers once, removes hand-coded descriptor walks +and literal membership tables for declared roles, and reuses persistent engine +recognition sequences for all growing compound child collections. Marker +receipts use an amortized constant-time two-stack FIFO. Complete control-flow +syntax and standalone Rust Command/Word execution remain open engine scopes; +this is not a claim of full Source AOT closure. + +The GCC qualification passes 61 cases in 14.86 seconds with explicit =OK=, +=HARNESS-RETURN status=0= and exit zero. All 65 complete ParseArtifacts match +an exact =edc53bfe9b9b500cff54953d154da12b45128013= parser control with the +same scanner, Word plan and digest. The control includes compound forms, +2,001-command pipelines, 2,000-element arrays and mixed here-document quote +policies. Temporary control sources are removed after qualification. + +Five alternating complete-parser samples record a 2,001-command pipeline +median of 99.674 ms before / 24.449 ms after. Plain 10,000-call medians are +177.565 / 168.232 ms and nested three-call medians are 6.208 / 6.213 ms. +The first run retained a plain-input negative sample of 173.030 / 204.214 ms; +role matchers now bind once rather than resolving role symbols per token. +These bounded measurements support the long-collection algorithm change, +not an overall or zero-regression performance claim. A failed first suite +referenced a nonexistent artifact-root accessor; the final test reads the +formal event API. No timeout or compiler choice was changed. + +The three scale regressions are also in the public =deflanguage-parser-tests= +macro suite; the final Bash/Command/topology gate passes in 7.58 seconds with +normal exit and harness status zero. + +** Control-program migration + +The =forms= slot of =CommandProfile.= now owns all compound control grammar. +The old if, loop, iteration, case, function, conditional, arithmetic, group +and array functions are removed in favor of one closed program executor. +Literal dispatch, bounded lookahead, field checks, reference/cycle checks and +consuming-loop admission are engine capabilities. Pipeline header FIRST sets +avoid running nullable programs for ordinary commands. Source identity includes +the owned forms without changing Source v1, Loader v2 or C ABI v2. +Independent Unicode loop declarations and empty inherited prefix programs +exercise POO override semantics. No former execution path or macro compatibility +mode is kept. Complete Rust control/Word execution remains open. + +The module-source generator previously printed its completion marker but +failed the five-second inactivity gate while exiting on macOS. It now flushes +and uses the existing finite-child normal exit binding. Local bounded native +regeneration exits zero in 8.22 seconds and matches the Rust artifact byte for +byte after formatting; neither timeout nor compiler selection changes. + +The first qualification passes 54 cases and 80 old-head controls in 15.02 +seconds. Accepted artifacts match in full; rejected artifacts match with only +the explanatory =message= field excluded, retaining all other diagnostics and +artifact fields. The reference is the exact =ff9e1e85c454f87f38090701608215169d3e3e14= +executor with the same scanner, Word plan and supplied digest. Its source stays +outside the production checkout after qualification. A later FIRST-set control +passes in 27.54 seconds; final native qualification also covers the three added +scale cases, empty-header inheritance and a 32-token lookahead admission bound. + +Five alternating complete-call samples after FIRST-set admission retain medians +of 691.763 / 649.101 ms for 10,000 plain calls, 157.004 / 137.931 ms for one +2,001-command pipeline and 10.273 / 7.245 ms for three real 512-deep parameter +calls. The initial plain sample is negative at 193.496 / 204.322 ms. These +wall-clock measurements remain sensitive to allocation/GC pressure; they do +not establish an overall speedup or zero regression. The structural change +removes dedicated grammar execution and skips nullable header interpretation +using an admitted FIRST set. + +A first suite fails its five-second inactivity gate at a test-local POO +=forms: forms= self-reference; the local variable is renamed and the failed +receipt retained. The successful suites require explicit =OK=, +=HARNESS-RETURN status=0= and process exit zero. The final scale/native suite +passes 56 cases in 12.64 seconds. + +Final admission review removes exponential call-path expansion with a +visiting/done DFS memo table. A 32-layer DAG with each callee referenced twice +now admits without rewalking completed forms; true cycles still reject. The +last GCC native gate passes 57 cases in 13.83 seconds with final =OK=, +=HARNESS-RETURN status=0= and normal exit. + +** Scheme Word projection closure + +=ResultProfile.= now owns closed =token=, =one= and =many= projection programs +with typed ordered captures. Projection-only declarations derive the node, +field and terminal catalog; Bash removes duplicated Word result catalog rows. +POO inheritance/replacement uses ordinary values and the same admission checks. +Word recognition retains span and boundary recognition, while field mapping, +optional emission and persistent token publication run in the shared engine. +All word, assignment, quote, substitution, parameter and subscript result +builders have migrated; their dedicated field/token assembly is removed. +Source v1, Loader v2 and C ABI v2 do not change. Complete standalone Rust Source +control/Word execution remains open; an IR recipe alone is not Rust execution. + +The exact reference is =753d70bd671e7dd94283882a8510195f69b433e3= with only +factory/module names changed for side-by-side Scheme execution. Eighty controls +compare complete accepted and rejected artifacts, including diagnostic message +text: no field is normalized away. Controls include Unicode, scoped array +subscripts, 512-deep parameters, long word parts and command structures. +Temporary reference sources remain outside the production checkout after +qualification. The final focused suite passes 56 cases in 11.25 seconds with +=OK=, =HARNESS-RETURN status=0= and normal exit. + +The first general-VM sample is negative for ordinary commands and pipelines: +five alternating complete-call medians are 171.719 / 188.388 ms for 10,000 +plain calls, 28.761 / 31.412 ms for one 2,001-command pipeline, and +12.883 / 8.551 ms for three 512-deep parameter calls (previous / migrated). +It remains recorded. Binding-time specialization of static single-token and +single-part-list programs removes dispatch and repeated catalog validation. +The repeated five-sample medians are 168.878 / 146.832 ms, 26.245 / 23.619 ms +and 6.256 / 5.848 ms respectively. These local wall-clock controls do not prove +an overall speedup or zero regression. + +GCC emits all changed Scheme engine modules. Runtime qualification retains +90-second total and five-second inactivity bounds; no widened timeout or +compiler switch is used. Independent projection-only and =@name= dialects +exercise POO replacement, inferred catalogs, Unicode offsets with nonzero +origins, rejected signatures, bounds and recipe-copy isolation. + +The combined native gate passes 82 cases in 9.98 seconds, including Source and +Loader contracts, language topology, command programs, Bash and native C ABI. +It ends with explicit =OK=, =HARNESS-RETURN status=0= and exit zero. + +The final cardinality audit includes =required=/=optional= in the bound engine +signature. A profile cannot make an optional recognition capture mandatory +without failing admission. Catalog publication copies its node and terminal +rows directly, avoiding two unnecessary copies of the entire projection IR. + +The subsequent combined qualification passes 85 cases and the same 80 exact +artifact controls in 20.67 seconds. Its ordinary-call median is negative at +123.355 / 185.287 ms; pipeline is 25.523 / 24.068 ms and nested parameters +6.339 / 5.818 ms. Alternating individual plain samples range from 115 to +311 ms across the two executors. This contradicts any claim of a stable plain +command improvement; the result remains recorded and runtime performance +qualification stays open. + +After the catalog-publication change, final native qualification passes 84 +cases plus the same 80 complete artifact comparisons in 14.09 seconds. This +last run excludes optional timing measurements and requires final =OK=, +=HARNESS-RETURN status=0= and exit zero. + diff --git a/docs/roadmap/history/source-design.org b/docs/roadmap/history/source-design.org new file mode 100644 index 00000000..91b31b08 --- /dev/null +++ b/docs/roadmap/history/source-design.org @@ -0,0 +1,217 @@ +#+title: Historical source design receipts + +This module preserves prior design and qualification records. Its status notes +are historical; the active execution plan is [[file:../0001-versioned-language-pack-closure.org][the convergence plan]]. + +** Bash authoring closure is one complete migration unit + +The authoring audit at =1d2d98a479f622c95d6754c2059f69b6b4b7e830= finds +18 lines in =languages/bash/grammar.ss= delegating execution to 1,353 lines: +126 scanner, 103 word-boundary scanner, 390 word decomposition and 734 source +parser/recognition lines. Registering those procedures with a loader does not +make the syntax declarative. Rust and Ubuntu native CI passed at this exact +head; that evidence qualifies the current implementation, not this migration. + +The previous shared-FIFO patch is a prerequisite. It does not satisfy the +language authoring boundary. Bash's four procedural modules must migrate as +one authoring obligation; individual helper moves cannot mark Bash converged. +The public pack should retain grammar declarations, its POO loader and declared +fixtures/tests/metadata. A downstream pack must not implement scanner traversal, +word slicing, worker state, artifact publication or C/Rust transport. + +Source-backed result publication now uses the common UTF-8 span context, +source-local nested boundary cache and persistent token sequence. ASCII skips +the index vector; ordinary words skip the region context. The shared boundary +suite also executes an independent declaration with overlapping quote/pair +entries, and Bash declares large Unicode/parameter controls through its test +macro. This removes three repeated-work paths inside Word execution; it does +not replace the remaining Word/Command recognition decisions. + +Result construction now uses a shared admitted POO =ResultProfile.=. Bash +Word/Command builders and the independent line source engine execute the same +indexed kind/field/terminal operations. Scanner terminals are derived from the +scanner slot; generated terminals and node fields are declared once. The native +catalog is derived from these operations, and source descriptors bind directly +to the language-independent length-delimited C ABI. Construction operations +participate in descriptor identity; schemas remain unchanged. S3 recognition +rules and S5 standalone Rust source recognition remain open; a Scheme engine +behind the C ABI does not close Rust portability. + +Part selection and literal boundaries now execute one =PartProfile.= plan. +Bash declares context/prefix/guard/operation rows; the engine compiles ordered +context/first-character dispatch. Quote, substitution, name and escape +selection no longer repeat conditions in two procedural paths. Assignment +values and here-document contents use the same plan. An independent =@name= +template configuration exercises the shared executor and POO rule override. +Admission checks result fields/terminals and region agreement before binding; +rule recipes enter source identity without a schema bump. This closes the +part-dispatch migration, not the internal parameter/assignment grammar, +Command recognition or standalone Rust Source AOT. + +Binding recognition now shares the existing TextProfile and literal-trie +algorithms. Bash declares simple names, parameter names/prefixes/operators, +assignment names/operators and array delimiters as POO =BindingProfile.= values. +The engine owns bounded matching, role admission, projection and source-local +scoped captures; hand-coded parameter/assignment name and operator matching +and the separate subscript traversal are removed. Independent assignment +syntax overrides operators through values. Rust =PreparedTextProfile= uses +the same existing matcher to execute production name IR with exact bounded +Unicode endpoint controls. This closes binding-name recognition and its AOT +control; Command/result projection and full standalone Source AOT stay open. + +Tree-sitter's public grammar DSL supplies sequence, choice, repetition, +precedence, aliases and fields, and generates ordinary lexical/parse execution. +Its external-scanner interface explicitly assigns special token recognition, +lifecycle and state serialization to C functions written by the grammar author; +Bash here-documents are an example in its documentation. Our authoring target +is more specific: built-in closed region/deferred-state operations should cover +these mechanisms so packs can declare them without implementing that C worker. +This is a target, not a claim of broader Bash coverage or measured superiority. + +References: +- [[https://tree-sitter.github.io/tree-sitter/creating-parsers/2-the-grammar-dsl.html][Tree-sitter grammar DSL]] +- [[https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners.html][Tree-sitter external scanners]] +- [[https://github.com/tree-sitter/tree-sitter-bash/blob/master/grammar.js][Bash grammar authoring example]] + +*** One authoring boundary, several engine representations + +| Current module | Language declaration that replaces it | Engine obligation | +|---+---+---| +| =scan-words.ss= | Quotes, opaque escapes, ordered substitution openers/closers, balancing and operator stops | Compile one shared region plan; match nested spans with progress/termination bounds and explicit quote context | +| =scanner.ss= | Command/body modes, comment/newline rules, deferred marker policy and transitions | Prepare a source worker from the admitted IR; own queue, source/worker identity, checkpoint serialization and token bytes | +| =word.ss= | Word-part rules with aliases/fields for quotes, parameters, assignments and substitutions | Lower embedded recognition/projection from closed data; retain exact CST field/span identity and source partition | +| =parser-core.ss= | Command, redirect, compound-command, list and pipeline rules plus precedence and fields | Use the grammar/recognition machine and common artifact admission; own failure frontiers and typed here-document links | + +These declarations belong to the existing syntax compiler and its shared +Scanner/Parser IR boundary. Do not introduce a =(syntax bash)= selector that +hides the old parser behind a language-name branch. Do not duplicate these +rules in a parallel source-parser DSL, or merely rename Bash procedures under +=src/=. Missing generic region, state or recognition operations must be added +to the engine and admitted before a pack can use them. + +The language loader derives its scanner services from the compiled grammar. +Authors must not supply a =make-*-scanner= factory, scanner callback or custom +serializer for these closed declarations. POO slots extend grammar/metadata/ +fixtures/test declarations as values; the compiler validates the effective +inherited value and rejects unknown operations, missing roles and incompatible +products before installation. The current raw source-worker bridge remains a +transitional Scheme capability until its last production pack is migrated. +The unreleased =gerbil-parser.language-entry.v2= schema remains unchanged. + +*** Structural work required during lowering + +Word decomposition currently computes each byte offset by encoding the entire +prefix substring again. The engine must prepare offsets in one forward pass +and materialize ordered spans from that index, preventing repeated prefix +allocation on words with many parts. Region matching must handle longest +openers such as =$((= versus =$(=, process substitutions shielding operator +stops, escaped delimiters, opaque single quotes/backticks and nested quoted +substitutions. Node/field identity is declared data, independent of scanning +control. Command recognition must also avoid repeatedly appending accumulated +children or pending marker lists; immutable checkpoints and ordered publication +remain mandatory. + +*** Whole-pack exit gates + +1. Bash grammar carries its real syntax/scanner/word declarations. No grammar + entry dispatches to a pack-owned procedural parser or scanner. +2. All four procedural language modules above and their imports/build entries + are removed after callers use the common engine; fixture and metadata + declarations remain in the three language interfaces. +3. The POO loader supplies parse, scanner, fixture and test services from the + admitted grammar without author-written worker/FFI functions. +4. A minimal independent downstream pack extends or overrides declared values, + compiles with the same public facade and consumes its C/Rust artifact without + importing engine internals. An unsupported target must reject explicitly; + =source-parser= metadata or a registered Scheme closure cannot certify Rust. +5. Current complete Bash acceptance/rejection, UTF-8 roundtrip, word/node/field + spans, deferred link ordering and checkpoint isolation remain controls. + Nested/ambiguous opener, malformed suffix and large-part inputs supplement + them; syntax coverage still does not exceed the qualified corpus. +6. Scheme producers deterministically emit the same closed product consumed by + Rust AOT and the language-independent C ABI. Each actual consumer must pass + before portable closure is reported. No per-language Rust/C shim is required. + +The dependency order is region IR -> source transitions -> embedded recognition +-> command grammar -> downstream products. These are internal implementation +steps of one Bash migration, not independent claims of authoring closure. + +** S2 source context contract and execution order + +The remaining work has three distinct input dependencies. They must share the +existing contextual Scanner IR and the loader's admitted scanner service rather +than gain independent language worker runtimes. + +| Pack | Remaining language-owned execution | Engine representation / migration gate | +|---+---+---| +| HL7 | Six external roles repeatedly derive MSH delimiters | Prepared prefix capture set, typed slot match / maximal exclusion run; remove all six callbacks after Scheme/Rust conformance | +| TLA+ SANY | Identifier and two proof roles are local; module text rescans source prefix | Close local profiles first; then prepare a nesting boundary index once and use predecessor lookup; keep proof policy a separate capability | +| Bash | Word/substitution recognizer, command token routing and source result assembly | Declarative balanced regions, command transitions and deferred obligations; migrate token routing before recognition/result assembly | +| FHIRPath | No external scanner roles after S1 migration | Keep full native parser and native-produced Rust AOT controls | +| HCL, HL7 segment/terminator, other grammar packs | Existing closed/generated lexing | Remain under common loader and product admission; generated strategies are S4, not source-worker migration | + +A source preparation call creates a worker bound to an immutable input revision, +its exact Scanner IR digest and its capture layout. A new parse or edit creates +a new worker. Captures are prepared once before scanning, never keyed by a +module-global last-source cache or inferred from pointer identity of mutable +strings. Existing source-worker callbacks are still an explicitly advertised +Scheme capability; registered factories alone do not establish closed IR. + +HL7's declaration needs: exact =MSH= prefix, five characters beginning at +character offset three, pairwise uniqueness and the producer's alphabetic/numeric +rejection predicate. Consumers refer to a named capture slot: matching one +captured character or a maximal run excluding the capture set plus CR/LF. +Capture preparation preserves the current rejection semantics, including +Unicode; Rust must consume an authoritative Scheme character class, not its +broader platform predicate. Captures are characters, source spans are UTF-8 +bytes in Rust and character/byte pairs in Scheme. Missing/invalid captures +reject before token execution. Equivalent declarations share a prepared capture +within one worker; distinct workers never share mutable state. + +TLA+ needs a single source pass over module opens/closes, nested comments, +quoted strings and line comments. Publish ordered boundary/depth records tied +to that source revision; predecessor lookup answers the depth at a candidate +offset. Repeated prefix rescans are removed only after arbitrary-offset, +nested-module, comment/string and checkpoint controls agree with the original +recognizer. Edits invalidate/rebuild the index before reusing any checkpoint. + +Bash now consumes the engine's existing immutable delimiter FIFO and marker +quote-removal operation. Enqueue allocates a constant number of cells instead +of copying all previous obligations with =append=. A linear drain reverses each +rear batch once, giving linear total queue work; replaying branches can repeat +a reversal, so no branching amortized bound is claimed. The public pending +value remains an ordered list projection. The scanner's own quote-removal loop +is removed; double-quoted backslashes before ordinary characters now retain +the backslash, matching the existing closed engine/Rust marker policy. +Command/substitution routing still remains in Bash. This shared state algorithm +is partial S2 implementation, not complete Bash Rust portability. + +Native-owned generated AOT tables belong in the existing =generated= ownership +boundary. The FHIRPath fixture moves there without changing its bytes. The +existing ASP policy is unchanged; native regeneration, byte comparison and +Rust parser consumers continue to check this product. + +Qualified local evidence for this state convergence slice: + +| Gate | Result | Bounded wall time | +|---+---+---| +| Source worker/queue suite | Six cases; rejected malformed obligations, branching/replay, Unicode spans and 128 Bash markers | 2.93 s | +| Complete Bash parser suite | Accepted/rejected syntax and lossless quote-removal regression cases pass | 4.28 s | +| Closed deferred contextual scanner suite | Queue/checkpoint restoration and shell/raw policies pass | 5.62 s | +| All ten public loaders | Metadata, fixture conformance and independent Bash workers pass | 5.46 s | +| Native FHIRPath AOT regeneration | Formatted bytes equal the moved product, SHA256 =8435e3e9186c2ecd28baf837706395e50c8a608c95274cc71c33511e075f8e2f= | 6.59 s before formatting | +| Rust complete engine unit suite | 88 tests pass; moved full FHIRPath parser product is consumed | 0.55 s | + +The first dependent Bash compilation imported an earlier native engine cache +and rejected its new export; the engine was rebuilt before the dependent. +The first full Bash test used a previously inlined heredoc-record accessor and +terminated abnormally. Rebuilding the unchanged parser caller from current +source restored the matched native record representation and the final suite +passed. These initial failures remain failures, not conformance evidence. +The local ASP policy test's cold dependency build hit the 30-second idle fence +in two attempts (86.29/50.97 seconds total). Its source-policy execution did not +run locally. No compiler profile or timeout was changed; the exact new PR head's +Ubuntu Rust CI still has to qualify this gate. The prior head's Ubuntu/macOS +native jobs passed, while its Rust source policy rejected the oversized fixture +outside the existing generated-table ownership boundary. + diff --git a/docs/roadmap/history/syntax-coverage.org b/docs/roadmap/history/syntax-coverage.org new file mode 100644 index 00000000..d0ba8a6b --- /dev/null +++ b/docs/roadmap/history/syntax-coverage.org @@ -0,0 +1,108 @@ +#+title: Historical syntax coverage receipts + +This module preserves prior design and qualification records. Its status notes +are historical; the active execution plan is [[file:../0001-versioned-language-pack-closure.org][the convergence plan]]. + +* Retained syntax-coverage milestones + +* Outcome + +Land GQL, openCypher, and TLA+ as immutable, source-owned language packs on the +one Gerbil parser compiler and runtime. A language is complete only when its +pinned upstream grammar, admitted corpus, generated artifacts, installed +consumer, negative contracts, and performance receipt are all exercised by the +single native package test entrypoint. + +* P0 Native package closure + +- Keep =build.ss= as a thin =std/build-script= entrypoint over one library + PackageSpec; build the optional =gparse= executable through the sibling + first-party =build-gparse.ss= entrypoint. +- Discover each production module once; never install =*-test.ss= modules. +- Inherit capacity only from =GERBIL_BUILD_CORES= and use the ASP native batch + testing interface without a package-local scheduler. +- CI must clean, build the package and command products, run the native + =gerbil test= interface + once, and exercise installed =gparse=. Build-spec duplication is a failing + contract. + +Status: locally closed. Remote Ubuntu CI is the remaining publication receipt. + +* P1 GQL ISO/IEC 39075:2024 closure + +- Preserve the byte- and digest-pinned OpenGQL 1.9.0 source at its declared + commit. +- Admit all 1,018 grammar rules: 574 parser rules and 444 lexer or fragment + rules, with no handwritten runtime language dispatch. +- Keep all 14 upstream samples, malformed-input failures, CST kinds, lowercase + keyword recognition, roundtrip, and installed sidecar consumption in the + native acceptance suite. + +Status: implementation and local acceptance closed; CI publication is pending. + +* P2 openCypher 2024.1 closure + +- Preserve the official 352-production WG3 BNF source and exact digest. +- Keep the 3,897-query TCK syntax audit reproducible from the pinned corpus; + expected-success and expected-runtime-error queries are syntax acceptance, + while expected compile errors remain an explicit negative set. +- Add the TCK audit to a bounded CI scenario instead of treating a checked-in + receipt as live evidence. + +Status: grammar, representative parser suite, and installed consumer are +closed; reproducible CI regeneration of the full TCK receipt remains. + +* P3 Native TLA+ syntax adaptation + +- Model the native =TLAPlusGrammar= productions from chapter 15 of + /Specifying Systems/ directly in the existing lexical and Grammar IR + algebras. Use the pinned official SANY =tla+.jj= source as the executable + conformance reference, never as a runtime parser dependency. +- Add only generally useful algebra operations to =src/language/= when a TLA+ + production cannot be represented. Every extension must retain canonical IR, + deterministic LALR construction, typed diagnostics, and byte roundtrip. +- Publish source-admission scenarios for unresolved productions, duplicate + identities, source-identity drift, deterministic canonical IR, and corpus + disagreement with SANY. + +Status: the digest-bound 99-production inventory and native-core v1 expression +slice are executable; production-to-GrammarIR closure remains. + +* P4 TLA+ syntax closure + +- Replace =tla-plus.native-core.v1= with a new explicit full-grammar contract; + do not silently widen the existing v1 identity. +- Admit every production and lexical constraint in the pinned native syntax + sources, including expression precedence, proof steps, indentation-sensitive + conjunction/disjunction lists, extramodular text, operator disambiguation, + and PlusCal boundaries. +- Run the official SANY syntax corpus plus normal, Unicode, proof, PlusCal, + rejected-syntax, and crash-regression cases. Preserve source bytes and + compare the required structural CST projection; never downgrade an + unsupported construct to an =unknown= acceptance path. + +Status: the native-core fixtures retain their original contract. A separate +layout contract admits aligned conjunction and disjunction lists, nested +column grouping, and the checked POO Flow temporal-model corpus. Full proof, +PlusCal, remaining layout constructs, and official-corpus conformance remain +open. + +* P5 Algorithm and performance admission + +- Keep LR(0) construction exactly-once and lookahead propagation delta-only. +- Establish cold expansion, warm artifact reuse, corpus parse, recovery, and + external-scanner scenarios with at least 40 samples where percentile gates + are used. +- Fix an algorithm or data structure when a gate is red; never reduce sample + counts or hard-code a smaller worker count to obtain a pass. + +* P6 Downstream and release closure + +- Link the completed parser into =gql-rust= for development, then pin the + published gerbil-parser PR revision through its real dependency graph. +- Prove the Rust GQL consumer uses the installed Gerbil artifacts and the same + canonical AST/ParseArtifact contract. +- Require clean-source rebuild, single-entry native tests, installed consumer + smoke tests, and green PR receipts before declaring the language packs + released. + diff --git a/docs/runtime-module-organization.org b/docs/runtime-module-organization.org new file mode 100644 index 00000000..bdc1cb20 --- /dev/null +++ b/docs/runtime-module-organization.org @@ -0,0 +1,59 @@ +#+title: Native runtime owner decomposition + +* Source and ownership + +This change starts from PR 13 revision +=f57d2aa6502bd39b62119cdd2e5a95469ef6fd9f=, not an older installed dependency. +The public =src/runtime/lr-parser.ss=, =incremental.ss= and =scan.ss= facades +retain their export lists. Internal owners separate private positional state, +runtime preparation, semantic reductions, recognition projection, checkpoint +execution, transfer, incremental reuse and pure lexical algorithms. + +No fallback evaluator, compatibility layer, policy exemption or timeout +increase is introduced. Existing POO dispatch remains at preparation boundaries; +hot algorithms remain named functions. Private =defstruct= execution state is +not converted into a new public extension API. + +* Correspondence and reproduction + +[[file:../t/runtime-module-organization-test.ss][Native source-form controls]] compare facade exports and the multiset of +all original execution forms against the named baseline Git revision. They +reject omitted, duplicated or changed forms. The baseline commit must be +available in the invoking Git clone. This checks source correspondence, not +compiler correctness or end-to-end performance. + +Use the current Scheme package environment, with installed dependencies at +the revisions declared in =gerbil.pkg=. Qualify production source policy with +=ASP_GERBIL_SCHEME_POLICY=1 gxpkg build github.com/tao3k/gerbil-parser=. +Then run =just test-runtime-organization= to prepare the existing qualification +FFI owner, link the declared native gxtest values and execute organization, semantic/action +preparation, stack reduction, fragment transfer and recovery/incremental. +The existing native process owner enforces 120-second total / five-second +idle budgets and checks a deliberate failed assertion exits 42. +The native closure also runs the test-style contract and the AOT lock control: +74 Cases passed locally. The lock control uses native assertions and an owned +fixture port, rather than console snapshots; dirty locks remain rejected and +their evidence remains unchanged. This repairs a baseline CI rejection without +exempting that control from the source contract. +Provide the current OpenSSL =pkg-config= metadata for AOT linking rather than +depending on a previously installed Homebrew version directory. +Build and test-closure preparation run +serially because they write the same package-native products. + +Keep the existing one-GiB runtime heap, five-second genuine-output watchdog +and 120-second overall native test budget. A source policy pass, build pass, +runtime suite pass and downstream consumer pass are distinct evidence. +Local monitoring output is not a retained evidence reference. + +* Exact-head CI input and cold-loading corrections + +The semantics job fetches repository history so the named source-form baseline +is available in the CI clone. A shallow checkout failed all three controls; +missing baseline input is not a reason to skip the comparisons. + +Module-loading controls initialize their existing preloader before importing +the two public compiler boundaries. The preloader reports real native module +and interface admission, with the original static imports and control bodies +unchanged. The controls passed locally in 9.75 seconds, retaining a five-second +idle limit, 90-second total limit and one-GiB heap. Interface tracing is not a +synthetic heartbeat or an initializer-failure exemption. diff --git a/formal/ParserEngine.lean b/formal/ParserEngine.lean new file mode 100644 index 00000000..7a7483f9 --- /dev/null +++ b/formal/ParserEngine.lean @@ -0,0 +1,158 @@ +import ParserEngine.Lookahead +import ParserEngine.Capture +import ParserEngine.Reuse +import ParserEngine.Recognition +import ParserEngine.Preparation +import ParserEngine.Sequence +import ParserEngine.Reduction +import ParserEngine.Examples +import ParserEngine.Fold +import ParserEngine.OwnedStack +import ParserEngine.Execution + +/-! Proofs for logical deterministic reduction and an explicit operation model. +No theorem asserts that Scheme execution refines this model or establishes CPU speed. -/ + +#print axioms ParserEngine.prepared_plan_equivalent +#print axioms ParserEngine.reverse_reduction_equivalent +#print axioms ParserEngine.sequence_visits_exact +#print axioms ParserEngine.copied_prefix_triangular +#print axioms ParserEngine.reduction_preserves_depth +#print axioms ParserEngine.complete_publication_lower_bound + +#print axioms ParserEngine.materialize_into_exact +#print axioms ParserEngine.prepared_reverse_reduction_equivalent + +#print axioms ParserEngine.freeze_stack_length +#print axioms ParserEngine.freeze_stack_ext +#print axioms ParserEngine.freeze_stack_roundtrip +#print axioms ParserEngine.frozen_frame_aligned +#print axioms ParserEngine.freeze_stack_push + +#print axioms ParserEngine.staged_action_equivalent + + +#print axioms ParserEngine.Reuse.admitted_identity +#print axioms ParserEngine.Reuse.retained_cost_not_more +#print axioms ParserEngine.Reuse.retained_cost_decomposition + +#print axioms ParserEngine.Reuse.checked_reuse_break_even + +#print axioms ParserEngine.Reuse.nested_commit_survives_outer +#print axioms ParserEngine.Reuse.failed_publication_preserves_history + +#print axioms ParserEngine.Capture.split_prefix_exact +#print axioms ParserEngine.Capture.capture_visits_strict + +#print axioms ParserEngine.Capture.token_count_order_independent + +#print axioms ParserEngine.Lookahead.admitted_projection +#print axioms ParserEngine.Lookahead.extent_break_even +#print axioms ParserEngine.Lookahead.complete_request_with_extra_work +#print axioms ParserEngine.Lookahead.extent_observations_equal +#print axioms ParserEngine.Lookahead.singleton_overhead_regresses + +#print axioms ParserEngine.Lookahead.unused_extent_can_pay_failed_admission +#print axioms ParserEngine.Lookahead.one_failed_attempt_charged + +#print axioms ParserEngine.Lookahead.extent_with_retries_break_even + +#print axioms ParserEngine.Reuse.failed_source_retries_original +#print axioms ParserEngine.Reuse.successful_source_reuses + +#print axioms ParserEngine.Capture.classified_metadata_exact +#print axioms ParserEngine.Capture.classified_consumer_exact +#print axioms ParserEngine.Capture.classification_break_even + +#print axioms ParserEngine.Capture.sparse_frontiers_exact +#print axioms ParserEngine.Capture.first_shift_skipped +#print axioms ParserEngine.Capture.fused_driver_order + +#print axioms ParserEngine.Lookahead.reductions_preserve_pending +#print axioms ParserEngine.Lookahead.reduced_shift_preserves_admission +#print axioms ParserEngine.Lookahead.per_query_conversion_regresses + +#print axioms ParserEngine.Lookahead.request_transport_regresses + +#print axioms ParserEngine.Lookahead.selector_transport_exact + +#print axioms ParserEngine.goto_column_exact +#print axioms ParserEngine.goto_plan_within_budget + +#print axioms ParserEngine.freeze_stack_pop +#print axioms ParserEngine.captured_entry_after_pop + +#print axioms ParserEngine.certified_fallback_loses +#print axioms ParserEngine.omitted_fallback_has_witness + +#print axioms ParserEngine.kernel_unroll_two +#print axioms ParserEngine.kernel_unroll_four +#print axioms ParserEngine.prepared_kernel_exact + +#print axioms ParserEngine.deferred_terminal_field_exact +#print axioms ParserEngine.restored_owned_snapshot +#print axioms ParserEngine.deferred_terminal_chain_exact + +#print axioms ParserEngine.singleton_state_elimination +#print axioms ParserEngine.prepared_class_membership + +#print axioms ParserEngine.prepared_start_query_exact +#print axioms ParserEngine.absent_literal_probe + +#print axioms ParserEngine.Capture.fused_yield_observation +#print axioms ParserEngine.Capture.fused_wide_decisions_strict +#print axioms ParserEngine.Capture.capture_cost_requires_all_charges + + +#print axioms ParserEngine.Capture.changed_boundary_rejected + +#print axioms ParserEngine.Capture.delayed_yield_first_exact +#print axioms ParserEngine.Capture.delayed_yield_warm_exact +#print axioms ParserEngine.Capture.fresh_yield_allocations_eliminated +#print axioms ParserEngine.Capture.forced_yield_allocations_retained + +#print axioms ParserEngine.Capture.capture_bounds_exact + +#print axioms ParserEngine.Capture.shared_context_roundtrip +#print axioms ParserEngine.Capture.shared_context_consumer +#print axioms ParserEngine.Capture.cached_context_admission +#print axioms ParserEngine.Capture.shared_context_slot_break_even + +#print axioms ParserEngine.retained_empty_digest + +#print axioms ParserEngine.Capture.prepared_row_context_admission + +#print axioms ParserEngine.Capture.unary_children_exact +#print axioms ParserEngine.Capture.unary_children_warm + +#print axioms ParserEngine.Capture.boundary_snapshot_exact + +#print axioms ParserEngine.Capture.prepared_capture_prefix_exact +#print axioms ParserEngine.Capture.prepared_capture_arguments_saved + +#print axioms ParserEngine.Capture.canonical_context_admission + +#print axioms ParserEngine.Capture.capture_row_partitions_disjoint +#print axioms ParserEngine.Capture.capture_row_custom_index_bounded + +#print axioms ParserEngine.Capture.deferred_prefix_children_exact +#print axioms ParserEngine.Capture.deferred_prefix_children_warm +#print axioms ParserEngine.Capture.deferred_prefix_no_demand +#print axioms ParserEngine.Capture.deferred_prefix_first_demand + +#print axioms ParserEngine.Capture.packed_children_exact +#print axioms ParserEngine.Capture.packed_children_warm +#print axioms ParserEngine.Capture.packed_children_exclude_suffix +#print axioms ParserEngine.Capture.packed_children_no_demand +#print axioms ParserEngine.Capture.packed_children_first_demand + +#print axioms ParserEngine.Capture.capture_two_static +#print axioms ParserEngine.Capture.capture_three_static +#print axioms ParserEngine.Capture.capture_four_static +#print axioms ParserEngine.Capture.capture_static_observation_order + +#print axioms ParserEngine.Capture.capture_countdown_stays_bounded +#print axioms ParserEngine.Capture.capture_countdown_index + +#print axioms ParserEngine.prepared_corpus_preserves +#print axioms ParserEngine.corpus_preparations_saved diff --git a/formal/ParserEngine/Capture.lean b/formal/ParserEngine/Capture.lean new file mode 100644 index 00000000..1c26c272 --- /dev/null +++ b/formal/ParserEngine/Capture.lean @@ -0,0 +1,572 @@ +namespace ParserEngine.Capture + +/-- Reverse the used stack prefix directly into source-order children while +retaining its suffix. Child values and their ownership are not changed. -/ +def splitPrefix (width : Nat) (stack acc : List α) : List α × List α := + match width, stack with + | 0, _ => (acc, stack) + | _ + 1, [] => (acc, []) + | n + 1, head :: tail => splitPrefix n tail (head :: acc) + +theorem split_prefix_exact (width : Nat) (stack acc : List α) : + splitPrefix width stack acc = ((stack.take width).reverse ++ acc, stack.drop width) := by + induction width generalizing stack acc with + | zero => simp [splitPrefix] + | succ n ih => + cases stack with + | nil => simp [splitPrefix] + | cons head tail => simp [splitPrefix, ih, List.reverse_cons, List.append_assoc] + +theorem source_order_preserved (width : Nat) (stack : List α) : + (splitPrefix width stack []).1 = (stack.take width).reverse := by + rw [split_prefix_exact] + simp + +theorem suffix_preserved (width : Nat) (stack : List α) : + (splitPrefix width stack []).2 = stack.drop width := by rw [split_prefix_exact] + +/-- Bounded static arity staging preserves the reversed stack prefix and its +suffix exactly. Metadata observations remain in stack order, before publication. -/ +theorem capture_two_static (last first : α) (tail : List α) : + splitPrefix 2 (last :: first :: tail) [] = ([first, last], tail) := by + simp [splitPrefix] + +theorem capture_three_static (last middle first : α) (tail : List α) : + splitPrefix 3 (last :: middle :: first :: tail) [] = ([first, middle, last], tail) := by + simp [splitPrefix] + +theorem capture_four_static (fourth third second first : α) (tail : List α) : + splitPrefix 4 (fourth :: third :: second :: first :: tail) [] = + ([first, second, third, fourth], tail) := by simp [splitPrefix] + +theorem capture_static_observation_order (observe : α → β) + (fourth third second first : α) (tail : List α) : + ((fourth :: third :: second :: first :: tail).take 4).map observe = + [observe fourth, observe third, observe second, observe first] := rfl + +/-- The prepared production width is a nonnegative machine-sized dimension. +Positions and terminal counts remain arbitrary integers. A positive countdown +step stays in the admitted range and is also the exact packed-array index. -/ +theorem capture_countdown_stays_bounded (left maximum : Int) + (positive : 0 < left) (bounded : left ≤ maximum) : + 0 ≤ left - 1 ∧ left - 1 ≤ maximum ∧ (left - 1) + 1 = left := by omega + +theorem capture_countdown_index (width left : Nat) + (positive : 0 < left) (bounded : left ≤ width) : left - 1 < width := by omega + +/-- Counts are additive metadata. The fused stack-order sum equals the +source-order sum; arbitrary effectful observers are not covered. -/ +theorem token_count_order_independent (stack : List α) (width : Nat) (count : α → Nat) : + (((stack.take width).reverse).map count).sum = ((stack.take width).map count).sum := by + simp [List.map_reverse] + +/-- Count list-cell inspections for take/drop/reverse, last, count fold and yield +fold. States, nested child getters, constructor and publication costs are separate. +For width zero the last traversal is omitted. -/ +def formerVisits (width : Nat) := 5 * width + (width - 1) +def fusedVisits (width : Nat) := if width = 1 then 1 else 2 * width + +theorem nullable_split_exact (stack : List α) : splitPrefix 0 stack [] = ([], stack) := rfl + +theorem unary_split_exact (head : α) (tail : List α) : + splitPrefix 1 (head :: tail) [] = ([head], tail) := rfl + +theorem capture_visits_not_more (width : Nat) : fusedVisits width ≤ formerVisits width := by + unfold fusedVisits formerVisits + split <;> omega + +theorem capture_visits_strict (width : Nat) (positive : 0 < width) : + fusedVisits width < formerVisits width := by + unfold fusedVisits formerVisits + split <;> omega + +/-- Scalar metadata with an opaque terminal yield. Yield relocation is supplied +by the existing owner; no associativity or canonicalization is assumed. -/ +structure Metadata (Yield : Type) where + start : Int + finish : Int + tokens : Nat + yield : Yield + +inductive Piece (Yield : Type) where + | token (start finish : Int) (yield : Yield) + | fragment (metadata : Metadata Yield) + | view (delta : Int) (base : Piece Yield) + +def pieceStart : Piece Yield → Int + | .token start _ _ => start + | .fragment m => m.start + | .view delta base => delta + pieceStart base + +def pieceEnd : Piece Yield → Int + | .token _ finish _ => finish + | .fragment m => m.finish + | .view delta base => delta + pieceEnd base + +def pieceTokens : Piece Yield → Nat + | .token _ _ _ => 1 + | .fragment m => m.tokens + | .view _ base => pieceTokens base + +def pieceYield (relocate : Yield → Int → Yield) : Piece Yield → Yield + | .token _ _ yield => yield + | .fragment m => m.yield + | .view delta base => relocate (pieceYield relocate base) delta + +def separateMetadata (relocate : Yield → Int → Yield) (piece : Piece Yield) : Metadata Yield := + ⟨pieceStart piece, pieceEnd piece, pieceTokens piece, pieceYield relocate piece⟩ + +/-- Reuse the existing four projections for a view, preserving its exact +relocation nesting. Closed base representations can be classified once. -/ +def classifiedMetadata (relocate : Yield → Int → Yield) : Piece Yield → Metadata Yield + | .token start finish yield => ⟨start, finish, 1, yield⟩ + | .fragment m => m + | .view delta base => separateMetadata relocate (.view delta base) + +theorem classified_metadata_exact (relocate : Yield → Int → Yield) (piece : Piece Yield) : + classifiedMetadata relocate piece = separateMetadata relocate piece := by + cases piece <;> rfl + +/-- The consumer receives the identical fields exactly once. The function is +pure; Scheme must establish stable local reads and absence of callbacks. -/ +theorem classified_consumer_exact (relocate : Yield → Int → Yield) (piece : Piece Yield) + (consume : Metadata Yield → Value) : + consume (classifiedMetadata relocate piece) = consume (separateMetadata relocate piece) := by + rw [classified_metadata_exact] + +/-- Count complete piece-kind decisions, not accessor checks, native instructions +or allocations. Each relocated root adds one decision before the old path. -/ +def separateDecisions (ordinary views : Nat) := 4 * (ordinary + views) +def classifiedDecisions (ordinary views : Nat) := ordinary + 5 * views + +theorem classification_break_even (ordinary views : Nat) : + classifiedDecisions ordinary views ≤ separateDecisions ordinary views ↔ views ≤ 3 * ordinary := by + unfold classifiedDecisions separateDecisions + omega + +-- Fresh capture benefits in this model; a view-only workload regresses. +example : classifiedDecisions 193 0 < separateDecisions 193 0 := by decide +example : separateDecisions 0 64 < classifiedDecisions 0 64 := by decide + +/-- Dense execution exposes every immutable frontier. Sparse execution constructs +only frontiers requested by the same shift-index predicate. Frame includes all +states, semantic values, recognition, mode and source coordinates. -/ +def numberedFrames (next : Nat) : List Frame → List (Nat × Frame) + | [] => [] + | frame :: rest => (next, frame) :: numberedFrames (next + 1) rest + +def selectedFrames (spacing next : Nat) : List Frame → List Frame + | [] => [] + | frame :: rest => + if next % spacing = 0 then frame :: selectedFrames spacing (next + 1) rest + else selectedFrames spacing (next + 1) rest + +def denseSelection (spacing next : Nat) (frames : List Frame) : List Frame := + (numberedFrames next frames).filterMap + (fun pair => if pair.1 % spacing = 0 then some pair.2 else none) + +theorem sparse_frontiers_exact (spacing next : Nat) (frames : List Frame) : + selectedFrames spacing next frames = denseSelection spacing next frames := by + induction frames generalizing next with + | nil => rfl + | cons frame rest ih => + simp only [selectedFrames, denseSelection, numberedFrames, List.filterMap_cons] + split <;> simp_all [denseSelection] + +theorem sparse_frontiers_not_more (spacing next : Nat) (frames : List Frame) : + (selectedFrames spacing next frames).length ≤ frames.length := by + induction frames generalizing next with + | nil => simp [selectedFrames] + | cons frame rest ih => + simp only [selectedFrames] + split <;> simp only [List.length_cons] + · exact Nat.add_le_add_right (ih (next + 1)) 1 + · exact Nat.le_trans (ih (next + 1)) (Nat.le_succ _) + +theorem first_shift_skipped (spacing : Nat) (wide : 1 < spacing) + (frame : Frame) (rest : List Frame) : + (selectedFrames spacing 1 (frame :: rest)).length < (frame :: rest).length := by + have nonzero : 1 % spacing ≠ 0 := by rw [Nat.mod_eq_of_lt wide]; decide + simp only [selectedFrames, nonzero, ↓reduceIte, List.length_cons] + change (selectedFrames spacing 2 rest).length < rest.length + 1 + exact Nat.lt_succ_of_le (sparse_frontiers_not_more spacing 2 rest) + +/-- Fusing driver calls preserves ordered transitions. State must include error +and observable callback effects; this is not a Scheme refinement theorem. -/ +theorem fused_driver_order (step : State → Action → State) + (chunks : List (List Action)) (initial : State) : + chunks.foldl (fun state chunk => chunk.foldl step state) initial = + chunks.flatten.foldl step initial := by + induction chunks generalizing initial with + | nil => rfl + | cons chunk rest ih => simp [List.foldl_cons, List.flatten_cons, List.foldl_append, ih] + +/-- A capture yield is opaque execution metadata. Only its ordered terminal +sequence is observed by transfer admission; binary association is not an ABI. -/ +inductive YieldTree (α : Type) where + | empty + | leaf (value : α) + | branch (left right : YieldTree α) + +def YieldTree.terminals : YieldTree α → List α + | .empty => [] + | .leaf value => [value] + | .branch left right => left.terminals ++ right.terminals + +def yieldJoin (left right : YieldTree α) : YieldTree α := + match left, right with + | .empty, right => right + | left, .empty => left + | left, right => .branch left right + +theorem yield_join_order (left right : YieldTree α) : + (yieldJoin left right).terminals = left.terminals ++ right.terminals := by + cases left <;> cases right <;> simp [yieldJoin, YieldTree.terminals] + +theorem left_yield_fold_order (pieces : List (YieldTree α)) (initial : YieldTree α) : + (pieces.foldl yieldJoin initial).terminals = + initial.terminals ++ (pieces.map YieldTree.terminals).flatten := by + induction pieces generalizing initial with + | nil => simp + | cons first rest ih => + simp [List.foldl_cons, ih, yield_join_order, List.append_assoc] + +theorem right_yield_fold_order (pieces : List (YieldTree α)) : + (pieces.foldr yieldJoin .empty).terminals = (pieces.map YieldTree.terminals).flatten := by + induction pieces with + | nil => rfl + | cons first rest ih => simp [List.foldr_cons, yield_join_order, ih] + +/-- Reverse-stack accumulation prepends each complete piece yield. It can fuse +capture's count/children traversal without changing any ordered terminal leaf. +Existing relocation subtrees are opaque pieces and are not distributed/reboxed. -/ +theorem fused_yield_observation (pieces : List (YieldTree α)) : + (pieces.foldr yieldJoin .empty).terminals = + (pieces.foldl yieldJoin .empty).terminals := by + rw [right_yield_fold_order, left_yield_fold_order] + rfl + +/-- This counts logical piece classifications, not native heap bytes or GC. +Published retained fragments, prefix frames and complete artifacts remain paid. +For positive widths the old algorithm inspects endpoints plus count/yield. -/ +def separateWideDecisions (width : Nat) := 2 * width + 2 +def fusedWideDecisions (width : Nat) := width + +theorem fused_wide_decisions_strict (width : Nat) : + fusedWideDecisions width < separateWideDecisions width := by + unfold fusedWideDecisions separateWideDecisions + omega + +/-- A whole-request cost model must retain allocation and tracing charges. +A reduction in logical visits alone places no bound on these supplied costs. -/ +structure CaptureCost where + transitions : Nat + allocated : Nat + retained : Nat + gc : Nat + +def CaptureCost.total (cost : CaptureCost) : Nat := + cost.transitions + cost.allocated + cost.retained + cost.gc + +theorem capture_cost_requires_all_charges (before after : CaptureCost) + (allocation : after.allocated ≤ before.allocated) + (retention : after.retained ≤ before.retained) + (collection : after.gc ≤ before.gc) + (execution : after.transitions < before.transitions) : + after.total < before.total := by + unfold CaptureCost.total + omega + +/-- A capture signature is a live reuse premise: identical consumed yield +alone does not certify an unchanged exit lexical boundary. -/ +structure BoundarySignature where + kind : Nat + lexeme : List Nat + deriving DecidableEq + +def boundaryAccepts (expected actual : Option BoundarySignature) : Bool := decide (expected = actual) + +theorem changed_boundary_rejected (expected actual : Option BoundarySignature) + (changed : expected ≠ actual) : boundaryAccepts expected actual = false := by + simp [boundaryAccepts, changed] + +example : boundaryAccepts (some ⟨1, [97]⟩) (some ⟨1, [98]⟩) = false := by decide + +/-- A memo belongs to one immutable grammar fragment. Token leaves are retained +by identity; relocation wrappers keep their original nesting. The producer +premise is essential: arbitrary writes to topology invalidate the memo. -/ +inductive YieldMemo (α : Type) where + | pending + | ready (value : α) + +def forceYield (produce : Unit → α) : YieldMemo α → α × YieldMemo α + | .pending => let value := produce (); (value, .ready value) + | .ready value => (value, .ready value) + +theorem delayed_yield_first_exact (produce : Unit → α) : + forceYield produce .pending = (produce (), .ready (produce ())) := rfl + +theorem delayed_yield_warm_exact (produce : Unit → α) : + forceYield produce (forceYield produce .pending).2 = forceYield produce .pending := rfl + +/-- Deferral removes fresh capture's parallel yield-tree allocations only when +no consumer requests the yield. Forced reuse still pays its full construction, +plus any memo traversal overhead; this is not free work or a GC-time proof. -/ +def eagerYieldAllocations (branches : Nat) := branches +def delayedYieldAllocations (demand : Bool) (branches : Nat) := if demand then branches else 0 + +theorem fresh_yield_allocations_eliminated (branches : Nat) : + delayedYieldAllocations false branches = 0 := rfl + +theorem forced_yield_allocations_retained (branches : Nat) : + delayedYieldAllocations true branches = eagerYieldAllocations branches := rfl + +/-- Fresh capture needs span/count metadata, not the deferred terminal index. +A span consumer must not accidentally force the derived yield while classifying. -/ +def captureBounds : Piece Yield → Int × Int × Nat + | .token start finish _ => (start, finish, 1) + | .fragment metadata => (metadata.start, metadata.finish, metadata.tokens) + | .view delta base => (delta + pieceStart base, delta + pieceEnd base, pieceTokens base) + +theorem capture_bounds_exact (piece : Piece Yield) : + captureBounds piece = (pieceStart piece, pieceEnd piece, pieceTokens piece) := by + cases piece <;> rfl + +/-- A prepared reduction context contains only transition identity. Payload +includes ordered children, exact lookahead, semantic value, bounds/count, anchor +and derived-index memo; none of those request-local values are interned. -/ +structure TransitionContext (Runtime Executor : Type) where + runtime : Runtime + executor : Executor + production : Nat + entry : Nat + target : Nat + lookaheadKind : Option Nat + +structure FlatCapture (Runtime Executor Payload : Type) where + runtime : Runtime + executor : Executor + production : Nat + entry : Nat + target : Nat + lookaheadKind : Option Nat + payload : Payload + +structure SharedCapture (Runtime Executor Payload : Type) where + context : TransitionContext Runtime Executor + payload : Payload + +def shareCapture (value : FlatCapture Runtime Executor Payload) : SharedCapture Runtime Executor Payload := + ⟨⟨value.runtime, value.executor, value.production, value.entry, value.target, value.lookaheadKind⟩, value.payload⟩ + +def expandCapture (value : SharedCapture Runtime Executor Payload) : FlatCapture Runtime Executor Payload := + ⟨value.context.runtime, value.context.executor, value.context.production, + value.context.entry, value.context.target, value.context.lookaheadKind, value.payload⟩ + +theorem shared_context_roundtrip (value : FlatCapture Runtime Executor Payload) : + expandCapture (shareCapture value) = value := by cases value; rfl + +theorem shared_context_consumer (value : FlatCapture Runtime Executor Payload) + (consume : FlatCapture Runtime Executor Payload → Result) : + consume (expandCapture (shareCapture value)) = consume value := by rw [shared_context_roundtrip] + +/-- Coordinate admission includes runtime, production and entry identity. +Mutable executor/target changes must replace a cached descriptor. -/ +theorem cached_context_admission (wanted cached : TransitionContext Runtime Executor) + (coordinates : cached.runtime = wanted.runtime ∧ cached.production = wanted.production ∧ + cached.entry = wanted.entry) + (live : cached.executor = wanted.executor ∧ cached.target = wanted.target ∧ + cached.lookaheadKind = wanted.lookaheadKind) : cached = wanted := by + cases wanted; cases cached + simp_all + +/-- Hoisted directory admission requires a row owned by this runtime and +production. Executor and target remain live checks on every reduction. -/ +theorem prepared_row_context_admission + (row : Nat → Option (TransitionContext Runtime Executor)) + (runtime : Runtime) (production : Nat) + (owned : ∀ entry context, row entry = some context → + context.runtime = runtime ∧ context.production = production ∧ context.entry = entry) + (wanted cached : TransitionContext Runtime Executor) + (found : row wanted.entry = some cached) + (sameRuntime : wanted.runtime = runtime) (sameProduction : wanted.production = production) + (live : cached.executor = wanted.executor ∧ cached.target = wanted.target ∧ + cached.lookaheadKind = wanted.lookaheadKind) : cached = wanted := by + apply cached_context_admission wanted cached + · obtain ⟨hr, hp, he⟩ := owned wanted.entry cached found + exact ⟨hr.trans sameRuntime.symm, hp.trans sameProduction.symm, he⟩ + · exact live + +/-- Engine-produced unary child storage is unambiguous: a token vector or +final fragment/view is not a Scheme list. The first observation materializes +its stable list; ordinary multi-child lists are already ready. -/ +inductive ChildStore (α : Type) where + | unary (child : α) + | ready (children : List α) + +def forceChildren : ChildStore α → List α × ChildStore α + | .unary child => ([child], .ready [child]) + | .ready children => (children, .ready children) + +theorem unary_children_exact (child : α) : + (forceChildren (.unary child)).1 = [child] := rfl + +theorem unary_children_warm (child : α) : + forceChildren (forceChildren (.unary child)).2 = forceChildren (.unary child) := rfl + +/-- Only a request with no child-list demand avoids the unary cons. A first +query pays one cell and memo publication; concurrent first queries may duplicate +construction. This is not a whole-request CPU or exactly-once theorem. -/ +def unaryChildrenCells (demand : Bool) : Nat := if demand then 1 else 0 + +theorem unary_children_no_demand : unaryChildrenCells false = 0 := rfl + +theorem unary_children_first_demand : unaryChildrenCells true = 1 := rfl + +/-- A deferred prefix borrows an engine-owned immutable recognition stack. +Its width belongs to the same immutable prepared production. This deliberately +retains the stack suffix; arbitrary pair mutation is outside the premise. -/ +inductive PrefixChildren (α : Type) where + | pending (stack : List α) + | ready (children : List α) + +def forcePrefixChildren (width : Nat) : PrefixChildren α → List α × PrefixChildren α + | .pending stack => let children := (stack.take width).reverse; (children, .ready children) + | .ready children => (children, .ready children) + +theorem deferred_prefix_children_exact (width : Nat) (stack : List α) : + (forcePrefixChildren width (.pending stack)).1 = (splitPrefix width stack []).1 := by + simp [forcePrefixChildren, source_order_preserved] + +theorem deferred_prefix_children_warm (width : Nat) (stack : List α) : + forcePrefixChildren width (forcePrefixChildren width (.pending stack)).2 = + forcePrefixChildren width (.pending stack) := rfl + +/-- Object counts only: one pending descriptor replaces width eager list cells. +The first demand subsequently pays those cells too. Retained prefix/suffix cells, +object headers, native allocation and collection are not erased by this count. -/ +def deferredPrefixObjects (width : Nat) (demand : Bool) : Nat := + 1 + if demand then width else 0 + +theorem deferred_prefix_no_demand (width : Nat) (wide : 1 < width) : + deferredPrefixObjects width false < width := by + simp [deferredPrefixObjects] + omega + +theorem deferred_prefix_first_demand (width : Nat) : + deferredPrefixObjects width true = width + 1 := by simp [deferredPrefixObjects, Nat.add_comm] + +/-- An owned packed array snapshots only the used prefix. It cannot retain +an unrelated recognition-stack suffix through list links. First observation +materializes the same source-order list; subsequent observations reuse it. -/ +inductive PackedChildren (α : Type) where + | pending (children : Array α) + | ready (children : List α) + +def packChildren (width : Nat) (stack : List α) : PackedChildren α := + .pending ((stack.take width).reverse.toArray) + +def forcePackedChildren : PackedChildren α → List α × PackedChildren α + | .pending children => (children.toList, .ready children.toList) + | .ready children => (children, .ready children) + +theorem packed_children_exact (width : Nat) (stack : List α) : + (forcePackedChildren (packChildren width stack)).1 = (splitPrefix width stack []).1 := by + simp [forcePackedChildren, packChildren, source_order_preserved] + +theorem packed_children_warm (width : Nat) (stack : List α) : + forcePackedChildren (forcePackedChildren (packChildren width stack)).2 = + forcePackedChildren (packChildren width stack) := by + simp [forcePackedChildren, packChildren] + +theorem packed_children_exclude_suffix (used suffix : List α) : + packChildren used.length (used ++ suffix) = .pending used.reverse.toArray := by + simp [packChildren] + +/-- Two cold objects (descriptor and array) replace width list cells. First +query additionally pays width list cells; actual bytes and CPU remain measured. -/ +def packedChildrenObjects (width : Nat) (demand : Bool) : Nat := + 2 + if demand then width else 0 + +theorem packed_children_no_demand (width : Nat) (wide : 3 < width) : + packedChildrenObjects width false < width := by + simp [packedChildrenObjects] + omega + +theorem packed_children_first_demand (width : Nat) : + packedChildrenObjects width true = width + 2 := by simp [packedChildrenObjects, Nat.add_comm] + +/-- Snapshot the live kind and lexeme before deferring their pair allocation. +Retaining a mutable token vector instead would not preserve this obligation. -/ +structure BoundarySnapshot (Lexeme : Type) where + kind : Nat + lexeme : Lexeme + +def expandBoundary (kind : Nat) (lexeme : Lexeme) : BoundarySnapshot Lexeme := ⟨kind, lexeme⟩ + +theorem boundary_snapshot_exact (snapshot : BoundarySnapshot Lexeme) : + expandBoundary snapshot.kind snapshot.lexeme = snapshot := by cases snapshot; rfl + +/-- Static capture input is prepared with the runtime's immutable grammar. +The row and every context retain the same runtime/production coordinate. -/ +structure CapturePlan (Runtime : Type) where + runtime : Runtime + production : Nat + width : Nat + +def preparedCapturePrefix (plan : CapturePlan Runtime) (stack : List α) := + splitPrefix plan.width stack [] + +theorem prepared_capture_prefix_exact (plan : CapturePlan Runtime) (width : Nat) + (ready : plan.width = width) (stack : List α) : + preparedCapturePrefix plan stack = ((stack.take width).reverse, stack.drop width) := by + unfold preparedCapturePrefix + rw [ready, split_prefix_exact] + simp + +/-- Explicit call-word accounting only. Compiler registers, continuation +frames, plan headers and native CPU require separate Scheme measurements. -/ +def formerCaptureArguments (nodes : Nat) := 10 * nodes +def preparedCaptureArguments (nodes : Nat) := 8 * nodes + +theorem prepared_capture_arguments_saved (nodes : Nat) : + formerCaptureArguments nodes = preparedCaptureArguments nodes + 2 * nodes := by + unfold formerCaptureArguments preparedCaptureArguments + omega + +/-- Only canonical transitions enter this cache. The compiled grammar's goto +mapping is immutable for this runtime/production/entry; arbitrary direct +executors use a disjoint checked cache partition and cannot violate these premises. -/ +theorem canonical_context_admission (wanted cached : TransitionContext Runtime Executor) + (canonical : Executor) (target : Nat) + (coordinates : cached.runtime = wanted.runtime ∧ cached.production = wanted.production ∧ + cached.entry = wanted.entry) + (cachedExecutor : cached.executor = canonical) (wantedExecutor : wanted.executor = canonical) + (cachedTarget : cached.target = target) (wantedTarget : wanted.target = target) + (kind : cached.lookaheadKind = wanted.lookaheadKind) : cached = wanted := by + apply cached_context_admission wanted cached coordinates + exact ⟨cachedExecutor.trans wantedExecutor.symm, cachedTarget.trans wantedTarget.symm, kind⟩ + +/-- Canonical and custom entries occupy disjoint halves of one bounded row. +The hot canonical index remains unchanged; the custom half retains exact checks. -/ +theorem capture_row_partitions_disjoint (states canonical custom : Nat) + (canonicalBound : canonical < states) : canonical ≠ states + custom := by + omega + +theorem capture_row_custom_index_bounded (states entry : Nat) + (entryBound : entry < states) : states + entry < 2 * states := by + omega + +/-- Context/node data slots only. Cold contexts and the bounded directory are +charged; child-list/prefix storage, object headers, lookup, allocator and GC are +separate costs. Deferred prefix descriptors are charged by deferredPrefixObjects. -/ +def flatCaptureSlots (nodes : Nat) := 13 * nodes +def sharedCaptureSlots (nodes contexts plans cells : Nat) := 9 * nodes + 6 * contexts + 4 * plans + cells + +theorem shared_context_slot_break_even (nodes contexts plans cells : Nat) : + sharedCaptureSlots nodes contexts plans cells < flatCaptureSlots nodes ↔ + 6 * contexts + 4 * plans + cells < 4 * nodes := by + unfold sharedCaptureSlots flatCaptureSlots + omega + +end ParserEngine.Capture diff --git a/formal/ParserEngine/Examples.lean b/formal/ParserEngine/Examples.lean new file mode 100644 index 00000000..2ab85fc8 --- /dev/null +++ b/formal/ParserEngine/Examples.lean @@ -0,0 +1,27 @@ +import ParserEngine.Reduction + +namespace ParserEngine.Examples + +private def token : Tree := .token "identifier" ⟨3, 7⟩ + +example : interpretActions [.field "name", .alias "expression"] ⟨[(none, token)], 9⟩ = + [(none, .node "expression" ⟨3, 7⟩ [(some "name", token)])] := rfl + +example : interpretActions [.alias "expression", .field "name"] ⟨[(none, token)], 9⟩ = + [(some "name", .node "expression" ⟨3, 7⟩ [(none, token)])] := rfl + +example : field "outer" [(some "inner", token)] 9 = + [(some "outer", .fragment ⟨3, 7⟩ [(some "inner", token)])] := rfl + +example : alias "empty" [] 17 = [(none, .node "empty" ⟨17, 17⟩ [])] := rfl + +example : reduceStack [⟨[.alias "later"], [], 11⟩, ⟨[.alias "earlier"], [], 5⟩] = + [(none, .node "earlier" ⟨5, 5⟩ []), (none, .node "later" ⟨11, 11⟩ [])] := rfl + +example : (Sequence.branch (.leaf [1, 2]) (.branch (.leaf []) (.leaf [3]))).materializeInto [4] = ([1, 2, 3, 4], 8) := rfl + +example : copiedPrefix 100 = 4950 := by decide + +example : ¬ (4 ≤ ([[] , []] : List Children).length) := by decide + +end ParserEngine.Examples diff --git a/formal/ParserEngine/Execution.lean b/formal/ParserEngine/Execution.lean new file mode 100644 index 00000000..6d4c59cb --- /dev/null +++ b/formal/ParserEngine/Execution.lean @@ -0,0 +1,125 @@ +import ParserEngine.Fold + +namespace ParserEngine + +/-- Whole-region execution: State includes all consumer slots, the ordered +event owner and rejection state. Fact is recognition over one immutable source +extent, not a cache of a mutable consumer or a request result. -/ +def repeatedRecognition {Source Fact State : Type} + (recognize : Source → Fact) (source : Source) + (consumers : List (Fact → State → State)) (state : State) : State := + consumers.foldl (fun current consume => consume (recognize source) current) state + +def sharedRecognition {Fact State : Type} (fact : Fact) + (consumers : List (Fact → State → State)) (state : State) : State := + consumers.foldl (fun current consume => consume fact current) state + +/-- Consumers may depend on preceding consumer writes; their original order +is retained. Only recognition is shared. Scheme must establish immutable extent +and absence of recognition effects; this theorem does not prove that analysis. -/ +theorem shared_recognition_ordered {Source Fact State : Type} + (recognize : Source → Fact) (source : Source) + (consumers : List (Fact → State → State)) (state : State) : + repeatedRecognition recognize source consumers state = + sharedRecognition (recognize source) consumers state := rfl + +/-- An invariant can survive writes (literal zero resets), rather than require +all state untouched. This applies to arbitrary ordered engine transitions. -/ +theorem transition_list_preserves {State : Type} (safe : State → Prop) + (transitions : List (State → State)) + (preserves : ∀ transition ∈ transitions, ∀ state, safe state → safe (transition state)) + (state : State) (entry : safe state) : + safe (transitions.foldl (fun current transition => transition current) state) := by + induction transitions generalizing state with + | nil => exact entry + | cons transition rest ih => + simp only [List.foldl_cons] + apply ih + · intro action member current valid + exact preserves action (List.mem_cons_of_mem transition member) current valid + · exact preserves transition (List.mem_cons_self ..) state entry + +/-- Logical repeated source work, not a CPU/GC/wall-time estimate. Real AOT +counters must bind consumers and scanned bytes before applying this count. -/ +def repeatedSourceVisits (consumers bytes : Nat) : Nat := consumers * bytes +def sharedSourceVisits (consumers bytes : Nat) : Nat := if consumers = 0 then 0 else bytes + +theorem shared_source_visits_not_more (consumers bytes : Nat) : + sharedSourceVisits consumers bytes ≤ repeatedSourceVisits consumers bytes := by + cases consumers with + | zero => simp [sharedSourceVisits, repeatedSourceVisits] + | succ n => + simp [sharedSourceVisits, repeatedSourceVisits, Nat.succ_mul] + +/-- A generated call boundary executes an ordered region on the same owner. +Splicing it into the caller must retain every prior/dependent effect. State +includes rejection and tape state; lexical alpha conversion is a separate +compiler obligation checked by directed Scheme controls, not this theorem. -/ +theorem splice_call_preserves_order {State : Type} + (before region after : List (State → State)) (state : State) : + (before ++ [fun owner => region.foldl (fun s action => action s) owner] ++ after).foldl + (fun s action => action s) state = + (before ++ region ++ after).foldl (fun s action => action s) state := by + simp [List.foldl_append] + +/-- Copying the used prefix and transferring a buffer with its logical length +reduced to used have identical observable byte sequences. This does not prove +the primitive's memory reclamation or the implementation's exclusive owner. -/ +theorem owned_publication_prefix {Byte : Type} (buffer : List Byte) (used : Nat) : + (buffer.take used).take used = buffer.take used := by + rw [List.take_take, Nat.min_self] + +/-- Carry is encoded by equality of suffix length and consumed extent, not by +passing the unbounded initial Scheme counter through a fixed-width ABI. Reuse +Fold's counter semantics; do not duplicate its refinement model here. -/ +def countSummarySized (summary : Bool × Nat) (consumed : Nat) : Prop := + (summary.1 = true ↔ summary.2 = consumed) ∧ summary.2 ≤ consumed + +theorem count_summary_extent_step (summary : Bool × Nat) (consumed : Nat) + (sized : countSummarySized summary consumed) (member : Bool) : + countSummarySized (countSummaryStep summary member) (consumed + 1) := by + rcases summary with ⟨carry, suffix⟩ + cases member <;> cases carry <;> + simp_all [countSummarySized, countSummaryStep] <;> omega + +theorem count_summary_extent (members : List Bool) (summary : Bool × Nat) + (consumed : Nat) (sized : countSummarySized summary consumed) : + countSummarySized (members.foldl countSummaryStep summary) (consumed + members.length) := by + induction members generalizing summary consumed with + | nil => simpa using sized + | cons member rest ih => + simp only [List.foldl_cons] + have result := ih (countSummaryStep summary member) (consumed + 1) + (count_summary_extent_step summary consumed sized member) + simpa [Nat.add_assoc, Nat.add_comm, Nat.add_left_comm] using result + +/-- This is the representation boundary used by the withdrawn span leaf. It +does not establish C storage, source eligibility or physical performance. -/ +theorem count_summary_length_encoding (members : List Bool) (initial : Nat) : + let summary := members.foldl countSummaryStep (true, 0) + applyCountSummary summary initial = + if summary.2 = members.length then initial + summary.2 else summary.2 := by + have extent := count_summary_extent members (true, 0) 0 + (by simp [countSummarySized]) + simp only [Nat.zero_add, countSummarySized] at extent + cases carry : (members.foldl countSummaryStep (true, 0)).1 with + | false => + have mismatch : (members.foldl countSummaryStep (true, 0)).2 ≠ members.length := by + intro equal + have contradiction := extent.1.mpr equal + simp [carry] at contradiction + simp [applyCountSummary, carry, mismatch] + | true => + have equal := extent.1.mp carry + simp [applyCountSummary, carry, equal] + +end ParserEngine + +#print axioms ParserEngine.shared_recognition_ordered +#print axioms ParserEngine.transition_list_preserves +#print axioms ParserEngine.shared_source_visits_not_more +#print axioms ParserEngine.splice_call_preserves_order +#print axioms ParserEngine.owned_publication_prefix +#print axioms ParserEngine.count_summary_extent_step +#print axioms ParserEngine.count_summary_extent +#print axioms ParserEngine.count_summary_length_encoding diff --git a/formal/ParserEngine/Fold.lean b/formal/ParserEngine/Fold.lean new file mode 100644 index 00000000..f2a060a0 --- /dev/null +++ b/formal/ParserEngine/Fold.lean @@ -0,0 +1,538 @@ +import Lean + +namespace ParserEngine + +/- Responsibility boundary: this file owns pure denotational refinement and +operation-count bounds. FoldPlan.qnt owns scan invalidation and consumer +interleaving histories. Neither layer establishes physical throughput, full +FFI ownership, or correctness of the compiler's complete IR analysis. -/ + +/-- Pure source classification for two predicates over one immutable frame. +The counter counts source visits, not CPU time, table lookups or allocations. -/ +def sharedProbe (p q : α → Bool) : List α → (Bool × Bool) × Nat + | [] => ((false, false), 0) + | head :: tail => + let rest := sharedProbe p q tail + ((p head || rest.1.1, q head || rest.1.2), rest.2 + 1) + +theorem shared_probe_exact (p q : α → Bool) (source : List α) : + sharedProbe p q source = ((source.any p, source.any q), source.length) := by + induction source with + | nil => rfl + | cons head tail ih => simp [sharedProbe, ih, List.any_cons] + +/-- Full scans only: separate early-exit probes may visit fewer than 2*n bytes. +This inequality must never be advertised as a guaranteed twofold speedup. -/ +theorem shared_probe_visit_bound (p q : α → Bool) (source : List α) : + (sharedProbe p q source).2 ≤ 2 * source.length := by + rw [shared_probe_exact] + simp only + omega + +/-- Consumers are arbitrary ordered state transitions; classification does +not run them or mutate their immutable source frame. Compiler eligibility +must separately establish this source/bounds stability assumption. -/ +theorem shared_probe_consumers (p q : α → Bool) (source : List α) + (first : Bool → σ → σ) (second : Bool → σ → σ) (state : σ) : + second (sharedProbe p q source).1.2 + (first (sharedProbe p q source).1.1 state) = + second (source.any q) (first (source.any p) state) := by + rw [shared_probe_exact] + +/-- A necessary first byte can precede total Boolean state reads. This does +not authorize moving queries, range errors, mutation or allocating helpers. -/ +theorem necessary_byte_guard (state first rest : Bool) : + (state && (first && rest)) = (first && (state && rest)) := by + cases state <;> cases first <;> cases rest <;> rfl + +def guardedStateReads (first : Bool) : Nat := if first then 1 else 0 + +theorem irrelevant_state_reads_eliminated : guardedStateReads false = 0 := rfl + +theorem relevant_state_read_retained : guardedStateReads true = 1 := rfl + +/-- Standard-library reuse is an implementation choice, not evidence that a +runtime gate passed. Order, effects, ownership and actual timing remain gates. -/ +structure LibraryAdmission where + standardLibrary : Bool + preservesOrder : Bool + preservesEffects : Bool + preservesOwnership : Bool + matchedRuntimeGate : Bool + +def LibraryAdmission.accepted (a : LibraryAdmission) : Bool := + a.preservesOrder && a.preservesEffects && a.preservesOwnership && a.matchedRuntimeGate + +theorem library_choice_cannot_replace_runtime_gate (a : LibraryAdmission) + (failed : a.matchedRuntimeGate = false) : a.accepted = false := by + simp [LibraryAdmission.accepted, failed] + +/-- Ordered emission has the List.flatten contract. This models sequence +semantics, not the Gerbil macro's heap implementation or elapsed time. -/ +theorem ordered_builder_contract (chunks : List (List α)) : + chunks.foldr List.append [] = chunks.flatten := by + induction chunks with + | nil => rfl + | cons head tail ih => simp [List.foldr, ih] + +/-- Every alternative retains its suffix test; membership is only necessary. -/ +theorem necessary_choice_guard (a b x y : Bool) : + ((a && x) || (b && y)) = ((a || b) && ((a && x) || (b && y))) := by + cases a <;> cases b <;> cases x <;> cases y <;> rfl + +/-- A cursor byte is loaded without a second range check only within the +validated loop frame. This does not justify reusing it in a nested frame. -/ +theorem validated_loop_byte (start cursor limit size : Nat) + (_lower : start ≤ cursor) (upper : cursor < limit) (extent : limit ≤ size) : + cursor < size := Nat.lt_of_lt_of_le upper extent + +/-- Dependence/effect summaries are compiler obligations, not facts inferred +from successful model tests. The implementation must compute these flags. -/ +def probeEligible (sameBounds boundWrite opaqueEffect helper frameChange : Bool) : Bool := + sameBounds && !boundWrite && !opaqueEffect && !helper && !frameChange + +theorem source_frame_change_blocks_shared_probe (sameBounds boundWrite opaqueEffect helper : Bool) : + probeEligible sameBounds boundWrite opaqueEffect helper true = false := by + simp [probeEligible] + +theorem bound_write_blocks_shared_probe (sameBounds opaqueEffect helper frameChange : Bool) : + probeEligible sameBounds true opaqueEffect helper frameChange = false := by + simp [probeEligible] + +theorem opaque_call_blocks_shared_probe (sameBounds boundWrite helper frameChange : Bool) : + probeEligible sameBounds boundWrite true helper frameChange = false := by + simp [probeEligible] + +/-- Code interning is not value reuse: invoking one shared procedure k times +still performs k scans. Counts here measure full-scan source visits only. -/ +def repeatedScanVisits (requests bytes : Nat) : Nat := requests * bytes + +theorem interned_code_does_not_reduce_scans (calls bytes : Nat) : + repeatedScanVisits calls bytes = calls * bytes := rfl + +/-- A dominating immutable scan shared by two ordered consumers removes one +full traversal. Early exits, lookup costs and clock time remain unmodeled. -/ +theorem dominating_scan_visit_saving (bytes : Nat) : + repeatedScanVisits 2 bytes - repeatedScanVisits 1 bytes = bytes := by + simp [repeatedScanVisits] + omega + +/-- Cross-consumer reuse supplies the same immutable observation to each ordered +consumer. Source/frame equality is a compiler obligation; ownership remains +in the state passed through the fold, not in the observation. -/ +theorem shared_consumer_observation {α β σ : Type} (observe : α → β) (source : α) + (consumers : List (β → σ → σ)) (state : σ) : + consumers.foldl (fun s consume => consume (observe source) s) state = + (let fact := observe source + consumers.foldl (fun s consume => consume fact s) state) := rfl + +/-- Representation changes must commute with every admitted transition. +This obligation includes writes, stack operations and parameter overrides; +it is not discharged by simply replacing vector reads with local variables. -/ +theorem frame_representation_refinement {σ τ α : Type} + (encode : σ → τ) (oldStep : σ → α → σ) (newStep : τ → α → τ) + (stepLaw : ∀ state action, encode (oldStep state action) = + newStep (encode state) action) + (actions : List α) (state : σ) : + encode (actions.foldl oldStep state) = actions.foldl newStep (encode state) := by + induction actions generalizing state with + | nil => rfl + | cons action rest ih => + simp only [List.foldl_cons] + rw [ih, stepLaw] + +-- Quint owns owner/frame lifecycle histories, not this transition algebra. +#print axioms ParserEngine.frame_representation_refinement + +/-- A typed Boolean subset with observable query traces. This is executable +model semantics, not semantics of every Fold instruction or Scheme exception. -/ +inductive FoldPredicate where + | literal : Bool → FoldPredicate + | slot : Nat → FoldPredicate + | byte : Nat → FoldPredicate + | query : Bool → Nat → FoldPredicate + | conjunction : FoldPredicate → FoldPredicate → FoldPredicate + | choice : FoldPredicate → FoldPredicate → FoldPredicate + +def FoldPredicate.evaluate (slots : Nat → Bool) (cursorByte : Nat) : + FoldPredicate → Bool × List Nat + | .literal value => (value, []) + | .slot key => (slots key, []) + | .byte expected => (cursorByte == expected, []) + | .query result tag => (result, [tag]) + | .conjunction left right => + let first := left.evaluate slots cursorByte + if first.1 then + let second := right.evaluate slots cursorByte + (second.1, first.2 ++ second.2) + else (false, first.2) + | .choice left right => + let first := left.evaluate slots cursorByte + if first.1 then (true, first.2) + else + let second := right.evaluate slots cursorByte + (second.1, first.2 ++ second.2) + +theorem byte_guard_preserves_query_trace (slots : Nat → Bool) (key expected byte : Nat) + (rest : FoldPredicate) : + (FoldPredicate.conjunction (.slot key) (.conjunction (.byte expected) rest)).evaluate slots byte = + (FoldPredicate.conjunction (.byte expected) (.conjunction (.slot key) rest)).evaluate slots byte := by + cases slotValue : slots key <;> cases byteValue : (byte == expected) <;> + simp [FoldPredicate.evaluate, slotValue, byteValue] + +/-- Mutation witness: a query cannot move behind a mismatching byte guard. -/ +theorem moving_query_is_observable : + (FoldPredicate.conjunction (.query true 7) (.byte 1)).evaluate (fun _ => false) 0 ≠ + (FoldPredicate.conjunction (.byte 1) (.query true 7)).evaluate (fun _ => false) 0 := by + decide + +/-- The validated byte can eliminate a mismatching state-dependent branch, +including arbitrary queries in its suffix, without executing those queries. -/ +theorem mismatching_byte_eliminates_state (slots : Nat → Bool) (key expected byte : Nat) + (rest : FoldPredicate) (mismatch : (byte == expected) = false) : + (FoldPredicate.conjunction (.slot key) (.conjunction (.byte expected) rest)).evaluate slots byte = + (false, []) := by + cases value : slots key <;> simp [FoldPredicate.evaluate, value, mismatch] + +/-- Cofactoring preserves a sequence of arbitrary state transitions once each +step has its independently established mismatch/effect obligation. This does +not prove that the compiler computed that obligation for every IR form. -/ +def executeSteps (steps : List (α → α)) (state : α) : α := + steps.foldl (fun current step => step current) state + +theorem ordered_region_refinement (steps : List (α → α)) + (transform : (α → α) → (α → α)) + (refinement : ∀ step ∈ steps, ∀ state, transform step state = step state) + (state : α) : executeSteps (steps.map transform) state = executeSteps steps state := by + induction steps generalizing state with + | nil => rfl + | cons step rest ih => + simp only [List.map_cons, executeSteps, List.foldl_cons] + rw [refinement step (by simp) state] + apply ih + intro next member current + exact refinement next (by simp [member]) current + +/-- A negative shared recognition result selects the original else transition +without evaluating irrelevant pure state guards. Necessity and effect freedom +must be established by IR admission; this is not a grammar-wide speed proof. -/ +theorem negative_line_consumer {σ : Type} (trigger condition : Bool) + (necessary : condition = true → trigger = true) (negative : trigger = false) + (yes no : σ → σ) (state : σ) : + (if condition then yes state else no state) = no state := by + cases condition with + | false => rfl + | true => have impossible := necessary rfl; simp [negative] at impossible + +/-- Scheme conditions consume truthiness, but Boolean slots publish actual +Booleans. This models that distinction without erasing observable traces. -/ +inductive ConditionValue where + | falseValue + | trueValue + | payload : Nat → ConditionValue + deriving DecidableEq + +def ConditionValue.truth : ConditionValue → Bool + | .falseValue => false + | _ => true + +def ConditionValue.normalize (value : ConditionValue) : ConditionValue := + if value.truth then .trueValue else .falseValue + +theorem condition_normalization_not_required (value : ConditionValue) : + value.normalize.truth = value.truth := by + cases value <;> rfl + +theorem condition_preserves_ordered_trace (value : ConditionValue) + (priorTrace onTrue onFalse : List Nat) : + priorTrace ++ (if value.normalize.truth then onTrue else onFalse) = + priorTrace ++ (if value.truth then onTrue else onFalse) := by + rw [condition_normalization_not_required] + +theorem boolean_publication_cannot_be_erased : + ConditionValue.normalize (.payload 7) ≠ ConditionValue.payload 7 := by + decide + +/-- The idle branch is a guarded specialization, never a cached result. +The compiler must prove equivalence for every state satisfying its guard; +outside that domain the original ordered transition is retained. This theorem +does not prove the Scheme IR transformation or physical timing. -/ +def guardedIdle (guard : σ → Bool) (specialized original : σ → σ) (state : σ) : σ := + if guard state then specialized state else original state + +theorem guarded_idle_refinement (guard : σ → Bool) (specialized original : σ → σ) + (eligible : ∀ state, guard state = true → specialized state = original state) + (state : σ) : guardedIdle guard specialized original state = original state := by + unfold guardedIdle + split + next h => exact eligible state h + next => rfl + +theorem zero_mode_guard (mode : Nat) (idle : mode = 0) : (0 < mode) = False := by + subst mode + simp + +/-- Consumer-local selector facts require an admitted unsigned cell. This +does not apply to a signed integer or an opaque predicate. -/ +theorem consumer_false_positive_zero (mode : Nat) (branch : ¬ 0 < mode) : mode = 0 := by + exact Nat.eq_zero_of_not_pos branch + +/-- Fact support is independent: refining one consumer never assigns the +other consumer's state. Runtime mutation/invalidation histories belong to Quint. -/ +theorem consumer_zero_support (mode other : Nat) (branch : ¬ 0 < mode) : + (mode, other) = (0, other) := by + rw [consumer_false_positive_zero mode branch] + +/-- Reusing a selector's branch truth requires its extracted read support to +be sound. This proves the disjoint-write rule, not the IR dependency extractor. -/ +theorem consumer_disjoint_write (reads : List Nat) (predicate : (Nat → Nat) → Bool) + (support : ∀ before after, (∀ name, name ∈ reads → before name = after name) → + predicate before = predicate after) + (state : Nat → Nat) (written value : Nat) (disjoint : written ∉ reads) : + predicate state = predicate (fun name => if name = written then value else state name) := by + apply support + intro name member + have different : name ≠ written := by + intro same + subst name + exact disjoint member + simp [different] + +/-- Removing internal owner-valued results preserves ordered effects when +each transition retains the same private owner. This is not a proof of mutable +alias admission, the generated compiler, or physical continuation allocation. -/ +def liftPrivateOwner {σ ω : Type} (effect : σ → σ) (owned : σ × ω) : σ × ω := + (effect owned.1, owned.2) + +theorem effect_only_owner_threading {σ ω : Type} (steps : List (σ → σ)) + (state : σ) (owner : ω) : + executeSteps (steps.map liftPrivateOwner) (state, owner) = + (executeSteps steps state, owner) := by + induction steps generalizing state with + | nil => rfl + | cons step rest ih => + simp only [List.map_cons, executeSteps, List.foldl_cons, liftPrivateOwner] + exact ih (step state) + +/-- A validated cursor and clamped forward steps stay ordered in its frame. +This arithmetic fact does not admit a reversed range or reuse across frames. -/ +theorem clamped_forward_range (start cursor stop : Nat) + (lower : start ≤ cursor) (upper : cursor ≤ stop) : + start ≤ cursor ∧ cursor ≤ min stop (cursor + 1) ∧ + min stop (cursor + 1) ≤ stop := by + omega + +/-- Erase a discarded condition only after its observation is total and pure. +The Scheme effect checker and its contextual admission remain separate duties. -/ +theorem discarded_pure_condition (observe : σ → Bool) (state : σ) : + (if observe state then state else state) = state := by + split <;> rfl + +/-- A fixed overwrite family depends on input, never on prior written cells. +The untouched context includes source, admission guards and the event owner. +This model does not prove the Scheme admission checker or compiler lowering. -/ +def overwriteOnly {κ ρ α : Type} (encode : α → ρ) (state : κ × ρ) (input : α) : κ × ρ := + (state.1, encode input) + +theorem overwrite_run_last {κ ρ α : Type} (encode : α → ρ) (state : κ × ρ) + (earlier : List α) (last : α) : + (earlier ++ [last]).foldl (overwriteOnly encode) state = + overwriteOnly encode state last := by + induction earlier generalizing state with + | nil => rfl + | cons head rest ih => + simp only [List.cons_append, List.foldl_cons] + rw [ih] + rfl + +/-- Independent consumer selectors read only untouched context. Active and +inactive consumers therefore coalesce without an all-idle assumption. The +Scheme checker must establish that context/write separation; it is not assumed +sound merely because this conditional transfer theorem is proven. -/ +def selectiveOverwrite {κ ρ α : Type} (enabled : κ → Bool) (encode : κ → α → ρ) + (state : κ × ρ) (input : α) : κ × ρ := + (state.1, if enabled state.1 then encode state.1 input else state.2) + +theorem selective_run_last {κ ρ α : Type} (enabled : κ → Bool) (encode : κ → α → ρ) + (state : κ × ρ) (earlier : List α) (last : α) : + (earlier ++ [last]).foldl (selectiveOverwrite enabled encode) state = + selectiveOverwrite enabled encode state last := by + induction earlier generalizing state with + | nil => rfl + | cons head rest ih => + simp only [List.cons_append, List.foldl_cons] + rw [ih] + cases h : enabled state.1 <;> simp [selectiveOverwrite, h] + +/-- Two independent consumers share one input traversal; neither needs the +other to be idle. This product theorem does not authorize shared write cells. -/ +def independentConsumerStep {κ ρ τ α : Type} + (first : κ → Bool) (firstEncode : κ → α → ρ) + (second : κ → Bool) (secondEncode : κ → α → τ) + (state : (κ × ρ) × (κ × τ)) (input : α) : (κ × ρ) × (κ × τ) := + (selectiveOverwrite first firstEncode state.1 input, + selectiveOverwrite second secondEncode state.2 input) + +theorem independent_consumer_run_last {κ ρ τ α : Type} + (first : κ → Bool) (firstEncode : κ → α → ρ) + (second : κ → Bool) (secondEncode : κ → α → τ) + (state : (κ × ρ) × (κ × τ)) (earlier : List α) (last : α) : + (earlier ++ [last]).foldl + (independentConsumerStep first firstEncode second secondEncode) state = + independentConsumerStep first firstEncode second secondEncode state last := by + induction earlier generalizing state with + | nil => rfl + | cons head rest ih => + simp only [List.cons_append, List.foldl_cons] + rw [ih] + apply Prod.ext + · simpa [independentConsumerStep] using + (selective_run_last first firstEncode state.1 [head] last) + · simpa [independentConsumerStep] using + (selective_run_last second secondEncode state.2 [head] last) + +/-- A scanner returns an ordinary prefix and leaves the first trigger for +the original transition. Refinement includes that residual transition; +dropping it would preserve final overwrite cells but lose observations. -/ +theorem overwrite_run_residual {κ ρ α : Type} (encode : α → ρ) + (state : κ × ρ) (earlier : List α) (last : α) + (residual : κ × ρ → κ × ρ) : + residual ((earlier ++ [last]).foldl (overwriteOnly encode) state) = + residual (overwriteOnly encode state last) := by + rw [overwrite_run_last] + +/-- The source partition is contiguous: scanning a run never consumes the +trigger at its limit. Actual byte membership and alias admission are Scheme +and complete AOT differential obligations, not consequences of this theorem. -/ +theorem run_partition (start stop limit : Nat) + (lower : start ≤ stop) (upper : stop ≤ limit) : + stop - start + (limit - stop) = limit - start := by omega + +/-- Pure contract of the bounded leaf: split at the first trigger, not after +it. Actual C byte classification and bounds remain native AOT obligations. -/ +def ordinaryPrefix (trigger : α → Bool) : List α → List α × List α + | [] => ([], []) + | head :: tail => + if trigger head then ([], head :: tail) + else let rest := ordinaryPrefix trigger tail + (head :: rest.1, rest.2) + +theorem ordinary_prefix_partition (trigger : α → Bool) (source : List α) : + (ordinaryPrefix trigger source).1 ++ (ordinaryPrefix trigger source).2 = source := by + induction source with + | nil => rfl + | cons head tail ih => + cases h : trigger head <;> simp [ordinaryPrefix, h, ih] + +theorem ordinary_prefix_leaves_trigger (trigger : α → Bool) (source : List α) : + match (ordinaryPrefix trigger source).2 with + | [] => True + | head :: _ => trigger head = true := by + induction source with + | nil => trivial + | cons head tail ih => + cases h : trigger head <;> simp [ordinaryPrefix, h, ih] + +/-- Environment projection has a different obligation from run coalescing. +This concrete expression fragment models reads and ordered composition; +compiler name resolution and effects still require complete AOT controls. -/ +inductive EnvironmentExpr (name value : Type) where + | literal : value → EnvironmentExpr name value + | read : name → EnvironmentExpr name value + | combine : (value → value → value) → EnvironmentExpr name value → + EnvironmentExpr name value → EnvironmentExpr name value + +def EnvironmentExpr.evaluate (env : name → value) : EnvironmentExpr name value → value + | .literal v => v + | .read n => env n + | .combine op left right => op (left.evaluate env) (right.evaluate env) + +def EnvironmentExpr.reads : EnvironmentExpr name value → List name + | .literal _ => [] + | .read n => [n] + | .combine _ left right => left.reads ++ right.reads + +theorem EnvironmentExpr.project_required (expr : EnvironmentExpr name value) + (original projected : name → value) + (same : ∀ n ∈ expr.reads, original n = projected n) : + expr.evaluate original = expr.evaluate projected := by + induction expr with + | literal v => rfl + | read n => exact same n (by simp [reads]) + | combine op left right ihLeft ihRight => + simp only [evaluate] + rw [ihLeft (fun n hn => same n (by simp [reads, hn])), + ihRight (fun n hn => same n (by simp [reads, hn]))] + +/-- Ordinary reset/increment machines have a constant-size run summary. +No event is published inside this fragment; the residual observes the final +counter after the owner commits it. Exact byte eligibility is an AOT duty. -/ +def runCountStep (count : Nat) (member : Bool) : Nat := + if member then count + 1 else 0 + +def countSummaryStep (summary : Bool × Nat) (member : Bool) : Bool × Nat := + if member then (summary.1, summary.2 + 1) else (false, 0) + +def applyCountSummary (summary : Bool × Nat) (initial : Nat) : Nat := + (if summary.1 then initial else 0) + summary.2 + +theorem count_summary_step (summary : Bool × Nat) (initial : Nat) (member : Bool) : + applyCountSummary (countSummaryStep summary member) initial = + runCountStep (applyCountSummary summary initial) member := by + cases member <;> simp [countSummaryStep, applyCountSummary, runCountStep, Nat.add_assoc] + +theorem count_summary_fold (members : List Bool) (summary : Bool × Nat) (initial : Nat) : + applyCountSummary (members.foldl countSummaryStep summary) initial = + members.foldl runCountStep (applyCountSummary summary initial) := by + induction members generalizing summary with + | nil => rfl + | cons head tail ih => + simp only [List.foldl_cons] + rw [ih, count_summary_step] + +theorem count_summary_identity (members : List Bool) (initial : Nat) : + applyCountSummary (members.foldl countSummaryStep (true, 0)) initial = + members.foldl runCountStep initial := by + simpa [applyCountSummary] using count_summary_fold members (true, 0) initial + +end ParserEngine + +#print axioms ParserEngine.count_summary_identity + +#print axioms ParserEngine.overwrite_run_residual +#print axioms ParserEngine.run_partition +#print axioms ParserEngine.ordinary_prefix_partition +#print axioms ParserEngine.ordinary_prefix_leaves_trigger +#print axioms ParserEngine.EnvironmentExpr.project_required + +#print axioms ParserEngine.shared_probe_exact +#print axioms ParserEngine.shared_probe_visit_bound +#print axioms ParserEngine.shared_probe_consumers +#print axioms ParserEngine.necessary_byte_guard +#print axioms ParserEngine.ordered_builder_contract +#print axioms ParserEngine.necessary_choice_guard +#print axioms ParserEngine.library_choice_cannot_replace_runtime_gate +#print axioms ParserEngine.validated_loop_byte +#print axioms ParserEngine.source_frame_change_blocks_shared_probe +#print axioms ParserEngine.bound_write_blocks_shared_probe +#print axioms ParserEngine.opaque_call_blocks_shared_probe +#print axioms ParserEngine.interned_code_does_not_reduce_scans +#print axioms ParserEngine.dominating_scan_visit_saving +#print axioms ParserEngine.byte_guard_preserves_query_trace +#print axioms ParserEngine.moving_query_is_observable +#print axioms ParserEngine.mismatching_byte_eliminates_state +#print axioms ParserEngine.ordered_region_refinement +#print axioms ParserEngine.condition_normalization_not_required +#print axioms ParserEngine.condition_preserves_ordered_trace +#print axioms ParserEngine.boolean_publication_cannot_be_erased +#print axioms ParserEngine.guarded_idle_refinement +#print axioms ParserEngine.zero_mode_guard +#print axioms ParserEngine.consumer_false_positive_zero +#print axioms ParserEngine.consumer_zero_support +#print axioms ParserEngine.consumer_disjoint_write +#print axioms ParserEngine.effect_only_owner_threading +#print axioms ParserEngine.clamped_forward_range +#print axioms ParserEngine.discarded_pure_condition +#print axioms ParserEngine.overwrite_run_last +#print axioms ParserEngine.selective_run_last +#print axioms ParserEngine.independent_consumer_run_last diff --git a/formal/ParserEngine/Lookahead.lean b/formal/ParserEngine/Lookahead.lean new file mode 100644 index 00000000..eb20d2ea --- /dev/null +++ b/formal/ParserEngine/Lookahead.lean @@ -0,0 +1,298 @@ +import LeanPoo.Prototype.DictionaryBridge + +namespace ParserEngine.Lookahead + +/-- Logical request-local scope, not implemented hashes or pointer equality. +The contract binds grammar, prepared plan and lexical policy. Extent changes +at input admission, including EOF; revision changes on metadata mutation. -/ +structure Key where + owner : Nat + source : Nat + contract : Nat + extent : Nat + revision : Nat + deriving DecidableEq, Repr + +structure Snapshot (Value : Type) where + basis : Key + value : Value + +def read (current : Key) (snapshot : Snapshot Value) : Option Value := + if snapshot.basis = current then some snapshot.value else none + +/-- Key equality is sufficient only with a producer premise binding the stored +value to a pure projection. It does not prove mutable Scheme tokens satisfy it. -/ +theorem admitted_projection (project : Key → Value) (current : Key) + (snapshot : Snapshot Value) (value : Value) + (producer : snapshot.value = project snapshot.basis) + (accepted : read current snapshot = some value) : value = project current := by + unfold read at accepted + split at accepted + · have keys : snapshot.basis = current := ‹snapshot.basis = current› + have values : snapshot.value = value := Option.some.inj accepted + rw [← values, producer, keys] + · contradiction + +theorem changed_key_rejects (current : Key) (snapshot : Snapshot Value) + (changed : snapshot.basis ≠ current) : read current snapshot = none := by + simp [read, changed] + +theorem mutation_rejects (current : Key) (snapshot : Snapshot Value) + (changed : snapshot.basis.revision ≠ current.revision) : read current snapshot = none := by + apply changed_key_rejects + exact fun equal => changed (congrArg Key.revision equal) + +theorem supersession_rejects (current : Key) (snapshot : Snapshot Value) + (changed : snapshot.basis.owner ≠ current.owner) : read current snapshot = none := by + apply changed_key_rejects + exact fun equal => changed (congrArg Key.owner equal) + +/-- Every pure consumer gets the same ordered metadata observations within +one stable extent. This does not preserve mutable pair identity or effects. -/ +theorem extent_observations_equal (project : Key → Value) (current : Key) + (snapshot : Snapshot Value) (reductions : Nat) + (producer : snapshot.value = project snapshot.basis) + (keys : snapshot.basis = current) : + List.replicate reductions snapshot.value = List.replicate reductions (project current) := by + rw [producer, keys] + +-- Equal identities alone cannot establish value equality: omitting the producer +-- premise admits stale metadata even when the read guard succeeds. +example (key : Key) : read key (⟨key, 0⟩ : Snapshot Nat) = some 0 ∧ (0 : Nat) ≠ 1 := by + simp [read] + +/-- Number of capture reductions while one admitted metadata extent is stable. +An extent is not an LR state or a source offset. These are weighted abstract +operations, not CPU instructions; a no-capture or unused extent has count zero. -/ +def repeatedMetadata (reductions materialize : Nat) := reductions * materialize +def scopedMetadata (reductions materialize check setup : Nat) := + if reductions = 0 then 0 else materialize + reductions * check + setup + +theorem unused_extent_free (materialize check setup : Nat) : + scopedMetadata 0 materialize check setup = 0 := rfl + +/-- Exact break-even, charging all checks and scope construction. In particular, +one reduction cannot benefit when either check or setup has positive cost. -/ +theorem extent_break_even (additional materialize check setup : Nat) : + scopedMetadata (additional + 1) materialize check setup ≤ + repeatedMetadata (additional + 1) materialize ↔ + (additional + 1) * check + setup ≤ additional * materialize := by + simp only [scopedMetadata, Nat.add_one_ne_zero, ↓reduceIte, repeatedMetadata, + Nat.add_mul, Nat.one_mul] + omega + +theorem singleton_overhead_regresses (materialize check setup : Nat) : + repeatedMetadata 1 materialize < scopedMetadata 1 materialize check setup ↔ + 0 < check + setup := by + simp [repeatedMetadata, scopedMetadata] + omega + +/-- Costs paid by both complete requests. Equality is an obligation, not an +inference from metadata parity; FFI and scheduling are explicit separate terms. -/ +structure SharedWork where + preparation : Nat + scan : Nat + lr : Nat + captureOther : Nat + validation : Nat + publication : Nat + ffi : Nat + scheduling : Nat + deriving Repr + +def shared (work : SharedWork) : Nat := + work.preparation + work.scan + work.lr + work.captureOther + + work.validation + work.publication + work.ffi + work.scheduling + +/-- Failed snapshot attempts must be charged even if no capture later uses +this extent. Each failed attempt materializes, validates and discards its work. +These costs are logical weights, not measurements or a bounded retry policy. -/ +def retryWork (failed materialize validate discard : Nat) := + failed * (materialize + validate + discard) + +theorem one_failed_attempt_charged (materialize validate discard : Nat) : + retryWork 1 materialize validate discard = materialize + validate + discard := by + simp [retryWork] + +theorem extent_with_retries_break_even (additional materialize check setup failed : Nat) : + scopedMetadata (additional + 1) materialize check setup + failed ≤ + repeatedMetadata (additional + 1) materialize ↔ + (additional + 1) * check + setup + failed ≤ additional * materialize := by + simp only [scopedMetadata, Nat.add_one_ne_zero, ↓reduceIte, repeatedMetadata, + Nat.add_mul, Nat.one_mul] + omega + +/-- Each extent has its own weights: EOF construction may be zero, and a +mutation/snapshot check may depend on lexeme size. Uniform weights are not +assumed when admitting the complete request. -/ +structure ExtentWork where + reductions : Nat + materialize : Nat + check : Nat + setup : Nat + failedAdmission : Nat := 0 + deriving Repr + +def repeatedTotal (extents : List ExtentWork) : Nat := + (extents.map (fun e => repeatedMetadata e.reductions e.materialize)).sum + +def scopedTotal (extents : List ExtentWork) : Nat := + (extents.map (fun e => scopedMetadata e.reductions e.materialize e.check e.setup + + e.failedAdmission)).sum + +def baseline (work : SharedWork) (extents : List ExtentWork) : Nat := + shared work + repeatedTotal extents + +def candidate (work : SharedWork) (extents : List ExtentWork) : Nat := + shared work + scopedTotal extents + +/-- Amortize over the whole request, not a chosen favorable extent. A costly +singleton extent may be offset by others; every extent remains in the sum. -/ +theorem complete_request_break_even (work : SharedWork) (extents : List ExtentWork) : + candidate work extents ≤ baseline work extents ↔ + scopedTotal extents ≤ repeatedTotal extents := by + simp [candidate, baseline] + +/-- Common work cannot be dropped if the proposal changes it. -/ +theorem complete_request_with_extra_work (work : SharedWork) (extents : List ExtentWork) + (extra : Nat) : + candidate work extents + extra ≤ baseline work extents ↔ + scopedTotal extents + extra ≤ repeatedTotal extents := by + unfold candidate baseline + omega + +theorem unused_extent_can_pay_failed_admission (e : ExtentWork) + (unused : e.reductions = 0) : scopedTotal [e] = e.failedAdmission := by + simp [scopedTotal, unused, scopedMetadata] + +-- A candidate beneficial without retries can lose after failed admission work. +example : scopedTotal [⟨8, 4, 1, 1, retryWork 5 4 1 1⟩] > + repeatedTotal [⟨8, 4, 1, 1, retryWork 5 4 1 1⟩] := by decide + +-- Retain singleton and EOF losses in a heterogeneous complete-request sum. +-- These are illustrative weights, not measured parser or CPU observations. +example : scopedTotal [⟨1, 4, 1, 1, 0⟩, ⟨8, 4, 1, 1, 0⟩, ⟨5, 0, 1, 0, 0⟩] < + repeatedTotal [⟨1, 4, 1, 1, 0⟩, ⟨8, 4, 1, 1, 0⟩, ⟨5, 0, 1, 0, 0⟩] := by decide + +example : repeatedTotal [⟨1, 4, 1, 1, 0⟩, ⟨1, 4, 1, 1, 0⟩, ⟨5, 0, 1, 0, 0⟩] < + scopedTotal [⟨1, 4, 1, 1, 0⟩, ⟨1, 4, 1, 1, 0⟩, ⟨5, 0, 1, 0, 0⟩] := by decide + +-- Both positive and negative regimes are part of the model. +example : scopedMetadata 8 4 1 1 < repeatedMetadata 8 4 := by decide +example : repeatedMetadata 1 4 < scopedMetadata 1 4 1 1 := by decide +example : repeatedMetadata 8 0 < scopedMetadata 8 0 1 0 := by decide +example : scopedMetadata 0 4 1 1 < 4 + 1 := by decide + +/-- A pending scanner admission owns its mode, even when reductions change +LR state before consumption. This is input transport, not metadata reuse. -/ +structure PendingInput (Token Mode : Type) where + token : Token + mode : Mode + deriving DecidableEq + +structure InputFrontier (Token Mode : Type) where + state : Nat + pending : Option (PendingInput Token Mode) + emitted : List (PendingInput Token Mode) + +def reduceFrontier (nextState : Nat) (frontier : InputFrontier Token Mode) := + { frontier with state := nextState } + +def shiftFrontier (frontier : InputFrontier Token Mode) := + { frontier with pending := none, emitted := frontier.emitted ++ frontier.pending.toList } + +/-- Capture the admitted token/mode, not the lexical mode of the reduced state. -/ +theorem reductions_preserve_pending (states : List Nat) (frontier : InputFrontier Token Mode) : + (states.foldl (fun current next => reduceFrontier next current) frontier).pending = + frontier.pending := by + induction states generalizing frontier with + | nil => rfl + | cons head tail ih => exact ih (reduceFrontier head frontier) + +theorem reduced_shift_preserves_admission (states : List Nat) + (frontier : InputFrontier Token Mode) : + (shiftFrontier (states.foldl (fun current next => reduceFrontier next current) frontier)).emitted = + frontier.emitted ++ frontier.pending.toList := by + have unchanged : (states.foldl (fun current next => reduceFrontier next current) frontier).emitted = + frontier.emitted := by + induction states generalizing frontier with + | nil => rfl + | cons head tail ih => exact ih (reduceFrontier head frontier) + simp only [shiftFrontier, unchanged, reductions_preserve_pending] + +/-- List-cell transport counts only. A scalar input saves the admission cell +only if action queries accept it directly. Converting back on each query pays +again for reductions and escapes; neither term may be omitted. -/ +def listInputCells (admissions : Nat) := admissions +def scalarInputCells (convertedQueries escapes : Nat) := convertedQueries + escapes + +theorem scalar_input_break_even (admissions queries escapes : Nat) : + scalarInputCells queries escapes ≤ listInputCells admissions ↔ + queries + escapes ≤ admissions := by rfl + +theorem per_query_conversion_regresses (admissions extraQueries escapes : Nat) + (more : 0 < extraQueries + escapes) : + listInputCells admissions < scalarInputCells (admissions + extraQueries) escapes := by + unfold listInputCells scalarInputCells + omega + +/-- Lift transport regression to complete logical work only when every other +request term is identical. This is not a physical CPU-cost assumption. -/ +theorem request_transport_regresses (work : SharedWork) (admissions extraQueries escapes : Nat) + (more : 0 < extraQueries + escapes) : + shared work + listInputCells admissions < + shared work + scalarInputCells (admissions + extraQueries) escapes := by + exact Nat.add_lt_add_left + (per_query_conversion_regresses admissions extraQueries escapes more) (shared work) + +-- Counterexample with nonempty queries: not a measured action trace. +example : listInputCells 192 < scalarInputCells (192 + 322) 0 := by decide + +/-- The streaming domain is empty or one admitted token. Borrowed multi-token +suffixes keep their separate identity-preserving list representation. -/ +abbrev SingleInput (Token : Type) := { xs : List Token // xs.length ≤ 1 } + +def encodeInput (input : Option Token) : SingleInput Token := + ⟨input.toList, by cases input <;> simp⟩ + +def decodeInput (input : SingleInput Token) : Option Token := input.val.head? + +def inputBridge (Token : Type) : + LeanPoo.Prototype.DictionaryBridge (Option Token) (SingleInput Token) where + toFields := encodeInput + fromFields := decodeInput + from_to := by intro input; cases input <;> rfl + to_from := by + intro input + apply Subtype.ext + rcases input with ⟨values, bounded⟩ + cases values with + | nil => rfl + | cons token tail => + have empty : tail = [] := by + have lengthZero : tail.length = 0 := by simp only [List.length_cons] at bounded; omega + cases tail with + | nil => rfl + | cons _ _ => simp at lengthZero + subst tail + rfl + +/-- Reuse the upstream inverse-bearing representation contract for every +state-specific selector, including EOF and effect-free literal precedence. -/ +theorem selector_transport_exact (selector : Nat → SingleInput Token → Row) + (state : Nat) (input : SingleInput Token) : + selector state ((inputBridge Token).toFields ((inputBridge Token).fromFields input)) = + selector state input := by rw [(inputBridge Token).to_from] + +/-- Rejected flat-selector experiment: token-fallback queries transfer through +three procedures; EOF and literal hits have separate costs. Flattening those +queries would remove two procedure transfers each. Private-row branches and body work are independent charged costs. -/ +def nestedSelectorCalls (queries : Nat) := 3 * queries +def directSelectorCalls (queries : Nat) := queries + +theorem direct_dispatch_transfers (queries : Nat) : + nestedSelectorCalls queries = directSelectorCalls queries + 2 * queries := by + unfold nestedSelectorCalls directSelectorCalls + omega + +end ParserEngine.Lookahead diff --git a/formal/ParserEngine/OwnedStack.lean b/formal/ParserEngine/OwnedStack.lean new file mode 100644 index 00000000..f4b24634 --- /dev/null +++ b/formal/ParserEngine/OwnedStack.lean @@ -0,0 +1,91 @@ +import ParserEngine.Reduction + +namespace ParserEngine + +/-- The private buffer's active prefix has the same newest-first order as an + immutable checkpoint stack. Capacity and inactive cells are not public data. -/ +def freezeStack (storage : Array α) (depth : Nat) : List α := + (storage.toList.take depth).reverse + +theorem freeze_stack_length (storage : Array α) (depth : Nat) + (fits : depth ≤ storage.size) : + (freezeStack storage depth).length = depth := by + simp [freezeStack, Nat.min_eq_left fits] + +/-- Growth/clearing may alter inactive cells only. This condition is discharged + by Scheme storage tests; it is not a proof of the native bulk primitives. -/ +theorem freeze_stack_ext (left right : Array α) (depth : Nat) + (same : left.toList.take depth = right.toList.take depth) : + freezeStack left depth = freezeStack right depth := by + exact congrArg List.reverse same + +theorem freeze_stack_roundtrip (stack : List α) : + freezeStack stack.reverse.toArray stack.length = stack := by + have all : stack.reverse.take stack.length = stack.reverse := by + simpa only [List.length_reverse] using (List.take_length : List.take stack.reverse.length stack.reverse = stack.reverse) + simp [freezeStack, all] + +/-- A checked LR frame owns one more active state than semantic values. -/ +theorem frozen_frame_aligned (states : Array Nat) (values : Array Children) (top : Nat) + (stateFits : top + 1 ≤ states.size) (valueFits : top ≤ values.size) : + (freezeStack states (top + 1)).length = (freezeStack values top).length + 1 := by + rw [freeze_stack_length states (top + 1) stateFits, + freeze_stack_length values top valueFits] + +/-- Writing one admitted value after the retained prefix publishes a cons onto + its immutable reverse-order frontier. No heap-safety claim is implied. -/ +theorem freeze_stack_push (before after : Array α) (top : Nat) (value : α) + (retained : after.toList.take (top + 1) = before.toList.take top ++ [value]) : + freezeStack after (top + 1) = value :: freezeStack before top := by + simp [freezeStack, retained, List.reverse_append] + +/-- Array indices in the private transition loop remain in the admitted capacity. + These arithmetic obligations do not certify native allocation or fixnum limits. -/ +theorem owned_reduction_index_fits (top width capacity : Nat) + (fits : top < capacity) (admitted : width ≤ top) : + top - width < capacity := by omega + +theorem owned_operand_index_fits (top width index capacity : Nat) + (fits : top < capacity) (admitted : width ≤ top) (operand : index < width) : + top - index - 1 < capacity := by omega + +theorem owned_shift_index_fits (top capacity : Nat) (fits : top < capacity) : + top + 1 < (if top + 1 < capacity then capacity else capacity * 2) := by + split <;> omega + +/-- One admitted unary transition replaces the active head and retains its + predecessor stack. Selection and semantic callbacks are assumptions here. -/ +def unaryStackStep (step : α → α) (stack : List α) : List α := + match stack with + | [] => [] + | head :: tail => step head :: tail + +/-- Carrying the current head through a chain and committing once is equivalent + to committing each selected unary transition. Native bounds, termination, + selector identity and CPU benefit remain separate executable obligations. -/ +theorem unary_chain_keeps_tail (steps : List (α → α)) (head : α) (tail : List α) : + steps.foldl (fun stack step => unaryStackStep step stack) (head :: tail) = + steps.foldl (fun value step => step value) head :: tail := by + induction steps generalizing head with + | nil => rfl + | cons step rest ih => + simpa only [List.foldl_cons, unaryStackStep] using ih (step head) + +/-- Captured execution may pop private storage while retaining the same frozen +immutable frontier. The active prefix must fit; inactive cells are irrelevant. -/ +theorem freeze_stack_pop (storage : Array α) (depth width : Nat) + (fits : depth ≤ storage.size) : + freezeStack storage (depth - width) = (freezeStack storage depth).drop width := by + simp only [freezeStack, List.drop_reverse, List.length_take, Array.length_toList, + Nat.min_eq_left fits, List.take_take] + congr 1 + congr 1 + omega + +theorem captured_entry_after_pop (storage : Array Nat) (depth width : Nat) + (fits : depth ≤ storage.size) : + (freezeStack storage (depth - width)).head? = + ((freezeStack storage depth).drop width).head? := by + rw [freeze_stack_pop storage depth width fits] + +end ParserEngine diff --git a/formal/ParserEngine/Preparation.lean b/formal/ParserEngine/Preparation.lean new file mode 100644 index 00000000..bbe1a390 --- /dev/null +++ b/formal/ParserEngine/Preparation.lean @@ -0,0 +1,224 @@ +import LeanPoo.Functional.Requirements +import ParserEngine.Recognition + +namespace ParserEngine + + +inductive Action where + | field (name : String) + | alias (kind : String) + deriving DecidableEq, Repr + +structure Context where + children : Children + offset : Nat + +def executeAction : Action → Context → Children + | .field name, ctx => field name ctx.children ctx.offset + | .alias kind, ctx => alias kind ctx.children ctx.offset + + +/-- Stage the action kind and declared name into a retained semantic closure. -/ +def stagedAction : Action → Context → Children + | .field name => fun ctx => field name ctx.children ctx.offset + | .alias kind => fun ctx => alias kind ctx.children ctx.offset + +/-- This is semantic staging, not a theorem about native call cost or allocation. -/ +theorem staged_action_equivalent (action : Action) (ctx : Context) : + stagedAction action ctx = executeAction action ctx := by + cases action <;> rfl + +def provider : LeanPoo.Functional.Provider Context Action (fun _ _ => Children) := + fun action => some (executeAction action) + +def interpretActions : List Action → Context → Children + | [], ctx => ctx.children + | action :: rest, ctx => + interpretActions rest { ctx with children := executeAction action ctx } + +abbrev Plan (actions : List Action) := LeanPoo.Functional.Requirements.Factories Context (fun _ _ => Children) actions + +/-- Compose selected factories in declaration order, preserving the request offset. -/ +def executePlan : {actions : List Action} → Plan actions → Context → Children + | [], _, ctx => ctx.children + | _ :: _, (factory, tail), ctx => + executePlan tail { ctx with children := factory ctx } + +theorem selected_plan_equivalent (actions : List Action) (plan : Plan actions) + (selected : LeanPoo.Functional.Requirements.Selected provider actions plan) (ctx : Context) : + executePlan plan ctx = interpretActions actions ctx := by + induction actions generalizing ctx with + | nil => rfl + | cons action rest ih => + rcases plan with ⟨factory, tail⟩ + rcases selected with ⟨head, selectedTail⟩ + have same : executeAction action = factory := Option.some.inj head + subst factory + exact ih tail selectedTail { ctx with children := executeAction action ctx } + +/-- Reuse LeanPoo's preparation provenance theorem, rather than postulating a plan. -/ +theorem prepared_plan_equivalent (actions : List Action) (plan : Plan actions) + (ready : LeanPoo.Functional.Requirements.prepare provider actions = .ok plan) (ctx : Context) : + executePlan plan ctx = interpretActions actions ctx := + selected_plan_equivalent actions plan ((LeanPoo.Functional.Requirements.prepare_ok_iff _ _ _).mp ready) ctx + +theorem preparation_total (actions : List Action) : + ∃ plan : Plan actions, LeanPoo.Functional.Requirements.prepare provider actions = .ok plan := by + induction actions with + | nil => exact ⟨PUnit.unit, rfl⟩ + | cons action rest ih => + obtain ⟨tail, ready⟩ := ih + exact ⟨(executeAction action, tail), by simp [LeanPoo.Functional.Requirements.prepare, provider, ready]⟩ + +/-- Count dynamic action-kind selections, not CPU instructions or factory-body work. -/ +def interpretedSelections : List Action → Nat + | [] => 0 + | _ :: rest => 1 + interpretedSelections rest + +theorem interpreted_selections_exact (actions : List Action) : + interpretedSelections actions = actions.length := by + induction actions with + | nil => rfl + | cons action rest ih => simp [interpretedSelections, ih, Nat.add_comm] + +/-- The retained executor calls factories directly; it performs no provider selection. -/ +def retainedSelections {actions : List Action} (_plan : Plan actions) : Nat := 0 + +theorem retained_selections_zero {actions : List Action} (plan : Plan actions) : + retainedSelections plan = 0 := rfl + +/-- Singleton prepending preserves ordered concatenation without copying its suffix. + Native tail sharing and allocation are checked separately in Scheme. -/ +theorem singleton_prepend_equivalent {α : Type} (child : α) (suffix : List α) : + [child] ++ suffix = child :: suffix := rfl + +variable {Key Target State : Type} + +/-- Transpose admitted goto rows once per distinct LHS. The source row remains +first-match authoritative, represented here as an already admitted function. -/ +def gotoColumn (rows : List (Key → Option Target)) (key : Key) : List (Option Target) := + rows.map (fun row => row key) + +theorem goto_column_exact (rows : List (Key → Option Target)) (key : Key) (state : Nat) : + (gotoColumn rows key)[state]? = (rows[state]?).map (fun row => row key) := by + simp [gotoColumn, List.getElem?_map] + +/-- Reject dense preparation before allocating columns when the product exceeds +its fixed cell budget. A rejected plan retains the original sparse lookup. -/ +def admitGotoColumns (states distinctKeys budget : Nat) : Bool := + states * distinctKeys ≤ budget + +theorem goto_plan_within_budget (states keys budget : Nat) + (accepted : admitGotoColumns states keys budget = true) : states * keys ≤ budget := by + simpa [admitGotoColumns] using accepted + +/-- Scalar matrix reads replace dynamic key search only under the actual +producer premise. A matching grammar key alone does not establish the cell. -/ +theorem retained_goto_cell (source retained : State → Key → Option Target) + (producer : ∀ state key, retained state key = source state key) (state : State) (key : Key) : + retained state key = source state key := producer state key + +/-- Lexical matches are ordered by exclusive end, precedence and declaration. +A pure fallback consumes one character; a certificate guarantees an admitted +nonfallback match of at least that width. No scanner callback is speculated. -/ +structure RankedMatch where + finish : Nat + rank : Int + ordinal : Nat + deriving DecidableEq, Repr + +def rankedBeats (winner loser : RankedMatch) : Prop := + loser.finish < winner.finish ∨ + (loser.finish = winner.finish ∧ + (loser.rank < winner.rank ∨ + (loser.rank = winner.rank ∧ winner.ordinal < loser.ordinal))) + +theorem certified_fallback_loses (start : Nat) (winner fallback : RankedMatch) + (fallbackWidth : fallback.finish = start + 1) + (guaranteed : start + 1 ≤ winner.finish) + (priority : fallback.rank < winner.rank ∨ + (fallback.rank = winner.rank ∧ winner.ordinal < fallback.ordinal)) : + rankedBeats winner fallback := by + unfold rankedBeats + by_cases longer : fallback.finish < winner.finish + · exact Or.inl longer + · exact Or.inr ⟨by omega, priority⟩ + +/-- Selecting a subset preserves the winner only with an actual admitted +witness for each omitted candidate, not a first-character hint alone. -/ +theorem omitted_fallback_has_witness (retained : List RankedMatch) (winner fallback : RankedMatch) + (admitted : winner ∈ retained) (beats : rankedBeats winner fallback) : + ∃ candidate ∈ retained, rankedBeats candidate fallback := ⟨winner, admitted, beats⟩ + +/-- In a singleton monotone DFA, every successful transition retains the +same state. Prepare the continuation predicate and winning rule once; only +source position remains dynamic. Numeric rollback states are excluded. -/ +def singletonRun (accept : Char → Bool) : List Char → Nat + | [] => 0 + | ch :: rest => if accept ch then 1 + singletonRun accept rest else 0 + +def monotoneSingletonRun (accept : Char → Bool) (active : Bool) : List Char → Nat + | [] => 0 + | ch :: rest => + if active && accept ch then 1 + monotoneSingletonRun accept true rest else 0 + +theorem singleton_state_elimination (accept : Char → Bool) (source : List Char) : + monotoneSingletonRun accept true source = singletonRun accept source := by + induction source with + | nil => rfl + | cons ch rest ih => + simp only [monotoneSingletonRun, Bool.true_and, singletonRun] + split <;> simp_all + +/-- A prepared ASCII class may replace the generic predicate only with exact +per-character membership. Unknown characters keep the original predicate. -/ +theorem prepared_class_membership (original prepared : Char → Bool) + (producer : ∀ ch, prepared ch = original ch) (source : List Char) : + singletonRun prepared source = singletonRun original source := by + have same : prepared = original := funext producer + rw [same] + +/-- Prepared initial product metadata stores candidate identities, not a cached +priority result. The query may read live candidate payloads through `consume`. -/ +structure ScanStart (Node Candidate : Type) where + node : Option Node + active : Nat + literal : Option Candidate + regular : Option Candidate + +def prepareScanStart (classify : Char → ScanStart Node Candidate) (ch : Char) : + (ScanStart Node Candidate → Query → Value) → Query → Value := + fun consume query => consume (classify ch) query + +theorem prepared_start_query_exact (classify : Char → ScanStart Node Candidate) + (ch : Char) (consume : ScanStart Node Candidate → Query → Value) (query : Query) : + prepareScanStart classify ch consume query = consume (classify ch) query := rfl + +/-- Eliminating an impossible literal probe needs a source-topology premise. +A missing first trie edge is stronger than losing a priority comparison. -/ +theorem absent_literal_probe (regular literal : Query → Option Candidate) + (choose : Option Candidate → Option Candidate → Option Candidate) + (leftOnly : ∀ value, choose value none = value) + (impossible : ∀ query, literal query = none) (query : Query) : + choose (regular query) (literal query) = regular query := by + rw [impossible, leftOnly] + +/-- A grammar-owned immutable empty-reference datum has one retained digest. +Every caller still receives an independent mutable string in Scheme; this +logical equality concerns contents, not native string aliasing or SHA proof. -/ +theorem retained_empty_digest (digest : List α → Value) (consume : Value → Result) : + consume (digest []) = consume (let retained := digest []; retained) := rfl + +/-- A corpus sweep retains one semantic preparation basis per immutable + grammar/capability pair. Fresh per-input state remains the executor's premise; + this equality does not authorize reuse of mutable token vectors or parse state. -/ +theorem prepared_corpus_preserves (prepare : Grammar → Runtime) + (execute : Runtime → Input → Output) (grammar : Grammar) (inputs : List Input) : + inputs.map (fun input => execute (prepare grammar) input) = + (let runtime := prepare grammar; inputs.map (execute runtime)) := rfl + +/-- Counts only the two parser preparations in an equivalence sweep. -/ +theorem corpus_preparations_saved (requests : Nat) (multiple : 1 < requests) : + 2 < 2 * requests := by omega + +end ParserEngine diff --git a/formal/ParserEngine/Recognition.lean b/formal/ParserEngine/Recognition.lean new file mode 100644 index 00000000..9570783a --- /dev/null +++ b/formal/ParserEngine/Recognition.lean @@ -0,0 +1,134 @@ +namespace ParserEngine + +/-- Byte coordinates are abstract naturals; source admission is a separate obligation. -/ +structure Span where + start : Nat + finish : Nat + deriving DecidableEq, Repr + +/-- Logical recognition trees, independent of their physical child-sequence storage. -/ +inductive Tree where + | token (kind : String) (span : Span) + | node (kind : String) (span : Span) (children : List (Option String × Tree)) + | fragment (span : Span) (children : List (Option String × Tree)) + deriving Repr + +def Tree.span : Tree → Span + | .token _ span | .node _ span _ | .fragment span _ => span + +abbrev Child := Option String × Tree +abbrev Children := List Child + +def bounds (children : Children) (offset : Nat) : Span := + match children with + | [] => ⟨offset, offset⟩ + | first :: rest => + ⟨first.2.span.start, ((first :: rest).getLast (by simp)).2.span.finish⟩ + +/-- Preserve existing field labels by placing them inside a fragment. -/ +def field (name : String) (children : Children) (offset : Nat) : Children := + match children with + | [] => [] + | [(none, value)] => [(some name, value)] + | _ => [(some name, .fragment (bounds children offset) children)] + +def alias (kind : String) (children : Children) (offset : Nat) : Children := + [(none, .node kind (bounds children offset) children)] + +@[simp] theorem field_empty (name : String) (offset : Nat) : field name [] offset = [] := rfl + +theorem field_unmarked_singleton (name : String) (value : Tree) (offset : Nat) : + field name [(none, value)] offset = [(some name, value)] := rfl + +theorem field_preserves_nested_label (name previous : String) (value : Tree) (offset : Nat) : + field name [(some previous, value)] offset = + [(some name, .fragment (bounds [(some previous, value)] offset) [(some previous, value)])] := rfl + +theorem empty_alias_anchor (kind : String) (offset : Nat) : + alias kind [] offset = [(none, .node kind ⟨offset, offset⟩ [])] := rfl + +theorem alias_children (kind : String) (children : Children) (offset : Nat) : + alias kind children offset = [(none, .node kind (bounds children offset) children)] := rfl + + +/-- The constructor continuation classifies a closed logical value and receives + scalar endpoints directly. This model does not describe native allocation. -/ +def withTreeBounds (tree : Tree) (consume : Nat → Nat → α) : α := + match tree with + | .token _ span | .node _ span _ | .fragment span _ => consume span.start span.finish + +theorem tree_bounds_continuation_equivalent (tree : Tree) (consume : Nat → Nat → α) : + withTreeBounds tree consume = consume tree.span.start tree.span.finish := by + cases tree <;> rfl + +/-- Empty and singleton inputs keep their original constructor boundary; wider + children retain first/last source endpoints and exact ordered child data. -/ +def withChildBounds (children : Children) (offset : Nat) (consume : Nat → Nat → α) : α := + match children with + | [] => consume offset offset + | [child] => withTreeBounds child.2 consume + | first :: second :: rest => + consume first.2.span.start + ((first :: second :: rest).getLast (by simp)).2.span.finish + +theorem child_bounds_continuation_equivalent (children : Children) (offset : Nat) + (consume : Nat → Nat → α) : + withChildBounds children offset consume = + consume (bounds children offset).start (bounds children offset).finish := by + cases children with + | nil => rfl + | cons first rest => + cases rest with + | nil => simp [withChildBounds, bounds, tree_bounds_continuation_equivalent] + | cons second rest => rfl + + +/-- Physical child storage separates a bare value from explicit field metadata. + The model concerns logical labels and tree identity, not native heap layout. -/ +inductive ChildStorage where + | unfielded (value : Tree) + | labelled (name : String) (value : Tree) + +def storeChild : Child → ChildStorage + | (none, value) => .unfielded value + | (some name, value) => .labelled name value + +def restoreChild : ChildStorage → Child + | .unfielded value => (none, value) + | .labelled name value => (some name, value) + +theorem compact_child_roundtrip (child : Child) : + restoreChild (storeChild child) = child := by + rcases child with ⟨name, value⟩ + cases name <;> rfl + +/-- Request-owned semantic slots can retain a shifted terminal without its +otherwise immediately discarded unfielded wrapper. Public boundaries restore +that wrapper; reduced nonterminal values remain canonical child sequences. -/ +inductive OwnedSemantic where + | terminal (value : Tree) + | canonical (children : Children) + +def restoreOwnedSemantic : OwnedSemantic → Children + | .terminal token => [(none, token)] + | .canonical children => children + +def fieldOwnedTerminal (name : String) (token : Tree) : Children := [(some name, token)] + +theorem deferred_terminal_field_exact (name : String) (token : Tree) (offset : Nat) : + fieldOwnedTerminal name token = field name (restoreOwnedSemantic (.terminal token)) offset := by + rfl + +theorem restored_owned_snapshot (slots : List OwnedSemantic) : + slots.map restoreOwnedSemantic = slots.map (fun slot => match slot with + | .terminal token => [(none,token)] + | .canonical children => children) := rfl + +/-- A following alias/field chain observes exactly the same first-field value; +no idempotence, shared mutable wrapper or changed action order is assumed. -/ +theorem deferred_terminal_chain_exact (consume : Children → α) (name : String) + (token : Tree) (offset : Nat) : + consume (fieldOwnedTerminal name token) = + consume (field name (restoreOwnedSemantic (.terminal token)) offset) := rfl + +end ParserEngine diff --git a/formal/ParserEngine/Reduction.lean b/formal/ParserEngine/Reduction.lean new file mode 100644 index 00000000..9876ac15 --- /dev/null +++ b/formal/ParserEngine/Reduction.lean @@ -0,0 +1,88 @@ +import ParserEngine.Preparation +import ParserEngine.Sequence + +namespace ParserEngine + +structure Operand where + actions : List Action + children : Children + offset : Nat + +def reduceSource (operands : List Operand) : Children := + (operands.map fun operand => interpretActions operand.actions ⟨operand.children, operand.offset⟩).flatten + +/-- A successful deterministic reduction; failure/admission precedes this operation. -/ +def reduceStack (operands : List Operand) : Children := + (stackSequence + (operands.map fun operand => interpretActions operand.actions ⟨operand.children, operand.offset⟩) + (.leaf [])).flatten + +theorem reverse_reduction_equivalent (source : List Operand) : + reduceStack source.reverse = reduceSource source := by + simp only [reduceStack, reduceSource, List.map_reverse] + exact reverse_stack_source_order _ + +/-- Prepared operand factories retain operand-local nullable anchors. -/ +def reducePreparedStack (operands : List Operand) + (plans : (operand : Operand) → Plan operand.actions) : Children := + (stackSequence + (operands.map fun operand => executePlan (plans operand) ⟨operand.children, operand.offset⟩) + (.leaf [])).flatten + +theorem prepared_reverse_reduction_equivalent (source : List Operand) + (plans : (operand : Operand) → Plan operand.actions) + (ready : ∀ operand, LeanPoo.Functional.Requirements.prepare provider operand.actions = + .ok (plans operand)) : + reducePreparedStack source.reverse plans = reduceSource source := by + have same : (fun operand : Operand => + executePlan (plans operand) ⟨operand.children, operand.offset⟩) = + (fun operand : Operand => + interpretActions operand.actions ⟨operand.children, operand.offset⟩) := by + funext operand + exact prepared_plan_equivalent operand.actions (plans operand) (ready operand) _ + simp only [reducePreparedStack, same] + exact reverse_reduction_equivalent source + +/-- LR holds one more state than semantic values. Width admission is explicit. -/ +theorem reduction_preserves_depth (states : List Nat) (values : List Children) + (width goto : Nat) (result : Children) + (aligned : states.length = values.length + 1) (fits : width ≤ values.length) : + (goto :: states.drop width).length = (result :: values.drop width).length + 1 := by + simp only [List.length_cons, List.length_drop] + omega + +theorem shift_preserves_depth (states : List Nat) (values : List Children) + (next : Nat) (token : Children) (aligned : states.length = values.length + 1) : + (next :: states).length = (token :: values).length + 1 := by + simp only [List.length_cons] + omega + +/-- A full flat publication charges every source byte and every event emission. -/ +def publicationWork (sourceBytes : Nat) (events : List α) : Nat := sourceBytes + events.length + +theorem complete_publication_lower_bound (bytes : Nat) (events : List α) : + bytes ≤ publicationWork bytes events ∧ events.length ≤ publicationWork bytes events := by + simp [publicationWork] + +/-- Fixed-arity semantic kernels retain the same right-to-left operand fold. +The accumulator is part of the step contract, including nullable anchors; +this does not assume commutativity or erase operand effects. -/ +def kernelFold (step : Kernel → Value → Value) : List Kernel → Value → Value + | [], value => value + | kernel :: rest, value => kernelFold step rest (step kernel value) + +theorem kernel_unroll_two (step : Kernel → Value → Value) (a b : Kernel) (initial : Value) : + kernelFold step [a,b] initial = step b (step a initial) := rfl + +theorem kernel_unroll_four (step : Kernel → Value → Value) (a b c d : Kernel) (initial : Value) : + kernelFold step [a,b,c,d] initial = step d (step c (step b (step a initial))) := rfl + +/-- Preparing a retained executor is semantically the same fold for every +request accumulator. Preparation and native closure cost need separate evidence. -/ +def prepareKernel (step : Kernel → Value → Value) (kernels : List Kernel) : Value → Value := + fun value => kernelFold step kernels value + +theorem prepared_kernel_exact (step : Kernel → Value → Value) (kernels : List Kernel) (initial : Value) : + prepareKernel step kernels initial = kernelFold step kernels initial := rfl + +end ParserEngine diff --git a/formal/ParserEngine/Reuse.lean b/formal/ParserEngine/Reuse.lean new file mode 100644 index 00000000..6419a0e3 --- /dev/null +++ b/formal/ParserEngine/Reuse.lean @@ -0,0 +1,151 @@ +namespace ParserEngine.Reuse + +/-- Logical identities, not hashes: implementations must establish that their +fingerprints bind these values. Equal keys are not a hash-collision proof. -/ +structure Key where + source : Nat + grammar : Nat + plan : Nat + deriving DecidableEq, Repr + +structure Receipt (Value : Type) where + basis : Key + value : Value + +/-- Admission checks identity and preserves the already computed value. +This does not establish that the value was computed correctly. -/ +def admit (current : Key) (receipt : Receipt Value) : Option Value := + if receipt.basis = current then some receipt.value else none + +theorem admitted_identity (current : Key) (receipt : Receipt Value) (value : Value) + (accepted : admit current receipt = some value) : + receipt.basis = current ∧ receipt.value = value := by + unfold admit at accepted + split at accepted + · exact ⟨‹receipt.basis = current›, Option.some.inj accepted⟩ + · contradiction + +theorem current_receipt (key : Key) (value : Value) : + admit key ⟨key, value⟩ = some value := by simp [admit] + +theorem source_change_rejects (current : Key) (receipt : Receipt Value) + (changed : receipt.basis.source ≠ current.source) : + admit current receipt = none := by + have different : receipt.basis ≠ current := fun equal => changed (congrArg Key.source equal) + simp [admit, different] + +theorem grammar_change_rejects (current : Key) (receipt : Receipt Value) + (changed : receipt.basis.grammar ≠ current.grammar) : + admit current receipt = none := by + have different : receipt.basis ≠ current := fun equal => changed (congrArg Key.grammar equal) + simp [admit, different] + +theorem plan_change_rejects (current : Key) (receipt : Receipt Value) + (changed : receipt.basis.plan ≠ current.plan) : + admit current receipt = none := by + have different : receipt.basis ≠ current := fun equal => changed (congrArg Key.plan equal) + simp [admit, different] + +/-- A source identity attests the current SDK context, not a historical success. +A reload can replace that context before evaluation fails. Therefore withdraw +identity before reload and restore it only after evaluation and selection. -/ +def sourceNeedsReload (admitted : Option Nat) (source : Nat) (explicit : Bool) : Bool := + explicit || admitted != some source + +def beginSourceReload (_previous : Option Nat) : Option Nat := none + +def finishSourceAdmission (source : Nat) (accepted : Bool) : Option Nat := + if accepted then some source else none + +theorem failed_source_retries_original (previous source : Nat) : + sourceNeedsReload (beginSourceReload (some previous)) source false = true := by + simp [sourceNeedsReload, beginSourceReload] + +theorem rejected_source_not_admitted (source : Nat) : + finishSourceAdmission source false = none := rfl + +theorem successful_source_reuses (source : Nat) : + sourceNeedsReload (finishSourceAdmission source true) source false = false := by + simp [sourceNeedsReload, finishSourceAdmission] + +theorem explicit_reload_not_skipped (admitted : Option Nat) (source : Nat) : + sourceNeedsReload admitted source true = true := by simp [sourceNeedsReload] + +-- Keeping the previous marker after a failed replacement wrongly skips reload. +example (source : Nat) : sourceNeedsReload (some source) source false = false := by + simp [sourceNeedsReload] + +/-- Repeated static preparation versus a retained plan. Counts are abstract +operations; hashing, eligibility checks, allocation and CPU require measurements. -/ +def repeatedCost (requests prepare execute : Nat) := requests * (prepare + execute) +def retainedCost (requests prepare execute : Nat) := + if requests = 0 then 0 else prepare + requests * execute + +theorem retained_cost_not_more (requests prepare execute : Nat) : + retainedCost requests prepare execute ≤ repeatedCost requests prepare execute := by + cases requests with + | zero => simp [retainedCost, repeatedCost] + | succ n => + simp only [retainedCost, Nat.succ_ne_zero, ↓reduceIte, repeatedCost, Nat.mul_add] + have h : prepare ≤ (n + 1) * prepare := by + rw [Nat.add_mul, Nat.one_mul] + omega + omega + +/-- Retaining a plan saves preparation exactly once per additional request. +This excludes revalidation cost and says nothing about wall-clock speed. -/ +theorem retained_cost_decomposition (additional prepare execute : Nat) : + repeatedCost (additional + 1) prepare execute = + retainedCost (additional + 1) prepare execute + additional * prepare := by + simp [repeatedCost, retainedCost, Nat.add_mul, Nat.mul_add] + omega + +/-- Include per-request eligibility work. Reuse is beneficial only if the +checks fit within saved preparation; identity correctness alone is insufficient. -/ +def checkedRetainedCost (requests prepare execute check : Nat) := + retainedCost requests prepare execute + requests * check + +theorem checked_reuse_break_even (additional prepare execute check : Nat) + (budget : (additional + 1) * check ≤ additional * prepare) : + checkedRetainedCost (additional + 1) prepare execute check ≤ + repeatedCost (additional + 1) prepare execute := by + rw [retained_cost_decomposition] + exact Nat.add_le_add_left budget _ + +example : checkedRetainedCost 64 100 20 5 < repeatedCost 64 100 20 := by decide +example : repeatedCost 1 100 20 < checkedRetainedCost 1 100 20 5 := by decide + +/-- Publication callbacks may re-enter on the same runtime owner. A failed +or superseded attempt cannot replace the current committed history. -/ +def commitHistory (current attempt : Nat) (live published : Bool) + (previous pending : History) : History := + if current = attempt ∧ live = true ∧ published = true then pending else previous + +theorem superseded_history_preserved (current attempt : Nat) (live published : Bool) + (previous pending : History) (obsolete : current ≠ attempt) : + commitHistory current attempt live published previous pending = previous := by + simp [commitHistory, obsolete] + +theorem failed_publication_preserves_history (current attempt : Nat) (live : Bool) + (previous pending : History) : + commitHistory current attempt live false previous pending = previous := by + simp [commitHistory] + +theorem released_owner_preserves_history (current attempt : Nat) (published : Bool) + (previous pending : History) : + commitHistory current attempt false published previous pending = previous := by + simp [commitHistory] + +theorem current_publication_commits (attempt : Nat) (previous pending : History) : + commitHistory attempt attempt true true previous pending = pending := by + simp [commitHistory] + +/-- A newer committed snapshot survives the completion of an older callback. +The caller supplies distinct generations; arithmetic overflow is not modeled. -/ +theorem nested_commit_survives_outer (outer newer : Nat) (old inner outerPending : History) + (distinct : newer ≠ outer) : + commitHistory newer outer true true + (commitHistory newer newer true true old inner) outerPending = inner := by + rw [current_publication_commits, superseded_history_preserved _ _ _ _ _ _ distinct] + +end ParserEngine.Reuse diff --git a/formal/ParserEngine/Sequence.lean b/formal/ParserEngine/Sequence.lean new file mode 100644 index 00000000..d5d2a8bb --- /dev/null +++ b/formal/ParserEngine/Sequence.lean @@ -0,0 +1,107 @@ +import Lean + +namespace ParserEngine + +/-- Logical proper-list leaves and concatenation records from runtime/funcs.ss. -/ +inductive Sequence (α : Type) where + | leaf (values : List α) + | branch (left right : Sequence α) + +def Sequence.flatten : Sequence α → List α + | .leaf values => values + | .branch left right => left.flatten ++ right.flatten + +/-- One structural visit per record/leaf, plus one emission per element. +This models a builder traversal, not the append-based reference above. -/ +def Sequence.visits : Sequence α → Nat + | .leaf values => 1 + values.length + | .branch left right => 1 + left.visits + right.visits + +def Sequence.records : Sequence α → Nat + | .leaf _ => 1 + | .branch left right => 1 + left.records + right.records + +theorem sequence_visits_exact (seq : Sequence α) : + seq.visits = seq.records + seq.flatten.length := by + induction seq with + | leaf values => rfl + | branch left right ihLeft ihRight => + simp only [Sequence.visits, Sequence.records, Sequence.flatten, List.length_append] + omega + +/-- Copy each leaf cell once into a suffix, counting the actual recursive emissions. -/ +def copyLeaf (values tail : List α) : List α × Nat := + match values with + | [] => (tail, 0) + | head :: rest => + let result := copyLeaf rest tail + (head :: result.1, result.2 + 1) + +theorem copy_leaf_exact (values tail : List α) : + copyLeaf values tail = (values ++ tail, values.length) := by + induction values with + | nil => rfl + | cons head rest ih => simp [copyLeaf, ih] + +/-- Executable suffix builder: traversal never copies an accumulated emitted. +This recursive specification does not model Scheme's bounded-stack visitor. -/ +def Sequence.materializeInto (seq : Sequence α) (tail : List α) : List α × Nat := + match seq with + | .leaf values => + let result := copyLeaf values tail + (result.1, result.2 + 1) + | .branch left right => + let suffix := right.materializeInto tail + let emitted := left.materializeInto suffix.1 + (emitted.1, 1 + emitted.2 + suffix.2) + +theorem materialize_into_exact (seq : Sequence α) (tail : List α) : + seq.materializeInto tail = (seq.flatten ++ tail, seq.records + seq.flatten.length) := by + induction seq generalizing tail with + | leaf values => simp [Sequence.materializeInto, copy_leaf_exact, + Sequence.flatten, Sequence.records, Nat.add_comm] + | branch left right ihLeft ihRight => + simp [Sequence.materializeInto, ihLeft, ihRight, Sequence.flatten, + Sequence.records, List.append_assoc] + omega + +/-- The output accumulator restores source order when consuming the reverse LR stack. -/ +def stackSequence (stack : List (List α)) (acc : Sequence α) : Sequence α := + match stack with + | [] => acc + | head :: rest => stackSequence rest (.branch (.leaf head) acc) + +theorem stack_sequence_order (stack : List (List α)) (acc : Sequence α) : + (stackSequence stack acc).flatten = stack.reverse.flatten ++ acc.flatten := by + induction stack generalizing acc with + | nil => simp [stackSequence] + | cons head rest ih => + simp [stackSequence, ih, Sequence.flatten, List.reverse_cons, List.append_assoc] + +theorem reverse_stack_source_order (source : List (List α)) : + (stackSequence source.reverse (.leaf [])).flatten = source.flatten := by + simp [stack_sequence_order, Sequence.flatten] + +/-- Copies made by repeatedly appending a singleton to an ordinary accumulated list. -/ +def copiedPrefix : Nat → Nat + | 0 => 0 + | n + 1 => copiedPrefix n + n + +theorem copied_prefix_triangular (n : Nat) : 2 * copiedPrefix n + n = n * n := by + induction n with + | zero => rfl + | succ n ih => + simp only [copiedPrefix, Nat.mul_add, Nat.mul_succ, Nat.succ_mul] + omega + +/-- A persistent join allocates one branch without copying either child list. -/ +def joinAllocations : Nat → Nat + | 0 => 0 + | n + 1 => joinAllocations n + 1 + +theorem join_allocations_linear (n : Nat) : joinAllocations n = n := by + induction n with + | zero => rfl + | succ n ih => simp [joinAllocations, ih] + +end ParserEngine diff --git a/formal/evidence/collect_engine.py b/formal/evidence/collect_engine.py new file mode 100644 index 00000000..579e89de --- /dev/null +++ b/formal/evidence/collect_engine.py @@ -0,0 +1,207 @@ +"""Execute model, Scheme and request checks; native ASCENT joins their receipts.""" +import argparse +import hashlib +import json +import os +from pathlib import Path + +from collect_models import model_checks +from parser_evidence import Check, NativeReview, REQUIREMENTS, observe, review + + +TESTS = ( + "t/lr-stack-reduction-test.ss", "t/fused-reduction-test.ss", + "t/lr-action-preparation-test.ss", "t/recognition-prepend-test.ss", + "t/recognition-sequence-test.ss", "t/artifact-publication-cost-test.ss", + "t/gql/benchmark-profile-test.ss", "t/language-abi-test.ss", + "t/source-recognition-test.ss", "t/event-program-test.ss", +) + + +def engine_inputs(root): + # Bind the declared source closure, including workload and admission owners. + paths = set() + for folder in ("src", "languages", "t", "scripts", "formal/evidence"): + paths.update(str(path.relative_to(root)) for path in (root / folder).rglob("*") + if path.is_file() and path.suffix in (".ss", ".py", ".toml", ".lock") + and "__pycache__" not in path.parts and ".venv" not in path.parts) + paths.update(str(path.relative_to(root)) for path in root.glob("*.ss")) + paths.add("gerbil.pkg") + return tuple(sorted(paths)) + + +def supervised(claim, name, command, inputs, markers=(), build=False): + budget, idle = (240, 30) if build else (120, 5) + argv = ("gerbil", "env", "gxi", "qualify.ss", "run", "--timeout", str(budget), + "--idle-timeout", str(idle), "--log", f".data/parser-evidence/{name}.log") + for marker in markers: + argv += ("--require", marker) + return Check(claim, argv + ("--",) + command, inputs, + ("PROCESS-RESULT",) + tuple(marker.strip("^$") for marker in markers), timeout=budget + 20) + + +# The encoder's caller and sampler must be the same native images in both roles. +MEASUREMENT_MODULES = ( + "gerbil-parser/t/benchmarks/lr-request-stack/sdk-control", + "gerbil-parser/t/benchmarks/lr-request-stack/record-access", + "gerbil-parser/t/benchmarks/parser-stage-cost/benchmark", +) + + +def shared_measurement_inputs(root, study, candidate_manifest): + baseline_path = study / "baseline.json" + baseline = json.loads(baseline_path.read_text())["selectedNativeModules"] + candidate = candidate_manifest["selectedNativeModules"] + inputs = [str(baseline_path.relative_to(root))] + for owner in MEASUREMENT_MODULES: + for module in (owner, owner + "~0"): + reference = baseline.get(module) + measured = candidate.get(module) + if reference is None or measured is None or reference["sha256"] != measured["sha256"]: + raise ValueError(f"measurement native image differs between roles: {module}") + path = study / "baseline/lib" / reference["file"] + if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != reference["sha256"]: + raise ValueError(f"measurement reference product changed: {module}") + inputs.append(str(path.relative_to(root))) + return tuple(inputs) + + +def measured_candidate_inputs(root, study): + candidate = tuple(sorted(str(path.relative_to(root)) + for path in (study / "candidate/lib").rglob("*") if path.is_file())) + manifest_path = study / "candidate.json" + manifest = json.loads(manifest_path.read_text()) + if not candidate or not manifest["sourceSha256"] or not manifest["selectedNativeModules"]: + raise ValueError("record admission requires a source-bound candidate product") + for source, expected in manifest["sourceSha256"].items(): + path = root / source + if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected: + raise ValueError(f"measured candidate source changed: {source}") + for product in manifest["selectedNativeModules"].values(): + path = study / "candidate/lib" / product["file"] + if hashlib.sha256(path.read_bytes()).hexdigest() != product["sha256"]: + raise ValueError(f"measured candidate product changed: {product['file']}") + return candidate + (str(manifest_path.relative_to(root)),) + shared_measurement_inputs(root, study, manifest) + + +def collect(root, record_study=None): + if record_study is not None: + measured_candidate_inputs(root, root / record_study) + inputs = engine_inputs(root) + observations = [] + dependencies = [] + + def execute(check): + item = observe(root, check) + item.save(root) + observations.append(item) + print(f"EVIDENCE {check.claim} {item.outcome} {item.identity}", flush=True) + return item + + for check in model_checks(root): + execute(check) + builds = ( + ("source-build", ("gxi", "build.ss", "compile")), + ("test-driver-build", ("gxi", "build-test-driver.ss", "compile")), + ("ffi-build", ("gxi", "build-ffi-tests.ss", "compile")), + ("request-build", ("gxi", "t/benchmarks/build.ss", "compile")), + # Static conformance preparation can leave source-only test interfaces. + # Reuse CI's std/make owner before source Suite admission. + ("gxtest-source-prepare", ("gxi", "-e", + '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!)', + "-e", '(call-with-compiled-interface-trace (lambda () ' + '(load "build-language-conformance.ss") (eval (quote (prepare-gxtest!))))) (exit 0)')), + ) + prepared = [] + for name, command in builds: + item = execute(supervised("scheme-build", name, command, inputs, build=True)) + prepared.append(item) + if item.outcome != "passed": + break + if len(prepared) == len(builds) and all(item.outcome == "passed" for item in prepared): + # Bind the actual prepared products as well as their source declarations. + products = tuple(sorted(str(path.relative_to(root)) + for path in (root / ".gerbil/lib").rglob("*") if path.is_file())) + runtime_inputs = inputs + products + # Independent native owner: callbacks and existing Suite workers must + # not share a handle runtime across different OS/VM owner threads. + history = execute(supervised("scheme-conformance", "history-commit", + ("gxi", "scripts/tests/history-commit-test.ss"), runtime_inputs, + ("HISTORY-COMMIT-CONTROLS-OK",))) + if history.outcome != "passed": + raise RuntimeError("history commit controls failed before semantic admission") + semantics = execute(supervised("scheme-conformance", "conformance", + ("gxi", "-e", '(load "t/fixtures/tla-sany-differential/preload.ss") ' + '(prefer-compiled-interfaces!) ' + '(preload-module "std/test") ' + '(preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process")', + "-e", '(import :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process) ' + '(load "t/fixtures/tla-sany-differential/test-suite.ss") ' + '(eval (quote (main ' + " ".join(json.dumps(file) for file in TESTS) + '))) ' + '(test-child-process-exit! 0)'), + runtime_inputs, (f"TEST-SUITE-OK modules={len(TESTS)}", "^OK$"))) + benchmark = execute(supervised("matched-performance", "requests", + ("gxi", "-e", '(load "t/fixtures/tla-sany-differential/preload.ss") ' + '(prefer-compiled-interfaces!) ' + '(preload-runtime-module "gerbil-parser/t/benchmarks/parser-evidence") ' + '(preload-runtime-module "gerbil-parser/t/benchmarks/lr-request-stack/request-profile") ' + '(preload-runtime-module "gerbil-parser/t/benchmarks/lr-request-stack/request-cost") ' + '(preload-runtime-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process")', + "-e", '(gerbil-parser/t/benchmarks/lr-request-stack/request-profile#profile-work) ' + '(gerbil-parser/t/benchmarks/parser-evidence#evidence-main) ' + '(gerbil-parser/t/benchmarks/lr-request-stack/request-cost#main) ' + '(gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process#test-child-process-exit! 0)'), runtime_inputs, + ("REQUEST-WORK-OK", "GQL-STAGES-OK", "LR-REQUEST-BACKENDS-OK", "PARSER-EVIDENCE-REQUESTS-OK", + "REQUEST-COST-BOUNDARIES-OK"))) + dependencies.append((semantics.identity, history.identity)) + dependencies.extend((semantics.identity, item.identity) for item in prepared) + dependencies.extend((benchmark.identity, item.identity) for item in (*prepared, semantics)) + if record_study is not None: + # Scheme recomputes admission from raw samples. Python transports + # receipts and both product closures, without computing statistics. + study = root / record_study + receipts = tuple(str((study / f"{name}.log").relative_to(root)) + for name in ("reference-a", "candidate-a", "candidate-b", "reference-b")) + baseline = tuple(sorted(str(path.relative_to(root)) + for path in (study / "baseline/lib").rglob("*") if path.is_file())) + candidate = measured_candidate_inputs(root, study) + if not baseline: + raise ValueError("record admission requires an independently retained baseline") + scope_path = study / "scope.json" + scope = json.loads(scope_path.read_text()) + # Roles are declared inputs; fixed margins and admission stay in Scheme. + scope_arguments = " " + json.dumps(scope["imageMarker"]) + for role in ("targets", "controls"): + scope_arguments += " (map string->symbol '(" + " ".join(json.dumps(name) for name in scope[role]) + "))" + # Shared measurement images also belong to the baseline closure. + # Bind each artifact once; snapshot rejects duplicate input paths. + admission_inputs = tuple(sorted(set(runtime_inputs + receipts + baseline + candidate + ( + str((study / "baseline.json").relative_to(root)), + str(scope_path.relative_to(root)))))) + admission = execute(supervised("record-cpu-improvement", "record-admission", + ("gxi", "-e", '(load "t/fixtures/tla-sany-differential/preload.ss") ' + '(prefer-compiled-interfaces!) ' + '(preload-runtime-module "gerbil-parser/t/benchmarks/lr-request-stack/record-evidence") ' + '(preload-runtime-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process")', + "-e", '(gerbil-parser/t/benchmarks/lr-request-stack/record-evidence#main ' + + " ".join(json.dumps(path) for path in receipts) + scope_arguments + ') ' + '(gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process#test-child-process-exit! 0)'), + admission_inputs, ("RECORD-CPU-ADMISSION-OK",))) + dependencies.extend((admission.identity, item.identity) for item in (*prepared, semantics, benchmark)) + engine = NativeReview(Path(os.environ["ASCENT_ENGINE_LIBRARY"])) + required = REQUIREMENTS + (("record-cpu-improvement",) if record_study is not None else ()) + result = review(engine, root, tuple(observations), required=required, dependencies=tuple(dependencies)) + output = root / ".data/parser-evidence/engine.json" + output.write_text(json.dumps(result, indent=2) + "\n") + rows = result["review"]["result"]["relations"] + print(json.dumps({"established": rows["Established"], "missing": rows["Missing"], + "executionAuthorized": result["review"]["executionAuthorized"]}, indent=2)) + if rows["Missing"] or any(item.outcome != "passed" for item in observations): + raise SystemExit(1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--record-study", help="repository-relative independent CPU receipt directory") + args = parser.parse_args() + collect(Path(__file__).resolve().parents[2], args.record_study) diff --git a/formal/evidence/collect_fold_controls.py b/formal/evidence/collect_fold_controls.py new file mode 100644 index 00000000..235a7e4a --- /dev/null +++ b/formal/evidence/collect_fold_controls.py @@ -0,0 +1,70 @@ +"""Join current Fold model and source-owned Scheme controls in native ASCENT. + +This collector deliberately supplies no performance observation: engine +differential controls cannot stand in for the real Org 10 ms preflight. +Python transports observations; Just owns compilation/tests and ASCENT owns +gap inference. It does not start a benchmark matrix or a second runtime. +""" +import argparse +import json +import os +from pathlib import Path + +from collect_engine import engine_inputs +from collect_models import model_checks +from parser_evidence import Check, NativeReview, observe, review, source_path + + +def collect(root: Path, output: str, load_path: str, real_product: str | None = None): + products = source_path(root, output) + if products.exists(): + raise ValueError("fresh Fold output required; do not reuse a running qualification") + inputs = engine_inputs(root) + ("justfile",) + recipe = "qualify-fold-plan" + arguments = (str(products), load_path) + if real_product is not None: + actual = source_path(root, real_product) + artifacts = tuple(str(path.relative_to(root)) for path in sorted(actual.iterdir()) if path.is_file()) + for name in ("org-fold", "org-fold-controls.ss", "fold-input-program.ss"): + if not (actual / name).is_file(): + raise ValueError(f"missing real AOT artifact: {name}") + inputs += artifacts + recipe = "qualify-fold-plan-and-real" + arguments += (str(actual),) + checks = model_checks(root) + ( + Check("scheme-conformance", + ("just", recipe) + arguments, + inputs, ("OK\n",), timeout=240), + ) + observations = [] + for check in checks: + item = observe(root, check) + item.save(root) + observations.append(item) + print(f"FOLD-EVIDENCE {check.claim} {item.outcome} {item.identity}", flush=True) + if item.outcome != "passed": + break + result = review(NativeReview(Path(os.environ["ASCENT_ENGINE_LIBRARY"])), + root, tuple(observations)) + result["scope"] = ("Fold compiler model and native differential controls only; " + "no real Org performance, FFI, Tokio or downstream acceptance.") + result["realAotControlsBound"] = real_product is not None + packet = root / ".data/parser-evidence/fold-controls.json" + packet.parent.mkdir(parents=True, exist_ok=True) + packet.write_text(json.dumps(result, indent=2) + "\n") + rows = result["review"]["result"]["relations"] + print(json.dumps({"established": rows["Established"], "missing": rows["Missing"], + "executionAuthorized": result["review"]["executionAuthorized"]}, indent=2)) + if any(item.outcome != "passed" for item in observations): + raise SystemExit(1) + if {row[1] for row in rows["Missing"]} != {"matched-performance"}: + raise RuntimeError("unexpected Fold evidence closure") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", help="fresh repository-relative product directory") + parser.add_argument("load_path", help="prepared compiler dependency search path") + parser.add_argument("--real-product", help="repository-local copied real AOT artifacts") + args = parser.parse_args() + collect(Path(__file__).resolve().parents[2], args.output, args.load_path, args.real_product) diff --git a/formal/evidence/collect_models.py b/formal/evidence/collect_models.py new file mode 100644 index 00000000..3a35ee7e --- /dev/null +++ b/formal/evidence/collect_models.py @@ -0,0 +1,103 @@ +"""Execute the model layer and ask native ASCENT for the remaining evidence gaps.""" +import json +import os +from pathlib import Path + +from parser_evidence import Check, NativeReview, observe, review + + +def model_checks(root: Path): + lean_inputs = tuple(str(path.relative_to(root)) for path in sorted((root / "formal").rglob("*.lean")) + if ".lake" not in path.parts) + lean_inputs += ("formal/lean-toolchain", "formal/lakefile.toml", "formal/lake-manifest.json", + "src/runtime/recognition.ss", "src/runtime/token.ss", "src/runtime/reduce.ss", "src/runtime/reduction-plan.ss", "src/runtime/funcs.ss") + lean_inputs += ("src/compiler/event-fold-analysis.ss", + "src/compiler/event-fold-call-plan.ss", "t/fold-call-plan-test.ss", + "src/compiler/event-fold-zero-state.ss", + "src/compiler/event-fold-byte-dispatch.ss", + "src/compiler/event-fold-prefix-family.ss", + "src/compiler/event-fold-shared-probes.ss", + "src/compiler/event-fold-scheme-context.ss", + "src/compiler/event-fold-scheme-modules.ss", + "src/compiler/event-fold-scheme-expressions.ss", + "src/compiler/event-fold-scheme-statements.ss") + lean_inputs += ("src/compiler/event-fold-ordinary-run.ss", + "src/compiler/event-fold-consumer-state.ss", "t/fold-consumer-state-test.ss", + "src/compiler/event-fold-scheme-runs.ss", + "src/compiler/event-fold-scheme.ss", + "src/runtime/byte-spans.ss", + "src/runtime/source-lines.ss", "src/runtime/fold-tape.ss", + "t/fold-buffer-consumer-test.ss", + "t/fold-call-environment-test.ss", + "t/fold-run-test.ss", + "t/fold-byte-dispatch-test.ss", + "t/fold-condition-test.ss", "t/fold-frame-facts-test.ss", "t/fold-statement-scans-test.ss") + # Materialized dependency contents are bound separately from the lockfile pin. + dependency = root / "formal/.lake/packages/LeanPoo/LeanPoo" + lean_inputs += tuple(str(path.relative_to(root)) for path in sorted(dependency.rglob("*.lean"))) + if not dependency.is_dir(): + raise RuntimeError("prepare the locked Lean dependency with just formal first") + # Bind actual owners and controls for traceability, not checked refinement. + bridge_inputs = ("t/benchmarks/lr-request-stack/lookahead.ss", "t/core-observability-test.ss", "t/language-abi-benchmark-test.ss", + "t/selective-glr-benchmark-test.ss", "t/lalr-fixed-point-benchmark-test.ss", + "t/recovery-frontier-benchmark-test.ss", "t/versioned-language-benchmark-test.ss", + "t/incremental-suffix-benchmark-test.ss", "t/gql/runtime-benchmark-test.ss", + "t/opencypher-large-query-benchmark-test.ss", "t/benchmark-test.ss", "t/fixtures/tla-sany-differential/worker-control.ss", + "t/fixtures/tla-sany-differential/worker-left.ss", "t/fixtures/tla-sany-differential/worker-right.ss", + "t/fixtures/tla-sany-differential/worker-exclusive.ss", + "t/fixtures/tla-sany-differential/affinity-exclusive.ss", + "t/fixtures/tla-sany-differential/worker-exclusive-invalid.ss", "test-processes.ss", "src/compiler/bound-ir.ss", "t/bound-ir-index-test.ss", "t/rust-runtime-generation-test.ss", + "src/runtime/scan/lexical.ss", "src/runtime/scan/literal.ss", "src/runtime/scan/regular.ss", "t/ranked-regular-scanner-test.ss", + "t/canonical-lr1-reference-test.ss", "src/ffi/language-handles.ss", "scripts/tests/history-commit-test.ss", + "src/runtime/lr-parser/recognition.ss", "src/runtime/lr-parser/checkpoint.ss", "src/runtime/lr-parser/entry.ss", + "src/runtime/lr-parser/transfer.ss", "src/runtime/lr-parser/prepare.ss", "src/runtime/parser.ss", + "src/runtime/lr-parser/batch.ss", "src/runtime/layout.ss", "src/runtime/lr-stack-storage.ss", + "t/fixtures/tla-sany-differential/test-suite.ss", "t/lr-input-protocol-test.ss", "t/lr-goto-plan-test.ss", + "t/lr-action-preparation-test.ss", "t/fixtures/lr-action-selection.ss", + "t/sparse-capture-test.ss", "t/lr-fragment-transfer-test.ss", "t/runtime-module-organization-test.ss", + "src/runtime/lr-parser/state.ss", "src/runtime/token.ss", + "src/language/module-input.ss", "src/ffi/rust-runtime-aot.ss", + "t/fixtures/tla-sany-differential/preload.ss", + "t/benchmarks/module-loading/controls.ss", "t/benchmarks/module-loading/benchmark.ss", + "t/benchmarks/module-loading/reference-preload.ss", + "t/benchmarks/capture-prefix/controls.ss", "t/benchmarks/capture-prefix/worker.ss", + "t/benchmarks/capture-prefix/benchmark.ss", "t/support/qualification/benchmark-qos.ss", + "t/benchmarks/capture-prefix/build-reference.ss", + "t/benchmarks/capture-prefix/phase-cost.ss", "t/benchmarks/parser-stage-cost/benchmark.ss", + "t/benchmarks/lr-request-stack/capture-reference.ss") + policy_inputs = (".github/workflows/ci.yml", ".github/workflows/scheme.yml", + "docs/research/dsl-test-preparation.org", "docs/research/lr-request-model-foundation.org", "docs/research/capture-fold-cost-model.org", + "formal/evidence/parser_evidence.py", "formal/evidence/collect_models.py", + "formal/evidence/pyproject.toml", "formal/evidence/uv.lock") + return ( + Check("lean-model", ("lake", "build"), lean_inputs + bridge_inputs + policy_inputs, + ("Build completed successfully",), cwd="formal"), + Check("quint-protocol", ("npm", "test"), + ("formal/quint/CaptureContext.qnt", "formal/quint/YieldMemo.qnt", "formal/quint/OwnedTerminal.qnt", "formal/quint/SparseCapture.qnt", "formal/quint/ModuleAdmission.qnt", "formal/quint/LookaheadScope.qnt", "formal/quint/CapturePlan.qnt", "formal/quint/HistoryCommit.qnt", "formal/quint/ReuseAdmission.qnt", "formal/quint/Publication.qnt", "formal/quint/TapeOwner.qnt", "formal/quint/FoldPlan.qnt", "formal/quint/IdlePlan.qnt", "formal/quint/RunPlan.qnt", "formal/quint/ConsumerScope.qnt", "formal/quint/package.json", + "formal/quint/package-lock.json", "formal/quint/node_modules/@informalsystems/quint/package.json") + bridge_inputs + policy_inputs, + ("11 passing", "16 passing", "23 passing", "8 passing", "7 passing", "12 passing", "4 passing", "5 passing", "6 passing", "No violation found"), cwd="formal/quint"), + ) + + +def collect(root: Path): + checks = model_checks(root) + observations = tuple(observe(root, check) for check in checks) + for item in observations: + item.save(root) + engine = NativeReview(Path(os.environ["ASCENT_ENGINE_LIBRARY"])) + result = review(engine, root, observations) + output = root / ".data/parser-evidence/models.json" + output.write_text(json.dumps(result, indent=2) + "\n") + rows = result["review"]["result"]["relations"] + print(json.dumps({"outcomes": [item.outcome for item in observations], + "established": rows["Established"], "missing": rows["Missing"], + "executionAuthorized": result["review"]["executionAuthorized"]}, indent=2)) + if any(item.outcome != "passed" for item in observations): + raise SystemExit(1) + # Passing models qualify only this layer; required runtime gaps remain visible. + if {row[1] for row in rows["Missing"]} != {"scheme-conformance", "matched-performance"}: + raise RuntimeError("unexpected evidence closure: inspect the native review") + + +if __name__ == "__main__": + collect(Path(__file__).resolve().parents[2]) diff --git a/formal/evidence/parser_evidence.py b/formal/evidence/parser_evidence.py new file mode 100644 index 00000000..96748cd2 --- /dev/null +++ b/formal/evidence/parser_evidence.py @@ -0,0 +1,205 @@ +"""Source-bound tool observations; native Scheme ASCENT owns inference. + +This host records completed commands, not parser semantics or performance +statistics. Supplied input sets and claim scopes still require human review. +""" +from dataclasses import dataclass +from hashlib import sha256 +import json +from pathlib import Path +import subprocess +import shutil + + +REQUIREMENTS = ( + "lean-model", "quint-protocol", "scheme-conformance", "matched-performance", +) + + +OPTIONAL_REQUIREMENTS = ("record-cpu-improvement",) + +def digest(value: bytes) -> str: + return sha256(value).hexdigest() + + +def canonical(value) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + +def source_path(root: Path, relative: str) -> Path: + name = Path(relative) + if name.is_absolute() or ".." in name.parts or not relative: + raise ValueError("repository-relative input required") + path = root / name + if not path.resolve().is_relative_to(root.resolve()): + raise ValueError("input escapes repository") + return path + + +def snapshot(root: Path, inputs: tuple[str, ...]) -> tuple[tuple[str, str], ...]: + if not inputs or len(set(inputs)) != len(inputs): + raise ValueError("nonempty distinct input set required") + return tuple((name, digest(source_path(root, name).read_bytes())) for name in sorted(inputs)) + + +def executable_identity(command: str) -> str: + path = shutil.which(command) + if path is None: + raise FileNotFoundError(command) + return digest(Path(path).resolve().read_bytes()) + + +@dataclass(frozen=True) +class Check: + claim: str + argv: tuple[str, ...] + inputs: tuple[str, ...] + markers: tuple[str, ...] + cwd: str = "." + timeout: int = 120 + + def __post_init__(self): + if self.claim not in (*REQUIREMENTS, *OPTIONAL_REQUIREMENTS, "scheme-build") or not self.argv or not self.markers: + raise ValueError("known claim, command and explicit success markers required") + if any(not marker for marker in self.markers) or self.timeout <= 0: + raise ValueError("nonempty markers and positive deadline required") + + +@dataclass(frozen=True) +class Observation: + check: Check + inputs: tuple[tuple[str, str], ...] + output: bytes + exit_code: int | None + outcome: str + executable: tuple[str, str] + + @property + def identity(self) -> str: + return digest(canonical(self.record())) + + def record(self): + return {"claim": self.check.claim, "argv": self.check.argv, + "cwd": self.check.cwd, "deadline": self.check.timeout, + "markers": self.check.markers, "inputs": self.inputs, + "outputSha256": digest(self.output), "exitCode": self.exit_code, + "outcome": self.outcome, "executable": self.executable} + + def save(self, root: Path) -> str: + relative = f".data/parser-evidence/{self.identity}.log" + path = source_path(root, relative) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(self.output) + path.with_suffix(".json").write_bytes(canonical(self.record()) + b"\n") + return relative + + +def observe(root: Path, check: Check) -> Observation: + """Run an explicit checker without a shell and retain failures/timeouts. + + This is a direct-child deadline, not a descendant-process supervisor. + Scheme callers use their existing qualification owner for process control. + Tool/dependency locks must be included in the supplied input set. + """ + before = snapshot(root, check.inputs) + tool = (check.argv[0], executable_identity(check.argv[0])) + try: + result = subprocess.run(check.argv, cwd=source_path(root, check.cwd), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=check.timeout, check=False) + output, code = result.stdout, result.returncode + outcome = "passed" if code == 0 and all( + marker.encode() in output for marker in check.markers) else "failed" + except subprocess.TimeoutExpired as failure: + output, code, outcome = failure.stdout or b"", None, "timeout" + try: + unchanged = snapshot(root, check.inputs) == before and executable_identity(check.argv[0]) == tool[1] + except FileNotFoundError: + unchanged = False + if not unchanged: + outcome = "changed-inputs" + return Observation(check, before, output, code, outcome, tool) + + +class NativeReview: + """Load one source-owner-built library after checking its artifact receipt.""" + def __init__(self, library: Path): + import ascent_engine + from ascent_engine import Engine + + sidecar = library.with_name(library.name + ".json") + metadata = json.loads(sidecar.read_bytes()) + library_digest = digest(library.read_bytes()) + if metadata.get("abi") != 1 or metadata.get("librarySha256") != library_digest: + raise ValueError("native library receipt mismatch") + self.runtime = Engine(str(library)) + if digest(library.read_bytes()) != library_digest: + raise ValueError("native library changed while loading") + binding = Path(ascent_engine.__file__).parent + self.identity = {"backend": self.runtime.backend, "librarySha256": library_digest, + "buildReceiptSha256": digest(sidecar.read_bytes()), + "bindingSources": {path.name: digest(path.read_bytes()) + for path in sorted(binding.glob("*.py"))}} + + +def review(engine, root: Path, observations: tuple[Observation, ...], + required: tuple[str, ...] = REQUIREMENTS, + dependencies: tuple[tuple[str, str], ...] = ()): + """Explain finite evidence gaps. No execution permission or speed assertion. + + Receipt dependencies use content IDs. Missing, stale, circular and failed + dependencies are derived by upstream EvidenceReview, never Python rules. + """ + import ascent_engine.review as native_review + EvidenceReview = native_review.EvidenceReview + + if not required or len(set(required)) != len(required) or any( + claim not in (*REQUIREMENTS, *OPTIONAL_REQUIREMENTS) for claim in required): + raise ValueError("explicit distinct known requirements required") + # Transport one canonical input-set commitment per observation. Detailed + # per-file digests remain in its record. Native inference checks this root + # exactly as it checks individual inputs; no host relational rules are added. + current = [] + for item in observations: + key = "inputs:" + digest(canonical(item.inputs)) + try: + refreshed = snapshot(root, tuple(name for name, _ in item.inputs)) + row = [key, digest(canonical(refreshed))] + if row not in current: + current.append(row) + except FileNotFoundError: + pass + for command in sorted({item.executable[0] for item in observations}): + try: + current.append(["tool:" + command, executable_identity(command)]) + except FileNotFoundError: + pass + candidate = digest(canonical(current)) + facts = {"Candidate": [[candidate, "parser-engine", candidate]], + "CurrentInput": current, "Passed": [], "ReceiptInput": [], + "Requires": [[candidate, claim] for claim in required], + "ReceiptDepends": [list(edge) for edge in dependencies]} + for item in observations: + receipt = item.identity + log = f".data/parser-evidence/{receipt}.log" + path = source_path(root, log) + if path.is_file(): + facts["CurrentInput"].append([log, digest(path.read_bytes())]) + facts["ReceiptInput"].append([receipt, "inputs:" + digest(canonical(item.inputs)), + digest(canonical(item.inputs))]) + facts["ReceiptInput"].append([receipt, log, digest(item.output)]) + metadata = f".data/parser-evidence/{receipt}.json" + metadata_path = source_path(root, metadata) + if metadata_path.is_file(): + facts["CurrentInput"].append([metadata, digest(metadata_path.read_bytes())]) + facts["ReceiptInput"].append([receipt, metadata, digest(canonical(item.record()) + b"\n")]) + facts["ReceiptInput"].append([receipt, "tool:" + item.executable[0], item.executable[1]]) + if item.outcome == "passed": + facts["Passed"].append([receipt, item.check.claim]) + with EvidenceReview(engine.runtime, facts) as evidence: + result = evidence.review() + # Serialize a traceable source-fact packet alongside the native derivation. + return {"reviewEngine": engine.identity, + "facts": facts, "review": result, + "observations": [{"id": item.identity, **item.record()} for item in observations], + "scope": "Finite host observations; not Scheme refinement, exhaustive protocol proof or speedup."} diff --git a/formal/evidence/pyproject.toml b/formal/evidence/pyproject.toml new file mode 100644 index 00000000..a1146b77 --- /dev/null +++ b/formal/evidence/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "parser-evidence" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["ascent-test-support", "cffi==2.1.1", "pycparser==3.11"] + +[tool.uv] +package = false + +[tool.uv.sources] +ascent-test-support = { git = "https://github.com/tao3k/gerbil-ascent.git", rev = "64742af763486f37e4318ed26f83b8bec46e4906", subdirectory = "python" } diff --git a/formal/evidence/test_collect_engine.py b/formal/evidence/test_collect_engine.py new file mode 100644 index 00000000..41691eb1 --- /dev/null +++ b/formal/evidence/test_collect_engine.py @@ -0,0 +1,62 @@ +"""Measured native candidates cannot be reused after source or product drift.""" +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +from collect_engine import MEASUREMENT_MODULES, measured_candidate_inputs, shared_measurement_inputs + + +class MeasuredCandidateInputs(unittest.TestCase): + def test_source_and_native_product_drift_are_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + source = root / "src/token.ss" + source.parent.mkdir() + source.write_text("original source") + study = root / "study" + product = study / "candidate/lib/module.o1" + product.parent.mkdir(parents=True) + product.write_bytes(b"original native product") + digest = lambda path: hashlib.sha256(path.read_bytes()).hexdigest() + manifest = {"sourceSha256": {"src/token.ss": digest(source)}, + "selectedNativeModules": {"module": {"file": "module.o1", + "sha256": digest(product)}}} + baseline_modules = {} + for owner in MEASUREMENT_MODULES: + for module in (owner, owner + "~0"): + for role in ("baseline", "candidate"): + image = study / role / "lib" / (module + ".o1") + image.parent.mkdir(parents=True, exist_ok=True) + image.write_bytes(b"shared measurement image") + record = {"file": module + ".o1", "sha256": digest(image)} + baseline_modules[module] = record.copy() + manifest["selectedNativeModules"][module] = record.copy() + (study / "baseline.json").write_text(json.dumps({"selectedNativeModules": baseline_modules})) + (study / "candidate.json").write_text(json.dumps(manifest)) + admitted = measured_candidate_inputs(root, study) + self.assertIn("study/candidate/lib/module.o1", admitted) + self.assertIn("study/candidate.json", admitted) + self.assertIn("study/baseline.json", admitted) + owner = MEASUREMENT_MODULES[0] + manifest["selectedNativeModules"][owner]["sha256"] = "different" + with self.assertRaisesRegex(ValueError, "differs between roles"): + shared_measurement_inputs(root, study, manifest) + manifest["selectedNativeModules"][owner]["sha256"] = baseline_modules[owner]["sha256"] + reference = study / "baseline/lib" / baseline_modules[owner]["file"] + reference.write_bytes(b"changed reference image") + with self.assertRaisesRegex(ValueError, "reference product changed"): + shared_measurement_inputs(root, study, manifest) + reference.write_bytes(b"shared measurement image") + source.write_text("later source") + with self.assertRaisesRegex(ValueError, "source changed"): + measured_candidate_inputs(root, study) + source.write_text("original source") + product.write_bytes(b"later native product") + with self.assertRaisesRegex(ValueError, "product changed"): + measured_candidate_inputs(root, study) + + +if __name__ == "__main__": + unittest.main() diff --git a/formal/evidence/test_model_inputs.py b/formal/evidence/test_model_inputs.py new file mode 100644 index 00000000..c1d932ff --- /dev/null +++ b/formal/evidence/test_model_inputs.py @@ -0,0 +1,79 @@ +"""Model observations bind each source and preserve model-only claim scope.""" +from pathlib import Path +import tempfile +import unittest + +from collect_models import model_checks +from parser_evidence import snapshot + + +class ModelInputControls(unittest.TestCase): + def test_reuse_models_are_bound_and_layers_stay_separate(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + dependency = root / "formal/.lake/packages/LeanPoo/LeanPoo" + dependency.mkdir(parents=True) + source = root / "formal/ParserEngine/Reuse.lean" + source.parent.mkdir(parents=True) + source.write_text("namespace Example\nend Example\n") + capture = root / "formal/ParserEngine/Capture.lean" + capture.write_text("namespace Capture\nend Capture\n") + lookahead = root / "formal/ParserEngine/Lookahead.lean" + lookahead.write_text("namespace Lookahead\nend Lookahead\n") + lean, quint = model_checks(root) + self.assertEqual((lean.claim, quint.claim), ("lean-model", "quint-protocol")) + self.assertIn("formal/ParserEngine/Reuse.lean", lean.inputs) + self.assertIn("formal/ParserEngine/Capture.lean", lean.inputs) + self.assertIn("formal/ParserEngine/Lookahead.lean", lean.inputs) + self.assertIn("formal/quint/LookaheadScope.qnt", quint.inputs) + self.assertIn("formal/quint/ModuleAdmission.qnt", quint.inputs) + self.assertIn("formal/quint/SparseCapture.qnt", quint.inputs) + self.assertIn("formal/quint/ReuseAdmission.qnt", quint.inputs) + self.assertIn("formal/quint/HistoryCommit.qnt", quint.inputs) + self.assertIn("formal/quint/CapturePlan.qnt", quint.inputs) + self.assertIn("formal/quint/OwnedTerminal.qnt", quint.inputs) + self.assertIn("formal/quint/YieldMemo.qnt", quint.inputs) + self.assertIn("formal/quint/CaptureContext.qnt", quint.inputs) + for check in (lean, quint): + self.assertIn("src/ffi/language-handles.ss", check.inputs) + self.assertIn("src/runtime/lr-parser/checkpoint.ss", check.inputs) + self.assertIn("src/runtime/lr-parser/prepare.ss", check.inputs) + self.assertIn("src/runtime/lr-parser/entry.ss", check.inputs) + self.assertIn("t/canonical-lr1-reference-test.ss", check.inputs) + self.assertIn("src/runtime/lr-stack-storage.ss", check.inputs) + self.assertIn("src/runtime/scan/regular.ss", check.inputs) + self.assertIn("t/ranked-regular-scanner-test.ss", check.inputs) + self.assertIn("t/fixtures/tla-sany-differential/test-suite.ss", check.inputs) + self.assertIn("t/lr-input-protocol-test.ss", check.inputs) + self.assertIn("t/lr-goto-plan-test.ss", check.inputs) + self.assertIn("t/lr-fragment-transfer-test.ss", check.inputs) + self.assertIn("t/benchmarks/capture-prefix/phase-cost.ss", check.inputs) + self.assertIn("t/benchmarks/parser-stage-cost/benchmark.ss", check.inputs) + self.assertIn("src/compiler/bound-ir.ss", check.inputs) + self.assertIn("t/bound-ir-index-test.ss", check.inputs) + self.assertIn("t/rust-runtime-generation-test.ss", check.inputs) + self.assertIn("test-processes.ss", check.inputs) + self.assertIn("src/runtime/token.ss", check.inputs) + self.assertIn("src/language/module-input.ss", check.inputs) + self.assertIn("t/fixtures/tla-sany-differential/preload.ss", check.inputs) + self.assertIn("scripts/tests/history-commit-test.ss", check.inputs) + self.assertNotIn("formal/quint/ReuseAdmission.qnt", lean.inputs) + for name in quint.inputs: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("input") + first = snapshot(root, quint.inputs) + (root / "formal/quint/ReuseAdmission.qnt").write_text("changed protocol") + self.assertNotEqual(first, snapshot(root, quint.inputs)) + for name in ("formal/quint/LookaheadScope.qnt", "formal/quint/ModuleAdmission.qnt", + "src/runtime/lr-parser/checkpoint.ss", "src/runtime/token.ss", + "src/language/module-input.ss", "t/fixtures/tla-sany-differential/preload.ss"): + with self.subTest(input=name): + before = snapshot(root, quint.inputs) + (root / name).write_text("changed extent premise") + self.assertNotEqual(before, snapshot(root, quint.inputs)) + self.assertEqual({lean.claim, quint.claim}, {"lean-model", "quint-protocol"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/formal/evidence/test_parser_evidence.py b/formal/evidence/test_parser_evidence.py new file mode 100644 index 00000000..8191bdac --- /dev/null +++ b/formal/evidence/test_parser_evidence.py @@ -0,0 +1,160 @@ +"""Admission controls use the real Scheme ASCENT C ABI, not Python inference.""" +import os +from pathlib import Path +import sys +import tempfile +import unittest + +from parser_evidence import Check, NativeReview, Observation, observe, review + + +class HostControls(unittest.TestCase): + def test_nonzero_exit_cannot_be_admitted_by_a_marker(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + (root / "source").write_text("revision") + check = Check("lean-model", (sys.executable, "-c", "print('OK'); exit(1)"), + ("source",), ("OK",)) + result = observe(root, check) + self.assertEqual((result.outcome, result.exit_code), ("failed", 1)) + self.assertEqual(result.output.strip(), b"OK") + + def test_missing_marker_and_source_mutation_are_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + (root / "source").write_text("revision") + for code, expected in (("print('partial')", "failed"), + ("open('source','w').write('new'); print('OK')", "changed-inputs")): + check = Check("quint-protocol", (sys.executable, "-c", code), ("source",), ("OK",)) + self.assertEqual(observe(root, check).outcome, expected) + + def test_timeout_is_retained(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + (root / "source").write_text("revision") + check = Check("lean-model", (sys.executable, "-c", "import time; time.sleep(2)"), + ("source",), ("OK",), timeout=1) + result = observe(root, check) + self.assertEqual((result.outcome, result.exit_code), ("timeout", None)) + + +class NativeReviewControls(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.engine = NativeReview(Path(os.environ["ASCENT_ENGINE_LIBRARY"])) + + def setUp(self): + self.folder = tempfile.TemporaryDirectory() + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name) + (self.root / "source").write_text("revision") + self.check = Check("lean-model", (sys.executable, "-c", "print('OK')"), + ("source",), ("OK",)) + self.item = observe(self.root, self.check) + self.item.save(self.root) + + def rows(self, items=None, dependencies=()): + return review(self.engine, self.root, items or (self.item,), + dependencies=dependencies)["review"]["result"]["relations"] + + def test_partial_bundle_exposes_all_missing_layers(self): + rows = self.rows() + self.assertEqual({row[1] for row in rows["Missing"]}, + {"quint-protocol", "scheme-conformance", "matched-performance"}) + self.assertEqual(rows["Reusable"], [[self.item.identity, "lean-model"]]) + + def test_source_and_log_mutations_invalidate_reuse(self): + (self.root / "source").write_text("new revision") + self.assertEqual(self.rows()["Reusable"], []) + (self.root / "source").write_text("revision") + (self.root / self.item.save(self.root)).write_bytes(b"tampered") + self.assertEqual(self.rows()["Reusable"], []) + + def test_metadata_mutation_invalidates_reuse(self): + metadata = self.root / f".data/parser-evidence/{self.item.identity}.json" + metadata.write_text("{}") + self.assertEqual(self.rows()["Reusable"], []) + + def test_cycle_and_missing_dependencies_invalidate_reuse(self): + for dependency in (self.item.identity, "absent"): + rows = self.rows(dependencies=((self.item.identity, dependency),)) + self.assertEqual(rows["Reusable"], []) + + def test_failed_dependency_invalidates_transitive_dependents(self): + failed = Observation(self.check, self.item.inputs, b"failure", 1, "failed", self.item.executable) + failed.save(self.root) + other = Observation(self.check, self.item.inputs, b"OK again", 0, "passed", self.item.executable) + other.save(self.root) + rows = self.rows((self.item, other, failed), + ((self.item.identity, other.identity), (other.identity, failed.identity))) + self.assertEqual(rows["Reusable"], []) + + def test_complete_runtime_bundle_requires_successful_build(self): + items = [] + for claim in ("lean-model", "quint-protocol", "scheme-build", + "scheme-conformance", "matched-performance"): + check = Check(claim, self.check.argv, self.check.inputs, self.check.markers) + item = observe(self.root, check) + item.save(self.root) + items.append(item) + build, semantics, measurement = items[2:] + edges = ((semantics.identity, build.identity), + (measurement.identity, semantics.identity)) + self.assertEqual(self.rows(tuple(items), edges)["Missing"], []) + failed = Observation(build.check, build.inputs, b"failure", 1, "failed", build.executable) + failed.save(self.root) + edges = ((semantics.identity, failed.identity), + (measurement.identity, semantics.identity)) + rows = self.rows(tuple(items[:2] + [failed, semantics, measurement]), edges) + self.assertEqual({row[1] for row in rows["Missing"]}, + {"scheme-conformance", "matched-performance"}) + + def test_measurement_receipt_cannot_establish_cpu_improvement(self): + check = Check("matched-performance", self.check.argv, self.check.inputs, self.check.markers) + measured = observe(self.root, check) + measured.save(self.root) + required = ("matched-performance", "record-cpu-improvement") + rows = review(self.engine, self.root, (measured,), required=required)["review"]["result"]["relations"] + self.assertEqual({row[1] for row in rows["Missing"]}, {"record-cpu-improvement"}) + check = Check("record-cpu-improvement", self.check.argv, self.check.inputs, self.check.markers) + admitted = observe(self.root, check) + admitted.save(self.root) + edges = ((admitted.identity, measured.identity),) + rows = review(self.engine, self.root, (measured, admitted), required=required, + dependencies=edges)["review"]["result"]["relations"] + self.assertEqual(rows["Missing"], []) + failed = Observation(check, admitted.inputs, b"rejected", 1, "failed", admitted.executable) + failed.save(self.root) + rows = review(self.engine, self.root, (measured, failed), required=required)["review"]["result"]["relations"] + self.assertEqual({row[1] for row in rows["Missing"]}, {"record-cpu-improvement"}) + + def test_models_and_conformance_cannot_hide_failed_performance(self): + """Admission mutation control, not a real parser benchmark result.""" + items = [] + for claim in ("lean-model", "quint-protocol", "scheme-build", "scheme-conformance"): + check = Check(claim, self.check.argv, self.check.inputs, self.check.markers) + item = observe(self.root, check) + item.save(self.root) + items.append(item) + check = Check("matched-performance", + (sys.executable, "-c", "print('BENCHMARK-COMPLETED'); exit(70)"), + self.check.inputs, ("BENCHMARK-COMPLETED",)) + failed = observe(self.root, check) + failed.save(self.root) + self.assertEqual((failed.outcome, failed.exit_code), ("failed", 70)) + result = review(self.engine, self.root, tuple(items + [failed]), + dependencies=((failed.identity, items[-1].identity),)) + rows = result["review"]["result"]["relations"] + self.assertEqual({row[1] for row in rows["Missing"]}, {"matched-performance"}) + self.assertNotIn("matched-performance", {row[0] for row in rows["Established"]}) + self.assertFalse(result["review"]["executionAuthorized"]) + + def test_review_never_authorizes_execution(self): + result = review(self.engine, self.root, (self.item,)) + self.assertFalse(result["review"]["executionAuthorized"]) + with self.assertRaises(ValueError): + review(self.engine, self.root, (self.item,), required=()) + + +if __name__ == "__main__": + unittest.main() diff --git a/formal/evidence/uv.lock b/formal/evidence/uv.lock new file mode 100644 index 00000000..7fbfb292 --- /dev/null +++ b/formal/evidence/uv.lock @@ -0,0 +1,147 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" + +[[package]] +name = "ascent-test-support" +version = "1" +source = { git = "https://github.com/tao3k/gerbil-ascent.git?subdirectory=python&rev=64742af763486f37e4318ed26f83b8bec46e4906#64742af763486f37e4318ed26f83b8bec46e4906" } +dependencies = [ + { name = "cffi" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/d2/2cde336b375f55c76ca670f0be3978cc048e31e24f3b4d7ce8473150a388/cffi-2.1.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be", size = 183779, upload-time = "2026-08-03T21:19:15.602Z" }, + { url = "https://files.pythonhosted.org/packages/94/1a/4b2f7c92293ba05cbd4a9a1b28faaf0326272d9488e6354657571c48a7aa/cffi-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b", size = 184178, upload-time = "2026-08-03T21:19:16.67Z" }, + { url = "https://files.pythonhosted.org/packages/17/0b/ba385d8ccedf926c3cd06e8e2f327027da5afe5f0eb30f1f7bc43ac55125/cffi-2.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004", size = 211037, upload-time = "2026-08-03T21:19:17.705Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b9/0f2e58b2cefa33255bff36935d42b13180fe559bba82596540eb404bde7d/cffi-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9", size = 218652, upload-time = "2026-08-03T21:19:18.735Z" }, + { url = "https://files.pythonhosted.org/packages/37/15/180e0dab27b9312c7479003d14c9e547634b7dcb934e2cc4650e1b131a7a/cffi-2.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98", size = 205422, upload-time = "2026-08-03T21:19:19.96Z" }, + { url = "https://files.pythonhosted.org/packages/18/d4/03026f0c850cbbaa9030750490225b4a7f4d524ea4df72c3cc740a90f4ef/cffi-2.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9", size = 205444, upload-time = "2026-08-03T21:19:21.246Z" }, + { url = "https://files.pythonhosted.org/packages/75/77/60bebf6f818bec84210ac5b6979ce4eeadce6fbbaabc9c7ab23e506d1ce5/cffi-2.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6", size = 218742, upload-time = "2026-08-03T21:19:22.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ae/679bf47e73fd77b352171727f07de559a003f14de5d02b904a6ec1fa73ca/cffi-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf", size = 221054, upload-time = "2026-08-03T21:19:23.694Z" }, + { url = "https://files.pythonhosted.org/packages/09/b8/eefc0e06913b70aa153bf74c946094a18f58fd4aff11b7f372bfdfdca050/cffi-2.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659", size = 213489, upload-time = "2026-08-03T21:19:24.922Z" }, + { url = "https://files.pythonhosted.org/packages/6f/13/4e56852824a03cdf68523a35686f1c28eacd4bd30a7b0a78e682e6e6e1d3/cffi-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9", size = 220241, upload-time = "2026-08-03T21:19:26.214Z" }, + { url = "https://files.pythonhosted.org/packages/99/7f/040f9e163e4acac3ee3d85b02d00b2576e7ca980d8785f0a3a5f1a9bf7f5/cffi-2.1.1-cp310-cp310-win32.whl", hash = "sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41", size = 174578, upload-time = "2026-08-03T21:19:27.338Z" }, + { url = "https://files.pythonhosted.org/packages/ba/0b/644a2ec1a4eaba49c2939410bb1eb1d25b09d6d0582f5d2f95c537043725/cffi-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1", size = 185082, upload-time = "2026-08-03T21:19:28.409Z" }, + { url = "https://files.pythonhosted.org/packages/70/d2/16d99a0c4948febc0ebd133a13b2f688ff7f8cb04da971e1128872ce0c03/cffi-2.1.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12", size = 183838, upload-time = "2026-08-03T21:19:29.637Z" }, + { url = "https://files.pythonhosted.org/packages/cd/95/31b535a9f0220ae9f357de4a08d57ce89cb417653c2fd9f075f50822a388/cffi-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1", size = 184168, upload-time = "2026-08-03T21:19:30.764Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5a/4707a0dc1f203f5dde5a907b0d4e3c25d71120241048bd5bc6f1bb9d4e71/cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0", size = 211805, upload-time = "2026-08-03T21:19:31.867Z" }, + { url = "https://files.pythonhosted.org/packages/ad/66/c19feabb28485b6e0bbaaafa90837a1ef5d302e90f2178bd33f17a49879b/cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813", size = 218716, upload-time = "2026-08-03T21:19:32.896Z" }, + { url = "https://files.pythonhosted.org/packages/a7/92/500760486c8baab49a7a8a58ba7fc3355ec3974b454b8a09e528efde9e1d/cffi-2.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990", size = 205569, upload-time = "2026-08-03T21:19:34.142Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a7/a67c733254d6e7373f7822f8082d8d6beade791e0cf12a7611f376fa61c7/cffi-2.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af", size = 204907, upload-time = "2026-08-03T21:19:35.174Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a4/4399daaf8f7dfee9d7c3327fdb0426ee041cc63edc358b93911ceb2bfc7a/cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632", size = 217807, upload-time = "2026-08-03T21:19:36.286Z" }, + { url = "https://files.pythonhosted.org/packages/28/f7/dabe6da2466ecbd82dc62e7342dc6b1065dad990c06f00f0ede9ebf2a0ed/cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd", size = 221252, upload-time = "2026-08-03T21:19:37.416Z" }, + { url = "https://files.pythonhosted.org/packages/ce/87/616202d8e51342c07d2534c510111c4cc37201775ce8f60802c9335d1edd/cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a", size = 214214, upload-time = "2026-08-03T21:19:38.507Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c6/ab025d75d2c26c19b087c0124e75ee31cb65032f4fe345d356d8c507ab97/cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa", size = 219408, upload-time = "2026-08-03T21:19:39.809Z" }, + { url = "https://files.pythonhosted.org/packages/db/e2/7e8109f65445bdc673a7b54f02c677de462db75674220fd1335efc8eb598/cffi-2.1.1-cp311-cp311-win32.whl", hash = "sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3", size = 174470, upload-time = "2026-08-03T21:19:41.246Z" }, + { url = "https://files.pythonhosted.org/packages/73/c0/77ba02423c2f7d7091143c45cd49e0e6575c4c1967394bb542bd923a9b74/cffi-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0", size = 185096, upload-time = "2026-08-03T21:19:42.615Z" }, + { url = "https://files.pythonhosted.org/packages/7c/47/9f1f85f9672ceda4984dc6c4f8824e8558992a2972c3d3c81fb8eb28d4ba/cffi-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455", size = 179941, upload-time = "2026-08-03T21:19:43.747Z" }, + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, + { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, + { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + +[[package]] +name = "parser-evidence" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "ascent-test-support" }, + { name = "cffi" }, + { name = "pycparser" }, +] + +[package.metadata] +requires-dist = [ + { name = "ascent-test-support", git = "https://github.com/tao3k/gerbil-ascent.git?subdirectory=python&rev=64742af763486f37e4318ed26f83b8bec46e4906" }, + { name = "cffi", specifier = "==2.1.1" }, + { name = "pycparser", specifier = "==3.11" }, +] + +[[package]] +name = "pycparser" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/a8/c5fdbeee588bb8ada9458774f43adf1bdd30bd59157055142183e769a024/pycparser-3.11.tar.gz", hash = "sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc", size = 113796, upload-time = "2026-10-09T12:56:59.539Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/11/0e6f11117525ff0eec40ebac3d313376f102df93ca44ad9e893ee85e4f89/pycparser-3.11-py3-none-any.whl", hash = "sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80", size = 51178, upload-time = "2026-10-09T12:56:58.131Z" }, +] diff --git a/formal/lake-manifest.json b/formal/lake-manifest.json new file mode 100644 index 00000000..c6d058b3 --- /dev/null +++ b/formal/lake-manifest.json @@ -0,0 +1,16 @@ +{"version": "1.2.0", + "packagesDir": ".lake/packages", + "packages": + [{"url": "https://github.com/tao3k/lean-poo.git", + "type": "git", + "subDir": null, + "scope": "", + "rev": "8561a3ef8120101f956855d128d44ff9507c6703", + "name": "LeanPoo", + "manifestFile": "lake-manifest.json", + "inputRev": "8561a3ef8120101f956855d128d44ff9507c6703", + "inherited": false, + "configFile": "lakefile.toml"}], + "name": "ParserEngine", + "lakeDir": ".lake", + "fixedToolchain": false} diff --git a/formal/lakefile.toml b/formal/lakefile.toml new file mode 100644 index 00000000..975da519 --- /dev/null +++ b/formal/lakefile.toml @@ -0,0 +1,14 @@ +name = "ParserEngine" +version = "0.1.0" +defaultTargets = ["ParserEngine"] + +[leanOptions] +warningAsError = true + +[[require]] +name = "LeanPoo" +git = "https://github.com/tao3k/lean-poo.git" +rev = "8561a3ef8120101f956855d128d44ff9507c6703" + +[[lean_lib]] +name = "ParserEngine" diff --git a/formal/lean-toolchain b/formal/lean-toolchain new file mode 100644 index 00000000..12359f92 --- /dev/null +++ b/formal/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.34.0 diff --git a/formal/quint/CaptureContext.qnt b/formal/quint/CaptureContext.qnt new file mode 100644 index 00000000..b38f4924 --- /dev/null +++ b/formal/quint/CaptureContext.qnt @@ -0,0 +1,54 @@ +// Transition descriptors are immutable. Reconfiguration replaces a cache cell; +// it cannot rewrite a context already retained by a grammar fragment. +module CaptureContext { + type State = {runtimeWidth:int,planWidth:int,usedWidth:int,kind:int,cachedKind:int,nodeKind:int,basisKind:int,oldKind:int,executor:int,target:int,cachedExecutor:int,cachedTarget:int,cached:bool, + nodeExecutor:int,nodeTarget:int,basisExecutor:int,basisTarget:int,node:bool, + directCached:bool,directKind:int,directExecutor:int,directTarget:int, + oldExecutor:int,oldTarget:int,old:bool,allocations:int} + var q:State + pure val initial:State = {runtimeWidth:1,planWidth:1,usedWidth:0,kind:0,cachedKind:0,nodeKind:0,basisKind:0,oldKind:0,executor:0,target:1,cachedExecutor:0,cachedTarget:1,cached:false, + nodeExecutor:0,nodeTarget:1,basisExecutor:0,basisTarget:1,node:false, + directCached:false,directKind:0,directExecutor:1,directTarget:1, + oldExecutor:0,oldTarget:1,old:false,allocations:0} + action init = q' = initial + action Capture = all {q.executor != 0 or q.target == 1, + q' = if(q.executor == 0) {...q,usedWidth:q.planWidth, + cachedKind:q.kind,nodeKind:q.kind,basisKind:q.kind,cached:true,cachedExecutor:0,cachedTarget:1, + node:true,nodeExecutor:0,nodeTarget:1,basisExecutor:0,basisTarget:1, + allocations:q.allocations + (if(q.cached and q.cachedKind == q.kind) 0 else 1)} + else {...q,usedWidth:q.planWidth,nodeKind:q.kind,basisKind:q.kind, + directCached:true,directKind:q.kind,directExecutor:q.executor,directTarget:q.target, + node:true,nodeExecutor:q.executor,nodeTarget:q.target,basisExecutor:q.executor,basisTarget:q.target, + allocations:q.allocations + (if(q.directCached and q.directKind == q.kind and q.directExecutor == q.executor and q.directTarget == q.target) 0 else 1)}} + action KeepOld = all {q.node,not(q.old),q' = {...q,oldKind:q.nodeKind,old:true,oldExecutor:q.nodeExecutor,oldTarget:q.nodeTarget}} + action ChangeExecutor = all {q.executor == 0,q' = {...q,executor:1}} + action RestoreCanonical = all {q.executor == 1,q' = {...q,executor:0,target:1}} + action ChangeTarget = all {q.target == 1,q' = {...q,target:2}} + action ChangeKind = all {q.kind == 0,q' = {...q,kind:1}} + action step = any {Capture,KeepOld,ChangeExecutor,RestoreCanonical,ChangeTarget,ChangeKind} + val ContextSafety = q.planWidth == q.runtimeWidth and q.allocations >= 0 + and (q.directCached implies q.directExecutor != 0) + and (q.cached implies (q.cachedExecutor == 0 and q.cachedTarget == 1)) + and (q.node implies (q.usedWidth == q.runtimeWidth and q.nodeKind == q.basisKind and q.nodeExecutor == q.basisExecutor and q.nodeTarget == q.basisTarget)) + action WrongCapture = all {q.cached,q' = {...q,node:true,nodeKind:q.cachedKind,basisKind:q.kind,nodeExecutor:q.cachedExecutor, + nodeTarget:q.cachedTarget,basisExecutor:q.executor,basisTarget:q.target}} + action WrongPlan = all {q.node,q' = {...q,usedWidth:q.planWidth + 1}} + run wrongWidthPlanRejectedTest = init.then(Capture).then(WrongPlan).expect(not(ContextSafety)) + run repeatedTransitionSharesTest = init.then(Capture).then(Capture) + .expect(ContextSafety and q.allocations == 1) + run executorChangeKeepsOldTest = init.then(Capture).then(KeepOld).then(ChangeExecutor).then(Capture) + .expect(ContextSafety and q.oldExecutor == 0 and q.nodeExecutor == 1 and q.allocations == 2) + run targetChangeKeepsOldTest = init.then(Capture).then(KeepOld).then(ChangeExecutor).then(ChangeTarget).then(Capture) + .expect(ContextSafety and q.oldTarget == 1 and q.nodeTarget == 2 and q.allocations == 2) + run kindChangeKeepsOldTest = init.then(Capture).then(KeepOld).then(ChangeKind).then(Capture) + .expect(ContextSafety and q.oldKind == 0 and q.nodeKind == 1 and q.allocations == 2) + run directExecutorCannotPolluteCanonicalCacheTest = init.then(Capture).then(ChangeExecutor).then(ChangeTarget).then(Capture) + .expect(ContextSafety and q.cachedExecutor == 0 and q.cachedTarget == 1 and q.nodeExecutor == 1 and q.nodeTarget == 2) + run directTransitionsShareAcrossCanonicalCallsTest = init.then(Capture).then(ChangeExecutor).then(Capture) + .then(RestoreCanonical).then(Capture).then(ChangeExecutor).then(Capture) + .expect(ContextSafety and q.allocations == 2) + run changedCanonicalTargetRejectedTest = init.then(Capture).then(ChangeTarget).then(WrongCapture) + .expect(not(ContextSafety)) + run staleDescriptorRejectedTest = init.then(Capture).then(ChangeExecutor).then(WrongCapture) + .expect(not(ContextSafety)) +} diff --git a/formal/quint/CapturePlan.qnt b/formal/quint/CapturePlan.qnt new file mode 100644 index 00000000..31c5c437 --- /dev/null +++ b/formal/quint/CapturePlan.qnt @@ -0,0 +1,44 @@ +// Lifecycle of an immutable capture extent. Lean owns list ordering and suffix +// equivalence; Scheme owns actual LR state, token counts and terminal yields. +module CapturePlan { + type State = {width:int, cursor:int, phase:str, epoch:int, basis:int, + work:int, output:bool} + var q: State + pure val initial:State = {width:3,cursor:0,phase:"idle",epoch:0,basis:0,work:0,output:false} + action init = q' = initial + action initUnary = q' = {...initial,width:1} + action initEmpty = q' = {...initial,width:0} + action Begin = all {q.phase == "idle",q' = {...q,phase:"prefix",basis:q.epoch}} + action Prefix = all {q.phase == "prefix",q.cursor < q.width, + q' = {...q,cursor:q.cursor + 1,work:q.work + 1}} + action BindYield = all {q.phase == "prefix",q.cursor == q.width,q.width != 1, + q' = {...q,phase:"yield",cursor:0}} + action Yield = all {q.phase == "yield",q.cursor < q.width, + q' = {...q,cursor:q.cursor + 1,work:q.work + 1}} + action Publish = all {q.phase == "yield",q.cursor == q.width,q.basis == q.epoch, + q' = {...q,phase:"published",output:true}} + action PublishUnary = all {q.phase == "prefix",q.width == 1,q.cursor == 1,q.basis == q.epoch, + q' = {...q,phase:"published",output:true}} + action Invalidate = all {q.epoch == 0, + q' = {...q,epoch:1,output:false,phase:"rejected"}} + action step = any {Begin,Prefix,BindYield,Yield,Publish,PublishUnary,Invalidate} + val CaptureSafety = q.cursor >= 0 and q.cursor <= q.width and q.work <= 2 * q.width + and (q.output implies (q.phase == "published" and q.basis == q.epoch + and q.cursor == q.width and q.work == (if (q.width == 1) 1 else 2 * q.width))) + run completeExtentTest = init.then(Begin).then(Prefix).then(Prefix).then(Prefix) + .then(BindYield).then(Yield).then(Yield).then(Yield).then(Publish).expect(CaptureSafety and q.output) + run unaryExtentTest = initUnary.then(Begin).then(Prefix).then(PublishUnary) + .expect(CaptureSafety and q.work == 1 and q.output) + run nullableExtentTest = initEmpty.then(Begin).then(BindYield).then(Publish) + .expect(CaptureSafety and q.work == 0 and q.output) + run mutationInvalidatesTest = init.then(Begin).then(Prefix).then(Invalidate) + .expect(CaptureSafety and not(q.output)) + run unaryInvalidationTest = initUnary.then(Begin).then(Prefix).then(Invalidate) + .expect(CaptureSafety and not(q.output)) + run publishedUnaryInvalidationTest = initUnary.then(Begin).then(Prefix).then(PublishUnary).then(Invalidate) + .expect(CaptureSafety and not(q.output)) + run nullableInvalidationTest = initEmpty.then(Begin).then(Invalidate) + .expect(CaptureSafety and not(q.output)) + run incompleteMutationTest = init.then(Begin).then(Prefix) + .expect(not({...q,output:true,phase:"published"}.work == 2 * q.width)) +} diff --git a/formal/quint/ConsumerScope.qnt b/formal/quint/ConsumerScope.qnt new file mode 100644 index 00000000..273f8073 --- /dev/null +++ b/formal/quint/ConsumerScope.qnt @@ -0,0 +1,26 @@ +// Lexical eligibility histories only. Lean owns unsigned-selector refinement. +// No runtime fact cache is implemented; epochs abstract the compiler's effects. +module ConsumerScope { + type State = {a: int, b: int, basisA: int, basisB: int, knownA: bool, knownB: bool} + var q: State + action init = q' = {a:0,b:0,basisA:-1,basisB:-1,knownA:false,knownB:false} + action SelectA = q' = {...q,basisA:q.a,knownA:true} + action SelectB = q' = {...q,basisB:q.b,knownB:true} + action WriteA = q' = {...q,a:q.a + 1,knownA:false} + action WriteB = q' = {...q,b:q.b + 1,knownB:false} + action Barrier = q' = {...q,a:q.a + 1,b:q.b + 1,knownA:false,knownB:false} + action NextByte = q' = {...q,knownA:false,knownB:false} + action step = any {SelectA,SelectB,WriteA,WriteB,Barrier,NextByte} + val ScopeSafety = (not(q.knownA) or q.basisA == q.a) and + (not(q.knownB) or q.basisB == q.b) + run independentWriteTest = init.then(SelectA).then(WriteB) + .expect(ScopeSafety and q.knownA and not(q.knownB)) + run dependentWriteTest = init.then(SelectA).then(SelectB).then(WriteA) + .expect(ScopeSafety and not(q.knownA) and q.knownB) + run barrierDropsBothTest = init.then(SelectA).then(SelectB).then(Barrier) + .expect(ScopeSafety and not(q.knownA) and not(q.knownB)) + run nextByteDropsBothTest = init.then(SelectA).then(SelectB).then(NextByte) + .expect(ScopeSafety and not(q.knownA) and not(q.knownB)) + run staleReuseRejectedTest = init.then(SelectA).then(WriteA) + .expect(not((q.basisA == q.a))) +} diff --git a/formal/quint/FoldPlan.qnt b/formal/quint/FoldPlan.qnt new file mode 100644 index 00000000..c1de7289 --- /dev/null +++ b/formal/quint/FoldPlan.qnt @@ -0,0 +1,63 @@ +// Shared-scan lifecycle histories only. Lean owns denotational refinement. +module FoldPlan { + type State = {epoch: int, basis: int, ready: bool, scans: int, + next: int, events: List[int]} + var q: State + pure val initial: State = {epoch:0,basis:-1,ready:false,scans:0,next:0,events:[]} + pure def invalidate(s: State): State = {...s,epoch:s.epoch + 1,ready:false} + pure def scan(s: State): State = {...s,basis:s.epoch,ready:true,scans:s.scans + 1} + pure def consume(s: State, consumer: int): State = + if (s.ready and s.basis == s.epoch and consumer == s.next) + {...s,next:s.next + 1,events:s.events.append(consumer)} else s + action init = q' = initial + action Scan = all {not(q.ready), q' = scan(q)} + action SourceEdit = q' = invalidate(q) + action BoundsChange = q' = invalidate(q) + action EffectBarrier = q' = invalidate(q) + action HelperBarrier = q' = invalidate(q) + // Generated consumers may borrow a fact only for the identical immutable + // extent. The event owner and next consumer stay with the caller. + action SameExtentConsumer = q' = q + // A proven zero-preserving local reset does not invalidate immutable source + // facts. This is a lifecycle abstraction, not a proof of the IR checker. + action LocalPreservingTransition = all {q.ready,q.basis == q.epoch,q' = q} + // Rejection neither publishes an event nor hands off the current owner. + action NegativeConsumer = all {q.ready, q.basis == q.epoch, q' = q} + action First = q' = consume(q,0) + action Second = q' = consume(q,1) + action step = any {Scan, SourceEdit, BoundsChange, EffectBarrier, HelperBarrier, SameExtentConsumer, LocalPreservingTransition, NegativeConsumer, First, Second} + val CurrentScan = not(q.ready) or q.basis == q.epoch + val OrderedConsumers = q.events == [] or q.events == [0] or q.events == [0,1] + val ExecutionSafety = CurrentScan and OrderedConsumers + // Epochs abstract frame identity. Detecting every implementation write, + // alias and helper effect remains a compiler-to-AOT refinement obligation. + run sharedScanLifecycleTest = init.then(Scan).then(First).then(Second) + .expect(ExecutionSafety and q.scans == 1 and q.events == [0,1]) + run sameExtentConsumerTest = init.then(Scan).then(First).then(SameExtentConsumer).then(Second) + .expect(ExecutionSafety and q.scans == 1 and q.events == [0,1]) + run localTransitionSharedSourceTest = init.then(Scan).then(First) + .then(LocalPreservingTransition).then(Second) + .expect(ExecutionSafety and q.scans == 1 and q.events == [0,1]) + run negativeConsumerTest = init.then(Scan).then(NegativeConsumer).then(First).then(Second) + .expect(ExecutionSafety and q.scans == 1 and q.events == [0,1]) + run sourceInvalidationTest = init.then(Scan).then(SourceEdit).then(First) + .expect(ExecutionSafety and q.events == []) + run boundsInvalidationTest = init.then(Scan).then(BoundsChange).then(First) + .expect(ExecutionSafety and q.events == []) + run effectInvalidationTest = init.then(Scan).then(EffectBarrier).then(First) + .expect(ExecutionSafety and q.events == []) + run helperInvalidationTest = init.then(Scan).then(HelperBarrier).then(First) + .expect(ExecutionSafety and q.events == []) + run rescanAfterChangeTest = init.then(Scan).then(SourceEdit).then(Scan).then(First) + .expect(ExecutionSafety and q.scans == 2 and q.events == [0]) + run consumerInterleavingTest = init.then(Scan).then(Second).then(First).then(Second) + .expect(ExecutionSafety and q.events == [0,1]) + // Call fusion borrows the same extent; a changed owner/frame cannot borrow + // the preceding recognition fact merely because a call boundary vanished. + run fusedCallInvalidationTest = init.then(Scan).then(First).then(BoundsChange) + .then(Second).then(Scan).then(Second) + .expect(ExecutionSafety and q.events == [0,1] and q.scans == 2) + // Mutation witness: retaining ready=true while advancing the epoch is unsafe. + run staleReuseMutationTest = init.then(Scan).expect( + not(not(q.ready) or q.basis == q.epoch + 1)) +} diff --git a/formal/quint/HistoryCommit.qnt b/formal/quint/HistoryCommit.qnt new file mode 100644 index 00000000..363ef866 --- /dev/null +++ b/formal/quint/HistoryCommit.qnt @@ -0,0 +1,46 @@ +// Callback histories only; Lean owns value-preserving commit admission. +// No independent parser, OS-thread scheduler or GC semantics are modeled. +module HistoryCommit { + type State = {generation:int, outer:int, inner:int, history:int, latest:int, + live:bool, outerOpen:bool, innerOpen:bool} + var q: State + pure val initial:State = {generation:0,outer:0,inner:0,history:0,latest:0, + live:true,outerOpen:false,innerOpen:false} + pure def commit(s:State,attempt:int,published:bool):State = + if (s.live and published and s.generation == attempt) + {...s,history:attempt,latest:attempt} else s + pure def valid(s:State):bool = s.history == s.latest and s.latest <= s.generation + action init = q' = initial + action Outer = all {q.live, q.generation == 0, + q' = {...q,generation:1,outer:1,outerOpen:true}} + action Inner = all {q.live,q.outerOpen,q.generation == 1, + q' = {...q,generation:2,inner:2,innerOpen:true}} + action CommitInner = all {q.innerOpen, + q' = {...commit(q,q.inner,true),innerOpen:false}} + action FailInner = all {q.innerOpen,q' = {...q,innerOpen:false}} + action CommitOuter = all {q.outerOpen,not(q.innerOpen), + q' = {...commit(q,q.outer,true),outerOpen:false}} + action FailOuter = all {q.outerOpen,not(q.innerOpen),q' = {...q,outerOpen:false}} + action Release = q' = {...q,live:false} + action step = any {Outer,Inner,CommitInner,FailInner,CommitOuter,FailOuter,Release} + val HistorySafety = valid(q) + pure val States = tuples(Set(0,1,2),Set(0,1,2),Set(0,1,2),Set(0,1,2),Set(0,1,2), + Set(false,true),Set(false,true),Set(false,true)) + .map(t => {generation:t._1,outer:t._2,inner:t._3,history:t._4,latest:t._5, + live:t._6,outerOpen:t._7,innerOpen:t._8}) + pure val Closed = States.forall(s => valid(s) implies + Set(false,true).forall(ok => + (valid(commit(s,s.outer,ok)) and valid(commit(s,s.inner,ok))))) + run finiteCommitPreservationTest = init.expect(Closed) + run ordinaryCommitTest = init.then(Outer).then(CommitOuter).expect(HistorySafety and q.history == 1) + run innerCommitSurvivesOuterTest = init.then(Outer).then(Inner).then(CommitInner).then(CommitOuter) + .expect(HistorySafety and q.history == 2) + run failedInnerSupersedesOuterTest = init.then(Outer).then(Inner).then(FailInner).then(CommitOuter) + .expect(HistorySafety and q.history == 0) + run failedOuterRetainsInnerTest = init.then(Outer).then(Inner).then(CommitInner).then(FailOuter) + .expect(HistorySafety and q.history == 2) + run releaseRejectsCommitTest = init.then(Outer).then(Release).then(CommitOuter) + .expect(HistorySafety and q.history == 0) + run missingGenerationMutationTest = init.then(Outer).then(Inner).then(CommitInner) + .expect(not(valid({...q,history:q.outer}))) +} diff --git a/formal/quint/IdlePlan.qnt b/formal/quint/IdlePlan.qnt new file mode 100644 index 00000000..b84bd8b9 --- /dev/null +++ b/formal/quint/IdlePlan.qnt @@ -0,0 +1,23 @@ +// Guard re-entry and invalidation histories. Lean owns transition equivalence. +// Abstract epochs are not an implementation alias/effect-analysis proof. +module IdlePlan { + type State = {epoch: int, basis: int, idle: bool, checked: bool, unsafe: bool} + var q: State + action init = q' = {epoch:0,basis:-1,idle:true,checked:false,unsafe:false} + action Check = q' = {...q,basis:q.epoch,checked:true} + action Activate = q' = {...q,epoch:q.epoch + 1,idle:false,checked:false} + action Deactivate = q' = {...q,epoch:q.epoch + 1,idle:true,checked:false} + action Barrier = q' = {...q,epoch:q.epoch + 1,checked:false} + action Fast = all {q.checked, q.basis == q.epoch, q.idle, + q' = {...q,checked:false}} + action Slow = q' = {...q,checked:false} + action step = any {Check, Activate, Deactivate, Barrier, Fast, Slow} + val GuardSafety = not(q.unsafe) and (not(q.checked) or q.basis == q.epoch) + run idleExecutionTest = init.then(Check).then(Fast).expect(GuardSafety and not(q.checked)) + run activationInvalidatesTest = init.then(Check).then(Activate) + .expect(GuardSafety and not(q.checked) and not(q.idle)) + run reentryChecksCurrentEpochTest = init.then(Activate).then(Deactivate).then(Check).then(Fast) + .expect(GuardSafety and q.epoch == 2 and not(q.checked)) + run barrierInvalidatesTest = init.then(Check).then(Barrier) + .expect(GuardSafety and not(q.checked)) +} diff --git a/formal/quint/LookaheadScope.qnt b/formal/quint/LookaheadScope.qnt new file mode 100644 index 00000000..854a461d --- /dev/null +++ b/formal/quint/LookaheadScope.qnt @@ -0,0 +1,102 @@ +// Scope safety only: Lean owns projection equality and complete-request costs. +// Key transitions abstract real admission/revision events, not pointer equality. +module LookaheadScope { + type Key = {owner:int,source:int,contract:int,extent:int,revision:int} + type State = {current:Key,basis:Key,active:bool,ready:bool,output:bool, + value:int,cached:int,eof:bool,cachedEof:bool,pending:bool, + observed:Key,observedValue:int,observedEof:bool} + var q:State + pure val key:Key = {owner:0,source:0,contract:0,extent:0,revision:0} + pure val initial:State = {current:key,basis:key,active:false,ready:false, + output:false,value:0,cached:0,eof:false,cachedEof:false,pending:false, + observed:key,observedValue:0,observedEof:false} + action init = q' = initial + action Begin = all {not(q.active),q.current.owner == 0, + q' = {...q,active:true,ready:false,pending:false,output:false}} + action Bind = all {q.active,q' = {...q,basis:q.current,ready:true, + cached:q.value,cachedEof:q.eof,output:false}} + // Atomic Bind remains the specification. These actions expose a staged + // snapshot's producer obligation instead of assuming construction atomic. + action Observe = all {q.active,not(q.pending), + q' = {...q,pending:true,ready:false,output:false,observed:q.current, + observedValue:q.value,observedEof:q.eof}} + action CommitObserved = all {q.active,q.pending,q.observed == q.current, + q' = {...q,pending:false,ready:true,basis:q.observed, + cached:q.observedValue,cachedEof:q.observedEof,output:false}} + action DiscardObserved = all {q.pending,q' = {...q,pending:false,output:false}} + action Read = all {q.active,q.ready,q.basis == q.current, + q' = {...q,output:true}} + // A reduction may change LR state without consuming this admitted token. + // No parser acceptance, shiftability or scan purity is inferred here. + action Reduce = all {q.active,q' = {...q,output:false}} + action Shift = all {q.active,q.current.extent == 0, + q' = {...q,current:{...q.current,extent:1},value:1,output:false}} + action AdmitEOF = all {q.active,q.current.extent == 0, + q' = {...q,current:{...q.current,extent:1},eof:true,output:false}} + action Mutate = all {q.active,q.current.revision == 0, + q' = {...q,current:{...q.current,revision:1},value:1 - q.value,output:false}} + action Edit = all {q.current.source == 0, + q' = {...q,current:{...q.current,source:1},active:false,output:false}} + action Reconfigure = all {q.current.contract == 0, + q' = {...q,current:{...q.current,contract:1},active:false,output:false}} + action Supersede = all {q.active,q.current.owner == 0, + q' = {...q,current:{...q.current,owner:1},ready:false,output:false}} + action Cancel = all {q.active,q' = {...q,active:false,output:false}} + action Pause = all {q.active,q' = {...q,active:false,ready:false,output:false}} + action Fallback = all {q.active,q' = {...q,active:false,ready:false,output:false}} + action Fail = all {q.active,q' = {...q,active:false,ready:false,output:false}} + action Finish = all {q.active,q' = {...q,active:false,ready:false,output:false}} + action step = any {Begin,Bind,Read,Reduce,Shift,AdmitEOF,Mutate,Edit, + Reconfigure,Supersede,Cancel,Pause,Fallback,Fail,Finish,Observe,CommitObserved,DiscardObserved} + pure def safe(s:State):bool = s.output implies (s.active and s.ready + and s.basis == s.current and s.cached == s.value and s.cachedEof == s.eof) + val ScopeSafety = safe(q) and (q.pending and q.observed == q.current implies + (q.observedValue == q.value and q.observedEof == q.eof)) + run stagedSnapshotTest = init.then(Begin).then(Observe).then(CommitObserved).then(Read) + .expect(ScopeSafety and q.output) + run reductionDuringSnapshotTest = init.then(Begin).then(Observe).then(Reduce) + .then(CommitObserved).then(Read).expect(ScopeSafety and q.output) + run mutationDuringSnapshotTest = init.then(Begin).then(Observe).then(Mutate) + .expect(ScopeSafety and q.observed != q.current and not(q.ready)) + run uncheckedCommitMutationWitnessTest = init.then(Begin).then(Observe).then(Mutate) + .expect(not(safe({...q,pending:false,ready:true,output:true,basis:q.current, + cached:q.observedValue,cachedEof:q.observedEof}))) + run snapshotRetryTest = init.then(Begin).then(Observe).then(Mutate).then(DiscardObserved) + .then(Observe).then(CommitObserved).then(Read).expect(ScopeSafety and q.output and q.cached == 1) + run eofDuringSnapshotTest = init.then(Begin).then(Observe).then(AdmitEOF) + .expect(ScopeSafety and q.observed != q.current and not(q.ready)) + run supersededSnapshotTest = init.then(Begin).then(Observe).then(Supersede) + .expect(ScopeSafety and q.observed.owner != q.current.owner and not(q.ready)) + run cancelledSnapshotTest = init.then(Begin).then(Observe).then(Cancel) + .expect(ScopeSafety and not(q.active) and not(q.output)) + run repeatedReductionTest = init.then(Begin).then(Bind).then(Read) + .then(Reduce).then(Read).expect(ScopeSafety and q.output) + run shiftedExtentTest = init.then(Begin).then(Bind).then(Shift) + .expect(ScopeSafety and q.basis != q.current and not(q.output)) + run rebindAfterShiftTest = init.then(Begin).then(Bind).then(Shift).then(Bind).then(Read) + .expect(ScopeSafety and q.output and q.cached == 1) + run eofAdmissionTest = init.then(Begin).then(Bind).then(AdmitEOF).then(Bind).then(Read) + .expect(ScopeSafety and q.output and q.cachedEof) + run mutationRequiresRebindTest = init.then(Begin).then(Bind).then(Mutate) + .expect(ScopeSafety and q.basis != q.current and q.cached != q.value) + run pointerOnlyMutationWitnessTest = init.then(Begin).then(Bind).then(Mutate) + .expect(not(safe({...q,output:true}))) + run eofSentinelMutationWitnessTest = init.then(Begin).then(Bind).then(AdmitEOF) + .expect(not(safe({...q,output:true}))) + run supersessionTest = init.then(Begin).then(Bind).then(Read).then(Supersede) + .expect(ScopeSafety and not(q.ready) and not(q.output)) + run cancelledScopeTest = init.then(Begin).then(Bind).then(Read).then(Cancel) + .expect(ScopeSafety and not(q.active) and not(q.output)) + run changedSourceTest = init.then(Begin).then(Bind).then(Read).then(Edit) + .expect(ScopeSafety and q.basis.source != q.current.source and not(q.output)) + run changedContractTest = init.then(Begin).then(Bind).then(Read).then(Reconfigure) + .expect(ScopeSafety and q.basis.contract != q.current.contract and not(q.output)) + run pauseRequiresFreshAdmissionTest = init.then(Begin).then(Bind).then(Read).then(Pause).then(Begin) + .expect(ScopeSafety and q.active and not(q.ready) and not(q.output)) + run fallbackWithdrawsScopeTest = init.then(Begin).then(Bind).then(Read).then(Fallback) + .expect(ScopeSafety and not(q.active) and not(q.ready) and not(q.output)) + run failedScopeTest = init.then(Begin).then(Bind).then(Read).then(Fail) + .expect(ScopeSafety and not(q.ready) and not(q.output)) + run finishedScopeTest = init.then(Begin).then(Bind).then(Read).then(Finish) + .expect(ScopeSafety and not(q.ready) and not(q.output)) +} diff --git a/formal/quint/ModuleAdmission.qnt b/formal/quint/ModuleAdmission.qnt new file mode 100644 index 00000000..23574eb7 --- /dev/null +++ b/formal/quint/ModuleAdmission.qnt @@ -0,0 +1,42 @@ +// Source identity is about the currently installed SDK context, not history. +// SDK runtime failure caching is owned by Gerbil; no loader is reimplemented. +module ModuleAdmission { + type State = {source:int,admitted:int,context:int,pending:bool,failed:bool,output:bool} + var q:State + pure val initial:State = {source:0,admitted:-1,context:-1,pending:false,failed:false,output:false} + pure def needsReload(s:State):bool = s.admitted != s.source + pure def safe(s:State):bool = + (s.admitted != -1 implies (s.admitted == s.context and not(s.failed) and not(s.pending))) + and (s.output implies (s.source == s.admitted and not(s.failed) and not(s.pending))) + action init = q' = initial + action BeginReload = all {not(q.pending), + q' = {...q,admitted:-1,context:q.source,pending:true,failed:false,output:false}} + action Accept = all {q.pending,q.context == q.source, + q' = {...q,admitted:q.source,pending:false,output:false}} + action Fail = all {q.pending,q' = {...q,pending:false,failed:true,output:false}} + action RejectSelection = all {q.pending,q' = {...q,pending:false,output:false}} + action Edit = all {not(q.pending),q.source == 0,q' = {...q,source:1,output:false}} + action Restore = all {not(q.pending),q.source == 1,q' = {...q,source:0,output:false}} + action Read = all {not(q.pending),not(q.failed),not(needsReload(q)),q' = {...q,output:true}} + action step = any {BeginReload,Accept,Fail,RejectSelection,Edit,Restore,Read} + val ModuleSafety = safe(q) + run unchangedReuseTest = init.then(BeginReload).then(Accept).then(Read) + .expect(ModuleSafety and q.output and not(needsReload(q))) + run changedSourceTest = init.then(BeginReload).then(Accept).then(Edit) + .expect(ModuleSafety and needsReload(q) and not(q.output)) + run failedReplacementRestoreTest = init.then(BeginReload).then(Accept).then(Edit) + .then(BeginReload).then(Fail).then(Restore) + .expect(ModuleSafety and needsReload(q) and not(q.output)) + run restoredSourceRecoveryTest = init.then(BeginReload).then(Accept).then(Edit) + .then(BeginReload).then(Fail).then(Restore).then(BeginReload).then(Accept).then(Read) + .expect(ModuleSafety and q.output and q.context == 0) + run explicitReloadFailureTest = init.then(BeginReload).then(Accept).then(BeginReload).then(Fail) + .expect(ModuleSafety and needsReload(q) and not(q.output)) + run rejectedSelectionTest = init.then(BeginReload).then(RejectSelection) + .expect(ModuleSafety and needsReload(q) and not(q.failed)) + run repeatedFailureTest = init.then(BeginReload).then(Fail).then(BeginReload).then(Fail) + .expect(ModuleSafety and needsReload(q) and not(q.output)) + run historicalMarkerMutationTest = init.then(BeginReload).then(Accept).then(Edit) + .then(BeginReload).then(Fail).then(Restore) + .expect(not(safe({...q,admitted:0})) and not(needsReload({...q,admitted:0}))) +} diff --git a/formal/quint/OwnedTerminal.qnt b/formal/quint/OwnedTerminal.qnt new file mode 100644 index 00000000..0c5ba5aa --- /dev/null +++ b/formal/quint/OwnedTerminal.qnt @@ -0,0 +1,23 @@ +// Private raw terminal publication and observer retention; not a Scheme heap model. +module OwnedTerminal { + type Slot = {raw:bool,published:bool,label:int,visible:int,observed:bool} + var slot:Slot + pure val initial:Slot = {raw:true,published:false,label:0,visible:0,observed:false} + action init = slot' = initial + action Publish = all {not(slot.published), + slot' = {...slot,raw:false,published:true,visible:slot.label}} + action Observe = all {slot.published, + slot' = {...slot,label:7,visible:7,observed:true}} + action ShiftAgain = slot' = initial + action step = any {Publish,Observe,ShiftAgain} + val PublicationSafety = (slot.published implies not(slot.raw)) + and (slot.observed implies slot.label == slot.visible) + // A discarded wrapper would erase an observer's label at the next reduction. + action WrongRebox = all {slot.published,slot.observed, + slot' = {...slot,label:0}} + run privateTerminalTest = init.expect(slot.raw and not(slot.published) and PublicationSafety) + run promotedObserverRetainedTest = init.then(Publish).then(Observe) + .expect(PublicationSafety and not(slot.raw) and slot.label == 7) + run reconstructedWrapperRejectedTest = init.then(Publish).then(Observe).then(WrongRebox) + .expect(not(PublicationSafety)) +} diff --git a/formal/quint/Publication.qnt b/formal/quint/Publication.qnt new file mode 100644 index 00000000..70960e9d --- /dev/null +++ b/formal/quint/Publication.qnt @@ -0,0 +1,40 @@ +// Verification-only publication protocol; not a parser implementation. +module Publication { + type State = {source: int, basis: int, phase: str, cancelled: bool, output: int} + var q: State + pure val initial: State = {source: 0, basis: 0, phase: "idle", cancelled: false, output: -1} + pure def prepare(s: State): State = {...s, basis: s.source, phase: "running", cancelled: false, output: -1} + pure def edit(s: State): State = {...s, source: s.source + 1, output: -1, + phase: if (s.phase == "published") "idle" else s.phase} + pure def cancel(s: State): State = {...s, cancelled: true, phase: "cancelled", output: -1} + pure def finish(s: State): State = + if (s.phase == "running" and not(s.cancelled) and s.basis == s.source) + {...s, phase: "published", output: s.basis} + else {...s, phase: "rejected", output: -1} + pure def valid(s: State): bool = s.output == -1 or + (s.phase == "published" and not(s.cancelled) and s.output == s.source and s.basis == s.source) + action init = q' = initial + action Prepare = all {q.phase != "running", q' = prepare(q)} + action Edit = all {q.source < 2, q' = edit(q)} + action Cancel = all {q.phase == "running", q' = cancel(q)} + action Finish = all {q.phase == "running", q' = finish(q)} + action step = any {Prepare, Edit, Cancel, Finish} + val CurrentPublication = valid(q) + + // Exhaustive one-step preservation over the explicit finite domain. + pure val States = tuples(Set(0,1,2), Set(0,1,2), + Set("idle","running","published","cancelled","rejected"), Set(false,true), Set(-1,0,1,2)) + .map(t => {source: t._1, basis: t._2, phase: t._3, cancelled: t._4, output: t._5}) + pure val Closed = States.forall(s => valid(s) implies + (valid(prepare(s)) and valid(cancel(s)) and valid(finish(s)) and + (s.source < 2 implies valid(edit(s))))) + run finiteClosureTest = init.expect(Closed) + run successfulPublicationTest = init.then(Prepare).then(Finish).expect(CurrentPublication and q.output == 0) + run staleRequestTest = init.then(Prepare).then(Edit).then(Finish).expect(q.phase == "rejected" and q.output == -1) + run cancelledRequestTest = init.then(Prepare).then(Cancel).expect(q.output == -1 and q.cancelled) + run editWithdrawsPublicationTest = init.then(Prepare).then(Finish).then(Edit).expect(q.output == -1) + // Mutation witnesses establish that the admission checks are consequential. + pure def uncheckedFinish(s: State): State = {...s, phase: "published", output: s.basis} + run staleMutationTest = init.then(Prepare).then(Edit).expect(not(valid(uncheckedFinish(q)))) + run cancelledMutationTest = init.then(Prepare).then(Cancel).expect(not(valid(uncheckedFinish(q)))) +} diff --git a/formal/quint/ReuseAdmission.qnt b/formal/quint/ReuseAdmission.qnt new file mode 100644 index 00000000..a864c7a9 --- /dev/null +++ b/formal/quint/ReuseAdmission.qnt @@ -0,0 +1,67 @@ +// Lifecycle safety only. Lean owns unbounded identity admission and cost laws. +// Integers abstract collision-free identities, not actual digests or parser data. +module ReuseAdmission { + type Key = {source:int, grammar:int, plan:int} + type State = {current:Key, basis:Key, ticket:int, owner:int, + phase:str, cancelled:bool, failed:bool, output:bool} + var q: State + pure val zero: Key = {source:0,grammar:0,plan:0} + pure val initial: State = {current:zero,basis:zero,ticket:0,owner:0, + phase:"idle",cancelled:false,failed:false,output:false} + pure def valid(s:State):bool = s.output implies + (s.phase == "published" and s.current == s.basis and s.ticket == s.owner + and not(s.cancelled) and not(s.failed)) + pure def begin(s:State):State = {...s,basis:s.current,owner:s.ticket, + phase:"running",cancelled:false,failed:false,output:false} + pure def finish(s:State):State = + if (s.phase == "running" and s.current == s.basis and s.ticket == s.owner + and not(s.cancelled) and not(s.failed)) + {...s,phase:"published",output:true} + else {...s,phase:"rejected",output:false} + pure def invalidate(s:State,k:Key):State = {...s,current:k,output:false} + pure def cancel(s:State):State = {...s,cancelled:true,output:false} + pure def failure(s:State):State = {...s,failed:true,output:false} + pure def supersede(s:State):State = {...s,ticket:s.ticket + 1,output:false} + action init = q' = initial + action Begin = all {q.phase != "running", q' = begin(q)} + action Finish = q' = finish(q) + action Edit = all {q.current.source < 1, + q' = invalidate(q,{...q.current,source:q.current.source + 1})} + action Grammar = all {q.current.grammar < 1, + q' = invalidate(q,{...q.current,grammar:q.current.grammar + 1})} + action Plan = all {q.current.plan < 1, + q' = invalidate(q,{...q.current,plan:q.current.plan + 1})} + action Cancel = q' = cancel(q) + action Fail = q' = failure(q) + action Supersede = all {q.ticket < 1, q' = supersede(q)} + action step = any {Begin,Finish,Edit,Grammar,Plan,Cancel,Fail,Supersede} + val ReuseSafety = valid(q) + + pure val Keys = tuples(Set(0,1),Set(0,1),Set(0,1)) + .map(t => {source:t._1,grammar:t._2,plan:t._3}) + pure val States = tuples(Keys,Keys,Set(0,1),Set(0,1), + Set("idle","running","published","rejected"),Set(false,true),Set(false,true),Set(false,true)) + .map(t => {current:t._1,basis:t._2,ticket:t._3,owner:t._4, + phase:t._5,cancelled:t._6,failed:t._7,output:t._8}) + // Exhaustive one-step invariant preservation, not a temporal liveness proof. + pure val Closed = States.forall(s => valid(s) implies + (valid(begin(s)) and valid(finish(s)) and valid(cancel(s)) and valid(failure(s)) + and (s.ticket < 1 implies valid(supersede(s))) and Keys.forall(k => valid(invalidate(s,k))))) + run finiteAdmissionClosureTest = init.expect(Closed) + run currentPublicationTest = init.then(Begin).then(Finish).expect(ReuseSafety and q.output) + run sourceInvalidationTest = init.then(Begin).then(Edit).then(Finish).expect(ReuseSafety and not(q.output)) + run grammarInvalidationTest = init.then(Begin).then(Grammar).then(Finish).expect(ReuseSafety and not(q.output)) + run planInvalidationTest = init.then(Begin).then(Plan).then(Finish).expect(ReuseSafety and not(q.output)) + run cancelledCompletionTest = init.then(Begin).then(Cancel).then(Finish).expect(ReuseSafety and not(q.output)) + run failedCompletionTest = init.then(Begin).then(Fail).then(Finish).expect(ReuseSafety and not(q.output)) + run olderOwnerCompletionTest = init.then(Begin).then(Supersede).then(Finish).expect(ReuseSafety and not(q.output)) + run recoveryTest = init.then(Begin).then(Fail).then(Finish).then(Begin).then(Finish).expect(ReuseSafety and q.output) + pure def unchecked(s:State):State = {...s,phase:"published",output:true} + run staleMutationTest = init.then(Begin).then(Edit).expect(not(valid(unchecked(q)))) + run grammarMutationTest = init.then(Begin).then(Grammar).expect(not(valid(unchecked(q)))) + run ownerMutationTest = init.then(Begin).then(Supersede).expect(not(valid(unchecked(q)))) + run planMutationTest = init.then(Begin).then(Plan).expect(not(valid(unchecked(q)))) + run cancellationMutationTest = init.then(Begin).then(Cancel).expect(not(valid(unchecked(q)))) + run editWithdrawsOutputTest = init.then(Begin).then(Finish).then(Edit).expect(ReuseSafety and not(q.output)) + run failureMutationTest = init.then(Begin).then(Fail).expect(not(valid(unchecked(q)))) +} diff --git a/formal/quint/RunPlan.qnt b/formal/quint/RunPlan.qnt new file mode 100644 index 00000000..7929aa1d --- /dev/null +++ b/formal/quint/RunPlan.qnt @@ -0,0 +1,42 @@ +// Lifecycle only; Lean owns overwrite/residual semantic refinement. +// Source immutability, compiler eligibility and FFI checks need AOT controls. +module RunPlan { + type State = {epoch: int, basis: int, phase: str, stop: int, + cursor: int, writes: int, residuals: int, activeA: bool, activeB: bool, membership: str} + var q: State + pure val initial: State = {epoch:0,basis:-1,phase:"idle",stop:0,cursor:0,writes:0,residuals:0, + activeA:false,activeB:false,membership:"unselected"} + pure def valid(s: State): bool = + (not(Set("borrowed", "scanned").contains(s.phase)) or (s.basis == s.epoch and + s.membership == (if (s.activeA or s.activeB) "wide" else "narrow"))) and + s.writes <= s.residuals + 1 and s.residuals <= s.writes + action init = q' = initial + action initActive = q' = {...initial,activeA:true,activeB:true} + action Borrow = all {q.phase == "idle", q' = {...q,basis:q.epoch,phase:"borrowed", + membership:if (q.activeA or q.activeB) "wide" else "narrow"}} + // The synchronous leaf releases every pointer before publishing summaries. + // No state write, callback or source mutation can interleave with its borrow. + action Scan = all {q.phase == "borrowed", q.basis == q.epoch, + q' = {...q,phase:"scanned",stop:q.cursor + 3}} + action Invalidate = all {q.phase != "borrowed",q.phase != "residual", + q' = {...q,epoch:q.epoch + 1,phase:"idle"}} + action Commit = all {q.phase == "scanned",q.basis == q.epoch, + q' = {...q,phase:"residual",cursor:q.stop,writes:q.writes + 1}} + action Residual = all {q.phase == "residual", + q' = {...q,phase:"idle",residuals:q.residuals + 1}} + action step = any {Borrow, Scan, Commit, Residual, + all {q.phase != "residual",Invalidate}} + val RunSafety = valid(q) + run activeConsumersCanCommitTest = initActive.then(Borrow).then(Scan).then(Commit).then(Residual) + .expect(RunSafety and q.activeA and q.activeB and q.membership == "wide" and q.writes == 1 and q.residuals == 1) + run orderedRunTest = init.then(Borrow).then(Scan).then(Commit).then(Residual) + .expect(RunSafety and q.membership == "narrow" and q.cursor == 3 and q.writes == 1 and q.residuals == 1) + run mutationInvalidatesTest = init.then(Borrow).then(Scan).then(Invalidate) + .expect(RunSafety and q.phase == "idle") + run residualCannotBeSkippedTest = init.then(Borrow).then(Scan).then(Commit) + .expect(RunSafety and q.phase == "residual" and q.residuals == 0) + run staleCommitMutationTest = init.then(Borrow).then(Scan).then(Invalidate) + .expect(not(valid({...q,phase:"scanned"}))) + run borrowHasNoPublishedEffectsTest = init.then(Borrow) + .expect(RunSafety and q.phase == "borrowed" and q.writes == 0 and q.residuals == 0) +} diff --git a/formal/quint/SparseCapture.qnt b/formal/quint/SparseCapture.qnt new file mode 100644 index 00000000..0e29c4dd --- /dev/null +++ b/formal/quint/SparseCapture.qnt @@ -0,0 +1,58 @@ +// Lifecycle and exact sparse observation only; this is not an LR parser oracle. +module SparseCapture { + type Frame = {shift:int,version:int,cursor:int,tokens:int} + type State = {spacing:int,shift:int,version:int,cursor:int,tokens:int, + pending:bool,pendingMode:int,mode:int,admitted:List[int],emitted:List[int],active:bool,output:bool,dense:List[Frame],sparse:List[Frame]} + var q:State + pure val first:Frame = {shift:0,version:0,cursor:0,tokens:0} + pure val initial:State = {spacing:2,shift:0,version:0,cursor:0,tokens:0, + pending:false,pendingMode:0,mode:0,admitted:List(),emitted:List(),active:true,output:false,dense:List(first),sparse:List(first)} + action init = q' = initial + action initEveryShift = q' = {...initial,spacing:1} + val canAdmit = q.active and not(q.pending) and q.cursor < 5 + action Trivia = all {canAdmit, + q' = {...q,cursor:q.cursor + 1,tokens:q.tokens + 1,admitted:q.admitted.append(q.mode),emitted:q.emitted.append(q.mode)}} + action Scan = all {canAdmit, + q' = {...q,cursor:q.cursor + 1,pending:true,pendingMode:q.mode,admitted:q.admitted.append(q.mode)}} + action Reduce = all {q.active,q.pending,q.version < 10, + q' = {...q,version:q.version + 1,mode:(q.mode + 1) % 3}} + action Shift = all {q.active,q.pending, + val frame = {shift:q.shift + 1,version:q.version + 1,cursor:q.cursor,tokens:q.tokens + 1} + q' = {...q,shift:q.shift + 1,version:q.version + 1,tokens:q.tokens + 1,pending:false,emitted:q.emitted.append(q.pendingMode), + dense:q.dense.append(frame), + sparse:if ((q.shift + 1) % q.spacing == 0) q.sparse.append(frame) else q.sparse}} + action Finish = all {q.active,not(q.pending),q' = {...q,active:false,output:true}} + action Fork = all {q.active,q.pending,q' = {...q,active:false,output:false}} + action Fail = all {q.active,q' = {...q,active:false,output:false}} + action step = any {Trivia,Scan,Reduce,Shift,Finish,Fork,Fail} + val SparseSafety = q.sparse == q.dense.select(frame => frame.shift % q.spacing == 0) + and (q.output implies not(q.pending)) + and q.admitted.length() == q.cursor and q.emitted.length() == q.tokens + and q.emitted == q.admitted.slice(0,q.tokens) + // Counterfactual implementation: exporting an aliased buffer rewrites a + // retained frame when a later reduction updates that private storage. + action AliasedSnapshotWrite = all {q.active,q.pending,q.sparse.length() > 1, + q' = {...q,sparse:q.sparse.foldl(List(), (frames,frame) => frames.append(if (frame.shift == 0) frame else {...frame,version:q.version}))}} + run aliasedBufferMustBeRejectedTest = init.then(Scan).then(Shift).then(Scan).then(Shift) + .then(Scan).then(Reduce).then(AliasedSnapshotWrite).expect(not(SparseSafety)) + run reducedModeDoesNotRewriteAdmissionTest = init.then(Scan).then(Reduce).then(Reduce).then(Shift) + .expect(SparseSafety and q.mode == 2 and q.emitted == List(0)) + run triviaWaitsForPendingShiftTest = init.then(Scan) + .expect(not(canAdmit) and SparseSafety) + run forkRetainsUnconsumedModeTest = init.then(Scan).then(Reduce).then(Fork) + .expect(SparseSafety and q.pending and q.pendingMode == 0 and q.mode == 1 and q.emitted == List()) + run sparseTwoShiftsTest = init.then(Scan).then(Shift).then(Scan).then(Shift).then(Finish) + .expect(SparseSafety and q.output and q.sparse.length() == 2 and q.dense.length() == 3) + run reductionBeforeShiftTest = init.then(Scan).then(Reduce).then(Shift).then(Scan) + .then(Reduce).then(Shift).expect(SparseSafety and q.sparse == List(first,{shift:2,version:4,cursor:2,tokens:2})) + run triviaDoesNotCaptureTest = init.then(Trivia).then(Trivia) + .expect(SparseSafety and q.sparse.length() == 1 and q.shift == 0) + run everyShiftTest = initEveryShift.then(Scan).then(Shift).then(Scan).then(Shift) + .expect(SparseSafety and q.sparse == q.dense) + run forkRetainsPrefixTest = init.then(Scan).then(Shift).then(Scan).then(Fork) + .expect(SparseSafety and not(q.output) and q.sparse.length() == 1) + run failureRetainsPrefixTest = init.then(Scan).then(Shift).then(Fail) + .expect(SparseSafety and not(q.output)) + run wrongShiftBoundaryMutationTest = init.then(Scan).then(Shift) + .expect(q.dense != q.sparse) +} diff --git a/formal/quint/TapeOwner.qnt b/formal/quint/TapeOwner.qnt new file mode 100644 index 00000000..7f39221a --- /dev/null +++ b/formal/quint/TapeOwner.qnt @@ -0,0 +1,33 @@ +// Lifecycle only: Lean owns exact-prefix semantics; AOT owns real bytes/GC. +module TapeOwner { + type State = {sealed: bool, writer: bool, output: bool} + var q: State + pure val initial: State = {sealed:false, writer:true, output:false} + pure def publish(s: State): State = + if (not(s.sealed) and s.writer) + {sealed:true, writer:false, output:true} else s + pure def write(s: State): State = s + pure def safe(s: State): bool = + (s.output implies (s.sealed and not(s.writer))) and + (s.sealed implies not(s.writer)) + action init = q' = initial + action Publish = q' = publish(q) + // Payload is outside this lifecycle model. A write is admitted only while + // the exclusive owner is open; rejected writes cannot reacquire ownership. + action Write = all {not(q.sealed), q.writer, q' = write(q)} + action RejectedWrite = all {q.sealed, q' = q} + action Collect = q' = q + action step = any {Publish, Write, RejectedWrite, Collect} + val OwnerSafety = safe(q) + pure val States = tuples(Set(false,true), Set(false,true), Set(false,true)) + .map(t => {sealed:t._1, writer:t._2, output:t._3}) + pure val Closed = States.forall(s => safe(s) implies + (safe(publish(s)) and safe(write(s)))) + run ownerFiniteClosureTest = init.expect(Closed) + run transferLifecycleTest = init.then(Write).then(Publish).then(Collect) + .expect(OwnerSafety and q.output and not(q.writer)) + run sealedReentryTest = init.then(Publish).then(RejectedWrite).then(Publish) + .expect(OwnerSafety and q == {sealed:true, writer:false, output:true}) + run aliasMutationWitnessTest = init.then(Publish) + .expect(not(safe({...q,writer:true}))) +} diff --git a/formal/quint/YieldMemo.qnt b/formal/quint/YieldMemo.qnt new file mode 100644 index 00000000..9dd6a8bc --- /dev/null +++ b/formal/quint/YieldMemo.qnt @@ -0,0 +1,27 @@ +// Derived child lists and terminal indexes over engine-owned immutable topology. +// No allocation/CPU claim: Lean owns cost obligations; Scheme measures them. +module YieldMemo { + type Memo = {epoch:int,basis:int,phase:str,work:int,visible:bool,cacheEpoch:int} + var q:Memo + pure val initial:Memo = {epoch:0,basis:0,phase:"pending",work:0,visible:false,cacheEpoch:0} + action init = q' = initial + action Begin = all {q.phase == "pending",q' = {...q,phase:"building",basis:q.epoch}} + action Build = all {q.phase == "building",q.work < 2,q' = {...q,work:q.work + 1}} + action Publish = all {q.phase == "building",q.work == 2,q.basis == q.epoch, + q' = {...q,phase:"ready",visible:true,cacheEpoch:q.epoch}} + action QueryWarm = all {q.phase == "ready",q' = q} + // Invalidating a topology is outside the producer premise. Do not expose + // partial output or admit a result from a different generation afterwards. + action Invalidate = all {q.epoch == 0,q' = {...q,epoch:1,phase:"invalid",visible:false}} + action step = any {Begin,Build,Publish,QueryWarm,Invalidate} + val MemoSafety = (q.work >= 0 and q.work <= 2) + and (q.visible implies (q.phase == "ready" and q.cacheEpoch == q.epoch and q.work == 2)) + action WrongPublish = all {q.phase == "building",q.work < 2,q' = {...q,visible:true,phase:"ready"}} + run freshCaptureDoesNotBuildTest = init.expect(MemoSafety and q.work == 0 and not(q.visible)) + run coldThenWarmTest = init.then(Begin).then(Build).then(Build).then(Publish).then(QueryWarm) + .expect(MemoSafety and q.visible and q.work == 2) + run failedProducerCannotPublishTest = init.then(Begin).then(Build).then(Invalidate) + .expect(MemoSafety and not(q.visible)) + run partialPublicationRejectedTest = init.then(Begin).then(Build).then(WrongPublish) + .expect(not(MemoSafety)) +} diff --git a/formal/quint/package-lock.json b/formal/quint/package-lock.json new file mode 100644 index 00000000..30d38989 --- /dev/null +++ b/formal/quint/package-lock.json @@ -0,0 +1,799 @@ +{ + "name": "parser-protocol-models", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "parser-protocol-models", + "dependencies": { + "@informalsystems/quint": "0.33.0" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.6", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.6.tgz", + "integrity": "sha512-SgC/B3/VZ8lTyOZ4Cwk6f1skWhmkNAaVfiX90qd5u1rRXGKk1//8FTUEN9GvOUq7rfWIqAYkA3CoyumYVpOS6g==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@informalsystems/quint": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/@informalsystems/quint/-/quint-0.33.0.tgz", + "integrity": "sha512-1hr+l+3wyKimjNXNl9lovpwmH5C9Ck898FL4yGbuEmFPiz7YRqvFvsB9hlHsaYukXqdUY5nuFKn/qiCon0O88g==", + "license": "Apache 2.0", + "dependencies": { + "@grpc/grpc-js": "^1.11.1", + "@grpc/proto-loader": "^0.7.7", + "@octokit/request": "^8.1.1", + "@sweet-monads/either": "~3.2.0", + "@sweet-monads/maybe": "~3.2.0", + "@types/line-column": "^1.0.0", + "@types/lodash.clonedeep": "4.5.0", + "@types/seedrandom": "^3.0.4", + "adm-zip": "^0.5.16", + "antlr4ts": "^0.5.0-alpha.4", + "chalk": "^4.1.2", + "cli-progress": "^3.12.0", + "cross-spawn": "^7.0.6", + "eol": "^0.9.1", + "immutable": "^4.3.0", + "json-bigint": "^1.0.0", + "line-column": "^1.0.2", + "lodash": "^4.17.21", + "lodash.clonedeep": "4.5.0", + "seedrandom": "^3.0.5", + "tar": "^7.5.13", + "yargs": "^17.7.2" + }, + "bin": { + "quint": "dist/src/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@octokit/endpoint": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", + "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "24.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", + "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", + "license": "MIT" + }, + "node_modules/@octokit/request": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", + "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==", + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^9.0.6", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/request-error": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz", + "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/types": { + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", + "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^24.2.0" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@sweet-monads/either": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@sweet-monads/either/-/either-3.2.0.tgz", + "integrity": "sha512-n+nR0b60GRTKb+D76qhTf4NEBXU9zfpigYYEtKtSYbV+5+i5gxr9jFd64pYkY2O7hVsb/G7nspbAeFni/i1ltA==", + "license": "MIT", + "dependencies": { + "@sweet-monads/interfaces": "^3.2.0" + } + }, + "node_modules/@sweet-monads/interfaces": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@sweet-monads/interfaces/-/interfaces-3.3.0.tgz", + "integrity": "sha512-66akGvjPD4lizQy+w4JSltJilc2w/QPdw8lPAniLJGHwyjmrw9xMJLx76Q/GDnbCU59Werses4aZJLWOlJrL5A==", + "license": "MIT" + }, + "node_modules/@sweet-monads/maybe": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@sweet-monads/maybe/-/maybe-3.2.0.tgz", + "integrity": "sha512-/t+K0D/kBfkYOkZaePEsrK868at0M9UIEVgehcM0xscrCSZhKWGteE41vl2XJQqh8WyiFo/mZ5y7eAPSYzS+pg==", + "license": "MIT", + "dependencies": { + "@sweet-monads/interfaces": "^3.2.0" + } + }, + "node_modules/@types/line-column": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/line-column/-/line-column-1.0.2.tgz", + "integrity": "sha512-099oFQmp/Tlf20xW5XI5R4F69N6lF/zQ09XDzc3R5BOLFlqIotgKoNIyj0HD4fQLWcGDreDJv8k/BkLJscrDrw==", + "license": "MIT" + }, + "node_modules/@types/lodash": { + "version": "4.17.26", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.26.tgz", + "integrity": "sha512-48NWfiY6oNcKOh47Z4h+sIvwcQR91h6KPhaYKgeMIrkT2jV82Gy90SErj60vW8jQbeMSOSbhIv3bZ7JoqPPVhA==", + "license": "MIT" + }, + "node_modules/@types/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@types/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-IHijjFVPJTvzvrNPz+6nQy5lZQb7uh807RfTIEaQBrZXrIGjZy0L2dEb3hju34J0eqbXLCY6Hub/g81Jl4pGCA==", + "license": "MIT", + "dependencies": { + "@types/lodash": "*" + } + }, + "node_modules/@types/node": { + "version": "26.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz", + "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/seedrandom": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-3.0.8.tgz", + "integrity": "sha512-TY1eezMU2zH2ozQoAFAQFOPpvP15g+ZgSfTZt31AUUH/Rxtnz3H+A/Sv1Snw2/amp//omibc+AEkTaA8KUeOLQ==", + "license": "MIT" + }, + "node_modules/adm-zip": { + "version": "0.5.18", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz", + "integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/antlr4ts": { + "version": "0.5.0-alpha.4", + "resolved": "https://registry.npmjs.org/antlr4ts/-/antlr4ts-0.5.0-alpha.4.tgz", + "integrity": "sha512-WPQDt1B74OfPv/IMS2ekXAKkTZIHl88uMetg6q3OTqgFxZ/dxDXI0EWLyZid/1Pe6hTftyg5N7gel5wNAGxXyQ==", + "license": "BSD-3-Clause" + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/cli-progress": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "license": "MIT", + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==", + "license": "ISC" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/eol": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/eol/-/eol-0.9.1.tgz", + "integrity": "sha512-Ds/TEoZjwggRoz/Q2O7SE3i4Jm66mqTDfmdHdq/7DKVk3bro9Q8h6WdXKdPqFLMoqxrDK5SVRzHVPOS6uuGtrg==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/immutable": { + "version": "4.3.9", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.9.tgz", + "integrity": "sha512-ObHy4YN7ycwZOUCLI1/6svfyAFu7vL8RhAvVu/bh/RZW9EPlOyDaQ9jDQWCtdqzaXUjgXZCW1migtHE7YI7UGQ==", + "license": "MIT" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "license": "MIT", + "dependencies": { + "isarray": "1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/line-column": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/line-column/-/line-column-1.0.2.tgz", + "integrity": "sha512-Ktrjk5noGYlHsVnYWh62FLVs4hTb8A3e+vucNZMgPeAOITdshMSgv4cCZQeRDjm7+goqmo6+liZwTXo+U3sVww==", + "license": "MIT", + "dependencies": { + "isarray": "^1.0.0", + "isobject": "^2.0.0" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT" + }, + "node_modules/universal-user-agent": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", + "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==", + "license": "ISC" + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + } + } +} diff --git a/formal/quint/package.json b/formal/quint/package.json new file mode 100644 index 00000000..64de03e4 --- /dev/null +++ b/formal/quint/package.json @@ -0,0 +1,10 @@ +{ + "name": "parser-protocol-models", + "private": true, + "dependencies": { + "@informalsystems/quint": "0.33.0" + }, + "scripts": { + "test": "quint test TapeOwner.qnt --backend typescript --seed 1 --max-samples 1 && quint run TapeOwner.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant OwnerSafety && quint test Publication.qnt --backend typescript --seed 1 --max-samples 1 && quint test FoldPlan.qnt --backend typescript --seed 1 --max-samples 1 && quint run FoldPlan.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant ExecutionSafety && quint test IdlePlan.qnt --backend typescript --seed 1 --max-samples 1 && quint run IdlePlan.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant GuardSafety && quint test RunPlan.qnt --backend typescript --seed 1 --max-samples 1 && quint run RunPlan.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant RunSafety && quint test ConsumerScope.qnt --backend typescript --seed 1 --max-samples 1 && quint run ConsumerScope.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant ScopeSafety && quint test ReuseAdmission.qnt --backend typescript --seed 1 --max-samples 1 && quint run ReuseAdmission.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant ReuseSafety && quint test HistoryCommit.qnt --backend typescript --seed 1 --max-samples 1 && quint run HistoryCommit.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant HistorySafety && quint test CapturePlan.qnt --backend typescript --seed 1 --max-samples 1 && quint run CapturePlan.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant CaptureSafety && quint test LookaheadScope.qnt --backend typescript --seed 1 --max-samples 1 && quint run LookaheadScope.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant ScopeSafety && quint test ModuleAdmission.qnt --backend typescript --seed 1 --max-samples 1 && quint run ModuleAdmission.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant ModuleSafety && quint test SparseCapture.qnt --backend typescript --seed 1 --max-samples 1 && quint run SparseCapture.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant SparseSafety && quint test OwnedTerminal.qnt --backend typescript --seed 1 --max-samples 1 && quint run OwnedTerminal.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant PublicationSafety && quint test YieldMemo.qnt --backend typescript --seed 1 --max-samples 1 && quint run YieldMemo.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant MemoSafety && quint test CaptureContext.qnt --backend typescript --seed 1 --max-samples 1 && quint run CaptureContext.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant ContextSafety" + } +} diff --git a/generate-rust-rowan.ss b/generate-rust-rowan.ss deleted file mode 100644 index e2f9074e..00000000 --- a/generate-rust-rowan.ss +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- -;;; Repository generation entrypoint for committed Rust/Rowan AOT products. - -(import (only-in :gerbil-parser/languages/arithmetic/v1/grammar - arithmetic-language-grammar) - (only-in :gerbil-parser/src/compiler/rust-rowan - generate-language-rust-rowan-module)) - -(def arguments (command-line)) -(def output - (if (> (length arguments) 2) - (car (reverse arguments)) - "rust/gerbil-parser-rowan-arithmetic/src/generated/mod.rs")) - -(generate-language-rust-rowan-module output arithmetic-language-grammar) diff --git a/generate-rust-runtime.ss b/generate-rust-runtime.ss new file mode 100644 index 00000000..a9135bf4 --- /dev/null +++ b/generate-rust-runtime.ss @@ -0,0 +1,16 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Repository generation entrypoint for committed Rust AOT products. + +(import (only-in :gerbil-parser/languages/arithmetic/parser + arithmetic-language-grammar) + (only-in :gerbil-parser/src/compiler/rust-runtime + generate-language-rust-runtime-module)) + +(def arguments (command-line)) +(def output + (if (> (length arguments) 2) + (car (reverse arguments)) + "rust/gerbil-parser-runtime-arithmetic/src/generated/mod.rs")) + +(generate-language-rust-runtime-module output arithmetic-language-grammar) diff --git a/gerbil.pkg b/gerbil.pkg index 854a4f0a..f3f1e781 100644 --- a/gerbil.pkg +++ b/gerbil.pkg @@ -1,5 +1,5 @@ (package: gerbil-parser depend: - ("github.com/tao3k/poo-flow-core@dc46034fbcc19262c82fb757bc57035305d595e3" - "github.com/tao3k/asp-gerbil-scheme@19bf659450f619a410cef46b093ccaa6bb203c03")) + ("github.com/tao3k/poo-flow-core@e85fd45303b208b23e5957fd316e7004994109a4" + "github.com/tao3k/asp-gerbil-scheme@f5b7c009d2cab61144a8af18008a0bfba3056584")) diff --git a/graph-projection-support.ss b/graph-projection-support.ss index 85f758eb..3ee0b392 100644 --- a/graph-projection-support.ss +++ b/graph-projection-support.ss @@ -5,11 +5,11 @@ GraphProjection. GraphNode. GraphField. make-graph-projection make-graph-node make-graph-field graph-projection?) - (only-in ./src/compiler/graph-projection-rowan + (only-in ./src/compiler/graph-projection-runtime graph-projection-digest - generate-graph-projection-rowan-module)) + generate-graph-projection-runtime-module)) (export GraphProjection. GraphNode. GraphField. make-graph-projection make-graph-node make-graph-field graph-projection? graph-projection-digest - generate-graph-projection-rowan-module) + generate-graph-projection-runtime-module) diff --git a/graph-query-support.ss b/graph-query-support.ss new file mode 100644 index 00000000..159b017c --- /dev/null +++ b/graph-query-support.ss @@ -0,0 +1,14 @@ +;;; -*- Gerbil -*- +;;; Public POO graph-query surface for language-owned graph views. + +(import (only-in ./src/modules/parser/graph-query-types + GraphQueryViewContract GraphQueryContextContract + graph-query-view? graph-query-context?) + (only-in ./src/modules/parser/graph-query-objects + GraphQueryContext. make-graph-query-context) + (only-in ./src/modules/parser/graph-query-funs + graph-query-map graph-query-lineage? graph-query-select)) +(export GraphQueryViewContract GraphQueryContextContract GraphQueryContext. + graph-query-view? graph-query-context? + make-graph-query-context + graph-query-map graph-query-lineage? graph-query-select) diff --git a/graph-syntax-support.ss b/graph-syntax-support.ss new file mode 100644 index 00000000..82bf06af --- /dev/null +++ b/graph-syntax-support.ss @@ -0,0 +1,3 @@ +;;; Public POO graph syntax shared by compatible language declarations. +(import ./src/modules/parser/graph-syntax) +(export (import: ./src/modules/parser/graph-syntax)) diff --git a/include/gerbil-parser/language.h b/include/gerbil-parser/language.h new file mode 100644 index 00000000..583284c1 --- /dev/null +++ b/include/gerbil-parser/language.h @@ -0,0 +1,52 @@ +#ifndef GERBIL_PARSER_LANGUAGE_H +#define GERBIL_PARSER_LANGUAGE_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +/* Handles are registered by a compiled language pack, never a name registry. + * All calls require the initialized Gerbil runtime and its owning thread. + * Release invalidates a handle; its numeric identity is never reused. + * The library/runtime must remain loaded through the last call and release. + * This API does not initialize/shutdown the runtime or admit arbitrary threads. */ +typedef uint64_t gerbil_parser_language; +typedef struct { int32_t status; uint8_t *payload; size_t length; } gerbil_parser_result; +/* Initialize once, release after use. Each call replaces the C-owned payload. + * Result payload contains GPA1 bytes (parse) or descriptor/error UTF-8 inert Scheme v1. + * Transport success is separate from accepted/rejected syntax in GPA1. + * Source is length-bearing: embedded NUL is retained; invalid UTF-8 fails. + * Maximum source length is 64 MiB. NULL source is allowed only for length zero. */ +/* GPA1 wire format (little endian, no C struct layout dependency): + * 80-byte header: magic[4], version:u32=1, syntax_status:u32 (0 accepted, + * 1 rejected), event_count:u32, grammar_sha256[32], source_sha256[32]. + * Exactly event_count 24-byte records follow: tag:u32, symbol:u32, id:u64, + * start:u32, end:u32. Tags 1/2 are node start/finish, 3/4 field start/finish, + * 5 token. Symbols index descriptor syntaxKinds/fields/terminals respectively. + * Node/token IDs are separate sequential streams; field IDs are zero. + * Start records have end=0; finish records have start=0. Token ranges are + * nonempty UTF-8 byte slices covering the source once in order. Rejected + * syntax retains lossless tokens and publishes no partial node/field tree. + * Contextual handles publish their complete product digest in grammar_sha256; + * bind it to the generated contextual spec, not merely its base grammar. + * Consumers must bind both digests, validate nesting and reserved positions, + * and keep the result alive until all borrowed payload views are dropped. + * Descriptor/error Scheme datum is not a GPA1 record; check transport status first. */ +/* Non-owner OS threads fail before entering the VM; initialization still + * belongs to the embedding host. This predicate is safe after module init. */ +int32_t gerbil_parser_language_is_owner_thread(void); +uint32_t gerbil_parser_language_abi_version(void); +/* Embedding shutdown barrier: owner thread only; 0 closes VM admission, + * -1 rejects thread/state, -2 retains admission while handles or allocated + * result payloads remain. After success no Scheme callback is admitted. + * This does not perform VM cleanup; runtime.h owns the standalone lifecycle. */ +int32_t gerbil_parser_language_runtime_detach(void); +void gerbil_parser_result_init(gerbil_parser_result *result); +void gerbil_parser_result_release(gerbil_parser_result *result); +int32_t gerbil_parser_language_descriptor(gerbil_parser_language language, gerbil_parser_result *result); +int32_t gerbil_parser_language_parse(gerbil_parser_language language, const uint8_t *source, size_t length, gerbil_parser_result *result); +int32_t gerbil_parser_language_release(gerbil_parser_language language); +#ifdef __cplusplus +} +#endif +#endif diff --git a/include/gerbil-parser/parse-artifact-v1.h b/include/gerbil-parser/parse-artifact-v1.h deleted file mode 100644 index 8ffe9354..00000000 --- a/include/gerbil-parser/parse-artifact-v1.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef GERBIL_PARSER_PARSE_ARTIFACT_V1_H -#define GERBIL_PARSER_PARSE_ARTIFACT_V1_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int32_t status; - uint8_t *payload; - size_t length; -} gerbil_parser_result_v1; - -void gerbil_parser_result_v1_init(gerbil_parser_result_v1 *result); -void gerbil_parser_result_v1_release(gerbil_parser_result_v1 *result); -uint32_t gerbil_parser_native_abi_version(void); -int32_t gerbil_parser_native_descriptor( - const char *language, - gerbil_parser_result_v1 *result); -int32_t gerbil_parser_native_parse( - const char *language, - const char *source, - gerbil_parser_result_v1 *result); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/include/gerbil-parser/runtime.c b/include/gerbil-parser/runtime.c new file mode 100644 index 00000000..3cb326cc --- /dev/null +++ b/include/gerbil-parser/runtime.c @@ -0,0 +1,37 @@ +/* Standalone lifecycle for the closure generated by Gerbil's compiler. + * See Gambit's official examples/pthread/pthread.c for setup/cleanup ownership. + * The build owner supplies GERBIL_PARSER_LINKER; no generated C is rewritten. */ +#include +#include +#include +#include +#include +#ifndef GERBIL_PARSER_LINKER +#error "The native library builder must supply its Gambit linker symbol" +#endif +extern ___mod_or_lnk GERBIL_PARSER_LINKER(___global_state); +static ___setup_params_struct params; /* reset_argv contains self-references */ +static _Atomic int gp_runtime_state = 0; /* fresh, starting, ready, closed, failed */ +int32_t gerbil_parser_runtime_init(void) { + int fresh = 0; + if (!atomic_compare_exchange_strong(&gp_runtime_state,&fresh,1)) return -3; + ___setup_params_reset(¶ms); + params.version = ___VERSION; + params.linker = GERBIL_PARSER_LINKER; + ___SCMOBJ status = ___setup(¶ms); + if (status != ___FIX(___NO_ERR)) { + atomic_store(&gp_runtime_state,4); + return -4; + } + atomic_store(&gp_runtime_state,2); + return 0; +} +int32_t gerbil_parser_runtime_shutdown(void) { + if (atomic_load(&gp_runtime_state) != 2) return -3; + if (!gerbil_parser_language_is_owner_thread()) return -1; + int32_t status = gerbil_parser_language_runtime_detach(); + if (status) return status; + ___cleanup(); + atomic_store(&gp_runtime_state,3); + return 0; +} diff --git a/include/gerbil-parser/runtime.h b/include/gerbil-parser/runtime.h new file mode 100644 index 00000000..c0d74031 --- /dev/null +++ b/include/gerbil-parser/runtime.h @@ -0,0 +1,20 @@ +#ifndef GERBIL_PARSER_RUNTIME_H +#define GERBIL_PARSER_RUNTIME_H +#include +#ifdef __cplusplus +extern "C" { +#endif +/* One standalone runtime bundle per process. Initialization chooses its owner + * OS thread and is allowed once, before any other Gambit VM is initialized. + * The bundle and all language modules remain loaded through shutdown. + * 0 success; -1 wrong thread; -2 live handles/results; -3 lifecycle state; + * -4 SDK setup failure (poisoned, no reinitialization). + * Shutdown retains a ready runtime on -2, so resources can be released and + * shutdown retried. Successful shutdown disables language callbacks and calls + * the SDK's normal cleanup. Restart and duplicate shutdown return -3. */ +int32_t gerbil_parser_runtime_init(void); +int32_t gerbil_parser_runtime_shutdown(void); +#ifdef __cplusplus +} +#endif +#endif diff --git a/include/gerbil-parser/rust-rowan-aot-v1.h b/include/gerbil-parser/rust-rowan-aot-v1.h deleted file mode 100644 index 6f387182..00000000 --- a/include/gerbil-parser/rust-rowan-aot-v1.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef GERBIL_PARSER_RUST_ROWAN_AOT_V1_H -#define GERBIL_PARSER_RUST_ROWAN_AOT_V1_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int32_t status; - uint8_t *payload; - size_t length; -} gerbil_parser_rowan_result_v1; - -void gerbil_parser_rowan_result_v1_init( - gerbil_parser_rowan_result_v1 *result); -void gerbil_parser_rowan_result_v1_release( - gerbil_parser_rowan_result_v1 *result); -uint32_t gerbil_parser_rowan_aot_abi_version(void); -int32_t gerbil_parser_rowan_compile( - const char *grammar_path, - gerbil_parser_rowan_result_v1 *result); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/include/gerbil-parser/rust-runtime-aot.h b/include/gerbil-parser/rust-runtime-aot.h new file mode 100644 index 00000000..2dcd788b --- /dev/null +++ b/include/gerbil-parser/rust-runtime-aot.h @@ -0,0 +1,33 @@ +#ifndef GERBIL_PARSER_RUST_RUNTIME_AOT_H +#define GERBIL_PARSER_RUST_RUNTIME_AOT_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Initialize before first use. Calls replace the owned payload, including + * on errors. Release after last use; repeated release and NULL are safe. + * Calls require the initialized Gerbil runtime and its calling thread. */ +typedef struct { + int32_t status; + uint8_t *payload; + size_t length; +} gerbil_parser_runtime_result; + +void gerbil_parser_runtime_result_init( + gerbil_parser_runtime_result *result); +void gerbil_parser_runtime_result_release( + gerbil_parser_runtime_result *result); +uint32_t gerbil_parser_runtime_aot_abi_version(void); +int32_t gerbil_parser_runtime_compile( + const char *grammar_path, + gerbil_parser_runtime_result *result); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/justfile b/justfile new file mode 100644 index 00000000..df7a1c81 --- /dev/null +++ b/justfile @@ -0,0 +1,616 @@ +set shell := ["sh", "-eu", "-c"] + +# Same native gxtest suite values, linked before the bounded runtime gate. +test-runtime-organization: + GAMBOPT=max-heap=1G,debug=q gxi build-qualification.ss compile --optimized + ASP_GERBIL_SCHEME_POLICY=1 GAMBOPT=max-heap=1G,debug=q gxi t/build-runtime-organization-tests.ss + GAMBOPT=max-heap=1G,debug=q gxi t/check-runtime-organization.ss + +# Public native declaration lowering; test-all discovers this suite once. +test-line-events output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/modules/parser/line-structure-objects.ss src/modules/parser/source-event-scope-types.ss src/modules/parser/source-event-scope-objects.ss src/modules/parser/source-event-scope-funs.ss src/modules/parser/line-event-funs.ss src/modules/parser/list-event-funs.ss src/modules/parser/inline-link-event-funs.ss src/modules/parser/source-boundary-types.ss src/modules/parser/source-boundary-objects.ss src/modules/parser/source-boundary-funs.ss src/modules/parser/interface.ss t/line-event-test.ss t/inline-link-event-test.ss t/source-boundary-test.ss t/source-event-scope-test.ss t/list-event-test.ss + GAMBOPT=max-heap=1G,debug=q GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/line-event-test.ss t/inline-link-event-test.ss t/source-boundary-test.ss t/source-event-scope-test.ss t/list-event-test.ss 2>&1 | tee "{{output}}/line-events-test.log" + rg --quiet '^HARNESS-OK' "{{output}}/line-events-test.log" + rg --quiet '^OK$' "{{output}}/line-events-test.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/line-events-test.log" + +test-source-priority: + GAMBOPT=max-heap=1G,debug=q gerbil test -v 3 t/source-priority-test.ss + +build-source-priority output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/modules/parser/source-pattern-funs.ss src/modules/parser/funcs.ss src/modules/parser/interface.ss + +build-process-fixture: build-qualification + mkdir -p .data/process-controls + ${CC:-cc} -std=c11 -Wall -Wextra -Werror t/fixtures/qualification/process-fixture.c -o .data/process-controls/fixture + +# Real initialization/load phases; never substitutes for strict controls. +diagnose-process-startup: build-process-fixture + gerbil env gxi t/fixtures/qualification/startup-controls.ss "$PWD/.data/process-controls/fixture" + +test-process-contracts: build-process-fixture + gerbil env gxi t/fixtures/qualification/process-controls.ss "$PWD/.data/process-controls/fixture" + +build-qualification: + gerbil env gxi build-qualification.ss compile + +# Focused native suites; test-all discovers the same suites once. +test-qualification-suites: build-qualification + GAMBOPT=max-heap=1G,debug=q gerbil test -v 3 t/qualification-plans-test.ss t/bootstrap-test.ss + +# Exercise the downstream admission caller after support-module relocation. +test-process-admission: build-qualification + GAMBOPT=max-heap=1G,debug=q gerbil env gxi scripts/tests/process-qualification-test.ss + +test-compiler-process: + GAMBOPT=max-heap=1G,debug=q gerbil env gxi scripts/tests/compiler-process-test.ss + +# Cache admission controls stay Scheme and share the CI owner. +test-preparation: + GAMBOPT=max-heap=1G,debug=q gerbil env gxi scripts/tests/preparation-test.ss + +# Build the shared helper through its compiled module owner before loading source suites. +test-workers output=".data/worker-controls": build-qualification + gerbil compile -O t/fixtures/tla-sany-differential/worker-control.ss + gerbil env gxi test-processes.ss workers "{{output}}" + +# Compile the suite's own fixtures, not only its import declarations. +build-fold-attribution output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/scan.ss src/runtime/byte-spans.ss src/compiler/event-list-marker.ss src/runtime/source-lines.ss src/runtime/event-fold-lines.ss src/compiler/event-fold-future.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/fold-cost.ss t/fold-cost-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-cost-test.ss > "{{output}}/fold-cost-test.log" 2>&1 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-cost-test.log" + rg --quiet '^OK$' "{{output}}/fold-cost-test.log" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-fold-analysis.ss src/compiler/event-fold-byte-set.ss src/compiler/event-fold-byte-dispatch.ss src/compiler/event-fold-zero-state.ss src/compiler/event-fold-consumer-state.ss src/compiler/event-fold-ordinary-run.ss src/compiler/event-fold-shared-probes.ss src/compiler/event-fold-scheme-context.ss src/compiler/event-fold-call-plan.ss src/compiler/event-fold-prefix-family.ss src/compiler/event-fold-scheme-expressions.ss src/compiler/event-fold-scheme-runs.ss src/compiler/event-fold-scheme-statements.ss src/compiler/event-fold-scheme.ss src/compiler/event-fold-scheme-modules.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-zero-state-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-zero-state-test.ss > "{{output}}/fold-zero-state-test.log" 2>&1 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-zero-state-test.log" + test "$(rg --count '^CASE-OK ' '{{output}}/fold-zero-state-test.log')" = 8 + rg --quiet '^OK$' "{{output}}/fold-zero-state-test.log" + +test-fold-consumer-state output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-consumer-state-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-consumer-state-test.ss > "{{output}}/fold-consumer-state-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-consumer-state-test.log')" = 5 + rg --quiet '^OK$' "{{output}}/fold-consumer-state-test.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/fold-consumer-state-test.log" + +test-fold-tape-tail output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-tape-tail-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-tape-tail-test.ss > "{{output}}/fold-tape-tail-test.log" 2>&1 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-tape-tail-test.log" + test "$(rg --count '^CASE-OK ' '{{output}}/fold-tape-tail-test.log')" = 2 + rg --quiet '^OK$' "{{output}}/fold-tape-tail-test.log" + +inspect-fold-call-sites product: + gxi t/benchmarks/parser-throughput/call-graph.ss "{{product}}" + +inspect-fold-ordinary-effects program load_path: + GERBIL_LOADPATH="{{load_path}}" gxi t/benchmarks/parser-throughput/ordinary-plan.ss "{{program}}" + +inspect-fold-runs program load_path: + GERBIL_LOADPATH="{{load_path}}" gxi t/benchmarks/parser-throughput/run-plan.ss "{{program}}" + +test-fold-runs output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-run-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-run-test.ss > "{{output}}/fold-run-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-run-test.log')" = 13 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-run-test.log" + rg --quiet '^OK$' "{{output}}/fold-run-test.log" + +test-fold-statement-scans output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-statement-scans-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-statement-scans-test.ss > "{{output}}/fold-statement-scans-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-statement-scans-test.log')" = 5 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-statement-scans-test.log" + rg --quiet '^OK$' "{{output}}/fold-statement-scans-test.log" + +# Rebuild package dependencies only; installed SDK and global libraries stay read-only. +test-fold-buffer-consumer output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/byte-spans.ss src/runtime/source-lines.ss src/runtime/fold-tape.ss t/event-strategy-fixture.ss t/fold-tape-test.ss t/fold-buffer-consumer-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-tape-test.ss t/fold-buffer-consumer-test.ss > "{{output}}/fold-buffer-consumer-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-buffer-consumer-test.log')" = 8 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-buffer-consumer-test.log" + rg --quiet '^OK$' "{{output}}/fold-buffer-consumer-test.log" + +prepare-fold-buffer-compiler output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/byte-spans.ss src/compiler/event-fold-scheme-expressions.ss src/compiler/event-fold-scheme-runs.ss src/compiler/event-fold-scheme-statements.ss src/compiler/event-fold-scheme.ss src/compiler/event-fold-scheme-modules.ss + +prepare-fold-dependency source output load_path: + mkdir -p "{{output}}" + cd "{{source}}" && GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil env gxi build.ss compile --optimized + +prepare-real-fold-inputs output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-fold-state-frame.ss src/compiler/event-fold-program.ss src/compiler/event-fold-runtime.ss src/compiler/event-fold-ir.ss src/runtime/fold-tape.ss t/fold-tape-oracle.ss src/compiler/event-fold-scheme-build.ss + +test-fold-inert output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-inert-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-inert-test.ss > "{{output}}/fold-inert-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-inert-test.log')" = 4 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-inert-test.log" + rg --quiet '^OK$' "{{output}}/fold-inert-test.log" + +link-fold-attribution output load_path: + gxi t/benchmarks/parser-throughput/link-locks.ss "{{output}}/lib/static" "{{output}}/../lib/static" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/fold-profile" t/benchmarks/parser-throughput/profile.ss + +run-fold-prepared-counts binary corpus digest receipt: + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{corpus}}" prepared-counts "{{digest}}" > "{{receipt}}" 2>&1 + rg --quiet '^DIAGNOSTIC-OK real-fold; NOT PERFORMANCE ACCEPTANCE' "{{receipt}}" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{receipt}}" + rg '^\(diagnostic-only|^FIELDS|^DIAGNOSTIC-OK' "{{receipt}}" + +link-real-fold-throughput output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/benchmarks/parser-throughput/protocol.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/real-fold" t/benchmarks/parser-throughput/real.ss + +link-real-fold-tape-throughput output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/benchmarks/parser-throughput/protocol.ss t/fold-tape-oracle.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/real-tape-fold" t/benchmarks/parser-throughput/real-tape.ss + +# Independent interpreter preparation is separated from the native measurement. +prepare-real-fold-tape-preflight fixtures program digest corpus load_path: + test ! -e "{{corpus}}" + GERBIL_LOADPATH="{{load_path}}" gxi t/benchmarks/parser-throughput/prepare-tape-preflight.ss "{{fixtures}}" "{{program}}" "{{digest}}" "{{corpus}}" + +prepare-fold-throughput-protocol output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/benchmarks/parser-throughput/protocol.ss + +link-prepared-fold-tape-preflight output load_path: + mkdir -p "{{output}}" + gxi t/benchmarks/parser-throughput/link-locks.ss "{{output}}/lib/static" "{{output}}/../lib/static" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/benchmarks/parser-throughput/protocol.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/prepared-tape-fold" t/benchmarks/parser-throughput/prepared-tape.ss + +link-fold-stalls output load_path: + mkdir -p "{{output}}" + gxi t/benchmarks/parser-throughput/link-locks.ss "{{output}}/lib/static" "{{output}}/../lib/static" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/fold-stalls" t/benchmarks/parser-throughput/stalls.ss + +link-fold-allocation-diagnostic output load_path: + mkdir -p "{{output}}" + gxi t/benchmarks/parser-throughput/link-locks.ss "{{output}}/lib/static" "{{output}}/../lib/static" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/allocation-diagnostic" t/benchmarks/parser-throughput/allocation.ss + +run-fold-stalls binary corpus receipt: + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{corpus}}" > "{{receipt}}" 2>&1 + rg --quiet '^FOLD-STALLS-OK' "{{receipt}}" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{receipt}}" + rg '^BATCH-DIAGNOSTIC|^FIELDS|^\(|^FOLD-STALLS-OK' "{{receipt}}" + +# One existing AOT process; OS sampling is diagnostic, never acceptance. +[script] +profile-fold-stacks binary corpus receipt sample: + #!/usr/bin/env bash + set -euo pipefail + test ! -e "{{receipt}}" + test ! -e "{{sample}}" + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{corpus}}" profile > "{{receipt}}" 2>&1 & + task_pid=$! + trap 'kill "$task_pid" 2>/dev/null || true' EXIT + ready=false + for attempt in {1..200}; do + if rg --quiet '^FOLD-PROFILE-READY' "{{receipt}}"; then ready=true; break; fi + kill -0 "$task_pid" + sleep 0.01 + done + test "$ready" = true + /usr/bin/sample "$task_pid" 1 1 -file "{{sample}}" + wait "$task_pid" + trap - EXIT + rg --quiet '^FOLD-PROFILE-OK' "{{receipt}}" + +run-fold-allocation-diagnostic binary corpus receipt: + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{corpus}}" > "{{receipt}}" 2>&1 + rg --quiet '^ALLOCATION-DIAGNOSTIC-OK' "{{receipt}}" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{receipt}}" + rg '^FIELDS|^\([0-2] |^ALLOCATION-DIAGNOSTIC-OK' "{{receipt}}" + +run-prepared-fold-tape-preflight binary corpus digest receipt: + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q /usr/bin/time {{if os() == "macos" { "-l" } else { "-v" }}} "{{binary}}" "{{corpus}}" "{{digest}}" > "{{receipt}}" 2>&1 + rg 'parser-throughput.v1|PARSER-THROUGHPUT-PREFLIGHT-OK' "{{receipt}}" + +# Admit an already linked actual-Org product; never rebuild it for a path fix. +test-real-fold-controls binary controls receipt: + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{controls}}" > "{{receipt}}" 2>&1 + test "$(rg --count '^CASE-OK native-org$' '{{receipt}}')" = 24 + rg --quiet '^OK$' "{{receipt}}" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{receipt}}" + +run-real-fold-throughput binary fixtures program digest receipt mode="preflight": + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{fixtures}}" "{{program}}" "{{digest}}" "{{mode}}" > "{{receipt}}" 2>&1 + rg '^PARSER-THROUGHPUT' "{{receipt}}" + +# Process RSS includes the retained independent oracle and input corpus. +# Preserve the failed preflight exit code; resource output is not admission. +run-real-fold-resources binary fixtures program digest receipt: + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q /usr/bin/time {{if os() == "macos" { "-l" } else { "-v" }}} "{{binary}}" "{{fixtures}}" "{{program}}" "{{digest}}" preflight > "{{receipt}}" 2>&1 + rg '^PARSER-THROUGHPUT' "{{receipt}}" + +run-real-fold-tape-resources binary fixtures program digest receipt consumer="direct": + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q /usr/bin/time {{if os() == "macos" { "-l" } else { "-v" }}} "{{binary}}" "{{fixtures}}" "{{program}}" "{{digest}}" preflight "{{consumer}}" > "{{receipt}}" 2>&1 + rg '^PARSER-THROUGHPUT' "{{receipt}}" + + +run-fold-attribution binary controls receipt mode="timed": + test ! -e "{{receipt}}" + GAMBOPT=max-heap=1G,debug=q /usr/bin/time {{if os() == "macos" { "-l" } else { "-v" }}} "{{binary}}" "{{controls}}" "{{mode}}" > "{{receipt}}" 2>&1 + rg '^\(diagnostic-only|^DIAGNOSTIC-OK|maximum resident|Maximum resident' "{{receipt}}" + +build-event-fold-controls output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/fold-tape.ss t/fold-tape-oracle.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-fold-analysis.ss src/compiler/event-fold-byte-dispatch.ss src/compiler/event-fold-zero-state.ss src/compiler/event-fold-consumer-state.ss src/compiler/event-fold-ordinary-run.ss src/compiler/event-fold-shared-probes.ss t/event-fold-analysis-test.ss t/fold-zero-state-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-zero-state-test.ss > "{{output}}/fold-zero-state-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-zero-state-test.log')" = 8 + rg --quiet '^OK$' "{{output}}/fold-zero-state-test.log" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/event-fold-analysis-test.ss > "{{output}}/fold-analysis-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-analysis-test.log')" = 3 + rg --quiet '^HARNESS-OK' "{{output}}/fold-analysis-test.log" + rg --quiet '^OK$' "{{output}}/fold-analysis-test.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/fold-analysis-test.log" + just test-fold-byte-dispatch "{{output}}" "{{load_path}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-shared-probes-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-shared-probes-test.ss > "{{output}}/fold-shared-probes-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-shared-probes-test.log')" = 2 + rg --quiet '^OK$' "{{output}}/fold-shared-probes-test.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/fold-shared-probes-test.log" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-fold-future.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/byte-spans.ss src/runtime/source-lines.ss src/runtime/event-fold-lines.ss src/compiler/event-strategy-aot.ss src/compiler/event-fold-program.ss src/compiler/event-fold-runtime.ss src/compiler/event-fold-ir.ss src/compiler/event-fold-scheme-context.ss src/compiler/event-fold-call-plan.ss src/compiler/event-fold-prefix-family.ss src/compiler/event-fold-scheme-expressions.ss src/compiler/event-fold-scheme-statements.ss src/compiler/event-fold-scheme.ss t/event-strategy-fixture.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-tape-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-tape-test.ss > "{{output}}/fold-tape-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-tape-test.log')" = 3 + rg --quiet '^OK$' "{{output}}/fold-tape-test.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/fold-tape-test.log" + just generate-event-fold-executable "{{output}}" "{{load_path}}" + +# One owner: any failed compile or semantic gate stops subsequent steps. +test-fold-byte-dispatch output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-byte-dispatch-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-byte-dispatch-test.ss > "{{output}}/fold-byte-dispatch-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-byte-dispatch-test.log')" = 2 + rg --quiet '^OK$' "{{output}}/fold-byte-dispatch-test.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/fold-byte-dispatch-test.log" + +qualify-fold-plan output load_path: + just test-aot-link-locks + just build-event-fold-controls "{{output}}" "{{load_path}}" + just test-fold-consumer-state "{{output}}" "{{load_path}}" + just test-fold-condition "{{output}}" "{{load_path}}" + just test-fold-call-environment "{{output}}" "{{load_path}}" + just test-fold-call-plan "{{output}}" "{{load_path}}" + just test-fold-frame-facts "{{output}}" "{{load_path}}" + just test-fold-statement-scans "{{output}}" "{{load_path}}" + just test-fold-tape-tail "{{output}}" "{{load_path}}" + just test-fold-inert "{{output}}" "{{load_path}}" + just test-fold-runs "{{output}}" "{{load_path}}" + just test-fold-source-controls "{{output}}" "{{load_path}}" + just link-event-fold-controls "{{output}}" "{{load_path}}" + just test-event-fold-controls "{{output}}" "{{load_path}}" + +# Copy immutable generated artifacts, not dependency manifests or SDK state. +test-fold-call-plan output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-fold-call-plan.ss t/fold-call-plan-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-call-plan-test.ss > "{{output}}/fold-call-plan-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-call-plan-test.log')" = 6 + rg --quiet '^OK$' "{{output}}/fold-call-plan-test.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/fold-call-plan-test.log" + +# Copy immutable generated artifacts, not dependency manifests or SDK state. +stage-real-fold-evidence source output: + test ! -e "{{output}}" + mkdir -p "{{output}}" + cp "{{source}}/org-fold" "{{source}}/org-fold-controls.ss" "{{source}}/fold-input-program.ss" "{{output}}/" + cp "{{source}}"/parse-org-compiled-events*.ss "{{output}}/" + +qualify-fold-plan-and-real output load_path actual: + just qualify-fold-plan "{{output}}" "{{load_path}}" + just test-real-fold-controls "{{actual}}/org-fold" "{{actual}}/org-fold-controls.ss" "{{output}}/actual-org-aot.log" + +generate-event-fold-executable output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil env gxi t/generate-event-fold-executable.ss "{{output}}/event-fold-generated.ss" + ! rg --quiet 'fold-statements|fold-predicate|fold-offset |fold-uint |run-event-fold|eval ' "{{output}}/event-fold-generated.ss" + +# Native std/test caller for the source-owned Fold, scanner and LR suites. +test-fold-source-controls output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-fold-scheme-modules.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/event-strategy-fixture.ss t/event-fold-scheme-test.ss + GAMBOPT=max-heap=1G,debug=q GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/event-fold-scheme-test.ss > "{{output}}/fold-source-controls.log" 2>&1 + rg '^CASE-OK |^OK$' "{{output}}/fold-source-controls.log" + test "$(rg --count '^CASE-OK ' '{{output}}/fold-source-controls.log')" = 6 + rg --quiet '^HARNESS-OK' "{{output}}/fold-source-controls.log" + rg --quiet '^OK$' "{{output}}/fold-source-controls.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/fold-source-controls.log" + +build-engine-controls output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/scan.ss src/compiler/event-list-marker.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/source-lines.ss src/compiler/event-fold-future.ss src/compiler/event-fold-scheme-expressions.ss src/compiler/event-fold-scheme.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/event-strategy-fixture.ss t/event-fold-scheme-test.ss t/ranked-regular-scanner-test.ss t/parser-runtime-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/engine-controls" t/engine-controls.ss + +test-engine-controls output: + GAMBOPT=max-heap=1G,debug=q "{{output}}/engine-controls" > "{{output}}/engine-controls.log" 2>&1 + rg '^CASE-OK |^OK$' "{{output}}/engine-controls.log" + test "$(rg --count '^MODULE-OK ' '{{output}}/engine-controls.log')" = 3 + test "$(rg --count '^CASE-OK ' '{{output}}/engine-controls.log')" = 22 + rg --quiet '^OK$' "{{output}}/engine-controls.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/engine-controls.log" + +build-event-fold-modules output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-fold-scheme-modules.ss src/compiler/event-fold-scheme-build.ss + +# Native engine-only frame costs; compilation is outside measured requests. +build-parser-throughput output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/parse-cost.ss t/benchmarks/parser-throughput/protocol.ss t/parser-throughput-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil env gxi t/benchmarks/parser-throughput/generate.ss "{{output}}/parser-throughput.ss" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/parser-throughput" "{{output}}/parser-throughput.ss" + +test-parser-throughput-protocol output load_path: + GAMBOPT=max-heap=1G,debug=q GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/parser-throughput-test.ss > "{{output}}/parser-throughput-test.log" 2>&1 + rg '^CASE-OK |^OK$' "{{output}}/parser-throughput-test.log" + test "$(rg --count '^CASE-OK ' '{{output}}/parser-throughput-test.log')" = 4 + rg --quiet '^HARNESS-OK' "{{output}}/parser-throughput-test.log" + rg --quiet '^OK$' "{{output}}/parser-throughput-test.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/parser-throughput-test.log" + +benchmark-parser-throughput binary mode="preflight" receipt="": + test ! -e "{{if receipt == "" { binary + "." + mode + ".log" } else { receipt }}}" + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{mode}}" > "{{if receipt == "" { binary + "." + mode + ".log" } else { receipt }}}" 2>&1 + rg '^PARSER-THROUGHPUT-' "{{if receipt == "" { binary + "." + mode + ".log" } else { receipt }}}" + +report-parser-throughput receipt: + gerbil env gxi t/benchmarks/parser-throughput/report.ss "{{receipt}}" + +build-event-fold-frame-benchmark output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/event-fold-frame-benchmark" t/benchmarks/event-fold-frames/benchmark.ss + +generate-prefix-family-benchmark output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil env gxi t/benchmarks/prefix-families/generate.ss "{{output}}/prefix-family-benchmark.ss" + +benchmark-prefix-families binary samples="8": + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{samples}}" 1000 2000 10000 + +benchmark-event-fold-frames binary samples="7": + GAMBOPT=max-heap=1G,debug=q "{{binary}}" "{{samples}}" 1000 2000 10000 + +link-event-fold-controls output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-list-marker.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/event-fold-controls" "{{output}}/event-fold-generated.ss" + +test-event-fold-controls output load_path: + GAMBOPT=max-heap=1G,debug=q GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" "{{output}}/event-fold-controls" > "{{output}}/event-fold-controls.log" 2>&1 + rg '^CASE-OK |^OK$' "{{output}}/event-fold-controls.log" + test "$(rg --count '^CASE-OK ' '{{output}}/event-fold-controls.log')" = 100 + test "$(rg --count '^CASE-TAPE-OK ' '{{output}}/event-fold-controls.log')" = 90 + rg --quiet '^OK$' "{{output}}/event-fold-controls.log" + ! rg --quiet 'ERROR|FAILED|FAILURE' "{{output}}/event-fold-controls.log" + +build-event-fold-compiled output load_path: + mkdir -p "{{output}}" + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/runtime/byte-spans.ss src/runtime/source-lines.ss src/runtime/event-fold-lines.ss src/compiler/event-strategy-aot.ss src/compiler/event-fold-state-frame.ss src/compiler/event-fold-program.ss src/compiler/event-fold-runtime.ss src/compiler/event-fold-aot.ss rust-runtime-event-support.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O src/compiler/event-fold-ir.ss src/compiler/event-fold-analysis.ss src/compiler/event-fold-byte-dispatch.ss src/compiler/event-fold-zero-state.ss src/compiler/event-fold-consumer-state.ss src/compiler/event-fold-ordinary-run.ss src/compiler/event-fold-shared-probes.ss src/compiler/event-fold-scheme-context.ss src/compiler/event-fold-call-plan.ss src/compiler/event-fold-prefix-family.ss src/compiler/event-fold-scheme-expressions.ss src/compiler/event-fold-scheme-runs.ss src/compiler/event-fold-scheme-statements.ss src/compiler/event-fold-scheme.ss src/compiler/event-fold-scheme-modules.ss t/event-strategy-fixture.ss t/event-fold-fixture.ss t/event-fold-test.ss t/event-fold-scheme-test.ss + +test-event-fold-compiled binary output: build-qualification + gerbil env gxi qualify.ss compiled-event-fold "{{binary}}" "{{output}}" + +# Fast lexical algorithm loop; keep the POO Flow Case heap fence pre-import. +test-lexer: + GAMBOPT=max-heap=1G,debug=q gerbil test -v 3 t/ranked-regular-scanner-test.ss + +# LR mode admission is checked separately from the inner scanner transition. +test-lexical-mode: + GAMBOPT=max-heap=1G,debug=q gerbil test -v 3 t/grammar-composition-contract-test.ss t/grammar-composition-execution-test.ss t/grammar-composition-lowering-test.ss + +# Full qualification after the focused algorithm loop. +test-build-product-contract output load_path: + GAMBOPT=max-heap=1G,debug=q GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 3 t/build-product-contract-test.ss + +test-all: build-qualification + GAMBOPT=max-heap=1G,debug=q gerbil test -v 3 t/... languages/... + +# All contracts execute from current source; every suite has a hard deadline. +test-performance-before-compile: build-qualification + gerbil env gxi qualify.ss performance + +# Reports each complete language batch sample without a shell timing wrapper. +benchmark-versioned-matched label="current" samples="20": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/versioned-languages/matched-batch.ss {{label}} {{samples}} + +# Compare long deterministic parses between separate v0.19 package revisions. +benchmark-arithmetic-matched terms="3200" samples="30": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/arithmetic-scale/matched-stream.ss {{terms}} {{samples}} + +# Parsed input has one addition term per line; the parser is already built. +benchmark-arithmetic-lines lines="1024" samples="30": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/arithmetic-scale/matched-stream.ss {{lines}} {{samples}} lines + +# Prepared-token stages show where CPU work goes; they are not streaming totals. +benchmark-arithmetic-stages terms="3200": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/arithmetic-scale/benchmark.ss {{terms}} + +# Complete LR construction at a scale where partition costs are visible. +benchmark-lr-grammar-construction contexts="256" samples="3": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/lr1-partition/context-scale.ss {{contexts}} {{samples}} + +# Complete immutable Examples syntax and lossless artifact receipt. +tla-sany-corpus corpus: build-qualification + GAMBOPT=max-heap=1G,debug=q GERBIL_PARSER_LR_TRACE=1 gerbil env gxi qualify.ss run --timeout 600 --idle-timeout 5 --log .data/tla-source.log -- gerbil env gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (preload-module "gerbil-parser/languages/tla-plus/sany-candidate")' t/fixtures/tla-sany-differential/corpus.ss {{quote(corpus)}} +# Current-source scanner execution; plan-batch includes one plan and requests bindings. +benchmark-contextual-scanner axes="16" samples="20" words="128" literals="0" phase="scan" requests="1": build-qualification + GERBIL_LOADPATH=.. GERBIL_PATH=$PWD/.gerbil GAMBOPT=max-heap=1G,debug=q gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 --log /private/tmp/parser-contextual-scanner-scale.log --require SCANNER-SCALE-OK -- gxi t/benchmarks/contextual-scanner/matched-scale.ss {{axes}} {{samples}} {{words}} {{literals}} {{quote(phase)}} {{requests}} + +# Complete contextual ParseArtifact pairs; plan creation is included per batch. +benchmark-contextual-parser requests="16" samples="5": build-qualification + GERBIL_LOADPATH=.. GERBIL_PATH=$PWD/.gerbil GERBIL_PARSER_LR_TRACE=1 GAMBOPT=max-heap=1G,debug=q gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 --log /private/tmp/parser-contextual-parser-scale.log --require CONTEXTUAL-PARSER-SCALE-OK -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (preload-module "gerbil/tools/gxtest")' t/benchmarks/contextual-scanner/matched-parser.ss {{requests}} {{samples}} + +# Complete deferred batches; each sample reports real token and byte coverage. +benchmark-contextual-deferred delimiters="512" samples="5": build-qualification + GERBIL_LOADPATH=.. GERBIL_PATH=$PWD/.gerbil GAMBOPT=max-heap=1G,debug=q gerbil env gxi qualify.ss run --timeout 120 --idle-timeout 5 --log /private/tmp/parser-contextual-deferred-scale.log --require DEFERRED-SCALE-OK -- gxi t/benchmarks/contextual-scanner/deferred-scale.ss {{delimiters}} {{samples}} + +# Host-native toolchain; compiler admission and real test progress stay separate. +test-local suite="rust" gerbil_path=".gerbil": build-qualification + gerbil env gxi qualify.ss local --suite {{quote(suite)}} --gerbil-path {{quote(gerbil_path)}} + +# Pure Rust projection semantics: no Scheme SDK compilation or runtime startup. +[private] +test-graph-projection: + cargo test --locked -p gerbil-parser-runtime --lib structural_lines_tests::graph_cases -- --list | rg --quiet 'prepared_graph_projection_reuses_admission_across_changing_documents: test' + cargo test --locked -p gerbil-parser-runtime --lib structural_lines_tests::graph_cases + +[private] +test-runtime-library: + cargo test --locked -p gerbil-parser-runtime +# Real TLA+ grammar through the default LALR route; complete LRSpec stability. +benchmark-lalr-construction samples="20": + gerbil env gxi -:max-heap=2G,debug=q t/benchmarks/lr1-partition/lalr-construction.ss {{samples}} + +# Complete output equality and inverse edits before CPU sampling. +benchmark-incremental-topology sizes="400 800 1600 3200": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss topology {{sizes}} + +# Independent siblings distinguish avoidable suffix replay from expression ancestry. +benchmark-incremental-siblings sizes="100 200 400 800": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss topology-hcl {{sizes}} + +# Capture gate reevaluates productions and compares both forward/inverse artifacts. +benchmark-incremental-capture sizes="2 400 800 1600 3200": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss topology-capture {{sizes}} + +benchmark-incremental-siblings-capture sizes="2 100 200 400 800": + gerbil env gxi -:max-heap=1G,debug=q t/benchmarks/incremental-session/benchmark.ss topology-hcl-capture {{sizes}} + +# Stage controls and uninstrumented public entry; 40 x 100, native-only. +benchmark-gql gerbil_path=".gerbil": build-qualification + gerbil env gxi qualify.ss local --suite gql-profile --gerbil-path {{quote(gerbil_path)}} + +# Optional application-owned actor library; 1/4 clients, 1/4 actors. +benchmark-gql-actors gerbil_path=".gerbil": build-qualification + gerbil env gxi qualify.ss local --suite gql-actors --gerbil-path {{quote(gerbil_path)}} + +# Real C ABI edits, full payload parity, natural GC; phase tracing is separate. +benchmark-source-edits samples="20" lines="128": build-qualification + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log /private/tmp/parser-source-edit-cost.log --require SOURCE-EDIT-BENCHMARK-OK -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (call-with-compiled-interface-trace (lambda () (eval (quote (import :gerbil-parser/t/benchmarks/source-edits/benchmark))) (eval (quote (main "{{samples}}" "{{lines}}")))))' + +# Verification-only model; production parser stays Scheme. +formal: + cd formal && lake build + +# Standalone Fold operation model; no dependency download or runtime build. +formal-fold: + cd formal && lean ParserEngine/Fold.lean + +test-fold-call-environment output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-call-environment-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-call-environment-test.ss > "{{output}}/fold-call-environment-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-call-environment-test.log')" = 4 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-call-environment-test.log" + rg --quiet '^OK$' "{{output}}/fold-call-environment-test.log" + +# Materialize the exact manifest pin from existing objects; no SDK/network build. +prepare-locked-lean-dependency source: + test ! -e formal/.lake/packages/LeanPoo + git -C "{{source}}" cat-file -e "$(jq -r '.packages[] | select(.name == "LeanPoo") | .rev' formal/lake-manifest.json)^{commit}" + git clone --no-hardlinks --no-checkout "{{source}}" formal/.lake/packages/LeanPoo + git -C formal/.lake/packages/LeanPoo remote set-url origin https://github.com/tao3k/lean-poo.git + git -C formal/.lake/packages/LeanPoo checkout --detach "$(jq -r '.packages[] | select(.name == "LeanPoo") | .rev' formal/lake-manifest.json)" + test "$(git -C formal/.lake/packages/LeanPoo rev-parse HEAD)" = "$(jq -r '.packages[] | select(.name == "LeanPoo") | .rev' formal/lake-manifest.json)" + +formal-run-plan: + quint test formal/quint/RunPlan.qnt --backend typescript --seed 1 --max-samples 1 + quint run formal/quint/RunPlan.qnt --backend typescript --seed 1 --max-samples 1000 --max-steps 20 --invariant RunSafety + +test-fold-condition output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-condition-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-condition-test.ss + +test-fold-frame-facts output load_path: + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O t/fold-frame-facts-test.ss + GERBIL_PATH="{{output}}" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil test -v 5 t/fold-frame-facts-test.ss > "{{output}}/fold-frame-facts-test.log" 2>&1 + test "$(rg --count '^CASE-OK ' '{{output}}/fold-frame-facts-test.log')" = 5 + rg '^CASE-OK |^HARNESS-OK|^OK$' "{{output}}/fold-frame-facts-test.log" + rg --quiet '^OK$' "{{output}}/fold-frame-facts-test.log" + +# Actual consumer declarations; engine owns generated-unit compilation. +prepare-org-fold-candidate consumer output load_path mode="tape-run": + test ! -e "{{output}}" + mkdir -p "{{output}}" + cd "{{consumer}}" && GERBIL_LOADPATH="{{load_path}}" gerbil env gxi t/generate-org-fold.ss "{{output}}/org-fold.ss" "{{output}}/org-fold-controls.ss" gerbil-parser/benchmarks/real-fold "{{mode}}" + GERBIL_PATH="{{output}}/sdk" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil env gxi "{{output}}/build-fold.ss" + just link-org-fold-candidate "{{output}}" "{{load_path}}" + +# Resume an existing generated closure without regenerating it or racing links. +test-aot-link-locks: + gxi t/benchmarks/parser-throughput/link-locks-test.ss + +link-org-fold-candidate output load_path: + gxi t/benchmarks/parser-throughput/link-locks.ss "{{output}}/lib/static" "{{output}}/sdk/lib/static" + GERBIL_PATH="{{output}}/sdk" GERBIL_LOADPATH="{{output}}/lib:{{load_path}}" gerbil compile -O -exe -o "{{output}}/org-fold" "{{output}}/org-fold.ss" + +# Native ASCENT is supplied by its upstream build owner, never rebuilt here. +evidence-controls: + uv run --project formal/evidence --locked python -m unittest discover -s formal/evidence -v + +evidence-models: + uv run --project formal/evidence --locked python formal/evidence/collect_models.py + +# Python joins actual Just-owned Fold controls; missing performance stays open. +evidence-fold-controls output load_path: + uv run --project formal/evidence --locked python formal/evidence/collect_fold_controls.py "{{output}}" "{{load_path}}" + +evidence-real-fold-controls output load_path actual: + uv run --project formal/evidence --locked python formal/evidence/collect_fold_controls.py "{{output}}" "{{load_path}}" --real-product "{{actual}}" + +evidence-engine: + uv run --project formal/evidence --locked python formal/evidence/collect_engine.py + +formal-protocol: + npm --prefix formal/quint test + +# SDK-owned module caching, source changes and initializer failure recovery. +test-module-loading: build-qualification + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log .data/module-loading-controls.log --require '^MODULE-LOADING-CONTROLS-OK$' -- gxi t/benchmarks/module-loading/controls.ss + +benchmark-module-loading mode="generation": build-qualification + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log .data/module-loading-{{mode}}.log --require '^MODULE-LOADING-CPU-PROOF-OK {{mode}}$' -- gxi t/benchmarks/module-loading/benchmark.ss {{mode}} + +prepare-formal-protocol: + npm --prefix formal/quint ci --offline --ignore-scripts + +# Same-owner callbacks cannot overwrite a newer committed Source history. +test-history-commit: build-qualification + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log .data/history-commit-controls.log --require HISTORY-COMMIT-CONTROLS-OK -- gxi scripts/tests/history-commit-test.ss + +# Frozen Git source, normal SDK compiler, shared immutable request dependencies. +prepare-capture-reference revision output=".data/selector-input/reference" scope="selector-input": + mkdir -p {{quote(output)}}/project + git archive {{quote(revision)}} gerbil.pkg src | tar -x -C {{quote(output)}}/project + cmp gerbil.pkg {{quote(output)}}/project/gerbil.pkg + gerbil env gxi t/benchmarks/capture-prefix/build-reference.ss {{quote(output)}}/project/src/runtime/lr-parser/recognition.ss {{quote(output)}}/lib {{quote(scope)}} + +benchmark-capture-prefix reference=".data/selector-input/reference/lib" scope="selector-input": build-qualification + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log .data/capture-prefix-cpu.log --require '^CAPTURE-COMPLETE-CPU-PROOF-OK$' -- gxi t/benchmarks/capture-prefix/benchmark.ss {{quote(reference)}} {{quote(scope)}} + +# Recorded lexical queries and LR replays diagnose costs; this is not a speedup gate. +diagnose-capture-cost: build-qualification + gerbil env gerbil compile -O t/benchmarks/capture-prefix/phase-cost.ss + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log .data/capture-phase-cost.log --require '^CAPTURE-COST-REPLAY-OK tokens=256 significant=256$' -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/benchmarks/capture-prefix/phase-cost") (preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") (eval (quote (import :gerbil-parser/t/benchmarks/capture-prefix/phase-cost :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process))) (eval (quote (main))) (eval (quote (test-child-process-exit! 0)))' + +# Inclusive diagnosis is separate from the matched complete-request CPU gate. +diagnose-capture-input: build-qualification + gerbil env gerbil compile -O t/benchmarks/capture-prefix/profile.ss + gerbil env gxi qualify.ss run --timeout 90 --idle-timeout 5 --log .data/capture-input-stages.log --require SELECTOR-STAGE-DIAGNOSIS-OK -- gxi -e '(load "t/fixtures/tla-sany-differential/preload.ss") (prefer-compiled-interfaces!) (preload-module "gerbil-parser/t/benchmarks/capture-prefix/profile") (preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") (eval (quote (import :gerbil-parser/t/benchmarks/capture-prefix/profile :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process))) (eval (quote (main))) (eval (quote (test-child-process-exit! 0)))' diff --git a/language-abi-support.ss b/language-abi-support.ss new file mode 100644 index 00000000..eb7c610d --- /dev/null +++ b/language-abi-support.ss @@ -0,0 +1,7 @@ +;;; -*- Gerbil -*- +;;; Public admission boundary for compiled language packs and the native language ABI. +(import (only-in ./src/ffi/language-handles register-language! release-language!)) +(export register-language! release-language!) + +(import (only-in ./src/compiler/language-pack generate-language-pack)) +(export generate-language-pack) diff --git a/language-build-support.ss b/language-build-support.ss new file mode 100644 index 00000000..844a7344 --- /dev/null +++ b/language-build-support.ss @@ -0,0 +1,12 @@ +;;; Public build strategy extension surface for downstream language repositories. +(import (only-in ./src/compiler/recursive-source RecursiveSourceStrategy. make-recursive-source-strategy) + (only-in ./src/compiler/build-strategy BuildStrategy. BuildStrategyContract make-bound-build-strategy emit-build-strategy) + (only-in ./src/compiler/runtime-strategy RustRuntimeStrategy. RustRuntimeStrategyContract make-rust-runtime-strategy) + (only-in ./src/compiler/fused-reduction FusedReductionStrategy. FusedReductionStrategyContract + make-fused-reduction-strategy) + (only-in ./language-support/development declare-language-build-strategy emit-language-build-strategy)) +(export RecursiveSourceStrategy. make-recursive-source-strategy + BuildStrategy. BuildStrategyContract make-bound-build-strategy emit-build-strategy + RustRuntimeStrategy. RustRuntimeStrategyContract make-rust-runtime-strategy + FusedReductionStrategy. FusedReductionStrategyContract make-fused-reduction-strategy + declare-language-build-strategy emit-language-build-strategy) diff --git a/language-source-support.ss b/language-source-support.ss new file mode 100644 index 00000000..deb9858b --- /dev/null +++ b/language-source-support.ss @@ -0,0 +1,8 @@ +;;; Public source parser services; author declarations use deflanguage. +(import (only-in ./src/language/source + deflanguage-parser-receipt + source-language? source-language-language source-language-version + source-language-contract source-language-digest source-language-result-catalog)) +(export deflanguage-parser-receipt + source-language? source-language-language source-language-version + source-language-contract source-language-digest source-language-result-catalog) diff --git a/language-support.ss b/language-support.ss index 05801210..0bcf169a 100644 --- a/language-support.ss +++ b/language-support.ss @@ -1,8 +1,13 @@ ;;; -*- Gerbil -*- ;;; Stable public facade for language implementations. -;;; Internal source-admission and fixture layout remains private under src/. +;;; Source adapters are common library modules; development services are opt-in. -(import (only-in ./src/language-support/antlr4-source +(import (only-in ./language-support/entry + deflanguage-parser-loader LanguageLoader. LanguageLoaderContract + +language-parser-entry-schema+ language-parser-entry-ref language-metadata-ref) + (only-in ./src/grammar/lexical-algebra deftext-profile) + (only-in ./src/runtime/region-scanner defregion-plan) + (only-in ./language-support/antlr4-source +antlr4-source-schema+ antlr4-rule? antlr4-rule-name @@ -29,7 +34,7 @@ antlr4-source-from-datum parse-antlr4-source parse-antlr4-source/expected) - (only-in ./src/language-support/javacc-source + (only-in ./language-support/javacc-source +javacc-source-schema+ javacc-production? javacc-production-name javacc-production-result javacc-production-line @@ -38,7 +43,7 @@ javacc-source-productions javacc-source-production javacc-source->datum javacc-source-from-datum parse-javacc-source parse-javacc-source/expected) - (only-in ./src/language-support/fixture + (only-in ./language-support/fixture defsyntax-fixture defsyntax-corpus syntax-fixture? @@ -51,14 +56,14 @@ syntax-fixture-expected-status syntax-fixture-root-kind syntax-fixture-required-kinds) - (only-in ./src/language-support/grammar-source + (only-in ./language-support/grammar-source defsyntax-iso-bnf-source defsyntax-antlr4-source - defsyntax-javacc-source - deflanguage-iso-bnf-grammar) - (only-in ./src/language-support/antlr4-language - deflanguage-antlr4-grammar) - (only-in ./src/language-support/iso-bnf + defsyntax-javacc-source) + (only-in ./language-support/iso-bnf-language iso-bnf) + (only-in ./language-support/antlr4-language + antlr4) + (only-in ./language-support/iso-bnf +iso-bnf-source-schema+ +iso-bnf-rule-overlay-schema+ iso-bnf-production? @@ -84,18 +89,21 @@ parse-iso-bnf-source/expected) (only-in ./src/language/grammar deflanguage - deflanguage-grammar defgrammar-syntax) - (only-in ./src/language/entry - +language-parser-entry-schema+ - deflanguage-parser - language-parser-entry-ref) + (only-in ./src/language/descriptor + language-grammar-with-parser-policy) + (only-in ./src/language/source + +source-language-schema+ + declare-source-language + source-language?) (only-in ./src/runtime/artifact parse-artifact-ref parse-artifact-roundtrip parse-artifact-success? parse-artifact-valid?)) -(export +antlr4-source-schema+ +(export deflanguage-parser-loader LanguageLoader. LanguageLoaderContract + defregion-plan deftext-profile + +antlr4-source-schema+ +javacc-source-schema+ +iso-bnf-source-schema+ +iso-bnf-rule-overlay-schema+ @@ -154,14 +162,16 @@ defsyntax-iso-bnf-source defsyntax-antlr4-source defsyntax-javacc-source - deflanguage-antlr4-grammar - deflanguage-iso-bnf-grammar + antlr4 + iso-bnf deflanguage - deflanguage-grammar defgrammar-syntax +language-parser-entry-schema+ - deflanguage-parser - language-parser-entry-ref + +source-language-schema+ + declare-source-language + language-grammar-with-parser-policy + source-language? + language-parser-entry-ref language-metadata-ref defsyntax-fixture defsyntax-corpus syntax-fixture? diff --git a/language-support/antlr4-language.ss b/language-support/antlr4-language.ss new file mode 100644 index 00000000..83340943 --- /dev/null +++ b/language-support/antlr4-language.ss @@ -0,0 +1,180 @@ +;;; -*- Gerbil -*- +;;; Expansion-time ANTLR4 language-pack compiler. + +(import (only-in ../src/grammar/lexical-algebra expand-text-lexical-rows) + (for-syntax :std/misc/ports + (only-in :gerbil-parser/src/compiler/language-artifact + make-language-declaration expand-language-declaration-syntax + materialize-compiled-language-artifact encode-compiled-language-artifact) + (only-in :gerbil-parser/src/compiler/parser-ir compile-parser) + (only-in :gerbil-parser/src/compiler/bound-ir bind-grammar-ir) + (only-in ./antlr4-source + antlr4-source->datum + antlr4-source-declaration-sources + antlr4-source-parser-grammar-rules + antlr4-source-parser-literals + antlr4-source-parser-syntax-kinds + parse-antlr4-source/expected)) + (only-in ./antlr4-source antlr4-source-from-datum) + (only-in ../src/runtime/language-artifact load-compiled-language-artifact/embedded) + (only-in :gerbil-parser/src/language/assembly assemble-language-parser) + (only-in :gerbil-parser/src/language/descriptor make-language-grammar)) +(export antlr4) + +;;; A language pack declares identity and pinned native grammar once; expansion +;;; materializes its catalog, Grammar IR, LALR table, and generated machine. +;; compile-antlr4-language +;; : (-> Syntax Syntax) +;; | doc m% +;; Lowers a pinned ANTLR4 grammar into the stable language v1 contract. +;; +;; # Examples +;; +;; ```scheme +;; (compile-antlr4-language language +;; (identity "lang" "v1" "contract.v1") +;; (reference "upstream-v1" "commit") +;; (digest "sha256:...") (source "grammar.g4") +;; (entrypoint program) (conflicts reject) (case-insensitive #f) +;; (lexical-profile +;; (token-bindings ("NAME" token identifier)) +;; (syntax-kinds (Name token (text)) (Punctuation token (text))) +;; (terminals (identifier Name) (punctuation Punctuation)) +;; (lexical-rules (identifier (identifier)) +;; (punctuation (source-literals))) +;; (extras))) +;; ;; => source catalog, grammar, Parser IR, machine, and descriptor bindings +;; ``` +;; Result: prefix-antlr4-source and parser products share one accepted source. +;; Runtime reconstructs the typed catalog from data; it never reparses source. +;; % +(defsyntax (compile-antlr4-language stx) + (syntax-case stx + (identity reference digest source entrypoint conflicts case-insensitive lexical-profile + token-bindings syntax-kinds terminals lexical-rules extras) + ((_ prefix (reference source-version source-commit) section ...) + (identifier? #'prefix) + (with-syntax ((language-name (datum->syntax #'prefix (symbol->string (syntax->datum #'prefix))))) + #'(compile-antlr4-language prefix (identity language-name #f #f) + (reference source-version source-commit) section ...))) + ((_ prefix + (identity language version contract) + (reference source-version source-commit) + (digest expected-digest) + (source path) + (entrypoint entry-name) + (conflicts conflict-policy) + (case-insensitive case-insensitive-value) + (lexical-profile (token-bindings binding ...) + (syntax-kinds kind ...) + (terminals terminal ...) + (lexical-rules lexical ...) + (extras extra ...))) + (and (identifier? #'prefix) + (stx-string? #'language) + (or (stx-string? #'version) (eq? (stx-e #'version) #f)) + (or (stx-string? #'contract) (eq? (stx-e #'contract) #f)) + (stx-string? #'source-version) + (stx-string? #'source-commit) + (stx-string? #'expected-digest) + (stx-string? #'path) + (identifier? #'entry-name)) + (let* ((resolved (gx#core-resolve-path #'path (stx-source stx))) + (content (call-with-input-file resolved read-all-as-string)) + (catalog + (parse-antlr4-source/expected + (stx-e #'language) (stx-e #'source-version) + (stx-e #'source-commit) (stx-e #'expected-digest) content)) + (bindings (syntax->datum #'(binding ...))) + (catalog-image + (materialize-compiled-language-artifact + `((schema . "gerbil-parser.source-catalog-image.v1") + (value . ,(antlr4-source->datum catalog))))) + (syntax-rows + (append (antlr4-source-parser-syntax-kinds catalog) + (syntax->datum #'(kind ...)))) + (rule-rows (antlr4-source-parser-grammar-rules catalog bindings)) + (literal-values (antlr4-source-parser-literals catalog bindings)) + (source-map + (antlr4-source-declaration-sources catalog (stx-e #'path)))) + ;; Submit the accepted source as a structured declaration. The shared + ;; compiler owns canonical admission, binding, publication and emission. + (let* ((sections + `((syntax-kinds ,@syntax-rows) + (terminals ,@(syntax->datum #'(terminal ...))) + (lexical-rules + ,@(map (lambda (row) + (if (and (list? row) (= (length row) 2) + (equal? (cadr row) '(source-literals))) + (list (car row) (cons 'literals literal-values)) + row)) + (syntax->datum #'(lexical ...)))) + (rules ,@rule-rows) + (extras ,@(syntax->datum #'(extra ...))) (keywords) + (parser-entrypoints (,(stx-e #'entry-name) parse pure)) + (recoveries + (,(stx-e #'entry-name) "GERBIL-PARSER-ANTLR4-SOURCE" preserve-source)) + (flow (source lexical) (lexical cst)))) + (declaration + (make-language-declaration + stx #'prefix (syntax->list #'(language version contract)) + (map (lambda (row) + (cons (car row) (datum->syntax #'prefix (cdr row)))) sections) + (syntax->datum #'conflict-policy) + (syntax->datum #'case-insensitive-value) + `(compile-antlr4-language ,(stx-e #'source-version) + ,(stx-e #'source-commit)) + source-map))) + (with-syntax ((catalog-binding + (datum->syntax #'prefix + (string->symbol + (string-append (symbol->string (stx-e #'prefix)) + "-antlr4-source")))) + (catalog-locator (datum->syntax #'prefix catalog-image)) + (catalog-payload (datum->syntax #'prefix (encode-compiled-language-artifact catalog-image))) + (profile-binding (datum->syntax #'prefix + (string->symbol (string-append (symbol->string (stx-e #'prefix)) + "-antlr4-token-bindings")))) + (profile-data bindings) + (compiled-declaration + (expand-language-declaration-syntax + declaration compile-parser bind-grammar-ir + #'assemble-language-parser #'make-language-grammar #'begin #'def))) + #'(begin + (def profile-binding 'profile-data) + (def catalog-binding + (antlr4-source-from-datum + (cdr (assq 'value + (load-compiled-language-artifact/embedded + "gerbil-parser.source-catalog-image.v1" 'catalog-locator 'catalog-payload))))) + compiled-declaration))))) + (_ (raise-syntax-error #f "invalid ANTLR4 language declaration" stx)))) + +;;; Token kinds, fields and terminal rows are derived from one lexical declaration. +(defsyntax (antlr4 stx) + (syntax-case stx (syntax rules reference digest source entrypoint conflicts case-insensitive token-bindings lex extras) + ((_ prefix + (syntax (reference version commit) (digest hash) (source path) + (entrypoint root) (conflicts policy) (case-insensitive insensitive) + (token-bindings binding ...) (lex lexical-row ...) (extras extra ...)) + (rules)) + (identifier? #'prefix) + (let* ((rows (expand-text-lexical-rows #'(lexical-row ...) stx)) + (datum (syntax->datum rows))) + (unless (and (list? datum) + (andmap (lambda (row) (and (list? row) (= (length row) 3) + (symbol? (car row)) (symbol? (cadr row)))) datum)) + (raise-syntax-error #f "antlr4 lexical declarations require token, kind and recognition" stx)) + (with-syntax (((kind ...) (datum->syntax #'prefix + (map (lambda (row) (list (cadr row) 'token '(text))) datum))) + ((terminal ...) (datum->syntax #'prefix + (map (lambda (row) (list (car row) (cadr row))) datum))) + ((lexical ...) (datum->syntax #'prefix + (map (lambda (row) (list (car row) (caddr row))) datum)))) + #'(compile-antlr4-language prefix + (reference version commit) (digest hash) (source path) + (entrypoint root) (conflicts policy) (case-insensitive insensitive) + (lexical-profile (token-bindings binding ...) (syntax-kinds kind ...) + (terminals terminal ...) (lexical-rules lexical ...) (extras extra ...)))))) + (_ (raise-syntax-error #f + "antlr4 vocabulary requires checked source syntax, one lexical declaration and an empty local rules block" stx)))) diff --git a/src/language-support/antlr4-source-lexer.ss b/language-support/antlr4-source-lexer.ss similarity index 100% rename from src/language-support/antlr4-source-lexer.ss rename to language-support/antlr4-source-lexer.ss diff --git a/src/language-support/antlr4-source.ss b/language-support/antlr4-source.ss similarity index 94% rename from src/language-support/antlr4-source.ss rename to language-support/antlr4-source.ss index e5e30138..99d74d07 100644 --- a/src/language-support/antlr4-source.ss +++ b/language-support/antlr4-source.ss @@ -401,23 +401,6 @@ (string-contains-from? text fragment 0 (string-length text) (string-length fragment))) -;; : (-> String Boolean) -(def (identifier-token-name? name) - (or (string-contains? name "IDENTIFIER") - (string=? name "PARAMETER_NAME"))) - -;; : (-> String Boolean) -(def (string-token-name? name) - (or (string-contains? name "CHARACTER_SEQUENCE") - (string-contains? name "STRING_LITERAL"))) - -;; : (-> String Boolean) -(def (number-token-name? name) - (or (string-contains? name "UNSIGNED_DECIMAL") - (string-contains? name "UNSIGNED_HEXADECIMAL") - (string-contains? name "UNSIGNED_OCTAL") - (string-contains? name "UNSIGNED_BINARY"))) - ;; : (-> (HashTable String Antlr4Rule) String (List String) (Maybe String)) (def (constant-lexer-expression rule-index name seen) (and (not (string-member? name seen)) @@ -435,14 +418,12 @@ (error "non-constant lexer rule" reference)))))))))) ;; : (-> (HashTable String Antlr4Rule) String GrammarExpr) -(def (parser-terminal-expression rule-index name) +(def (parser-terminal-expression rule-index name token-bindings) (def (lower name seen) (cond ((string=? name "EOF") (antlr4-empty)) ((constant-lexer-expression rule-index name '()) => values) - ((identifier-token-name? name) '(token identifier)) - ((string-token-name? name) '(token string)) - ((number-token-name? name) '(token number)) + ((assoc name token-bindings) => cdr) ((string-member? name seen) (error "cyclic ANTLR4 lexer rule" name)) (else @@ -451,7 +432,7 @@ (error "ANTLR4 lexer rule expected" name)) (with-catch (lambda (_) - (error "OpenGQL lexer token requires an admitted scanner primitive" + (error "ANTLR4 lexer token requires an explicit lexical binding" name)) (lambda () (parse-rule-grammar-expression @@ -462,8 +443,8 @@ ;; ANTLR alternatives are ordered decisions, including outside direct left ;; recursion. Project every alternative rank into the native LR precedence -;; model so the adapter preserves upstream decision semantics instead of -;; manufacturing an unordered selective-GLR fork. +;; model used by this adapter. This projection avoids an unordered fork for +;; the admitted controls; it is not a general proof of ANTLR decision equivalence. ;; : (-> (List GrammarExpr) Integer (List GrammarExpr) (List GrammarExpr)) (def (apply-antlr-precedence-from rest rank found) (if (null? rest) @@ -537,15 +518,24 @@ (cons 'choice (map inline-alternative (cdr expression))) (inline-alternative expression))) -;; : (-> Antlr4Source (List GrammarRule)) -(def (antlr4-source-parser-grammar-rules source) +;; : (-> Antlr4Source (Optional (List TerminalBinding)) (List GrammarRule)) +;; Nonliteral terminals require the importing language's explicit profile. +(def (antlr4-source-parser-grammar-rules source (token-bindings '())) + (unless (and (list? token-bindings) + (andmap (lambda (row) (and (pair? row) (string? (car row)) + (pair? (cdr row)))) token-bindings) + (let loop ((rows token-bindings) (seen '())) + (or (null? rows) + (and (not (member (caar rows) seen)) + (loop (cdr rows) (cons (caar rows) seen)))))) + (error "invalid or duplicate ANTLR4 lexical bindings" token-bindings)) (let ((rule-index (antlr4-source-rule-index source)) (terminal-cache (make-table test: equal?))) (def (resolve reference) (if (uppercase-rule-name? reference) (or (table-ref terminal-cache reference #f) (let (expression - (parser-terminal-expression rule-index reference)) + (parser-terminal-expression rule-index reference token-bindings)) (table-set! terminal-cache reference expression) expression)) (list 'reference (string->symbol reference)))) @@ -588,12 +578,12 @@ (grammar-literals (cadddr expression))) (else '()))) -;; : (-> Antlr4Source (List String)) -(def (antlr4-source-parser-literals source) +;; : (-> Antlr4Source (Optional (List TerminalBinding)) (List String)) +(def (antlr4-source-parser-literals source (token-bindings '())) (unique-strings (apply append (map (lambda (row) (grammar-literals (cadr row))) - (antlr4-source-parser-grammar-rules source))))) + (antlr4-source-parser-grammar-rules source token-bindings))))) ;;; ANTLR rule names remain native grammar names in Bound Grammar IR. The ;;; catalog currently owns rule identity but not token offsets, so location is diff --git a/language-support/command-grammar.ss b/language-support/command-grammar.ss new file mode 100644 index 00000000..df692127 --- /dev/null +++ b/language-support/command-grammar.ss @@ -0,0 +1,197 @@ +;;; Source command grammar elaborates to the admitted command core. +;;; FIRST and continuations are local to calls, never a global guessed stop set. +(import (only-in :std/list/list delete-duplicates/hash)) +(export lower-command-grammar command-grammar-entrypoints) +;;; Entry positions are semantic capabilities, independent of private rule names. +(def (command-grammar-entrypoints rules (diagnose #f)) + (filter-map (lambda (row) + (let (expression (cadr row)) + (and (pair? expression) (eq? (car expression) 'at) + (begin + (unless (and (= (length expression) 3) + (memq (cadr expression) '(assignment-tail pipeline-head))) + (if diagnose (diagnose "unknown command entry position" expression) (error "unknown command entry position" expression))) + (list (cadr expression) (car row)))))) rules)) +(def (lower-command-grammar rules (diagnose #f)) + (def (fail message value) + (if diagnose (diagnose message value) (error message value))) + (def (unique values) (delete-duplicates/hash values)) + (unless (and (list? rules) (pair? rules) + (andmap (lambda (row) (and (list? row) (= (length row) 2) (symbol? (car row)))) rules) + (= (length rules) (length (unique (map car rules))))) + (fail "source grammar requires distinct named command rules" rules)) + (def entries (command-grammar-entrypoints rules fail)) + (unless (= (length entries) (length (unique (map car entries)))) + (fail "duplicate command entry position" entries)) + (set! rules (map (lambda (row) + (let (expression (cadr row)) + (list (car row) (if (and (pair? expression) (eq? (car expression) 'at)) + (caddr expression) expression)))) rules)) + (def (arity expression n) + (unless (= (length expression) n) (fail "invalid source grammar constructor arity" expression))) + (def (union triggers) + (def (flatten trigger) + (if (and trigger (eq? (car trigger) 'or)) (apply append (map flatten (cdr trigger))) + (if trigger (list trigger) '()))) + (let* ((items (unique (apply append (map flatten triggers)))) + (groups '())) + (for-each (lambda (item) + (let (prior (find (lambda (row) (and (eq? (car row) (car item)) (memq (car row) '(word operator)))) groups)) + (if prior + (set-cdr! prior (if (or (null? (cdr prior)) (null? (cdr item))) '() (unique (append (cdr prior) (cdr item))))) + (set! groups (append groups (list (map values item))))))) items) + (cond ((null? groups) #f) ((null? (cdr groups)) (car groups)) (else (cons 'or groups))))) + (def (nullable? expression) + (and (pair? expression) + (case (car expression) + ((optional repeat) #t) + ((seq) (andmap nullable? (cdr expression))) + ((choice) (ormap nullable? (cdr expression))) + ((field) (nullable? (caddr expression))) (else #f)))) + (def (sequence expression) + (if (and (pair? expression) (eq? (car expression) 'seq)) (cdr expression) (list expression))) + (def (first-sequence body seen) + (if (null? body) #f + (union (list (first (car body) seen) + (and (nullable? (car body)) (first-sequence (cdr body) seen)))))) + (def (first expression (seen '())) + (cond ((string? expression) (list 'word expression)) + ((symbol? expression) + (case expression + ((word command command-list) '(word)) + (else (fail "unresolved source grammar reference" expression)))) + ((not (and (list? expression) (pair? expression))) (fail "invalid source grammar expression" expression)) + (else + (case (car expression) + ((word operator role newline) expression) + ((adjacent) expression) + ((field) (arity expression 3) (first (caddr expression) seen)) + ((node) (first-sequence (cddr expression) seen)) + ((seq) (first-sequence (cdr expression) seen)) + ((choice) (union (map (lambda (child) (first child seen)) (cdr expression)))) + ((optional repeat empty-allowed) (arity expression 2) (first (cadr expression) seen)) + ((balanced) (arity expression 4) (first (cadr expression) seen)) + ((reference) + (arity expression 2) + (let* ((name (cadr expression)) (rule (assq name rules))) + (unless rule (fail "unresolved source grammar reference" expression)) + (when (memq name seen) (fail "source grammar FIRST cycle" name)) + (first (cadr rule) (cons name seen)))) + (else (fail "unknown source grammar constructor" expression)))))) + (def (overlap? left right) + (and left right + (cond ((eq? (car left) 'or) (ormap (lambda (child) (overlap? child right)) (cdr left))) + ((eq? (car right) 'or) (overlap? right left)) + ((and (eq? (car left) (car right)) (memq (car left) '(word operator))) + (or (null? (cdr left)) (null? (cdr right)) + (ormap (lambda (value) (member value (cdr right))) (cdr left)))) + ((eq? (car left) 'role) #t) + ((eq? (car right) 'role) #t) + (else (equal? left right))))) + (def (continuation rest follow) + (union (list (first-sequence rest '()) (and (andmap nullable? rest) follow)))) + (def (lower-body body follow) + (if (null? body) '() + (append (lower (car body) (continuation (cdr body) follow)) + (lower-body (cdr body) follow)))) + (def (lower-field field expression follow) + (unless (symbol? field) (fail "source field requires an identifier" field)) + (cond + ((string? expression) (list (list 'take field (list 'word expression)))) + ((symbol? expression) + (case expression + ((word command) (list (list expression field))) + ((command-list) + (unless follow (fail "command-list requires an explicit continuation" expression)) + (list (list 'list field follow #f))) + (else (fail "unresolved source grammar reference" expression)))) + ((not (and (list? expression) (pair? expression))) (fail "invalid source field expression" expression)) + (else + (case (car expression) + ((word operator role newline) + (if (and (memq (car expression) '(word operator role newline)) + (null? (cdr expression))) + (list (list 'raw field)) + (list (list 'take field expression)))) + ((reference) (arity expression 2) (first expression) (list (list 'call field (cadr expression)))) + ((empty-allowed) + (arity expression 2) + (unless (and (eq? (cadr expression) 'command-list) follow) + (fail "empty-allowed requires a command-list with a continuation" expression)) + (list (list 'list field follow #t))) + ((node) (list (append (list 'node field (cadr expression)) (lower-body (cddr expression) follow)))) + ((adjacent) (apply append (map (lambda (item) (lower-field field item follow)) (cdr expression)))) + (else (fail "unsupported source field expression" expression)))))) + (def (lower expression follow) + (unless (and (list? expression) (pair? expression)) + (fail "command grammar requires field-labelled expressions" expression)) + (case (car expression) + ((field) (arity expression 3) (lower-field (cadr expression) (caddr expression) follow)) + ((seq) (lower-body (cdr expression) follow)) + ((node) (append (list (list 'as (cadr expression))) (lower-body (cddr expression) follow))) + ((optional repeat) + (arity expression 2) + (let* ((child (cadr expression)) (trigger (first child))) + (unless (and trigger (not (nullable? child))) + (fail "source repetition/optional body must consume input" expression)) + (if (eq? (car expression) 'optional) + (list (cons 'optional (cons trigger (lower child follow)))) + (let (stops? (overlap? trigger follow)) + (list (cons (if stops? 'until 'many) + (cons (if stops? follow trigger) (lower child (if stops? follow (union (list trigger follow))))))))))) + ((choice) + (unless (pair? (cdr expression)) (fail "source choice requires alternatives" expression)) + (let (alternatives (cdr expression)) + (when (ormap nullable? alternatives) (fail "source choice requires consuming alternatives" expression)) + (if (and (= (length alternatives) 2) (pair? (car alternatives)) (eq? (caar alternatives) 'node)) + (list (list 'branch (first (car alternatives)) + (lower (car alternatives) follow) (lower (cadr alternatives) follow))) + (list (cons 'choose (map (lambda (child) (cons (first child) (lower child follow))) alternatives)))))) + ((balanced) + (arity expression 4) + (let ((open (cadr expression)) (close (caddr expression)) (body (cadddr expression))) + (unless (and (pair? open) (eq? (car open) 'field) (pair? close) (eq? (car close) 'field) + (pair? (caddr open)) (eq? (caaddr open) 'adjacent) + (pair? (caddr close)) (eq? (caaddr close) 'adjacent)) + (fail "balanced grammar requires field-labelled adjacent delimiters" expression)) + (let ((opens (cdaddr open)) (closes (cdaddr close))) + (unless (and (pair? opens) (= (length opens) (length closes)) + (andmap (lambda (item) (equal? item (car opens))) opens) + (andmap (lambda (item) (equal? item (car closes))) closes)) + (fail "balanced grammar requires repeated matching delimiter tokens" expression)) + (append (lower open follow) + (list (append (list 'balance (car opens) (car closes) (length opens) (cadr open) (cadr close)) + (lower body follow))))))) + (else (fail "unknown source grammar constructor" expression)))) + (map (lambda (row) + (let* ((name (car row)) (expression (cadr row))) + (when (getenv "GERBIL_PARSER_SOURCE_TRACE" #f) + (displayln "SOURCE-COMMAND-LOWER " name) (force-output)) + (unless (and (list? expression) (>= (length expression) 3) (eq? (car expression) 'node)) + (fail "command rule requires a node constructor" row)) + (let* ((kind (cadr expression)) (body (cddr expression)) + ;; Infer a token prefix up to the first contextual operation. + (prefix (let loop ((remaining body) (result '())) + (if (null? remaining) (reverse result) + (let* ((item (car remaining)) + (leaf (and (pair? item) (eq? (car item) 'field) + (begin (arity item 3) (caddr item)))) + (selector (and leaf + (cond ((string? leaf) (list 'word leaf)) + ((eq? leaf 'word) '(word)) + ((and (pair? leaf) (memq (car leaf) '(word operator))) leaf) + (else #f))))) + (if selector (loop (cdr remaining) (cons selector result)) (reverse result)))))) + (position (find (lambda (entry) (eq? (cadr entry) name)) entries)) + (trigger (cond + (position (if (eq? (car position) 'assignment-tail) + (list 'manual (first-sequence body '())) '(manual))) + ((> (length prefix) 1) (cons 'lookahead prefix)) + (else (first-sequence body '())))) + (_prefix (unless trigger (fail "source entry requires a token prefix" expression))) + ;; Numeric encoding of (specific head, prefix width), never a field + ;; or rule-name policy. Prefix length is bounded by admission. + (head (if (memq (car trigger) '(lookahead adjacent)) (cadr trigger) trigger)) + (width (if (memq (car trigger) '(lookahead adjacent)) (length (cdr trigger)) 1)) + (rank (if position 0 (+ (if (and (memq (car head) '(word operator)) (pair? (cdr head))) 33 0) width)))) + (append (list name trigger rank kind) (lower-body body #f))))) rules)) diff --git a/language-support/development.ss b/language-support/development.ss new file mode 100644 index 00000000..1db4db49 --- /dev/null +++ b/language-support/development.ss @@ -0,0 +1,387 @@ +;;; -*- Gerbil -*- +;;; Optional corpus, scanner and build services for language development. + +(import ./loader-identity + (only-in ../src/language/entry parse-language-source call-with-language-parser-policy) + (only-in ./fixture + syntax-fixture? syntax-fixture-copy syntax-fixture-source-digest syntax-fixture-id syntax-fixture-source + syntax-fixture-language syntax-fixture-version syntax-fixture-contract + syntax-fixture-expected-status syntax-fixture-root-kind syntax-fixture-required-kinds) + (only-in :clan/poo/object .o .ref .cc .slot? object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ../src/runtime/identity sha256-text) + (only-in ../src/runtime/source-scanner source-scanner-for-source?) + (only-in ../src/compiler/machine parser-machine-grammar-digest parser-machine-direct-source) + (only-in ../src/compiler/build-strategy BuildStrategyContract emit-build-strategy) + (only-in ../src/runtime/parser parse-source) + (only-in ../src/runtime/lr-parser current-lr-branch-budget) + (only-in ../src/runtime/cst parse-artifact->cst) + (only-in ../src/runtime/token make-token) + (only-in ../src/runtime/artifact + parse-artifact-success? parse-artifact-valid? parse-artifact-ref + parse-artifact-roundtrip parse-artifact-status event-kind + parse-artifact-events token-event? token-event-token-kind + token-event-lexeme event-start event-end make-failure-parse-artifact) + (only-in ../src/language/descriptor + language-grammar? language-grammar-contract language-grammar-language + language-grammar-machine language-grammar-observability + language-grammar-version language-grammar-parser-policy + language-parser-policy-branch-budget language-parser-policy-cst-validator) + (only-in ../src/language/source + parse-source-language source-language? + source-language-contract source-language-language + source-language-version source-language-digest source-language-scanner-factory)) +(export deflanguage-development-loader LanguageDevelopmentLoader. LanguageDevelopmentLoaderContract + +language-parser-entry-schema+ language-parser-entry-ref language-metadata-ref parse-language-source + language-loader-fixtures language-loader-fixture-count language-loader-fixture + check-language-loader-fixtures! call-with-language-parser-policy + declare-language-source-scan-worker make-language-scan-worker + declare-language-fixture-test run-language-test + declare-language-build-strategy emit-language-build-strategy) + +;;; Fixture indexes are an engine-derived cached POO slot. Authors provide only +;;; the fixture value list. A new .cc/.o object recomputes its effective index. +(defstruct fixture-catalog (descriptor origin all accepted rejected by-id counts)) +(def (prepare-loader-fixtures descriptor fixtures) + (unless (list? fixtures) (error "language loader fixtures must be a list")) + (let ((seen (make-hash-table)) (all '()) (accepted '()) (rejected '())) + (for-each (lambda (fixture) + (unless (syntax-fixture? fixture) (error "language loader requires syntax fixtures")) + (let ((id (syntax-fixture-id fixture)) (source (syntax-fixture-source fixture)) + (status (syntax-fixture-expected-status fixture)) (kinds (syntax-fixture-required-kinds fixture))) + (unless (and (string? id) (positive? (string-length id)) (not (hash-get seen id)) + (string? source) (equal? (sha256-text source) (syntax-fixture-source-digest fixture)) + (equal? (syntax-fixture-language fixture) (descriptor-ref descriptor 'language)) + (equal? (syntax-fixture-version fixture) (descriptor-ref descriptor 'version)) + (equal? (syntax-fixture-contract fixture) (descriptor-ref descriptor 'contract)) + (list? kinds) (every symbol? kinds) + (case status + ((accepted) (symbol? (syntax-fixture-root-kind fixture))) + ((rejected) (and (not (syntax-fixture-root-kind fixture)) (null? kinds))) + (else #f))) + (error "invalid, foreign or duplicate language fixture" id)) + (let (owned (syntax-fixture-copy fixture)) + (hash-put! seen (syntax-fixture-id owned) owned) + (set! all (cons owned all)) + (if (eq? status 'accepted) (set! accepted (cons owned accepted)) (set! rejected (cons owned rejected)))))) fixtures) + (make-fixture-catalog descriptor fixtures (reverse all) (reverse accepted) (reverse rejected) seen (vector (length all) (length accepted) (length rejected))))) +(def (loader-fixture-catalog loader) + (unless (object? loader) (error "fixture query requires a language loader")) + (let (catalog (.ref loader 'fixture-catalog)) + (unless (and (fixture-catalog? catalog) + (eq? (fixture-catalog-descriptor catalog) (.ref loader 'descriptor)) + (eq? (fixture-catalog-origin catalog) (.ref loader 'fixtures))) + (error "fixture catalog does not belong to the effective loader")) + catalog)) +(def (catalog-fixtures catalog status) + (case status + ((#f) (fixture-catalog-all catalog)) + ((accepted) (fixture-catalog-accepted catalog)) + ((rejected) (fixture-catalog-rejected catalog)) + (else (error "unknown fixture status" status)))) +(def (language-loader-fixtures loader (status #f)) + (map syntax-fixture-copy (catalog-fixtures (loader-fixture-catalog loader) status))) +(def (language-loader-fixture-count loader (status #f)) + (let (catalog (loader-fixture-catalog loader)) + (vector-ref (fixture-catalog-counts catalog) + (case status ((#f) 0) ((accepted) 1) ((rejected) 2) (else (error "unknown fixture status" status)))))) +(def (language-loader-fixture loader id) + (unless (string? id) (error "fixture lookup requires a string id" id)) + (let (fixture (hash-get (fixture-catalog-by-id (loader-fixture-catalog loader)) id)) + (and fixture (syntax-fixture-copy fixture)))) + +;;; Test declarations carry an engine opcode and exact descriptor identity. +;;; Their constructor is private; a language cannot substitute a success callback. +(defstruct language-fixture-test (descriptor)) + +(def (declare-language-fixture-test descriptor) + (unless (or (language-grammar? descriptor) (source-language? descriptor)) + (error "fixture test requires a language descriptor")) + (make-language-fixture-test descriptor)) + +(def (declared-test-services? rows descriptor) + (and (list? rows) + (let loop ((remaining rows) (seen '())) + (or (null? remaining) + (let (row (car remaining)) + (and (pair? row) (symbol? (car row)) + (not (memq (car row) seen)) + (language-fixture-test? (cdr row)) + (eq? descriptor (language-fixture-test-descriptor (cdr row))) + (loop (cdr remaining) (cons (car row) seen)))))))) + +(def (run-language-test loader name) + (validate LanguageDevelopmentLoaderContract loader) + (unless (and (object? loader) (symbol? name)) + (error "invalid language test request" name)) + (let* ((descriptor (.ref loader 'descriptor)) (rows (.ref loader 'tests))) + (unless (declared-test-services? rows descriptor) + (error "language tests are not declared for this descriptor")) + (unless (assq name rows) (error "unknown declared language test" name)) + (check-language-loader-fixtures! loader))) + +;;; Source workers are derived from the admitted engine binding; no user factory. +(defstruct language-source-scan-worker (descriptor factory)) + +(def (declare-language-source-scan-worker descriptor) + (unless (source-language? descriptor) + (error "source scan worker requires a source language declaration")) + (let (factory (source-language-scanner-factory descriptor)) + (unless (procedure? factory) (error "source engine has no scanner factory")) + (make-language-source-scan-worker descriptor factory))) + +(def (declared-scan-workers? rows descriptor) + (and (list? rows) + (let loop ((remaining rows) (seen '())) + (or (null? remaining) + (let (row (car remaining)) + (and (pair? row) (symbol? (car row)) + (not (memq (car row) seen)) + (language-source-scan-worker? (cdr row)) + (eq? descriptor (language-source-scan-worker-descriptor (cdr row))) + (loop (cdr remaining) (cons (car row) seen)))))))) + +(def (make-language-scan-worker loader name source) + (unless (and (object? loader) (symbol? name) (string? source)) + (error "invalid language scan worker request" name)) + (let* ((descriptor (.ref loader 'descriptor)) (rows (.ref loader 'scan-workers))) + (unless (declared-scan-workers? rows descriptor) + (error "language scan workers are not declared for this descriptor")) + (let (row (assq name rows)) + (unless row (error "unknown declared language scan worker" name)) + (let (worker ((language-source-scan-worker-factory (cdr row)) source)) + (unless (source-scanner-for-source? worker source) + (error "declared scan worker returned an invalid or foreign source executor" name)) + worker)))) + +;;; Optional generated-recognizer test profile is a POO value, bound to the +;;; effective Grammar IR machine. It describes existing generated routes; +;;; it cannot replace the Loader's parse method or fixture test service. +(def (loader-native-test-profile? candidate descriptor) + (or (not (.slot? candidate 'native-test-profile)) + (let (profile (.ref candidate 'native-test-profile)) + (and (language-grammar? descriptor) (object? profile) + (andmap (lambda (slot) (.slot? profile slot)) '(profile digest source lexer)) + (eq? (.ref profile 'profile) 'recursive-source) + (let (machine (language-grammar-machine descriptor)) + (and (equal? (.ref profile 'digest) (parser-machine-grammar-digest machine)) + (procedure? (.ref profile 'source)) + (eq? (.ref profile 'source) (parser-machine-direct-source machine)) + (procedure? (.ref profile 'lexer)))))))) + +(def (declare-language-build-strategy name strategy) + (unless (symbol? name) (error "build strategy name must be a symbol" name)) + (validate BuildStrategyContract strategy) + (cons name strategy)) + +;;; Build strategies are admitted once with the Loader. No build method is +;;; invoked during parser dispatch, and no language-specific route is selected. +(def (declared-build-strategies? rows descriptor) + (with-catch (lambda (_) #f) + (lambda () + (and (list? rows) + (let loop ((remaining rows) (seen '())) + (or (null? remaining) + (let (row (car remaining)) + (and (pair? row) (symbol? (car row)) (not (memq (car row) seen)) + (begin (validate BuildStrategyContract (cdr row)) #t) + (eq? (.ref (cdr row) 'descriptor) descriptor) + (loop (cdr remaining) (cons (car row) seen)))))))))) + +(def (emit-language-build-strategy loader name port) + (validate LanguageDevelopmentLoaderContract loader) + (let (row (assq name (.ref loader 'build-strategies))) + (unless row (error "unknown declared language build strategy" name)) + (emit-build-strategy (cdr row) port))) + +(def (loader-shape? candidate) + (and (object? candidate) + (andmap (lambda (slot) (.slot? candidate slot)) + '(schema descriptor language version contract capabilities .parse + grammars metadata fixtures fixture-catalog tests scan-workers build-strategies)) + (equal? (.ref candidate 'schema) +language-parser-entry-schema+) + (let (descriptor (.ref candidate 'descriptor)) + (and (or (language-grammar? descriptor) (source-language? descriptor)) + (andmap (lambda (slot) + (let (value (descriptor-ref descriptor slot)) + (and (string? value) (positive? (string-length value)) + (equal? value (.ref candidate slot))))) + '(language version contract)) + (equal? (.ref candidate 'capabilities) + (descriptor-capabilities descriptor)) + (procedure? (.ref candidate '.parse)) + (let (grammars (.ref candidate 'grammars)) + (and (list? grammars) (memq descriptor grammars) + (every (lambda (grammar) + (or (language-grammar? grammar) (source-language? grammar))) grammars))) + (loader-metadata? (.ref candidate 'metadata) descriptor) + (loader-native-test-profile? candidate descriptor) + (with-catch (lambda (_) #f) (lambda () (loader-fixture-catalog candidate) #t)) + (declared-test-services? (.ref candidate 'tests) descriptor) + (declared-scan-workers? (.ref candidate 'scan-workers) descriptor) + (declared-build-strategies? (.ref candidate 'build-strategies) descriptor))))) + +(define-type (LanguageDevelopmentLoaderContract @ PooFlowContract.) + identity: 'gerbil-parser/language-loader + .classify: (lambda (candidate context) + (let (accepted? (loader-shape? candidate)) + (poo-flow-classification-evidence + 'gerbil-parser/language-loader candidate accepted? + (if accepted? '() '((expected gerbil-parser/language-loader))) + context)))) + +;;; Slot inheritance is resolved once during loading; parsing uses the bound +;;; engine method without per-call POO contract admission or compiler work. +(def LanguageDevelopmentLoader. + (.o (:: self) + schema: +language-parser-entry-schema+ + descriptor: #f + ;; Language-facing extension slots are inherited POO values. Admission + ;; checks fixture data without parsing it or executing test/scan workers. + (grammars (list (.ref self 'descriptor))) + metadata: (.o) + fixtures: '() + (fixture-catalog (prepare-loader-fixtures (.ref self 'descriptor) (.ref self 'fixtures))) + (tests (list (cons 'fixtures (declare-language-fixture-test (.ref self 'descriptor))))) + scan-workers: '() + build-strategies: '() + (language (descriptor-ref (.ref self 'descriptor) 'language)) + (version (descriptor-ref (.ref self 'descriptor) 'version)) + (contract (descriptor-ref (.ref self 'descriptor) 'contract)) + (capabilities (descriptor-capabilities (.ref self 'descriptor))) + (.parse (let (descriptor (.ref self 'descriptor)) + (lambda (source) (parse-language-source descriptor source)))))) + +;;; Public slot extensions are ordinary POO expressions and may inherit from +;;; a user prototype. Engine identity/dispatch slots are sealed by this macro. +;;; Additional contracts validate the effective object after inheritance. +(defsyntax (deflanguage-development-loader stx) + (syntax-case stx (@ descriptor parse slots contracts grammar source) + ((_ (binding marker loader-self prototype) + (descriptor descriptor-value) + (parse parse-binding) + (slots slot ...) + (contracts extension-contract ...)) + (and (identifier? #'binding) (identifier? #'loader-self) + (eq? (syntax->datum #'marker) '::) + (identifier? #'parse-binding)) + (let () + (def (check-slot! name location) + (when (memq name '(schema descriptor language version contract + capabilities .parse fixture-catalog)) + (raise-syntax-error #f + "language loader extension overrides an engine slot" location))) + (let loop ((rows (stx-map (lambda (row) row) #'(slot ...)))) + (unless (null? rows) + (let* ((row (car rows)) (datum (syntax->datum row))) + (if (keyword? datum) + (begin + (check-slot! (string->symbol (keyword->string datum)) row) + (when (null? (cdr rows)) + (raise-syntax-error #f "language loader slot requires a value" row)) + (loop (cddr rows))) + (begin + (syntax-case row () + ((name . _) + (begin + (unless (identifier? #'name) + (raise-syntax-error #f "language loader requires POO slot forms" row)) + (check-slot! (syntax->datum #'name) row))) + (_ (raise-syntax-error #f "language loader requires POO slot forms" row))) + (loop (cdr rows))))))) + #'(begin + (def binding + (let* ((author-extensions (.o (:: loader-self prototype) slot ...)) + (parser-descriptor-value + (bind-parser-identity descriptor-value (.ref author-extensions 'metadata))) + (candidate + (.o (:: loader-self author-extensions) + descriptor: parser-descriptor-value + ;; Bind these methods at the leaf; a prototype cannot + ;; replace identity or dispatch through inherited slots. + schema: +language-parser-entry-schema+ + (language (descriptor-ref (.ref loader-self 'descriptor) 'language)) + (version (descriptor-ref (.ref loader-self 'descriptor) 'version)) + (contract (descriptor-ref (.ref loader-self 'descriptor) 'contract)) + (capabilities (descriptor-capabilities (.ref loader-self 'descriptor))) + (.parse (let (descriptor (.ref loader-self 'descriptor)) + (lambda (source) + (parse-language-source descriptor source)))) + (fixture-catalog (prepare-loader-fixtures (.ref loader-self 'descriptor) (.ref loader-self 'fixtures))) + ))) + (let* ((metadata (.ref candidate 'metadata)) + (admitted (.cc candidate 'metadata + (bind-loader-metadata (.ref candidate 'descriptor) metadata)))) + (validate LanguageDevelopmentLoaderContract admitted) + (validate extension-contract admitted) ... + admitted))) + (def parse-binding (.ref binding '.parse))))) + ((_ binding (kind descriptor-value) (parse parse-binding) section ...) + (let () + (unless (identifier? #'parse-binding) + (raise-syntax-error #f "language loader parse binding must be an identifier" #'parse-binding)) + (def normalized-binding + (if (identifier? #'binding) + #'(binding :: self LanguageDevelopmentLoader.) + (syntax-case #'binding (@) + ((name @ prototype) + (identifier? #'name) + #'(name :: self prototype)) + ((name marker self prototype) + (and (identifier? #'name) (identifier? #'self) + (eq? (syntax->datum #'marker) '::)) + #'(name :: self prototype)) + (_ (raise-syntax-error #f "invalid language loader binding" #'binding))))) + (def normalized-descriptor + (case (syntax->datum #'kind) + ((descriptor) #'descriptor-value) + ((grammar) #'(require-grammar-descriptor descriptor-value)) + ((source) #'(require-source-descriptor descriptor-value)) + (else (raise-syntax-error #f "invalid language loader descriptor kind" #'kind)))) + (def slot-section #'(slots)) + (def contract-section #'(contracts)) + (def seen-slots? #f) + (def seen-contracts? #f) + (for-each + (lambda (row) + (syntax-case row (slots contracts) + ((slots slot ...) + (begin + (when (or seen-slots? seen-contracts?) + (raise-syntax-error #f "duplicate or out-of-order language loader slots" row)) + (set! seen-slots? #t) (set! slot-section row))) + ((contracts contract ...) + (begin + (when seen-contracts? + (raise-syntax-error #f "duplicate language loader contracts" row)) + (set! seen-contracts? #t) (set! contract-section row))) + (_ (raise-syntax-error #f "unknown language loader section" row)))) + (stx-map (lambda (row) row) #'(section ...))) + (with-syntax ((normalized-binding normalized-binding) + (normalized-descriptor normalized-descriptor) + (slot-section slot-section) (contract-section contract-section)) + #'(deflanguage-development-loader normalized-binding + (descriptor normalized-descriptor) (parse parse-binding) + slot-section contract-section)))) + (_ (raise-syntax-error #f "invalid language loader declaration" stx)))) + +;;; Named test services accept a loader. Fixture values are ordinary POO slots; +;;; the engine verifies statuses, lossless source and required public node kinds. +(def (check-language-loader-fixtures! loader) + (let (fixtures (fixture-catalog-all (loader-fixture-catalog loader))) + (map + (lambda (fixture) + (let* ((source (syntax-fixture-source fixture)) + (artifact ((.ref loader '.parse) source)) + (kinds (filter-map + (lambda (event) + (and (eq? (event-kind event) 'start-node) (vector-ref event 2))) + (parse-artifact-events artifact)))) + (unless (and (eq? (parse-artifact-status artifact) (syntax-fixture-expected-status fixture)) + (equal? (parse-artifact-roundtrip artifact) source) + (or (not (parse-artifact-success? artifact)) + (and (pair? kinds) (eq? (car kinds) (syntax-fixture-root-kind fixture)))) + (every (lambda (kind) (memq kind kinds)) (syntax-fixture-required-kinds fixture))) + (error "language fixture conformance failed" (syntax-fixture-id fixture))) + artifact)) fixtures))) diff --git a/language-support/entry.ss b/language-support/entry.ss new file mode 100644 index 00000000..63f99c0e --- /dev/null +++ b/language-support/entry.ss @@ -0,0 +1,210 @@ +;;; -*- Gerbil -*- +;;; Common parsing entry without corpus, test or build dependencies. + +(import ./loader-identity + (only-in ./source-schema source-schema? source-schema-receipt) + (only-in ../src/language/entry parse-language-source call-with-language-parser-policy) + (only-in :clan/poo/object .o .ref .cc .slot? object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ../src/compiler/machine parser-machine-grammar-digest install-parser-machine-backends!) + (only-in ../src/language/descriptor + language-grammar? language-grammar-contract language-grammar-language + language-grammar-machine language-grammar-observability + language-grammar-version language-grammar-parser-policy + language-parser-policy-branch-budget language-parser-policy-cst-validator) + (only-in ../src/language/source + parse-source-language source-language? + source-language-contract source-language-language + source-language-version source-language-digest source-language-scanner-factory)) +(export deflanguage-parser-loader LanguageLoader. LanguageLoaderContract + +language-parser-entry-schema+ language-parser-entry-ref language-metadata-ref parse-language-source + language-parser-source-receipt) + +(def (loader-shape? candidate) + (and (object? candidate) + (andmap (lambda (slot) (.slot? candidate slot)) + '(schema descriptor language version contract capabilities .parse + grammars metadata)) + (equal? (.ref candidate 'schema) +language-parser-entry-schema+) + (let (descriptor (.ref candidate 'descriptor)) + (and (or (language-grammar? descriptor) (source-language? descriptor)) + (andmap (lambda (slot) + (let (value (descriptor-ref descriptor slot)) + (and (string? value) (positive? (string-length value)) + (equal? value (.ref candidate slot))))) + '(language version contract)) + (equal? (.ref candidate 'capabilities) + (descriptor-capabilities descriptor)) + (procedure? (.ref candidate '.parse)) + (let (grammars (.ref candidate 'grammars)) + (and (list? grammars) (memq descriptor grammars) + (every (lambda (grammar) + (or (language-grammar? grammar) (source-language? grammar))) grammars))) + (or (not (.slot? candidate 'source-schema)) + (source-schema? (.ref candidate 'source-schema))) + (loader-metadata? (.ref candidate 'metadata) descriptor) +)))) + +(define-type (LanguageLoaderContract @ PooFlowContract.) + identity: 'gerbil-parser/language-loader + .classify: (lambda (candidate context) + (let (accepted? (loader-shape? candidate)) + (poo-flow-classification-evidence + 'gerbil-parser/language-loader candidate accepted? + (if accepted? '() '((expected gerbil-parser/language-loader))) + context)))) + +;;; Development services belong to the development declaration, including inherited slots. +(def (require-production-entry! candidate) + (for-each (lambda (slot) + (when (.slot? candidate slot) + (error "production parser entry contains a development service" slot))) + '(fixtures fixture-catalog tests scan-workers build-strategies native-test-profile)) + candidate) + +;;; Slot inheritance is resolved once during loading; parsing uses the bound +;;; engine method without per-call POO contract admission or compiler work. +(def LanguageLoader. + (.o (:: self) + schema: +language-parser-entry-schema+ + descriptor: #f + (grammars (list (.ref self 'descriptor))) + metadata: (.o) + (language (descriptor-ref (.ref self 'descriptor) 'language)) + (version (descriptor-ref (.ref self 'descriptor) 'version)) + (contract (descriptor-ref (.ref self 'descriptor) 'contract)) + (capabilities (descriptor-capabilities (.ref self 'descriptor))) + (.parse (let (descriptor (.ref self 'descriptor)) + (lambda (source) (parse-language-source descriptor source)))))) + +;;; Public slot extensions are ordinary POO expressions and may inherit from +;;; a user prototype. Engine identity/dispatch slots are sealed by this macro. +;;; Additional contracts validate the effective object after inheritance. +(defsyntax (deflanguage-parser-loader stx) + (syntax-case stx (@ descriptor parse slots contracts grammar source backends) + ((_ binding (grammar definition) (parse parse-binding) + (slots slot ...) (backends machine (kind digest procedure) ...) section ...) + (identifier? #'machine) + #'(begin + (install-parser-machine-backends! machine + (list (list 'kind digest procedure) ...)) + (deflanguage-parser-loader binding + (grammar definition) (parse parse-binding) + (slots slot ...) section ...))) + ((_ (binding marker loader-self prototype) + (descriptor descriptor-value) + (parse parse-binding) + (slots slot ...) + (contracts extension-contract ...)) + (and (identifier? #'binding) (identifier? #'loader-self) + (eq? (syntax->datum #'marker) '::) + (identifier? #'parse-binding)) + (let () + (def (check-slot! name location) + (when (memq name '(fixtures fixture-catalog tests scan-workers build-strategies native-test-profile)) + (raise-syntax-error #f + "development service requires deflanguage-development-loader" location)) + (when (memq name '(schema descriptor language version contract + capabilities .parse)) + (raise-syntax-error #f + "language loader extension overrides an engine slot" location))) + (let loop ((rows (stx-map (lambda (row) row) #'(slot ...)))) + (unless (null? rows) + (let* ((row (car rows)) (datum (syntax->datum row))) + (if (keyword? datum) + (begin + (check-slot! (string->symbol (keyword->string datum)) row) + (when (null? (cdr rows)) + (raise-syntax-error #f "language loader slot requires a value" row)) + (loop (cddr rows))) + (begin + (syntax-case row () + ((name . _) + (begin + (unless (identifier? #'name) + (raise-syntax-error #f "language loader requires POO slot forms" row)) + (check-slot! (syntax->datum #'name) row))) + (_ (raise-syntax-error #f "language loader requires POO slot forms" row))) + (loop (cdr rows))))))) + #'(begin + (def binding + (let* ((author-extensions (.o (:: loader-self prototype) slot ...)) + (parser-descriptor-value + (bind-parser-identity descriptor-value (.ref author-extensions 'metadata))) + (candidate + (.o (:: loader-self author-extensions) + descriptor: parser-descriptor-value + ;; Bind these methods at the leaf; a prototype cannot + ;; replace identity or dispatch through inherited slots. + schema: +language-parser-entry-schema+ + (language (descriptor-ref (.ref loader-self 'descriptor) 'language)) + (version (descriptor-ref (.ref loader-self 'descriptor) 'version)) + (contract (descriptor-ref (.ref loader-self 'descriptor) 'contract)) + (capabilities (descriptor-capabilities (.ref loader-self 'descriptor))) + (.parse (let (descriptor (.ref loader-self 'descriptor)) + (lambda (source) + (parse-language-source descriptor source)))) + ))) + (let* ((metadata (.ref candidate 'metadata)) + (admitted (.cc candidate 'metadata + (bind-loader-metadata (.ref candidate 'descriptor) metadata)))) + (require-production-entry! admitted) + (validate LanguageLoaderContract admitted) + (validate extension-contract admitted) ... + admitted))) + (def parse-binding (.ref binding '.parse))))) + ((_ binding (kind descriptor-value) (parse parse-binding) section ...) + (let () + (unless (identifier? #'parse-binding) + (raise-syntax-error #f "language loader parse binding must be an identifier" #'parse-binding)) + (def normalized-binding + (if (identifier? #'binding) + #'(binding :: self LanguageLoader.) + (syntax-case #'binding (@) + ((name @ prototype) + (identifier? #'name) + #'(name :: self prototype)) + ((name marker self prototype) + (and (identifier? #'name) (identifier? #'self) + (eq? (syntax->datum #'marker) '::)) + #'(name :: self prototype)) + (_ (raise-syntax-error #f "invalid language loader binding" #'binding))))) + (def normalized-descriptor + (case (syntax->datum #'kind) + ((descriptor) #'descriptor-value) + ((grammar) #'(require-grammar-descriptor descriptor-value)) + ((source) #'(require-source-descriptor descriptor-value)) + (else (raise-syntax-error #f "invalid language loader descriptor kind" #'kind)))) + (def slot-section #'(slots)) + (def contract-section #'(contracts)) + (def seen-slots? #f) + (def seen-contracts? #f) + (for-each + (lambda (row) + (syntax-case row (slots contracts) + ((slots slot ...) + (begin + (when (or seen-slots? seen-contracts?) + (raise-syntax-error #f "duplicate or out-of-order language loader slots" row)) + (set! seen-slots? #t) (set! slot-section row))) + ((contracts contract ...) + (begin + (when seen-contracts? + (raise-syntax-error #f "duplicate language loader contracts" row)) + (set! seen-contracts? #t) (set! contract-section row))) + (_ (raise-syntax-error #f "unknown language loader section" row)))) + (stx-map (lambda (row) row) #'(section ...))) + (with-syntax ((normalized-binding normalized-binding) + (normalized-descriptor normalized-descriptor) + (slot-section slot-section) (contract-section contract-section)) + #'(deflanguage-parser-loader normalized-binding + (descriptor normalized-descriptor) (parse parse-binding) + slot-section contract-section)))) + (_ (raise-syntax-error #f "invalid language loader declaration" stx)))) + +;;; Source schema is admitted with the entry, separate from parser dispatch. +(def (language-parser-source-receipt entry provenance) + (let (configuration (language-parser-entry-ref entry 'source-schema)) + (unless configuration (error "parser entry has no source schema" entry)) + (source-schema-receipt configuration provenance))) diff --git a/src/language-support/fixture.ss b/language-support/fixture.ss similarity index 56% rename from src/language-support/fixture.ss rename to language-support/fixture.ss index 4b9bffc3..a046bca1 100644 --- a/src/language-support/fixture.ss +++ b/language-support/fixture.ss @@ -5,6 +5,7 @@ :gerbil-parser/src/runtime/identity) (export defsyntax-fixture defsyntax-corpus + syntax-fixture-copy syntax-fixture? syntax-fixture-id syntax-fixture-language @@ -29,6 +30,14 @@ expected-status root-kind required-kinds) transparent: #t) +(def (syntax-fixture-copy fixture) + (unless (syntax-fixture? fixture) (error "expected syntax fixture" fixture)) + (make-syntax-fixture (string-copy (syntax-fixture-id fixture)) + (string-copy (syntax-fixture-language fixture)) (string-copy (syntax-fixture-version fixture)) + (string-copy (syntax-fixture-contract fixture)) (string-copy (syntax-fixture-source-digest fixture)) + (string-copy (syntax-fixture-source fixture)) (syntax-fixture-expected-status fixture) + (syntax-fixture-root-kind fixture) (map values (syntax-fixture-required-kinds fixture)))) + ;;; Expansion resolves the source relative to its declaration. Runtime parsing ;;; therefore has no filesystem dependency, and content identity excludes paths. ;; defsyntax-fixture @@ -48,32 +57,30 @@ ;; Result: the binding contains source bytes and their SHA-256 identity. ;; % (defsyntax (defsyntax-fixture stx) - (syntax-case stx (identity source expect) - ((_ binding - (identity fixture-id language version contract) - (source path) - (expect expected-status root-kind (required-kind ...))) - (and (identifier? #'binding) - (stx-string? #'fixture-id) - (stx-string? #'path) - (memq (stx-e #'expected-status) '(accepted rejected)) - (if (eq? (stx-e #'expected-status) 'accepted) - (identifier? #'root-kind) - (eq? (stx-e #'root-kind) #f))) - (let* ((resolved (gx#core-resolve-path #'path (stx-source stx))) - (content (call-with-input-file resolved read-all-as-string))) - (with-syntax ((fixture-content content)) + (def (emit binding identity-clause expected-clause content) + (with-syntax ((binding binding) (identity-clause identity-clause) + (expected-clause expected-clause) (fixture-content content)) + (syntax-case #'(identity-clause expected-clause) (identity expect) + (((identity fixture-id language version contract) + (expect expected-status root-kind (required-kind ...))) + (and (identifier? #'binding) (stx-string? #'fixture-id) + (memq (stx-e #'expected-status) '(accepted rejected)) + (if (eq? (stx-e #'expected-status) 'accepted) + (identifier? #'root-kind) (eq? (stx-e #'root-kind) #f))) #'(def binding - (make-syntax-fixture - fixture-id - language - version - contract - (sha256-text fixture-content) - fixture-content - 'expected-status - 'root-kind - '(required-kind ...)))))) + (make-syntax-fixture fixture-id language version contract + (sha256-text fixture-content) fixture-content + 'expected-status 'root-kind '(required-kind ...)))) + (_ (raise-syntax-error #f "invalid native-syntax fixture declaration" stx))))) + (syntax-case stx (source text) + ((_ binding identity-clause (source path) expected-clause) + (stx-string? #'path) + (let (resolved (gx#core-resolve-path #'path (stx-source stx))) + (emit #'binding #'identity-clause #'expected-clause + (call-with-input-file resolved read-all-as-string)))) + ((_ binding identity-clause (text content) expected-clause) + (stx-string? #'content) + (emit #'binding #'identity-clause #'expected-clause (stx-e #'content))) (_ (raise-syntax-error #f "invalid native-syntax fixture declaration" stx)))) ;;; A corpus is an expansion-time manifest: paths stay reviewable in source, @@ -95,6 +102,12 @@ ;; Result: accepted rows precede rejected rows exactly as declared. ;; % (defsyntax (defsyntax-corpus stx) + (def (source-clause source-stx) + (with-syntax ((value source-stx)) + (if (stx-string? source-stx) #'(source value) + (syntax-case source-stx (text) + ((text content) (stx-string? #'content) #'(text content)) + (_ (raise-syntax-error #f "corpus source requires a path or (text string)" stx source-stx)))))) (syntax-case stx (identity accepted rejected) ((_ binding (identity language-value version-value contract-value) @@ -103,15 +116,17 @@ (required-kind ...)) ...) (rejected (rejected-id rejected-binding rejected-path) ...)) - #'(begin + (with-syntax (((accepted-source ...) (stx-map source-clause #'(path ...))) + ((rejected-source ...) (stx-map source-clause #'(rejected-path ...)))) + #'(begin (defsyntax-fixture accepted-binding (identity fixture-id language-value version-value contract-value) - (source path) + accepted-source (expect accepted root-kind (required-kind ...))) ... (defsyntax-fixture rejected-binding (identity rejected-id language-value version-value contract-value) - (source rejected-path) + rejected-source (expect rejected #f ())) ... (def binding - (list accepted-binding ... rejected-binding ...)))) + (list accepted-binding ... rejected-binding ...))))) (_ (raise-syntax-error #f "invalid native-syntax corpus declaration" stx)))) diff --git a/src/language-support/grammar-source.ss b/language-support/grammar-source.ss similarity index 85% rename from src/language-support/grammar-source.ss rename to language-support/grammar-source.ss index 326d1773..57c3e10e 100644 --- a/src/language-support/grammar-source.ss +++ b/language-support/grammar-source.ss @@ -12,14 +12,14 @@ (for-syntax (only-in ./javacc-source javacc-source->datum parse-javacc-source/expected)) - (only-in ./iso-bnf parse-iso-bnf-source/expected) + (for-syntax (only-in ./iso-bnf + parse-iso-bnf-source/expected iso-bnf-source->datum)) + (only-in ./iso-bnf iso-bnf-source-from-datum) (only-in ./antlr4-source antlr4-source-from-datum) - (only-in ./javacc-source javacc-source-from-datum) - (only-in ./iso-bnf-language deflanguage-iso-bnf-grammar)) + (only-in ./javacc-source javacc-source-from-datum)) (export defsyntax-iso-bnf-source defsyntax-antlr4-source - defsyntax-javacc-source - deflanguage-iso-bnf-grammar) + defsyntax-javacc-source) ;;; JavaCC source is admitted and reduced to an immutable production inventory ;;; during expansion. Generated Java and Java action blocks never enter the @@ -71,13 +71,16 @@ (identity language version commit) (digest expected-digest) (source path)) - (and (identifier? #'binding) (stx-string? #'path)) + (and (identifier? #'binding) (stx-string? #'language) + (stx-string? #'version) (stx-string? #'commit) + (stx-string? #'expected-digest) (stx-string? #'path)) (let* ((resolved (gx#core-resolve-path #'path (stx-source stx))) - (content (call-with-input-file resolved read-all-as-string))) - (with-syntax ((grammar-content content)) - #'(def binding - (parse-iso-bnf-source/expected - language version commit expected-digest grammar-content))))) + (content (call-with-input-file resolved read-all-as-string)) + (catalog (parse-iso-bnf-source/expected + (stx-e #'language) (stx-e #'version) (stx-e #'commit) + (stx-e #'expected-digest) content))) + (with-syntax ((materialized-catalog (iso-bnf-source->datum catalog))) + #'(def binding (iso-bnf-source-from-datum 'materialized-catalog))))) (_ (raise-syntax-error #f "invalid ISO BNF source declaration" stx)))) ;;; ANTLR text is parsed and digest-checked by the expander; runtime code only diff --git a/language-support/grammar.ss b/language-support/grammar.ss new file mode 100644 index 00000000..6864c40d --- /dev/null +++ b/language-support/grammar.ss @@ -0,0 +1,4 @@ +;;; Public Scheme grammar frontend without pinned-source adapter dependencies. +(import (only-in ../src/language/grammar deflanguage defgrammar-syntax) + (only-in ../src/grammar/lexical-algebra deftext-profile)) +(export deflanguage defgrammar-syntax deftext-profile) diff --git a/src/language-support/iso-bnf-language.ss b/language-support/iso-bnf-language.ss similarity index 75% rename from src/language-support/iso-bnf-language.ss rename to language-support/iso-bnf-language.ss index d8ce73d9..94f8e321 100644 --- a/src/language-support/iso-bnf-language.ss +++ b/language-support/iso-bnf-language.ss @@ -2,6 +2,7 @@ ;;; Expansion-time ISO BNF compiler for runtime-only language packs. (import (for-syntax :std/misc/ports + (only-in ../src/compiler/concise-language lower-source-rule-overlays lower-source-rule-precedences) (only-in ./iso-bnf iso-bnf-source-declaration-sources iso-bnf-source-declaration-sources/overrides @@ -12,13 +13,13 @@ iso-bnf-source-literals iso-bnf-source-syntax-kinds parse-iso-bnf-source/expected)) - :gerbil-parser/src/language/grammar) -(export deflanguage-iso-bnf-grammar) + :gerbil-parser/src/compiler/language-expander) +(export iso-bnf) ;;; ISO WG3 BNF uses prose for lexical character classes. The source adapter ;;; lowers those named classes to the shared scanner token contract while all ;;; parser productions remain represented in the immutable Grammar IR. -;; deflanguage-iso-bnf-grammar +;; compile-iso-bnf-language ;; : (-> Syntax Syntax) ;; | doc m% ;; Lowers a pinned ISO WG3 BNF catalog through the stable language v1 @@ -27,7 +28,7 @@ ;; # Examples ;; ;; ```scheme -;; (deflanguage-iso-bnf-grammar language +;; (compile-iso-bnf-language language ;; (identity "lang" "v1" "contract.v1") ;; (reference "source-v1" "commit") ;; (digest "sha256:...") (source "grammar.bnf") @@ -37,48 +38,16 @@ ;; Result: Runtime receives only immutable v1 data; native BNF names and ;; source coordinates remain available in the Bound Grammar IR sidecar. ;; % -(defsyntax (deflanguage-iso-bnf-grammar stx) +(defsyntax (compile-iso-bnf-language stx) (syntax-case stx (identity reference digest source rule-overrides rule-precedences entrypoint conflicts case-insensitive) - ((_ prefix - (identity language version contract) - (reference source-version source-commit) - (digest expected-digest) - (source path) - (entrypoint entry-name) - (conflicts conflict-policy) - (case-insensitive case-insensitive-value)) - #'(deflanguage-iso-bnf-grammar prefix - (identity language version contract) - (reference source-version source-commit) - (digest expected-digest) - (source path) - (rule-overrides) - (rule-precedences) - (entrypoint entry-name) - (conflicts conflict-policy) - (case-insensitive case-insensitive-value))) - ((_ prefix - (identity language version contract) - (reference source-version source-commit) - (digest expected-digest) - (source path) - (rule-overrides override-row ...) - (entrypoint entry-name) - (conflicts conflict-policy) - (case-insensitive case-insensitive-value)) - #'(deflanguage-iso-bnf-grammar prefix - (identity language version contract) - (reference source-version source-commit) - (digest expected-digest) - (source path) - (rule-overrides override-row ...) - (rule-precedences) - (entrypoint entry-name) - (conflicts conflict-policy) - (case-insensitive case-insensitive-value))) + ((_ prefix (reference source-version source-commit) section ...) + (identifier? #'prefix) + (with-syntax ((language-name (datum->syntax #'prefix (symbol->string (syntax->datum #'prefix))))) + #'(compile-iso-bnf-language prefix (identity language-name #f #f) + (reference source-version source-commit) section ...))) ((_ prefix (identity language version contract) (reference source-version source-commit) @@ -91,8 +60,8 @@ (case-insensitive case-insensitive-value)) (and (identifier? #'prefix) (stx-string? #'language) - (stx-string? #'version) - (stx-string? #'contract) + (or (stx-string? #'version) (eq? (stx-e #'version) #f)) + (or (stx-string? #'contract) (eq? (stx-e #'contract) #f)) (stx-string? #'source-version) (stx-string? #'source-commit) (stx-string? #'expected-digest) @@ -142,7 +111,7 @@ (((literal-value ...)) (list literal-values)) (((overlay-id ...)) (list overlay-lineage)) (source-map-value source-map)) - #'(deflanguage-grammar prefix + #'(compile-language prefix (identity language version contract) (syntax-kinds syntax-row ...) (terminals @@ -175,8 +144,25 @@ (conflicts conflict-policy) (case-insensitive case-insensitive-value) (source-ownership source-map-value) - (lineage deflanguage-iso-bnf-grammar source-version source-commit + (lineage compile-iso-bnf-language source-version source-commit overlay-id ...) (flow (source lexical) (lexical cst)))))) (_ (raise-syntax-error #f "invalid ISO BNF language declaration" stx)))) + +;;; Source-backed rules retain their checked upstream witness and provenance. +(defsyntax (iso-bnf stx) + (syntax-case stx (syntax rules reference digest source rule-precedences entrypoint conflicts case-insensitive) + ((_ prefix + (syntax (reference version commit) (digest hash) (source path) + (rule-precedences precedence ...) + (entrypoint root) (conflicts policy) (case-insensitive insensitive)) + (rules overlay ...)) + (identifier? #'prefix) + (with-syntax (((lowered ...) (lower-source-rule-overlays #'(overlay ...) stx #'prefix #'version #'commit)) + ((lowered-precedence ...) (lower-source-rule-precedences #'(precedence ...) stx #'prefix #'version #'commit))) + #'(compile-iso-bnf-language prefix + (reference version commit) (digest hash) (source path) + (rule-overrides lowered ...) (rule-precedences lowered-precedence ...) + (entrypoint root) (conflicts policy) (case-insensitive insensitive)))) + (_ (raise-syntax-error #f "iso-bnf vocabulary requires checked source syntax and rule overlays" stx)))) diff --git a/src/language-support/iso-bnf.ss b/language-support/iso-bnf.ss similarity index 94% rename from src/language-support/iso-bnf.ss rename to language-support/iso-bnf.ss index aa35ab09..83e3a997 100644 --- a/src/language-support/iso-bnf.ss +++ b/language-support/iso-bnf.ss @@ -14,6 +14,7 @@ iso-bnf-production-line iso-bnf-production-references iso-bnf-source? + iso-bnf-source->datum iso-bnf-source-from-datum iso-bnf-source-language iso-bnf-source-version iso-bnf-source-commit @@ -304,6 +305,37 @@ expected-digest (iso-bnf-source-digest catalog))) catalog)) +;;; Source declarations freeze the admitted catalog at expansion time. Runtime +;;; materialization reconstructs its records without scanning the BNF again. +(def (iso-bnf-source->datum source) + (list (iso-bnf-source-schema source) + (iso-bnf-source-language source) (iso-bnf-source-version source) + (iso-bnf-source-commit source) (iso-bnf-source-digest source) + (map (lambda (row) + (list (iso-bnf-production-name row) + (iso-bnf-production-expression row) + (iso-bnf-production-ast row) (iso-bnf-production-line row) + (iso-bnf-production-references row))) + (iso-bnf-source-productions source)))) + +(def (iso-bnf-source-from-datum value) + (unless (and (list? value) (= (length value) 6) + (andmap string? (take value 5)) + (string=? (car value) +iso-bnf-source-schema+) + (list? (list-ref value 5)) (pair? (list-ref value 5)) + (andmap + (lambda (row) + (and (list? row) (= (length row) 5) + (string? (car row)) (string? (cadr row)) + (pair? (caddr row)) (exact-integer? (cadddr row)) + (positive? (cadddr row)) + (list? (list-ref row 4)) (andmap string? (list-ref row 4)))) + (list-ref value 5))) + (error "invalid materialized ISO BNF source v1" value)) + (make-iso-bnf-source + (car value) (cadr value) (caddr value) (cadddr value) (list-ref value 4) + (map (lambda (row) (apply make-iso-bnf-production row)) (list-ref value 5)))) + ;; : (-> IsoBnfSource String (Maybe IsoBnfProduction)) (def (iso-bnf-source-production source name) (find (lambda (production) diff --git a/src/language-support/javacc-source.ss b/language-support/javacc-source.ss similarity index 100% rename from src/language-support/javacc-source.ss rename to language-support/javacc-source.ss diff --git a/language-support/loader-identity.ss b/language-support/loader-identity.ss new file mode 100644 index 00000000..77521be5 --- /dev/null +++ b/language-support/loader-identity.ss @@ -0,0 +1,113 @@ +;;; -*- Gerbil -*- +;;; Private common identity utilities shared by public and development entries. +(import (only-in ../src/language/entry parse-language-source call-with-language-parser-policy) + (only-in :clan/poo/object .o .ref .cc .slot? object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ../src/compiler/machine parser-machine-grammar-digest) + (only-in ../src/language/descriptor + language-grammar? language-grammar-contract language-grammar-language + language-grammar-machine language-grammar-observability + language-grammar-version language-grammar-parser-policy + language-parser-policy-branch-budget language-parser-policy-cst-validator language-grammar-with-identity) + (only-in ../src/language/source + parse-source-language source-language? + source-language-contract source-language-language + source-language-version source-language-digest source-language-scanner-factory source-language-with-identity)) +(export +language-parser-entry-schema+ language-parser-entry-ref descriptor-capabilities + require-grammar-descriptor require-source-descriptor descriptor-ref descriptor-digest + descriptor-digest-kind bind-loader-metadata loader-metadata? language-metadata-ref bind-parser-identity) + +(def +language-parser-entry-schema+ "gerbil-parser.language-entry.v2") + +(def (language-parser-entry-ref entry key) + (and (object? entry) (.slot? entry key) (.ref entry key))) + +(def (descriptor-capabilities descriptor) + (cond ((source-language? descriptor) '(scheme-source)) + ((language-grammar-parser-policy descriptor) + '(grammar-ir parser-ir scheme scheme-policy)) + (else '(grammar-ir parser-ir scheme)))) + +(def (require-grammar-descriptor descriptor) + (unless (language-grammar? descriptor) + (error "grammar loader requires a generated language descriptor" descriptor)) + descriptor) + +(def (require-source-descriptor descriptor) + (unless (source-language? descriptor) + (error "source loader requires a source language descriptor" descriptor)) + descriptor) + +(def (descriptor-ref descriptor field) + (if (source-language? descriptor) + (case field + ((language) (source-language-language descriptor)) + ((version) (source-language-version descriptor)) + ((contract) (source-language-contract descriptor))) + (case field + ((language) (language-grammar-language descriptor)) + ((version) (language-grammar-version descriptor)) + ((contract) (language-grammar-contract descriptor))))) + +(def (descriptor-digest descriptor) + (if (source-language? descriptor) (source-language-digest descriptor) + (parser-machine-grammar-digest (language-grammar-machine descriptor)))) + +(def (descriptor-digest-kind descriptor) + (if (source-language? descriptor) 'source-identity 'parser-ir)) + +;;; Metadata admits native POO objects or ordinary association-list values. +;;; POO inheritance remains native; value users do not construct an object. +(def (metadata-value? value) + (and (list? value) + (let loop ((rows value) (seen '())) + (or (null? rows) + (and (pair? (car rows)) (symbol? (caar rows)) + (not (memq (caar rows) seen)) + (loop (cdr rows) (cons (caar rows) seen))))))) +(def (require-metadata value) + (unless (or (object? value) (metadata-value? value)) + (error "metadata requires a POO object or distinct symbol-keyed association list" value)) + value) +(def (language-metadata-ref value key) + (require-metadata value) + (if (object? value) + (and (.slot? value key) (.ref value key)) + (let (row (assq key value)) (and row (cdr row))))) +(def (bind-parser-identity definition metadata) + (require-metadata metadata) + (if (and (descriptor-ref definition 'language) + (descriptor-ref definition 'version) (descriptor-ref definition 'contract)) + definition + (let ((language-value (language-metadata-ref metadata 'language)) + (version-value (language-metadata-ref metadata 'version)) + (contract-value (language-metadata-ref metadata 'contract))) + (unless (andmap (lambda (value) (and (string? value) (positive? (string-length value)))) + (list language-value version-value contract-value)) + (error "parser metadata requires nonempty language, version and contract")) + (if (source-language? definition) + (source-language-with-identity definition language-value version-value contract-value) + (language-grammar-with-identity definition language-value version-value contract-value))))) +(def (bind-loader-metadata descriptor inherited) + (require-metadata inherited) + (let ((language-value (descriptor-ref descriptor 'language)) + (version-value (descriptor-ref descriptor 'version)) + (contract-value (descriptor-ref descriptor 'contract)) + (digest-value (descriptor-digest descriptor)) + (digest-kind-value (descriptor-digest-kind descriptor))) + (if (object? inherited) + (.o (:: self inherited) + language: language-value version: version-value contract: contract-value + digest: digest-value digest-kind: digest-kind-value) + (append (list (cons 'language language-value) (cons 'version version-value) + (cons 'contract contract-value) (cons 'digest digest-value) + (cons 'digest-kind digest-kind-value)) + (filter (lambda (row) (not (memq (car row) '(language version contract digest digest-kind)))) inherited))))) +(def (loader-metadata? metadata descriptor) + (and (or (object? metadata) (metadata-value? metadata)) + (andmap (lambda (key) + (equal? (language-metadata-ref metadata key) (descriptor-ref descriptor key))) + '(language version contract)) + (equal? (language-metadata-ref metadata 'digest) (descriptor-digest descriptor)) + (eq? (language-metadata-ref metadata 'digest-kind) (descriptor-digest-kind descriptor)))) diff --git a/language-support/projection.ss b/language-support/projection.ss new file mode 100644 index 00000000..78667cd0 --- /dev/null +++ b/language-support/projection.ss @@ -0,0 +1,3 @@ +;;; Public POO record recipe declarations; execution belongs to the engine. +(import (only-in ../src/runtime/record-projection RecordProjection. RecordProjectionContract deflanguage-projection)) +(export RecordProjection. RecordProjectionContract deflanguage-projection) diff --git a/language-support/shell-grammar.ss b/language-support/shell-grammar.ss new file mode 100644 index 00000000..82e6364d --- /dev/null +++ b/language-support/shell-grammar.ss @@ -0,0 +1,172 @@ +;;; Common source-library frontend. Language packages declare intent, not plans. +(import (only-in :clan/poo/object .o) + (only-in :std/list/list delete-duplicates/hash) + (only-in ./command-grammar lower-command-grammar command-grammar-entrypoints) + (only-in ../src/language/command-profile CommandProfile. command-node-schemas) + (only-in ../src/language/part-profile BindingProfile. PartProfile.) + (only-in ../src/language/result-profile ResultProfile.) + (only-in ../src/language/scanner-profile ScannerProfile.) + (only-in ../src/runtime/region-scanner prepare-region-plan region-plan-specification) + (only-in ../src/runtime/source-engines ShellSourceStrategy.)) +(export compile-shell-grammar) +(def (trace step) + (when (getenv "GERBIL_PARSER_SOURCE_TRACE" #f) + (displayln "SOURCE-LOWER " step) (force-output))) +(def (compile-shell-grammar declaration) + + (trace "declaration") + (let () + (def (distinct? values) (= (length values) (length (delete-duplicates/hash values)))) + (def (section key) + (let (row (assq key declaration)) + (unless row (error "missing shell grammar section" key)) (cdr row))) + (unless (and (list? declaration) + (andmap (lambda (row) (and (list? row) (pair? row) (memq (car row) '(operators regions bindings words commands)))) declaration) + (= (length declaration) 5) (distinct? (map car declaration))) + (error "shell grammar requires operators, regions, bindings, words and commands exactly once")) + (let* ((operators (section 'operators)) (region-rows (section 'regions)) + (binding-rows (section 'bindings)) (word-rows (section 'words)) + (command-rows (section 'commands))) + (unless (and (pair? operators) (distinct? operators) + (andmap (lambda (value) (and (string? value) (> (string-length value) 0))) operators)) + (error "shell operators must be distinct nonempty strings")) + (unless (and (list? region-rows) + (andmap (lambda (row) (and (list? row) (>= (length row) 5) + (memq (car row) '(quote pair)) (symbol? (cadr row)) (symbol? (caddr row)) + (string? (cadddr row)) (> (string-length (cadddr row)) 0))) region-rows) + (distinct? (map cadr region-rows))) + (error "shell regions require distinct names and semantic delimiters")) + (def (region name) + (or (find (lambda (row) (eq? (cadr row) name)) region-rows) (error "unknown shell region" name))) + (def (opening row) (cadddr row)) + (def (closing row) + (if (eq? (car row) 'quote) + (string (string-ref (opening row) (- (string-length (opening row)) 1))) + (list-ref row 4))) + (for-each (lambda (row) + (if (eq? (car row) 'quote) + (unless (memq (list-ref row 4) '(opaque escaped)) (error "unknown quote policy" row)) + (unless (and (= (length row) 5) (string? (closing row)) (> (string-length (closing row)) 0) + (andmap (lambda (ch) (char=? ch (string-ref (closing row) 0))) (string->list (closing row)))) + (error "pair closing delimiter must repeat one character" row)))) region-rows) + (let* ((quotes (foldl (lambda (row result) + (if (eq? (car row) 'quote) + (let (quote-row (list (string-ref (closing row) 0) + (eq? (list-ref row 4) 'escaped) (list-tail row 5))) + (let (prior (assv (car quote-row) result)) + (when (and prior (not (equal? prior quote-row))) (error "conflicting quote region" row)) + (if prior result (append result (list quote-row))))) result)) '() region-rows)) + (pairs (filter-map (lambda (row) + (and (eq? (car row) 'pair) + (list (opening row) (string-ref (opening row) (- (string-length (opening row)) 1)) + (string-ref (closing row) 0) (string-length (closing row))))) region-rows)) + (region-plan-value (prepare-region-plan (list operators quotes pairs #t))) + (binding-profile-values + (map (lambda (row) + (unless (and (list? row) (= (length row) 5) (symbol? (car row))) + (error "binding requires name, prefixes, operators and subscript semantics" row)) + (cons (car row) (.o (:: self BindingProfile.) name: (cadr row) + prefixes: (caddr row) operators: (cadddr row) subscript: (list-ref row 4)))) binding-rows))) + (trace "regions-and-bindings") + (unless (and (distinct? (map car binding-profile-values)) + (equal? (map car binding-profile-values) '(simple parameter assignment))) + (error "shell grammar requires simple, parameter and assignment binding semantics")) + (def (part row) + (unless (and (list? row) (= (length row) 2) (list? (car row)) (pair? (car row)) + (distinct? (car row)) (andmap symbol? (car row)) (list? (cadr row)) (pair? (cadr row))) + (error "word grammar requires contexts and a semantic constructor" row)) + (let* ((form (cadr row)) (op (car form))) + (case op + ((quoted substitution parameter) + (unless (= (length form) 2) (error "invalid word region reference" form)) + (let* ((r (region (cadr form))) (prefix (opening r)) (kind (caddr r)) + (action (case op + ((quoted) + (unless (eq? (car r) 'quote) (error "quoted word requires a quote region" form)) + (list 'quote kind (string-length prefix) 1 (string-ref (closing r) 0))) + ((substitution) + (if (eq? (car r) 'quote) + (list 'quoted-body kind (string-length prefix) 1 (string-ref (closing r) 0)) + (list 'pair kind (string-length prefix) (string-length (closing r))))) + (else + (unless (and (eq? kind 'ParameterExpansion) (equal? prefix "${") (equal? (closing r) "}")) + (error "parameter word requires the admitted parameter delimiters" form)) '(parameter))))) + (list (car row) (list 'prefix prefix 'any) action))) + ((name) + (unless (and (= (length form) 4) (string? (caddr form))) (error "invalid name word" form)) + (list (car row) (list 'prefix (caddr form) 'next) (list 'name (cadr form) (cadddr form)))) + ((escape) + (unless (and (= (length form) 3) (or (eq? (caddr form) 'any) (string? (caddr form)))) + (error "invalid escape word" form)) + (list (car row) (list 'prefix "\\" (caddr form)) (list 'escape (cadr form)))) + (else (error "unknown word grammar constructor" form))))) + (let* ((part-rules (map part word-rows)) + (context-values (append '(word) (filter (lambda (kind) (not (eq? kind 'CommandSubstitution))) + (delete-duplicates/hash (filter-map (lambda (row) (and (eq? (car row) 'quote) (caddr row))) region-rows))) '(HereDocument))) + (part-profile-value (.o (:: self PartProfile.) contexts: context-values bindings: binding-profile-values rules: part-rules literal: 'LiteralPart)) + (region-spec (region-plan-specification region-plan-value)) + (word-matcher (list 'region-word (append (take region-spec 3) '(#t)))) + (scanner-profile-value (.o (:: self ScannerProfile.) modes: '(command marker body) initial-mode: 'command + tokens: '(horizontal-whitespace newline line-continuation comment word operator heredoc-marker heredoc-content heredoc-end) + rules: (list + '(space command horizontal-whitespace (horizontal-whitespace+) 0 keep) + '(newline command newline (literal "\n") 0 (activate-next body)) + '(continuation command line-continuation (literal "\\\n") 10 keep) + '(comment command comment (line-prefix "#" "\n") 0 keep) + '(strip command operator (literal "<<-") 30 (expect-marker-in #t marker)) + '(plain command operator (literal "<<") 30 (expect-marker-in #f marker)) + (list 'operator 'command 'operator (list 'literals (filter (lambda (op) (not (member op '("<<" "<<-")))) operators)) 0 'keep) + (list 'word 'command 'word (list 'unless-prefix '(" " "\t" "\n" "#" "\\\n" ";" "&" "|" "(" ")" "<" ">") '("<(" ">(") word-matcher) 0 'keep) + '(marker-space marker horizontal-whitespace (horizontal-whitespace+) 0 keep) + '(marker-continuation marker line-continuation (literal "\\\n") 10 keep) + '(marker-newline marker newline (literal "\n") 10 (activate-next body)) + (list 'marker-word 'marker 'heredoc-marker (list 'unless-prefix '(" " "\t" "\n" "\\\n") '() word-matcher) 0 '(enqueue-marker-in shell-quote-removal command)) + '(end body heredoc-end (marker-line-at "\n") 10 (finish-marker command body)) + '(body body heredoc-content (body-line-at "\n") 0 keep)))) + (projection-rows '())) + (trace "scanner-and-parts") + ;; Capture signatures are properties of the source executor operations. + ;; The frontend derives programs; authors do not allocate capture frames. + (def (projection id fields) + (append (list id id (map (lambda (field) (list (car field) (cadr field))) fields)) + (map (lambda (field) + (case (cadr field) + ((span) (list 'token (caddr field) (cadddr field) (car field) (list-ref field 4))) + ((node) (list 'one (caddr field) (car field) (cadddr field))) + ((parts) (list 'many (caddr field) (car field))))) fields))) + (def (add! id fields) + (let (row (projection id fields)) + (let (prior (assq id projection-rows)) + (when (and prior (not (equal? prior row))) (error "conflicting word result kind" id)) + (unless prior (set! projection-rows (append projection-rows (list row))))))) + (add! 'LiteralPart '((text span text LiteralPart required))) + (for-each (lambda (op) + (for-each (lambda (row) + (let (form (caddr row)) + (when (eq? (car form) op) + (let (kind (cadr form)) (add! kind (list (list 'text 'span 'text kind 'required))))))) part-rules)) '(escape name)) + (add! 'Word '((parts parts part))) (add! 'HereDocumentLine '((parts parts part))) + (for-each (lambda (r) + (when (and (eq? (car r) 'quote) (not (eq? (caddr r) 'CommandSubstitution))) + (add! (caddr r) '((open span open quote-open required) (parts parts part) (close span close quote-close required))))) region-rows) + (for-each (lambda (row) + (let (form (caddr row)) + (when (memq (car form) '(pair quoted-body)) + (add! (cadr form) '((open span open substitution-open required) (body span body substitution-body optional) (close span close substitution-close required)))))) part-rules) + (add! 'ArraySubscript '((open span open subscript-open required) (parts parts index) (close span close subscript-close required))) + (add! 'ParameterExpansion '((open span open parameter-open required) (prefix span prefix parameter-prefix optional) + (name span name parameter-name optional) (subscript node subscript optional) (operator span operator parameter-operator optional) + (parts parts operand) (close span close parameter-close required))) + (add! 'Assignment '((name span name assignment-name required) (operator span operator assignment-operator required) (parts parts value))) + (unless (and (= (length command-rows) 3) + (equal? (map car command-rows) '(roles descriptor rules))) + (error "commands require roles, descriptor and rules exactly once")) + (trace "result-projections") + (let* ((result-profile-value (.o (:: self ResultProfile.) scanner: scanner-profile-value nodes: (command-node-schemas) + tokens: '(unparsed-source) projections: projection-rows)) + (command-profile-value (.o (:: self CommandProfile.) roles: (cdar command-rows) + texts: (list (cons 'descriptor (cdadr command-rows))) + ports: (command-grammar-entrypoints (cdr (assq 'rules command-rows))) + forms: (lower-command-grammar (cdr (assq 'rules command-rows)))))) + (trace "command-plans") + (.o (:: self ShellSourceStrategy.) regions: region-plan-value scanner: scanner-profile-value parts: part-profile-value results: result-profile-value commands: command-profile-value))))))) diff --git a/language-support/shell.ss b/language-support/shell.ss new file mode 100644 index 00000000..9776dfb8 --- /dev/null +++ b/language-support/shell.ss @@ -0,0 +1,14 @@ +;;; Author vocabulary entry; profiles and execution recipes are compiler products. +(import (only-in ./shell-grammar compile-shell-grammar) + (only-in ../src/language/source declare-source-syntax) + (for-syntax (only-in ../src/compiler/source-language expand-shell-grammar-syntax))) +(export shell) +(defsyntax (shell stx) + (syntax-case stx (syntax rules) + ((_ binding (syntax clause ...) (rules rule ...)) + (identifier? #'binding) + (with-syntax ((strategy + (expand-shell-grammar-syntax #'(shell clause ... (rules rule ...)) + #'compile-shell-grammar))) + #'(def binding (declare-source-syntax strategy)))) + (_ (raise-syntax-error #f "shell vocabulary requires syntax and rules blocks" stx)))) diff --git a/language-support/source-schema.ss b/language-support/source-schema.ss new file mode 100644 index 00000000..ce1dabb0 --- /dev/null +++ b/language-support/source-schema.ss @@ -0,0 +1,45 @@ +;;; -*- Gerbil -*- +;;; Declarative source receipt templates; configuration is a value or native POO. +(import (only-in :std/list/list delete-duplicates/hash) + (only-in :clan/poo/object object? .ref .slot?)) +(export source-schema? source-schema-receipt) + +(def (source-schema-ref value key) + (if (object? value) + (and (.slot? value key) (.ref value key)) + (let (row (assq key value)) (and row (cdr row))))) + +(def (nonempty-string? value) + (and (string? value) (positive? (string-length value)))) + +(def (source-schema? value) + (and (or (object? value) + (and (list? value) + (every (lambda (row) + (and (pair? row) (memq (car row) + '(schema namespace sourceVersion upstreamCommit)))) value) + (= (length value) 4) + (= (length (delete-duplicates/hash (map car value))) 4))) + (nonempty-string? (source-schema-ref value 'schema)) + (symbol? (source-schema-ref value 'namespace)) + (nonempty-string? (source-schema-ref value 'sourceVersion)) + (nonempty-string? (source-schema-ref value 'upstreamCommit)))) + +;;; Only name and optional normalization/disambiguation vary per receipt. +;;; Published records remain ordinary values, independent of POO construction. +(def (source-schema-receipt configuration provenance) + (unless (source-schema? configuration) + (error "source schema requires schema, namespace, sourceVersion and upstreamCommit" configuration)) + (let* ((qualified? (and (list? provenance) (= (length provenance) 2) + (memq (car provenance) '(normalization disambiguation)) + (symbol? (cadr provenance)))) + (name (if qualified? (cadr provenance) provenance))) + (unless (symbol? name) + (error "source receipt requires an identifier or typed normalization/disambiguation identifier" provenance)) + (append + (list (cons 'schema (source-schema-ref configuration 'schema)) + (cons 'namespace (source-schema-ref configuration 'namespace)) + (cons 'name name)) + (if qualified? (list (cons 'kind (car provenance))) '()) + (list (cons 'sourceVersion (source-schema-ref configuration 'sourceVersion)) + (cons 'upstreamCommit (source-schema-ref configuration 'upstreamCommit)))))) diff --git a/language-support/structured.ss b/language-support/structured.ss new file mode 100644 index 00000000..ef3ae594 --- /dev/null +++ b/language-support/structured.ss @@ -0,0 +1,4 @@ +;;; Language recipes declare values; the engine binds and validates their behavior. +(import (only-in :gerbil-parser/src/runtime/structured + StructuredProofPolicy. StructuredProofPolicyContract bind-structured-proof-policy)) +(export StructuredProofPolicy. StructuredProofPolicyContract bind-structured-proof-policy) diff --git a/language-test-support.ss b/language-test-support.ss new file mode 100644 index 00000000..e0654368 --- /dev/null +++ b/language-test-support.ss @@ -0,0 +1,4 @@ +;;; -*- Gerbil -*- +;;; Public constrained language-test syntax; test-only dependency surface. +(import ./src/testing/language-parser) +(export deflanguage-parser-tests) diff --git a/languages/README.org b/languages/README.org new file mode 100644 index 00000000..2f8185ac --- /dev/null +++ b/languages/README.org @@ -0,0 +1,734 @@ +#+title: Language pack boundary + +Each language pack has three Scheme interfaces: + +| Interface | Responsibility | +|-----------+----------------| +| =grammar.ss= | Declarative syntax, pinned source metadata and corpus declarations | +| =parser.ss= | Production POO Loader entry and parser exports | +| =parser-test.ss= | Corpus declarations, development Loader and native conformance tests | + +Raw upstream grammar and corpus files remain resources. Scanning, state, +recognition, source offsets, backend generation and FFI execution belong to the +engine. =deflanguage-parser-loader= admits the effective POO object, its grammar +and extension values before publishing the entry. Inheritance, overrides and +extension contracts use the same admission boundary. Authors supply fixture +values directly; the engine owns evaluation and service binding. + +All eight builtin language directories now follow this contract. TLA+ declares +its core, layout and candidate scopes in the same grammar and parser interfaces. +Generated products, scanners, projection execution and benchmark infrastructure +belong to the engine or test owners. No compatibility modules remain. + +#+begin_src scheme +(import (only-in :gerbil-parser/language-support + deflanguage-parser-loader) + ./grammar) +(export example-language parse-example) + +(deflanguage-parser-loader example-language + (grammar example-syntax) + (parse parse-example) + (slots metadata: '((language . "example") (version . "v1") + (contract . "example.syntax.v1")))) +#+end_src + +The loader is a POO object, admitted when the module loads. =parse-example= binds +the engine method once. Parsing neither rebuilds the loader nor repeats POO +contract admission. Engine slots bind schema, descriptor, language, version, +syntax contract, capabilities and dispatch; extension declarations cannot +replace them. Parent prototypes cannot replace the bound engine slots either. + +* Grammar declaration contract + +=deflanguage= accepts exactly =(syntax (vocabulary ...))= and =(rules ...)=. +Builtin lexical grammars use =lexical=; pinned source grammars use their imported +=antlr4= or =iso-bnf= vocabulary; Bash uses =shell=. All share the same author +boundary and canonical engine admission. The flat form and old source-adapter +macro exports are removed, so a retired declaration cannot publish a parser. + +#+begin_src scheme +;; grammar.ss +(deflanguage example + (syntax (lexical (root name) + (lex (identifier Identifier (identifier))))) + (rules (name (node Name (field value identifier))))) +#+end_src + +Token kinds and node fields are derived from these declarations. Grammar rules +use =node=, =seq= and =prec= where explicit construction is needed; engine +=alias/sequence/precedence= encodings are not author constructors. Strings and +unambiguous declared names need no literal/token/reference wrapper. Hygienic +=defgrammar-syntax= abstractions preserve author binding and source diagnostics. + +Release metadata belongs to the parser Loader. A generated execution backend is +bound through the parser declaration's =backends= section, where engine admission +checks its kind, grammar digest and procedure before publication. Grammar authors +cannot provide =flow= or =backends= clauses. Corpus and benchmarks remain under +the test and development interfaces. + +* Extension slots and contracts + +Use ordinary POO inheritance for language options and tooling services. Supply +POO contracts when an extension needs additional constraints. These contracts +check the effective inherited object before its parse entry is published. + +#+begin_src scheme +(import (only-in :clan/poo/object .o .ref) + (only-in :clan/poo/mop define-type) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence)) + +(def ExampleDialect. + (.o (:: @ LanguageLoader.) dialect: 'example)) + +(define-type (ExampleDialectContract @ PooFlowContract.) + identity: 'example/dialect + .classify: (lambda (candidate context) + (let (accepted? (eq? (.ref candidate 'dialect) 'example)) + (poo-flow-classification-evidence + 'example/dialect candidate accepted? + (if accepted? '() '((invalid dialect))) context)))) + +(deflanguage-parser-loader (example-language :: self ExampleDialect.) + (descriptor example-language-grammar) + (parse parse-example) + (slots display-name: "Example" + (summary (string-append (.ref self 'display-name) " dialect"))) + (contracts ExampleDialectContract)) +#+end_src + +The explicit =self= name binds computations in the extension slots. The shorter +=(name @ prototype)= form is available when slots do not need a self binding. + +The grammar descriptor remains the compiler's admission boundary. Slots do not +make arbitrary parser callbacks into a portable Scanner/Parser IR, or qualify a +backend automatically. A source parser such as Bash uses the same loader but +advertises =scheme-source=; generated grammar entries advertise +=grammar-ir=, =parser-ir= and =scheme=. Rust AOT and C ABI products retain their +own artifact/capability checks. + +* Grammar DSL + +Native grammar packs use =deflanguage=: =root=, =lex= and =rules= +define the syntax. =extras=, =keywords= and =recoveries= default to empty; +=conflicts= defaults to =reject= and =case-insensitive= to =#f=. Optional sections +can be declared in any order; unknown or duplicate sections reject. +The engine infers syntax kinds, fields, terminals and lexical +rules. Alternatives of the same named node contribute to one field catalog. +Use explicit =(token name)= or =(reference name)= when both namespaces contain +the same name; an ambiguous bare identifier is rejected. + +=node-fields= declares optional public fields required by an existing syntax +contract, without repeating the complete inferred catalog. =flow= can retain a +pack's phase names; the default is source → lexical → parser → CST. The engine +validates both through the same canonical Grammar IR compiler. + +A minimal =grammar.ss= is: + +#+begin_src scheme +(import (only-in :gerbil-parser/language-support/grammar deflanguage)) +(export example-syntax) + +(deflanguage example + (root document) + (lex (word Identifier (identifier))) + (rules (document (node SourceFile (field name word))))) +#+end_src + +The =parser.ss= example above imports =./grammar= and exports +=example-language= and =parse-example=. The compiler generates +=example-syntax=; parser metadata binds =example-language-grammar=. The pack does not write its own terminal catalog, +Parser IR assembler, backend installer or Rust/C bridge. Product generation +still uses the engine's build entrypoints and each target's capability admission. + +Imported ANTLR/ISO-BNF grammars keep their source-admission DSL; they produce the +same generated descriptor and use the same loader. Bash's compiled source +plans and TLA+'s multiple grammar scopes remain explicit implementation +boundaries. Their shared-IR migration is not implied by entry unification. + +** Strict quoted text in shared lexical IR + +Use =(quoted-string-profile delimiter escapes unicode-width)= for quoted text +whose backslash escapes must be admitted explicitly. For example, FHIRPath +strings and delimited identifiers declare: + +#+begin_src scheme +(lex + (string StringToken (quoted-string-profile "'" "`'\\/fnrt" 4)) + (delimited-identifier DelimitedIdentifierToken + (quoted-string-profile "`" "`'\\/fnrt" 4))) +#+end_src + +=delimiter= is exactly one character. =escapes= lists accepted single-character +backslash escapes. =unicode-width= is an exact integer from 0 to 8; zero disables +special =\u= handling. Simple escapes take priority, so including =u= in +=escapes= makes it a one-character escape. Otherwise =\u= requires exactly +=unicode-width= numeric or ASCII A–F/a–f source characters. Recognition preserves +source spelling without decoding Unicode values. The numeric class retains the +existing Scheme scanner contract, including non-ASCII decimal digits. Rust uses +a generated range table from the pinned native =char-numeric?= implementation; +superscripts, fractions and Roman numeral characters remain rejected. + +Raw newlines remain accepted; adjacent quoted values stay separate. Unknown +escapes, incomplete hex sequences and missing closing delimiters fail closed. +The descriptor owns these rules: loader extension slots and scan workers cannot +redefine an admitted lexical profile. Scheme executes the compiled profile with +character endpoints; generated Rust executes the same data with UTF-8 byte +endpoints. Grammar identity includes the declaration. + +FHIRPath uses shared quote and closed text profiles for strings, identifiers, +numbers and temporal lexemes. Each portable product still requires its own +producer and consumer qualification. +=t/quoted-string-profile-test.ss= owns Scheme semantics and the portable grammar +witness. =t/generate-quoted-string-profile.ss OUTPUT.rs= emits that witness for +Rust; CI regenerates it and Cargo tests its accepted/rejected source behavior. + + +* POO pack metadata and services + +Public entries use native POO prototypes: =LanguageLoader.= for production parsing +and =LanguageDevelopmentLoader.= for optional corpus, scanner and build services. +Both use the explicit =(name :: self prototype)= +form. =slots= are ordinary POO slots: a downstream prototype can inherit or +replace the language-facing services, and =contracts= validate the effective +object. They are not a second parser registry. + +| Slot | Default | Admission / interface | +|---+---+---| +| =grammars= | List containing the configured =descriptor= | List of grammar/source descriptors; must contain the entry descriptor. Multiple scopes may be exposed here. | +| =metadata= | Normal value or native POO object | parser.ss declares language/version/contract; the engine binds syntax and derives digest/digest-kind. Native inheritance is optional; extension contracts constrain extra fields. | +| =fixtures= (development only) | Empty list | Ordinary POO value containing native =syntax-fixture= data. Override with a new corpus or extend with =append=; no callback wrapper. Loader admission checks the value shape; the common development test service checks identity and conformance. | +| =tests= (development only) | Common fixture conformance service | Unique named, descriptor-bound =declare-language-fixture-test= declarations. =run-language-test= executes the common conformance protocol; bare procedures, duplicate names and foreign descriptors reject. | +| =scan-workers= (development only) | Empty list | Unique named declarations bound to the exact source descriptor; bare procedures and foreign descriptors reject. Engine creation checks the executor/source binding. | + +For example, the HCL development entry binds its existing corpus and the common +conformance service through POO rather than another language-specific wrapper: + +#+begin_src scheme +(deflanguage-development-loader hcl-test-language + (grammar hcl-language-grammar) + (parse parse-hcl-test) + (slots metadata: '((grammar-format . concise-dsl)) + fixtures: hcl-official-fixtures)) + +(run-language-test hcl-test-language 'fixtures) +#+end_src + +Production pack entries import =language-support/entry=. The optional +=parser-test.ss= interface imports =language-support/development= and +=language-support/fixture= directly. This removes their direct dependency +on optional ANTLR/BNF/JavaCC authoring importers in the aggregate +=language-support= facade; the facade exports the production entry API; optional development operations +are imported from =language-support/development=. +Native grammar modules import the narrow =language-support/grammar= frontend; +ANTLR and ISO-BNF modules import their respective source adapters. + +The unreleased =gerbil-parser.language-entry.v2= contract is corrected in place; +this source migration does not create a new schema or release version. +Language modules and public bindings use stable names (=languages/hcl/parser=, +=hcl-language=, =parse-hcl=). Edition/version data belongs in Loader metadata. + +The development prototype using =gerbil-parser.language-entry.v2= derives the default fixture-test +declaration from the effective descriptor through the inherited POO slot. +Fixture values participate in ordinary POO inheritance. Users assign a new list +for replacement, or a computed value slot such as +=(fixtures (append (.ref BaseLanguage. 'fixtures) extra-fixtures))= for extension. +POO handles value-slot evaluation; callers never invoke the fixture slot. + +Additional names may use =declare-language-fixture-test= with that descriptor. +A new service kind requires an engine opcode and input/result contract. + +=check-language-loader-fixtures!= first admits the whole corpus: fixture IDs +must be unique and every fixture's language/version/contract must match the +loader. A foreign fixture or duplicate ID rejects before any source is parsed. +It then owns status, roundtrip, root and required-kind checks and returns the +admitted artifacts. Lexical rejection retains the complete source bytes as an +opaque token when the lexer cannot publish a token stream. This makes negative +fixtures lossless even when a language has no fallback lexical rule. +The fixture slot must contain syntax-fixture data; a mismatch raises an error. +Loader admission reads the value and checks its shape without running fixture +parses, test services or scan workers. Ordinary parsing does not +resolve these services. Corpus declarations belong to =parser-test.ss=, which +imports the production parser; production modules never import their tests. + +All shipped development entries register their corpus conformance service, +including HL7, FHIRPath, Bash, TLA layout and the policy-bound SANY candidate. +The services are available through =language-support/development=. A short +fixture is declared inside =parser-test.ss= without a fourth interface: + +#+begin_src scheme +(defsyntax-corpus example-fixtures + (identity "example" "v1" "example.syntax.v1") + (accepted ("example/word" example-word (text "hello") SourceFile ())) + (rejected ("example/empty" example-empty (text "")))) +#+end_src + +A corpus source is either a path string relative to the declaration or +=(text "source bytes")=. Both produce the same immutable =syntax-fixture= +representation and SHA-256 source identity. =defsyntax-fixture= likewise accepts +=(source "path")= or =(text "source bytes")=. Providers are still ordinary POO +slots and are invoked only when the test service is explicitly run; file sources +are embedded at expansion, so runtime conformance has no filesystem dependency. +These syntax fixtures complement language-specific projection, concurrency and +incremental tests; they do not stand in for those assertions. + +Bash registers its =command= source worker using +=(declare-language-source-scan-worker bash-source-language)=. +Create it with =(make-language-scan-worker loader 'command source)=. This is an +explicit transitional source-parser declaration, not a closed Scanner IR product. +The loader rejects raw factories, duplicate names and foreign descriptors; +creation rejects malformed or foreign-source executors. Checkpoints bind the +creating worker even when two workers receive the exact same string object. +The engine's +=source-scanner-tokens= owns traversal, checkpoint progression, UTF-8 offsets and +token construction through =source-scanner-step=. The shell engine owns word +recognition and deferred here-document transitions. HL7 uses portable closed +header rules; FHIRPath uses portable local text profiles. Structured TLA+ module +text and proof scanners are bound from POO recipes in the Scheme engine. +Those recipes do not yet have a standalone Rust consumer. + +* Shared closed text profiles + +Local lexemes use =(text-profile expression)= inside =deflanguage='s =lex= +section. The engine prepares the Scheme matcher once when constructing its +lexer and emits the same closed data into Rust AOT. Language packs supply +syntax; neither target requires a language callback. + +#+begin_src scheme +(import (only-in :gerbil-parser/language-support deflanguage deftext-profile)) +(deftext-profile fractional-part + (optional (seq (literal ".") (run (numeric) 1 #f)))) +;; Within a lex row: +;; (number Number (text-profile +;; (seq (run (numeric) 1 #f) (ref fractional-part)))) +#+end_src + +=deftext-profile= declares a reusable syntax fragment. =(ref fragment)= expands +hygienically during declaration compilation; fragment names do not survive in +Grammar/Parser IR and cannot become runtime lookups. Cyclic or excessively deep +expansion rejects before publication. Direct profile data remains available in +the canonical IR; fragments are not runtime scanner procedures. + +| Operation | Semantics | +|---+---| +| =(literal text)= | Consume nonempty exact text | +| =(run class minimum maximum)= | Consume a maximal class run within bounds; =#f= means input-bounded maximum | +| =(seq expression ...)= | Require every step in order | +| =(optional expression)= | Restore the entry offset when the whole child fails | +| =(if-next class yes [no])= | Choose by the next character; a chosen branch must succeed | +| =(not-next class)= | Reject a matching next character without consuming input | + +Classes are =(numeric)=, =(ascii-letter)=, =(characters text)= and +=(union class ...)=. Numeric matches Scheme's Unicode decimal-digit semantics, +including Arabic and full-width digits; general Unicode number categories are +not interchangeable. Widths count characters in both engines. Scheme returns +character positions; Rust returns UTF-8 byte spans. Counts and summed minimum +widths fit an unsigned 32-bit value. Profiles and classes have depth below 64. +Empty character sets, malformed bounds, unknown operations and nullable roots +reject at compilation/product admission. Interior zero-width guards are valid; +there is no repetition operator over a nullable profile. + +Choose rollback explicitly: =1.= retains the integer token, while a date/month +prefix such as =@2024-= must reject after the hyphen selects its required +month branch. FHIRPath's date/time grammar declares these decisions directly. +Source-dependent delimiters, nesting and deferred obligations require shared +source context/state IR; a local text profile does not encode those mechanisms. + +* Source-dependent scanners and migration boundary + +=deflanguage-parser-loader= admits the worker declaration against its exact +source descriptor. A worker's checkpoints belong to the creating worker, even +when another worker scans the same string. Source callbacks retain their Scheme +capability classification until the language's execution is represented by +closed Scanner IR; loader registration alone does not confer Rust portability. + +Bash uses the engine's deferred delimiter queue and quote-removal semantics. +Its language context retains syntax routing and exposes pending obligations as +an ordered value; insertion does not rebuild that list. Closed contextual +Scanner IR uses the same queue and marker operation. Engine admission rejects +unknown marker policies, malformed values and foreign obligation types. + +The next source declarations require worker-local prepared captures (HL7), a +source revision bound nesting index (TLA+) and closed region/command transitions +(Bash). These are implementation work, not currently available authoring slots. +The detailed contract and closure gates are in +[[file:../docs/roadmap/0001-versioned-language-pack-closure.org][the language convergence roadmap]]. + +Bash authoring now declares =ShellSourceStrategy.= and a closed region value. +The engine owns scanners, word recognition, span links and artifacts. Full +standalone Rust execution still requires command/state IR and an equivalent +consumer; source strategy registration does not grant that capability. + +* Published catalog projection + +New concise DSL packs infer their kinds, fields and terminals. An existing pack +may supply =(catalog (syntax-kinds ...) (terminals ...))= to preserve its public +ABI. The engine validates it against the inferred declaration: every emitted +kind/category/field remains present; terminal mappings remain exact; kind, +field and terminal identities remain unique. Additional reserved nodes and +fields are allowed. Unused token kinds require the explicit optional +=(reserved-token-kinds ...)= section inside =catalog=; reservations must name +unique unused token rows. Undeclared extra tokens or reservations of active +lexer kinds are rejected. Public row and +field order is retained, and lexical priority still follows =lex= order. + +The first published kind must be a node and retains the existing Parser IR +=root-kind= identity. It may differ from the entry rule's constructed node in a +legacy catalog: the SANY candidate historically starts with =SelectorArguments=. +The projection preserves that ABI rather than silently resetting it during +migration. Core, layout and SANY candidate now use =deflanguage= with their +published catalogs; syntax-scope and proof-policy boundaries stay explicit. +The candidate also retains its unused =SeparatorLine= token through an explicit +reservation. The candidate projection exposed a historical metadata omission: =TerminalProof= +emits =item= through =fact-list=, while its catalog declared only =fact/definition=. +Its catalog now includes =item=. Kind IDs and recognition rules are retained; +the field catalog and grammar digest change deliberately. Consumers must use the +regenerated descriptor/product identity for this draft scope. + +* Ownership + +- =grammar.ss=: declarative syntax, language profiles and pinned metadata. +- =parser.ss=: the only public loader entry, including declared POO extensions. +- =parser-test.ss=: corpus values, development Loader and conformance assertions. +- Resource files: pinned upstream grammar and corpus source text. +- Engine: scanner algorithms, state/checkpoints, recognition, byte offsets, + parser assembly, generated backends and C/Rust products. Existing handwritten + language modules and checked-in specializations remain migration debt until + this execution is derived from admitted declarations. + +* =deflanguage-parser-loader= contract + +This is the production parsing macro. Optional development services use +=deflanguage-development-loader= and =LanguageDevelopmentLoader.=. The former =deflanguage-parser= and +=deflanguage-loader= names have been removed from this development branch; +existing packs and the external record-pack fixture use the canonical name. +=LanguageLoader.= is its POO prototype and =LanguageLoaderContract= is the +common parsing-entry admission contract. + +The declaration defines two bindings: the admitted loader object and the +procedure named in =(parse ...)=. It does not read files, resolve package names, +start a VM, emit C or Rust, or compile a grammar on a parse request. + +The following forms show the arguments after =deflanguage-parser-loader=. + +| Form | Required input | Result | +|---+---+---| +| =(name (grammar descriptor) (parse procedure))= | Generated =language-grammar= descriptor | Grammar/Parser IR Scheme entry | +| =(name (source descriptor) (parse procedure))= | =source-language= descriptor | Scheme source entry | +| =(name (descriptor descriptor) (parse procedure))= | Either descriptor type | Descriptor-selected entry | +| =((name @ prototype) (descriptor descriptor) (parse procedure) (slots ...) (contracts ...))= | POO parent plus either descriptor | Inherited extension object, then engine and extension admission | +| =((name :: self prototype) (descriptor descriptor) (parse procedure) (slots ...) (contracts ...))= | Same, with an explicit self binding | Computed extension slots can refer to the effective object | + +For both =@= and explicit =self=, =slots= and =contracts= may be omitted, +including a contracts-only declaration. When both are supplied, =slots= precedes +=contracts=. =grammar= and =source= may replace =descriptor= in any prototype +form and reject a descriptor of the other kind. All forms normalize to the same +admission and dispatch implementation. + +Protected slots are =schema=, =descriptor=, =language=, =version=, =contract=, +=capabilities=, and =.parse=. The macro rejects direct extension overrides and +binds these slots at the leaf so inherited methods cannot replace them. +User slots can describe dialects, editor services, query/projection services, +and other language-specific options; additional POO contracts constrain these +slots after inheritance. Adding a slot alone does not make the recognition +engine consume it. New recognition policies require an engine-owned consumer +and a defined artifact/capability contract. + +=language-parser-entry-ref= reads metadata. The exported parse binding captures +the descriptor once; it performs no per-call POO lookup or extension admission. +A prototype's own =.parse= cannot override engine recognition. General POO +updates to a separate object do not rebind an already exported parse procedure; +create a new loader declaration when defining a different language entry. + +The language-independent C ABI currently admits descriptors through +=register-language!=. Its current codec requires a generated +=language-grammar= descriptor; Bash source descriptors are not admitted by that +codec. It does not serialize loader slots or POO callbacks. +An opaque lexical rejection uses the =unknown= token. GPA1 serialization requires +that token in the descriptor terminal catalog; descriptors without it receive an +explicit catalog error. Scheme fixture conformance alone does not qualify that +C ABI rejection path. +Standalone Rust AOT consumes admitted portable IR. Neither path automatically +inherits arbitrary POO extension behavior. Descriptor-bound Scheme policies +are consumed by the loader and native C ABI; standalone Rust AOT rejects them +before emission until a portable policy representation exists. + +* Parser policy and capability admission + +Use =language-grammar-with-parser-policy= from the public language-support facade +when a generated language needs a recognition budget and a successful-CST +validation rule. Configure the descriptor before declaring the loader; export +that configured descriptor from the public entry module. + +#+begin_src scheme +(def example-language-grammar + (language-grammar-with-parser-policy + recognition-language-grammar "example.validation.v1" 4096 example-cst-diagnostic)) + +(deflanguage-parser-loader (example-language :: self ExampleDialect.) + (grammar example-language-grammar) + (parse parse-example) + (slots display-name: "Example")) +#+end_src + +=recognition-language-grammar= is the raw grammar descriptor imported under a +local name. The policy identity is a nonempty, versioned author-owned string; +the budget must be a positive fixnum and the checker a procedure. The checker +receives a successful CST and returns =#f= to accept or a valid +=gerbil-parser.diagnostic.v1= diagnostic to reject. Recognition failures skip +the checker. Exceptions propagate, and an invalid diagnostic cannot publish an +artifact. The engine retains all token spans and source bytes in a rejection. + +The budget is dynamically scoped to recognition and validation, including +exceptional exits. =language-grammar-with-observability= preserves this policy. +The loader advertises =scheme-policy= in addition to its generated-grammar +capabilities. Extension contracts still run only during loader admission; +the CST checker runs once for each successful recognition. + +The configured descriptor is the product boundary: pass it to +=register-language!= or export it from the module named by +=generate-language-pack=. A prepared contextual native parser applies +the same policy; its deferred event publication cannot bypass the CST checker. +Native descriptor metadata contains =parserPolicy= with identity, branch +budget, =schemeCstValidation=true= and =portable=false=. The recognition grammar +digest remains the syntax identity; it does not fingerprint callback code. +Version the policy identity when the rule changes. + +All standalone Rust descriptor products delegate to the engine's +=language-rust-runtime-module-source= admission boundary: the ordinary module +generator, the combined contextual Scanner/Parser IR generator, and the build +ABI. They reject this Scheme policy before product preparation or writing output. Generating raw recognition IR is a narrower +syntax-only product and cannot establish policy equivalence. A Rust caller using +the native C ABI does execute the Scheme policy through that ABI. Do not +advertise a standalone Rust policy capability until its shared representation, +lowering and parity tests exist. + +* A downstream language repository + +The minimum authored files are =gerbil.pkg=, =grammar.ss= and =parser.ss=; +conformance fixtures belong alongside them. Choose a package namespace and pin +engine dependencies through the normal Gerbil package workflow. Neither a new +language name in the engine nor a custom C dispatcher is required. + +1. Declare syntax in =grammar.ss= using the minimal DSL example above. Keep its + language, version and syntax-contract identities explicit. +2. Declare the public loader in =parser.ss=. Export its parse procedure, loader + object and configured descriptor when native consumers need it. Import only + the required names from =:gerbil-parser/language-support=. +3. Validate accepted and rejected source, lossless roundtrip, UTF-8/CRLF spans, + field catalogs and any language policy using Scheme tests. POO extension + tests must include inherited values, invalid values and sealed-slot rejection. +4. Select a product using the table below. Generate it through the engine and + run downstream semantic parity; a successful loader admission alone does not + qualify a target. + +| Product | Engine admission | Downstream responsibility | +|---+---+---| +| Scheme | =deflanguage-parser-loader= | Syntax declarations, optional POO services/contracts, conformance fixtures | +| Native C ABI / Rust native wrapper | =generate-language-pack= and =register-language!= | Export the configured descriptor, link the generated factory with the normal SDK builder, use generic handle/result ownership | +| Independent Rust AOT | =generate-language-rust-runtime-module=, =generate-contextual-language-rust-runtime-module=, or the engine build ABI | Supply an admitted portable descriptor, consume generated Rust with =gerbil_parser_runtime=, compare accepted/rejected artifacts | +| Engine-bound source parser | =deflanguage= syntax provider, then its configured parser descriptor | Native C ABI publishes the engine result catalog; independent AOT requires an explicitly admitted source product | + +The C factory generator takes output Scheme/header paths, the generated module +identity, the descriptor-exporting module, its descriptor name and a C symbol +prefix. Its optional contextual export must come from that same module. It +emits integration code; the normal Gerbil builder owns compilation and linking. +Create a language handle once and reuse it across parses; initialize the runtime +once and release results and handles through the generic ABI. Per-call VM or +process creation is outside the loader contract. + +For working native ownership examples see +[[file:../rust/gerbil-parser-ffi/README.org][Rust FFI wrapper guide]]. For the standalone build ABI, portable scanner +admission and external consumer witness see +[[file:../docs/architecture/0006-rust-aot.org][Rust AOT design]]. + +* Generated backend declarations + +=deflanguage= accepts an optional =backends= section for existing generated +specializations. Packs declare what they supply; the engine installs it. + +#+begin_src scheme +(backends + (step direct-grammar-digest direct-step) + (source direct-hcl-grammar-digest direct-parse-hcl) + (event-step direct-grammar-digest direct-event-step)) +#+end_src + +The closed kinds are =drive=, =source=, =step=, and =event-step=. The engine +checks every row's kind, exact Parser IR digest, procedure, duplicate kind, and +occupied slot before installing any row. A rejected later row cannot leave an +earlier backend installed. This check establishes identity and assembly safety; +semantic equivalence still requires regeneration and language conformance tests. +The declaration is excluded from Grammar/Parser IR, so installing the same +specialization does not change syntax identity. New ordinary language packs do +not need a specialization or this section. + +* Migration audit and remaining engine work + +The complete dependency plan, all-eight-family inventory and stage exit criteria +are maintained in [[file:../docs/roadmap/0001-versioned-language-pack-closure.org][the language-pack convergence plan]]. + +This inventory describes implementation boundaries, not equivalent syntax +coverage across the languages. + +| Pack | Authoring and engine ownership | Capability boundary | +|---+---+---| +| Arithmetic | Three interfaces; generated drive in =src/compiler/arithmetic-drive.ss= | Reviewed specialization; shared reduction and Rust build providers | +| HCL | Three interfaces; source generator and generated products in =src/compiler/= | Recursive source provider admits the reviewed lexical/grammar corridor | +| HL7 | Three interfaces; closed header rules and generic record projection | Independent record recipe validates projection reuse; Rust header consumer remains qualified separately | +| FHIRPath | Three interfaces; shared quote/text profile execution | Portable products retain their independent admission gates | +| openCypher | Three interfaces; pinned ISO-BNF grammar admission | Shared graph syntax emits the constrained MATCH/WHERE/RETURN subset | +| GQL | Three interfaces; pinned ANTLR4 admission | Graph syntax in the engine; actor/profile/benchmark tests under =t/= | +| TLA+ | Three interfaces contain all three scopes; structured lexeme/proof recipes | Scheme execution; proof policy still rejects standalone Rust AOT | +| Bash | Three interfaces; =deflanguage= syntax/rules compile through the common shell vocabulary | Scheme source engine; full shell grammar is not standalone Rust Scanner/Parser IR | + +* Source author contract and reusable engine contracts + +Source-backed authors use =deflanguage= with one =syntax= block and one =rules= +block. Release language/version/contract are declared once in parser.ss metadata. Bash uses =shell= from +=:gerbil-parser/language-support/shell= and native =defgrammar-syntax= vocabulary. +The common library derives scanner, capture and command plans, then binds the +checked source descriptor. + +=SourceStrategy.=, its profiles and =declare-source-language= are implementation +interfaces, not an alternative author entry. The old =deflanguage-source= entry +has been removed. =language-source-support= supplies source inspection and +=deflanguage-parser-receipt= for parser.ss. See the complete +[[file:../docs/research/scheme-bash-source-frontend.org][Bash frontend implementation and admission study]]. + +Module framing is declared as closed lexical data: + +#+begin_src scheme +(module-text "----" "MODULE" "====" "(*" "*)" "\\*" "_") +#+end_src + +The seven values specify the opening border, header word, closing border, nested +comment delimiters, line-comment prefix and extra name character. Borders must +repeat a single character; values are nonempty and at most 128 characters. The +engine prepares compact header intervals and nested-depth events once per source. +Scheme and Rust execute the same declaration. No scanner closure is supplied by +the pack; framing spelling participates in the grammar digest. + +=language-support/structured= exposes =StructuredProofPolicy.= with its POO +contract. Policies constrain CST kinds, field names, terminal commands and +forbidden subproof commands/kinds. Independent checklist tests exercise inherited +overrides with different CST fields. Effective values are validated and copied +before binding. This proof policy remains a Scheme CST service; portable proof +policy execution is a separate implementation obligation. + +=language-support/projection= provides =RecordProjection.= and +=deflanguage-projection=. Projection columns are closed =record=, =split=, =text=, +=concat= and =list= selectors. Prefix, delimiter count, expectations and constant +fields are ordinary values. The engine checks accepted artifact ownership and +publishes source/grammar digests. HL7 and an independent Unicode header record +use this protocol; columns cannot substitute arbitrary projection callbacks. + +=graph-syntax-support= exposes shared =GraphSyntax*= POO prototypes for a +constrained graph pattern. GQL and openCypher both parse the generated source. +This contract does not describe all graph language syntax or domain semantics. + +=RecursiveSourceStrategy.= uses the common =BuildStrategy.= protocol. Its closed +HCL corridor remains explicit; moving its generator does not claim universal +recursive lowering. The generic generation entry is: + +#+begin_src sh +gerbil env gxi scripts/generate-source-parser.ss check :gerbil-parser/languages/hcl/grammar src/compiler/hcl-source.ss +#+end_src + +A module exporting several descriptors must select an export for build input: +=languages/tla-plus/parser.ss#tla-plus-sany-candidate-language-grammar=. +The same selector is used by the build C ABI and generator CLIs. Ambiguous, +missing or non-descriptor exports reject before code emission. + +=t/language-topology-test.ss= checks all builtin packs for exactly three Scheme +interfaces and mandatory Loader/test declarations. The Loader schema remains +=gerbil-parser.language-entry.v2=. There are no versioned namespace aliases or +compatibility adapters. Shared Scheme recipes, portable Scanner/Parser IR and +actual C/Rust products retain separate qualification boundaries. + +* Authoring design research + +[[file:../docs/research/scheme-language-pack-design.org][Scheme language pack design]] records paper-based research into macro +composition, binding, diagnostics and derived language products. It is a research +proposal; the authoring contract above describes the implemented interfaces. + +* Fixture metadata through the Loader + +For every language, declare corpus rows in =parser-test.ss= and assign their list to the +development Loader's =fixtures= value slot. The common development library +derives =fixture-catalog= internally; +this slot is sealed by =deflanguage-development-loader=. Authors write neither a +filter nor a factory/thunk for status groups or indexes. + +#+begin_src scheme +(deflanguage-development-loader (example-test-language :: self LanguageDevelopmentLoader.) + (grammar example-language-grammar) + (parse parse-example-test) + (slots fixtures: example-fixtures)) + +(language-loader-fixture-count example-test-language) ; total +(language-loader-fixture-count example-test-language 'accepted) +(language-loader-fixtures example-test-language 'rejected) +(language-loader-fixture example-test-language "example/valid") ; fixture or #f +#+end_src + +These functions are exported from =language-support/development=. Only =accepted=, +=rejected= or omitted status is admitted. Loading validates descriptor identity, +unique nonempty IDs, source digests and expectation shapes, then owns copies of +the corpus and derives its partitions and ID index once. Counting is constant +time; ID lookup uses the index. Published fixture values are copies and cannot +change the admitted snapshot. Selection preserves declaration order, including +interleaved statuses. Parsing/conformance remains an explicit test service; +metadata admission does not parse every fixture. + +POO replacement/extension of =fixtures= through =.cc= or =.o= derives a new +index for that effective object. Existing metadata versions and Loader v2 stay +unchanged. Mutating input strings after admission does not replace the bound +snapshot; use a new POO value override to rebind. Pack-specific accepted and +rejected aliases have been removed; test and benchmark consumers use this same +engine interface. Benchmark corpora prepare outside the timed parse region. + + +* Portable result construction + +Declare constructors with =defresult-profile= and POO inheritance. The engine's +=result-profile-rust-source= lowers the admitted profile to immutable Rust +projection tables; =PreparedResultProfile= validates them and executes typed +captures through bound projection entries. Fields and terminal names remain +grammar data, shared with Scheme. Downstream authors do not implement tree +building or token accumulation in Rust. + +This generic construction capability is available independently of syntax +recognition. Complete standalone Rust Source/Command recognition remains +an engine implementation boundary; emitting result tables alone does not +publish a standalone language parser. + + +* Portable Word/Binding engine + +PartProfile and BindingProfile POO slots now lower through +=part-profile-rust-source= to a shared Rust =PreparedPartProfile=. An inherited +profile can replace rules, binding prefixes/operators and subscript delimiters; +the same engine executes those admitted values. Languages supply no Rust Word +scanner or callback. Recognition accepts a full source and bounded UTF-8 range, +and composes captured nodes through the declared ResultProfile instructions. +Production Bash and an inherited Unicode configuration have independent +Scheme/Rust event, field and lossless-text controls. Complete Command/Source AOT +and full ParseArtifact parity still require the next shared engine boundary. + +* Production and development responsibilities + +Every shipped =parser.ss= imports =language-support/entry= and exports a +corpus-free production Loader. Each =parser-test.ss= owns its corpus and declares +a =LanguageDevelopmentLoader.= with =deflanguage-development-loader=. This +contract covers all eight packages and ten production entries. +The engine's =src/language/entry= exports descriptor dispatch and generic policy +execution; it does not supply fixture or Loader development services. + +ANTLR source adaptation requires an explicit =lexical-profile= in the language +grammar. GQL declares exact token bindings, scanner expressions, terminal kinds +and trivia there. The common adapter does not classify tokens by GQL name patterns. +See [[file:../docs/research/scheme-engine-library-boundary.org][the ownership contract and paper-to-code rationale]] for the full boundary, +verification route and remaining semantic limits. + +See [[file:../docs/research/scheme-engine-library-boundary.org::*Complete language alignment][Complete language alignment]] for the per-entry corpus inventory and verification obligations. diff --git a/languages/arithmetic/v1/corpus/basic.expr b/languages/arithmetic/corpus/basic.expr similarity index 100% rename from languages/arithmetic/v1/corpus/basic.expr rename to languages/arithmetic/corpus/basic.expr diff --git a/languages/arithmetic/grammar.ss b/languages/arithmetic/grammar.ss new file mode 100644 index 00000000..b5679208 --- /dev/null +++ b/languages/arithmetic/grammar.ss @@ -0,0 +1,63 @@ +;;; -*- Gerbil -*- +;;; Canonical left-recursive arithmetic grammar for LR precedence admission. + +(import (only-in :gerbil-parser/language-support/grammar deflanguage)) +(export arithmetic-syntax + arithmetic-grammar + arithmetic-parser-ir + arithmetic-parser) + + +(deflanguage arithmetic + (syntax + (lexical + (root source-file) + (lex + (whitespace Whitespace (whitespace+)) + (number Number (decimal-digit+)) + (identifier Identifier (identifier)) + (punctuation Punctuation (literals "+" "-" "*" "**" "/" "(" ")")) + (unknown Unknown (fallback))) + (extras whitespace) + (keywords) + (recoveries + (expression "GERBIL-PARSER-EXPRESSION" preserve-source)) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile (field expression expression))) + (expression + (choice + (prec left 10 + (node Expression + (seq + (field left expression) + (field operator (choice (literal "+") (literal "-"))) + (field right expression)))) + (prec left 20 + (node Expression + (seq + (field left expression) + (field operator (choice (literal "*") (literal "/"))) + (field right expression)))) + (prec right 30 prefix-expression) + grouped-expression + name-expression + number-expression)) + (grouped-expression + (node GroupedExpression + (seq + (literal "(") + (field expression expression) + (literal ")")))) + (prefix-expression + (prec right 30 + (node PrefixExpression + (seq + (field operator (choice (literal "+") (literal "-"))) + (field operand expression))))) + (name-expression + (node NameExpression (field name identifier))) + (number-expression + (node NumberExpression (field value number))))) diff --git a/languages/arithmetic/parser-test.ss b/languages/arithmetic/parser-test.ss new file mode 100644 index 00000000..e44bb877 --- /dev/null +++ b/languages/arithmetic/parser-test.ss @@ -0,0 +1,47 @@ +;;; -*- Gerbil -*- +(import :gerbil-parser/language-test-support + (only-in :gerbil-parser/src/compiler/machine parser-machine-direct-drive) + ./parser) +(import (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/fixture defsyntax-fixture) + (only-in :gerbil-parser/language-support/development deflanguage-development-loader LanguageDevelopmentLoader.) + (only-in :gerbil-parser/language-build-support declare-language-build-strategy make-fused-reduction-strategy make-rust-runtime-strategy)) +(export arithmetic-basic-fixture arithmetic-incomplete-fixture arithmetic-test-language) + +(defsyntax-fixture arithmetic-basic-fixture + (identity "arithmetic/v1/basic" + "arithmetic" + "v1" + "arithmetic-expression.v1") + (source "corpus/basic.expr") + (expect accepted SourceFile (Expression))) +(defsyntax-fixture arithmetic-incomplete-fixture + (identity "arithmetic/v1/incomplete" "arithmetic" "v1" "arithmetic-expression.v1") + (text "1 +") + (expect rejected #f ())) +(deflanguage-development-loader arithmetic-test-language + (grammar arithmetic-language-grammar) + (parse parse-arithmetic-test) + (slots metadata: '((grammar-format . concise-dsl)) + build-strategies: (list (declare-language-build-strategy 'fused-reductions + (make-fused-reduction-strategy arithmetic-language-grammar)) + (declare-language-build-strategy 'rust-runtime + (make-rust-runtime-strategy arithmetic-language-grammar))) + fixtures: (list arithmetic-basic-fixture arithmetic-incomplete-fixture))) + +(deflanguage-parser-tests arithmetic-parser-test "arithmetic v1 language pack" + (loader arithmetic-test-language) + (property "compiled machine installs its direct LR driver" + (bindings) + (equal (procedure? (parser-machine-direct-drive arithmetic-parser)) #t)) + (identity "declarative entry identity" + (language "arithmetic") (version "v1") (contract "arithmetic-expression.v1") + (schema "gerbil-parser.language-entry.v2")) + (fixtures "colocated native fixtures") + (accepted "streaming LR preserves the 1024-line source" + (string-join (make-list 1024 "1") "+\n")) + (ast "canonical AST shape is exact" "1" + (node SourceFile 0 1 + (field expression 0 1 + (node NumberExpression 0 1 + (field value 0 1 (token number "1" 0 1))))))) diff --git a/languages/arithmetic/parser.ss b/languages/arithmetic/parser.ss new file mode 100644 index 00000000..2e6ae6c2 --- /dev/null +++ b/languages/arithmetic/parser.ss @@ -0,0 +1,22 @@ +(import (only-in :gerbil-parser/src/compiler/arithmetic-drive direct-drive direct-grammar-digest)) +;;; -*- Gerbil -*- +;;; Canonical public parser entry for the arithmetic v1 reference language. + +(import (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-entry-ref language-metadata-ref) + ./grammar) +(export +arithmetic-language-version+ +arithmetic-syntax-contract+ (import: ./grammar) arithmetic-language-grammar arithmetic-language parse-arithmetic) + +(deflanguage-parser-loader arithmetic-language + (grammar arithmetic-syntax) + (parse parse-arithmetic) + (slots metadata: '((language . "arithmetic") + (version . "v1") + (contract . "arithmetic-expression.v1") + (grammar-format . concise-dsl))) + (backends arithmetic-parser (drive direct-grammar-digest direct-drive))) + +(def arithmetic-language-grammar (language-parser-entry-ref arithmetic-language 'descriptor)) + +(def +arithmetic-language-version+ (language-metadata-ref (language-parser-entry-ref arithmetic-language 'metadata) 'version)) + +(def +arithmetic-syntax-contract+ (language-metadata-ref (language-parser-entry-ref arithmetic-language 'metadata) 'contract)) diff --git a/languages/arithmetic/v1/fixtures.ss b/languages/arithmetic/v1/fixtures.ss deleted file mode 100644 index 2065f121..00000000 --- a/languages/arithmetic/v1/fixtures.ss +++ /dev/null @@ -1,13 +0,0 @@ -;;; -*- Gerbil -*- -;;; Native source fixtures for the arithmetic v1 reference language. - -(import (only-in :gerbil-parser/language-support defsyntax-fixture)) -(export arithmetic-v1-basic-fixture) - -(defsyntax-fixture arithmetic-v1-basic-fixture - (identity "arithmetic/v1/basic" - "arithmetic reference language" - "v1" - "arithmetic-expression.v1") - (source "corpus/basic.expr") - (expect accepted SourceFile (Expression))) diff --git a/languages/arithmetic/v1/grammar.ss b/languages/arithmetic/v1/grammar.ss deleted file mode 100644 index 4676c91d..00000000 --- a/languages/arithmetic/v1/grammar.ss +++ /dev/null @@ -1,67 +0,0 @@ -;;; -*- Gerbil -*- -;;; Canonical left-recursive arithmetic grammar for LR precedence admission. - -(import (only-in :gerbil-parser/language-support deflanguage)) -(export +arithmetic-language-version+ - +arithmetic-syntax-contract+ - arithmetic-language-grammar - arithmetic-grammar - arithmetic-parser-ir - arithmetic-parser) - -(def +arithmetic-language-version+ "v1") -(def +arithmetic-syntax-contract+ "arithmetic-expression.v1") - -(deflanguage arithmetic - (identity "arithmetic" +arithmetic-language-version+ - +arithmetic-syntax-contract+) - (root source-file) - (lex - (whitespace Whitespace (whitespace+)) - (number Number (decimal-digit+)) - (identifier Identifier (identifier)) - (punctuation Punctuation (literals "+" "-" "*" "**" "/" "(" ")")) - (unknown Unknown (fallback))) - (rules - (source-file - (node SourceFile (field expression expression))) - (expression - (choice - (prec left 10 - (node Expression - (seq - (field left expression) - (field operator (choice (literal "+") (literal "-"))) - (field right expression)))) - (prec left 20 - (node Expression - (seq - (field left expression) - (field operator (choice (literal "*") (literal "/"))) - (field right expression)))) - (prec right 30 prefix-expression) - grouped-expression - name-expression - number-expression)) - (grouped-expression - (node GroupedExpression - (seq - (literal "(") - (field expression expression) - (literal ")")))) - (prefix-expression - (prec right 30 - (node PrefixExpression - (seq - (field operator (choice (literal "+") (literal "-"))) - (field operand expression))))) - (name-expression - (node NameExpression (field name identifier))) - (number-expression - (node NumberExpression (field value number)))) - (extras whitespace) - (keywords) - (recoveries - (expression "GERBIL-PARSER-EXPRESSION" preserve-source)) - (conflicts reject) - (case-insensitive #f)) diff --git a/languages/arithmetic/v1/parser-test.ss b/languages/arithmetic/v1/parser-test.ss deleted file mode 100644 index 1a5d07ac..00000000 --- a/languages/arithmetic/v1/parser-test.ss +++ /dev/null @@ -1,41 +0,0 @@ -;;; -*- Gerbil -*- -;;; Arithmetic language-pack entry and native fixture contract. - -(import (only-in :std/test check test-case test-suite) - (only-in :gerbil-parser/src/language/entry - +language-parser-entry-schema+ language-parser-entry-ref) - (only-in :gerbil-parser/src/runtime/artifact - parse-artifact-ref parse-artifact-roundtrip - parse-artifact-success?) - (only-in :gerbil-parser/language-support - syntax-fixture-source syntax-fixture-source-digest) - (only-in :gerbil-parser/src/testing/parser-ast check-parser-ast) - (only-in ./fixtures arithmetic-v1-basic-fixture) - (only-in ./parser arithmetic-v1-language parse-arithmetic-v1)) -(export arithmetic-v1-parser-test) - -(def arithmetic-v1-parser-test - (test-suite "arithmetic v1 language pack" - (test-case "the declarative entry parses its colocated native fixture" - (let* ((fixture arithmetic-v1-basic-fixture) - (source (syntax-fixture-source fixture)) - (artifact (parse-arithmetic-v1 source))) - (check (language-parser-entry-ref arithmetic-v1-language 'schema) - => +language-parser-entry-schema+) - (check (language-parser-entry-ref arithmetic-v1-language 'language) - => "arithmetic") - (check (language-parser-entry-ref arithmetic-v1-language 'version) - => "v1") - (check (language-parser-entry-ref arithmetic-v1-language 'contract) - => "arithmetic-expression.v1") - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-ref artifact 'sourceDigest) - => (syntax-fixture-source-digest fixture)) - (check (parse-artifact-roundtrip artifact) => source))) - (test-case "canonical AST shape is an exact parser contract" - (check-parser-ast (parse-arithmetic-v1 "1") "1" - (node SourceFile 0 1 - (field expression 0 1 - (node NumberExpression 0 1 - (field value 0 1 - (token number "1" 0 1))))))))) diff --git a/languages/arithmetic/v1/parser.ss b/languages/arithmetic/v1/parser.ss deleted file mode 100644 index a9cc40a2..00000000 --- a/languages/arithmetic/v1/parser.ss +++ /dev/null @@ -1,23 +0,0 @@ -;;; -*- Gerbil -*- -;;; Canonical public parser entry for the arithmetic v1 reference language. - -(import (only-in :gerbil-parser/src/language/entry deflanguage-parser) - (only-in ./grammar - +arithmetic-language-version+ - +arithmetic-syntax-contract+ - arithmetic-language-grammar - arithmetic-grammar - arithmetic-parser-ir - arithmetic-parser)) -(export +arithmetic-language-version+ - +arithmetic-syntax-contract+ - arithmetic-language-grammar - arithmetic-grammar - arithmetic-parser-ir - arithmetic-parser - arithmetic-v1-language - parse-arithmetic-v1) - -(deflanguage-parser arithmetic-v1-language - (grammar arithmetic-language-grammar) - (parse parse-arithmetic-v1)) diff --git a/languages/bash/README.org b/languages/bash/README.org new file mode 100644 index 00000000..bab8dedd --- /dev/null +++ b/languages/bash/README.org @@ -0,0 +1,69 @@ +#+title: Bash 5.3 source parser + +This language directory publishes the preserved Scheme source-parser implementation. +It builds with the normal Gerbil package and exposes a versioned parser entry: + +#+begin_src scheme +(import :gerbil-parser/languages/bash/parser) +(parse-bash "printf %s \"${name:-world}\"\n") +#+end_src + +The result is a lossless =ParseArtifact=, with structured word/expansion, +assignment, pipeline, compound-command and redirection nodes. Here-document +receipts bind queued markers to body spans. Parsing does not execute shell code +or evaluate expansions. + +| Capability | Current evidence | +|---+---| +| Scheme source parser and public entry | 17 native regression cases pass | +| UTF-8 round trips and nested word/parameter syntax | Covered by selected corpus | +| Here-document ordering and compound statements | Covered by selected corpus | +| Complete Bash grammar or editor recovery | Not qualified | +| Declared nested region scanner | Native Scheme and generated Rust traces pass; explicit stack handles 5000 levels | +| Complete Grammar/Parser/Scanner IR | Command transitions and structured recognition still require migration | +| Generated pure Rust AOT language | Not supported by this source declaration | +| Standalone native C library / Rust FFI language crate | Packaging remains required | + +The descriptor is a =source-language=, not a =LanguageGrammar= generated by +=deflanguage=. Its digest binds the declared language/version/contract labels; +it does not by itself fingerprint the handwritten parser implementation. +Scanner checkpoints are source-bound scan state, not incremental parse reuse +receipts. The release design will additionally bind implementation artifacts +and admit supported backend capabilities explicitly. + +[[file:../../docs/architecture/0018-downstream-language-products.org][Downstream language products and Bash admission]] +defines the shared IR port, stable field granularity, generated Cargo/C products, +length-bearing C ABI and runtime ownership requirements. + +* Closed region declarations + +=grammar.ss= declares =bash-word-regions= through the public +=defregion-plan= macro. Stops, quote escape policy, quote-visible substitutions, +paired delimiters and initial depth are values. For example, the engine selects +=$((= before =$(= and shields =<(= from the overlapping =<= operator. + +#+begin_src scheme +(defregion-plan word-regions + (stops ";" "<" ">") + (quotes (#\' #f ()) (#\" #t ("${" "$(" "$((")) (#\` #t ())) + (pairs ("${" #\{ #\} 1) ("$(" #\( #\) 1) + ("$((" #\( #\) 2) ("<(" #\( #\) 1) (">(" #\( #\) 1)) + (consume-initial-stop #f)) +#+end_src + +The engine owns plan admission, copied declaration storage, operator preparation, +forward traversal and explicit nesting frames. Its closed =region-word= opcode +is admitted and generated by the existing contextual Scanner IR. Scheme offsets +are scalar positions; the Rust consumer publishes matching UTF-8 byte spans. +=consume-initial-stop= controls whether a word may consume its initial stop; +Bash's transitional marker reader preserves its existing =#t= policy, while +ordinary closed command-word rules use =#f=. + +The deleted =scan-words.ss= has no replacement procedural language module. +=scanner.ss=, =word.ss= and =parser-core.ss= still own command transitions, +word projection and recognition respectively. Full Bash Rust/C products remain +unqualified until those declarations migrate and their callbacks are removed. +Fixtures live in =grammar.ss=. The 17-case =parser-test.ss= now uses +=deflanguage-parser-tests=: source obligations and node/token declarations +replace local artifact traversal and assertion loops. Two engine contract and +UTF-8 checkpoint controls live under =t/= with all original assertions. diff --git a/languages/bash/grammar.ss b/languages/bash/grammar.ss new file mode 100644 index 00000000..a3193f36 --- /dev/null +++ b/languages/bash/grammar.ss @@ -0,0 +1,106 @@ +;;; -*- Gerbil -*- +;;; Bash author grammar. The source library derives and admits execution plans. +(import (only-in :gerbil-parser/language-support/grammar deflanguage defgrammar-syntax) + (only-in :gerbil-parser/language-support/shell shell)) +(export bash-syntax) + +(defgrammar-syntax (compound-loop kind opener) + (node kind (field keyword opener) (field condition command-list) + (field keyword "do") (field body command-list) (field keyword "done"))) + +(deflanguage bash-syntax + (syntax + (shell + (operators ";;&" "&>>" "<<-" "<<<" "&&" "||" "|&" ";;" ";&" + "<<" ">>" "<>" "<&" ">&" ">|" "&>" ";" "&" "|" "(" ")" "<" ">") + (regions + (quote single SingleQuoted "'" opaque) + (quote double DoubleQuoted "\"" escaped "${" "$(" "$((") + (quote backtick CommandSubstitution "`" escaped) + (pair parameter ParameterExpansion "${" "}") + (pair command CommandSubstitution "$(" ")") + (pair arithmetic ArithmeticExpansion "$((" "))") + (pair input ProcessSubstitution "<(" ")") + (pair output ProcessSubstitution ">(" ")") + (quote ansi AnsiCString "$'" opaque)) + (bindings + (simple (if-next (union (numeric) (characters "?@*#$!-_0")) + (run (union (numeric) (characters "?@*#$!-_0")) 1 1) + (run (union (alphabetic) (numeric) (characters "_")) 1 #f)) () () #f) + (parameter (if-next (characters "@*?#$!-") (run (characters "@*?#$!-") 1 1) + (if-next (numeric) (run (numeric) 1 #f) + (run (union (alphabetic) (numeric) (characters "_")) 1 #f))) + ("#" "!") (":-" ":=" ":+" ":?" "##" "%%" "//" "^^" ",," "~~" + ":" "-" "=" "+" "?" "#" "%" "/" "@" "^" "," "~") ("[" #\])) + (assignment (seq (run (union (alphabetic) (characters "_")) 1 1) + (run (union (alphabetic) (numeric) (characters "_")) 0 #f)) () ("=" "+=") #f)) + (words + ((word) (quoted ansi)) ((word) (quoted single)) ((word) (quoted double)) + ((word DoubleQuoted HereDocument) (parameter parameter)) + ((word DoubleQuoted HereDocument) (substitution arithmetic)) + ((word DoubleQuoted HereDocument) (substitution command)) + ((word) (substitution input)) ((word) (substitution output)) + ((word DoubleQuoted HereDocument) (substitution backtick)) + ((word DoubleQuoted HereDocument) (name SimpleParameter "$" simple)) + ((word DoubleQuoted AnsiCString) (escape EscapeSequence any)) + ((HereDocument) (escape EscapeSequence "$`\\\n"))) + (commands + (roles + (trivia (horizontal-whitespace) (comment) (line-continuation)) + (separator (newline) (operator ";" "&")) + (redirect (operator "<" ">" ">>" "<>" "<<" "<<-" "<<<" "<&" ">&" ">|" "&>" "&>>")) + (here-redirect (operator "<<" "<<-")) (strip-redirect (operator "<<-")) + (reserved (word "if" "then" "elif" "else" "fi" "while" "until" "do" "done" "for" "select" "case" "in" "esac" "function" "{" "}")) + (conditional-operator (word "==" "=" "!=" "=~" "-eq" "-ne" "-lt" "-le" "-gt" "-ge" "-z" "-n" "!")) + (pipeline (operator "|" "|&")) (and-or (operator "&&" "||")) (case-end (operator ";;" ";&" ";;&"))) + (descriptor (if-next (numeric) (run (numeric) 1 #f) + (seq (literal "{") (run (union (alphabetic) (characters "_")) 1 1) + (run (union (alphabetic) (numeric) (characters "_")) 0 #f) (literal "}")))) + ))) + (rules + (conditional-branch + (node IfCommand (field keyword "if") (field condition command-list) (field keyword "then") + (field body command-list) + (repeat (seq (field keyword "elif") (field condition command-list) (field keyword "then") (field body command-list))) + (optional (seq (field keyword "else") (field else-body command-list))) (field keyword "fi"))) + (while-loop (compound-loop WhileCommand "while")) + (until-loop (compound-loop UntilCommand "until")) + (for-loop + (node ForCommand (field keyword "for") + (choice (node ArithmeticForCommand (field header (reference arithmetic))) + (seq (field variable word) (optional (seq (field keyword "in") (repeat (field item word)))))) + (field separator (role separator)) (field keyword "do") (field body command-list) (field keyword "done"))) + (select-loop + (node SelectCommand (field keyword "select") (field variable word) + (optional (seq (field keyword "in") (repeat (field item word)))) + (field separator (role separator)) (field keyword "do") (field body command-list) (field keyword "done"))) + (case-selection + (node CaseCommand (field keyword "case") (field subject word) (field keyword "in") + (repeat (choice (field separator (role separator)) + (field clause (node CaseClause (optional (field open (operator "("))) + (field pattern word) (repeat (seq (field alternate (operator "|")) (field pattern word))) + (field close (operator ")")) (field body (empty-allowed command-list)) + (optional (field terminator (role case-end))))))) + (field keyword "esac"))) + (named-function + (node FunctionDefinition (field keyword "function") (field name word) + (optional (seq (field open (operator "(")) (field close (operator ")")))) (field body command))) + (function-header + (node FunctionDefinition (field name word) (field open (operator "(")) (field close (operator ")")) (field body command))) + (conditional + (node ConditionalCommand (field open "[[") + (repeat (choice (field operator (role conditional-operator)) (field operand word) (field operator (operator)))) + (field close "]]"))) + (arithmetic + (node ArithmeticCommand + (balanced (field open (adjacent (operator "(") (operator "("))) + (field close (adjacent (operator ")") (operator ")"))) + (choice (field expression word) (field operator (operator)))))) + (brace-group (node BraceGroup (field open "{") (field body command-list) (field close "}"))) + (subshell (node Subshell (field open (operator "(")) (field body command-list) (field close (operator ")")))) + (array-tail + (at assignment-tail (node ArrayAssignment (field open (operator "(")) + (repeat (choice (field separator (newline)) (field element word))) (field close (operator ")"))))) + (pipeline-prefix + (at pipeline-head (node Pipeline (optional (seq (field keyword "time") (optional (field option "-p")))) + (optional (field negate "!"))))))) diff --git a/languages/bash/parser-test.ss b/languages/bash/parser-test.ss new file mode 100644 index 00000000..1252d57f --- /dev/null +++ b/languages/bash/parser-test.ss @@ -0,0 +1,115 @@ +;;; -*- Gerbil -*- +;;; Bash syntax cases are values; execution and invariant checks belong to the engine. +(import :gerbil-parser/language-test-support + (only-in :gerbil-parser/language-support parse-artifact-success? parse-artifact-roundtrip) + ./parser) +(import (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/fixture defsyntax-corpus) + (only-in :gerbil-parser/language-support/development deflanguage-development-loader LanguageDevelopmentLoader. declare-language-source-scan-worker)) +(export bash-fixtures bash-test-language) + +(defsyntax-corpus bash-fixtures + (identity "bash" "5.3" "bash-5.3-structured-source.v1") + (accepted + ("bash/heredoc" bash-heredoc (text "cat <out\n" + (nodes ConditionalCommand ArithmeticCommand RedirectedCommand)) + (accepted "file descriptors and read-write redirects" + "{fd}<>data echo ok 2>>errors\n" (counts (Redirection 2))) + (rejected "unterminated compound has diagnostics" "case x in a) echo a\n" (diagnostics)) + (rejected-many "empty required lists and missing targets" + '("if true; then fi\n" "while true; do done\n" "{ ; }\n" "echo >\n" "cat < CSTValue (List Symbol)) -(def (cst-node-kinds value) - (cond - ((syntax-node? value) - (cons (syntax-node-kind value) - (apply append (map cst-node-kinds (syntax-node-children value))))) - ((syntax-field? value) - (apply append (map cst-node-kinds (syntax-field-children value)))) - (else '()))) - -;; : (forall (a) (-> [(Pair Symbol a)] Symbol a)) -;; : (-> Alist Symbol Datum) -(def (bound-row-ref row key) - (alet (entry (assq key row)) - (cdr entry))) - -;; : (-> Fixnum String) -(def (repeated-boolean-list-source count) - (string-append - "RETURN [" (string-join (make-list count "true") ", ") "]")) - -;; : (-> Datum Boolean) -(def (fork-action-entry? entry) - (eq? (cadr entry) 'fork)) - -;;; This suite couples catalog completeness to native Bound IR identities and -;;; lossless runtime artifacts, so a parse total alone cannot satisfy it. -;; : TestSuite -(def opencypher-2024-1-parser-test - (test-suite "openCypher 2024.1 generated parser" - (test-case "the complete BNF lowers through Parser IR and Bound IR" - (check (length (parser-ir-ref opencypher-2024-1-parser-ir 'rules)) => 352) - (check (parser-ir-ref opencypher-2024-1-parser-ir 'root-rule) => 'program) - (check (parser-ir-ref opencypher-2024-1-parser-ir 'materialization) - => 'aot-expansion) - (check (bound-grammar-ir-ref - opencypher-2024-1-bound-grammar-ir 'bindingCount) - => 736) - (let* ((binding - (bound-grammar-ir-binding - opencypher-2024-1-bound-grammar-ir 'rule 'program)) - (source (bound-grammar-ir-ref binding 'source))) - (check (bound-row-ref source 'path) - => "grammar-source/openCypher.bnf") - (check (bound-row-ref source 'line) => 3) - (check (bound-grammar-ir-ref binding 'references) - => '(((package . gerbil-parser) - (grammar . opencypher-2024-1-grammar) - (namespace . rule) - (name . |procedure specification|)) - ((package . gerbil-parser) - (grammar . opencypher-2024-1-grammar) - (namespace . rule) - (name . |standalone procedure call|))))) - (let* ((binding - (bound-grammar-ir-binding - opencypher-2024-1-bound-grammar-ir 'rule - '|linear statement|)) - (source (bound-grammar-ir-ref binding 'source))) - (check (bound-row-ref source 'compatibilityOverlay) - => +opencypher-linear-result-overlay+) - (check (string-prefix? - "sha256:" - (bound-row-ref source 'sourceExpressionDigest)) - => #t) - (check (string-prefix? - "sha256:" - (bound-row-ref source 'replacementExpressionDigest)) - => #t)) - (for-each - (lambda (expectation) - (let* ((binding - (bound-grammar-ir-binding - opencypher-2024-1-bound-grammar-ir - 'rule (car expectation))) - (source (bound-grammar-ir-ref binding 'source))) - (check (bound-row-ref source 'compatibilityOverlay) - => (cdr expectation)) - (check (string-prefix? - "sha256:" - (bound-row-ref source 'replacementExpressionDigest)) - => #t))) - (list - (cons '|arithmetic unary| +opencypher-nullable-prefix-normalization+) - (cons '|boolean factor| +opencypher-nullable-prefix-normalization+) - (cons '|pattern source| +opencypher-nullable-prefix-normalization+) - (cons '|list value constructor| - +opencypher-nullable-prefix-normalization+) - (cons '|list literal| +opencypher-nullable-prefix-normalization+) - (cons '|general literal| +opencypher-constructor-deduplication+) - (cons '|boolean literal| +opencypher-literal-keyword-preference+) - (cons '|null literal| +opencypher-literal-keyword-preference+) - (cons '|graph reference| +opencypher-qualified-reference-normalization+) - (cons '|procedure reference| - +opencypher-qualified-reference-normalization+) - (cons '|function reference| - +opencypher-qualified-reference-normalization+))) - (let* ((binding - (bound-grammar-ir-binding - opencypher-2024-1-bound-grammar-ir - 'rule '|non-reserved word|)) - (source (bound-grammar-ir-ref binding 'source))) - (check (bound-row-ref source 'precedenceOverlay) - => +opencypher-non-reserved-word-preference+) - (check (string-prefix? - "sha256:" - (bound-row-ref source 'replacementExpressionDigest)) - => #t)) - (let* ((spec (parser-ir-ref opencypher-2024-1-parser-ir 'lr-spec)) - (productions (lr-spec-ref spec 'productions)) - (actions (lr-spec-ref spec 'actions)) - (nullable-prefixes - '("$arithmetic unary" "$boolean factor" "$pattern source" - "$list value constructor" "$list literal")) - (nullable-prefix-helpers - (filter - (lambda (production) - (and (null? (production-rhs production)) - (ormap - (lambda (prefix) - (string-prefix? - prefix - (symbol->string (production-lhs production)))) - nullable-prefixes))) - productions)) - (fork-cells - (apply + - (map - (lambda (row) - (length (filter fork-action-entry? row))) - (vector->list actions)))) - (initial-actions (vector-ref actions 0))) - (check (lr-spec-ref spec 'state-count) => 1317) - (check nullable-prefix-helpers => '()) - ;; The source-owned non-reserved-word precedence resolves 129 static - ;; cells; remaining runtime forks retain selective-GLR admission. - (check fork-cells => 321) - (for-each - (lambda (literal) - (let (entry - (assoc (list 'terminal 'literal literal) initial-actions)) - (check (and entry (eq? (cadr entry) 'shift)) => #t))) - '("UNWIND" "WITH" "RETURN")))) - (test-case "representative and negative fixtures publish typed artifacts" - (for-each - (lambda (fixture) - (let* ((source (syntax-fixture-source fixture)) - (artifact (parse-opencypher-2024-1 source)) - (expected (syntax-fixture-expected-status fixture)) - (accepted? (parse-artifact-success? artifact))) - (check (list (syntax-fixture-id fixture) accepted?) - => (list (syntax-fixture-id fixture) - (eq? expected 'accepted))) - (check (parse-artifact-valid? artifact) => #t) - (when accepted? - (let* ((root (parse-artifact->cst artifact)) - (kinds (cst-node-kinds root))) - (check (syntax-node-kind root) - => (syntax-fixture-root-kind fixture)) - (check (parse-artifact-roundtrip artifact) => source) - (for-each - (lambda (required-kind) - (check (member required-kind kinds) ? values)) - (syntax-fixture-required-kinds fixture)))))) - opencypher-2024-1-fixtures)) - ;; This checks the source-format boundary itself: openCypher admits its - ;; native backtick/radix forms while GQL-only binary and trailing-dot forms - ;; remain typed rejections rather than leaking through shared scanners. - (test-case "native delimited identifiers and numeric profile remain isolated" - (for-each - (lambda (source) - (let (artifact (parse-opencypher-2024-1 source)) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source))) - '("MATCH (`a``b`) RETURN `a``b`\n" - "RETURN 1 AS one\n" - "RETURN $1 AS value\n" - "MERGE (n:L) ON CREATE SET n.x = 1 ON MATCH SET n.x = 2 RETURN n\n" - "MERGE (a)-[r:R]-(b) RETURN r\n" - "RETURN 1 AS n UNION ALL RETURN 2 AS n\n" - "RETURN false = true IS NULL AS value\n" - "UNWIND [0xFF, 0o77, .5, 1.25e+2, 3.0F] AS n RETURN n\n")) - (for-each - (lambda (source) - (let (artifact (parse-opencypher-2024-1 source)) - (check (parse-artifact-success? artifact) => #f) - (check (parse-artifact-valid? artifact) => #t))) - '("UNWIND [0b101] AS n RETURN n\n" - "UNWIND [1.] AS n RETURN n\n"))) - (test-case "static conflict normalization removes boolean path explosion" - (let* ((source (repeated-boolean-list-source 300)) - (tokens - (parser-significant-tokens - opencypher-2024-1-parser - (lex-source opencypher-2024-1-parser source))) - (spec - (bound-row-ref - (parser-machine-ir opencypher-2024-1-parser) 'lr-spec))) - (let-values (((_root rest receipt) - (lr-parse/receipt spec tokens 1))) - (check rest => '()) - (check (bound-row-ref receipt 'branchesExplored) => 0) - (check (bound-row-ref receipt 'speculativeBranchesExplored) => 0) - (check (bound-row-ref receipt 'maxSpeculativeDepth) => 0) - (check (bound-row-ref receipt 'successfulCompletions) => 1)))))) diff --git a/languages/cypher/opencypher-2024-1/parser.ss b/languages/cypher/opencypher-2024-1/parser.ss deleted file mode 100644 index 78f922c0..00000000 --- a/languages/cypher/opencypher-2024-1/parser.ss +++ /dev/null @@ -1,12 +0,0 @@ -;;; -*- Gerbil -*- -;;; Canonical public parser entry for openCypher 2024.1. - -(import :gerbil-parser/src/language/entry - ./grammar) -(export (import: ./grammar) - opencypher-2024-1-language - parse-opencypher-2024-1) - -(deflanguage-parser opencypher-2024-1-language - (grammar opencypher-2024-1-language-grammar) - (parse parse-opencypher-2024-1)) diff --git a/languages/cypher/opencypher-2024-1/source.ss b/languages/cypher/opencypher-2024-1/source.ss deleted file mode 100644 index 3cd56125..00000000 --- a/languages/cypher/opencypher-2024-1/source.ss +++ /dev/null @@ -1,15 +0,0 @@ -;;; -*- Gerbil -*- -;;; Authoring-only typed source catalog for openCypher 2024.1. - -(import (only-in :gerbil-parser/src/language-support/grammar-source - defsyntax-iso-bnf-source) - (only-in ./grammar - +opencypher-version+ - +opencypher-commit+ - +opencypher-bnf-digest+)) -(export opencypher-2024-1-bnf) - -(defsyntax-iso-bnf-source opencypher-2024-1-bnf - (identity "opencypher" +opencypher-version+ +opencypher-commit+) - (digest +opencypher-bnf-digest+) - (source "grammar-source/openCypher.bnf")) diff --git a/languages/cypher/parser-test.ss b/languages/cypher/parser-test.ss new file mode 100644 index 00000000..f038f22e --- /dev/null +++ b/languages/cypher/parser-test.ss @@ -0,0 +1,90 @@ +;;; -*- Gerbil -*- +;;; Declarative BNF/Bound IR identity, static conflict controls and syntax. +(import :gerbil-parser/language-test-support ./parser + (only-in :gerbil-parser/language-support/entry language-parser-source-receipt)) +(import (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/fixture defsyntax-corpus) + (only-in :gerbil-parser/language-support/development deflanguage-development-loader LanguageDevelopmentLoader.)) +(export opencypher-fixtures opencypher-test-language) + +(defsyntax-corpus opencypher-fixtures + (identity "opencypher" "2024.1" "opencypher-2024.1-syntax.v1") + (accepted + ("opencypher/2024.1/match-return" opencypher-match-return + "corpus/representative/match-return.cypher" program + (|match statement| |return statement|)) + ("opencypher/2024.1/create-return" opencypher-create-return + "corpus/representative/create-return.cypher" program + (|create statement| |return statement|)) + ("opencypher/2024.1/path-filter" opencypher-path-filter + "corpus/representative/path-filter.cypher" program + (|match statement| |where clause| |return statement|)) + ("opencypher/2024.1/unwind" opencypher-unwind + "corpus/representative/unwind.cypher" program + (|unwind statement| |return statement|))) + (rejected + ("opencypher/2024.1/invalid/match-only" opencypher-match-only + "corpus/invalid/match-only.cypher"))) + +(deflanguage-development-loader opencypher-test-language + (grammar opencypher-language-grammar) + (parse parse-opencypher-test) + (slots metadata: `((grammar-format . iso-bnf) (reference-commit . ,+opencypher-commit+) (source-digest . ,+opencypher-bnf-digest+)) + fixtures: opencypher-fixtures)) + +(deflanguage-parser-tests opencypher-parser-test "openCypher 2024.1 generated parser" + (loader opencypher-test-language) + (parser-ir "complete frozen BNF" opencypher-parser-ir + (rule-count 352) (root-rule 'program) (materialization 'aot-expansion)) + (bound-ir "bound declaration catalog" opencypher-bound-grammar-ir (binding-count 736)) + (bound-rules "source and normalization identities" opencypher-bound-grammar-ir + (program (source path "grammar-source/openCypher.bnf") (source line 3) + (references '(((package . gerbil-parser) (grammar . opencypher-grammar) + (namespace . rule) (name . |procedure specification|)) + ((package . gerbil-parser) (grammar . opencypher-grammar) + (namespace . rule) (name . |standalone procedure call|))))) + (|linear statement| (source compatibilityOverlay (language-parser-source-receipt opencypher-language 'linear-result-only)) + (source-prefix sourceExpressionDigest "sha256:") + (source-prefix replacementExpressionDigest "sha256:")) + (|arithmetic unary| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization nullable-prefix-left-factoring))) + (source-prefix replacementExpressionDigest "sha256:")) + (|boolean factor| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization nullable-prefix-left-factoring))) + (source-prefix replacementExpressionDigest "sha256:")) + (|pattern source| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization nullable-prefix-left-factoring))) + (source-prefix replacementExpressionDigest "sha256:")) + (|list value constructor| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization nullable-prefix-left-factoring))) + (source-prefix replacementExpressionDigest "sha256:")) + (|list literal| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization nullable-prefix-left-factoring))) + (source-prefix replacementExpressionDigest "sha256:")) + (|general literal| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization value-constructor-deduplication))) + (source-prefix replacementExpressionDigest "sha256:")) + (|boolean literal| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(disambiguation literal-keyword-preference))) + (source-prefix replacementExpressionDigest "sha256:")) + (|null literal| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(disambiguation literal-keyword-preference))) + (source-prefix replacementExpressionDigest "sha256:")) + (|graph reference| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization qualified-reference-left-factoring))) + (source-prefix replacementExpressionDigest "sha256:")) + (|procedure reference| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization qualified-reference-left-factoring))) + (source-prefix replacementExpressionDigest "sha256:")) + (|function reference| (source compatibilityOverlay (language-parser-source-receipt opencypher-language '(normalization qualified-reference-left-factoring))) + (source-prefix replacementExpressionDigest "sha256:")) + (|non-reserved word| (source precedenceOverlay (language-parser-source-receipt opencypher-language '(disambiguation non-reserved-word-preference))) + (source-prefix replacementExpressionDigest "sha256:"))) + (lr "static normalization removes nullable prefixes and retains admitted forks" opencypher-parser-ir + (state-count 1317) (fork-count 321) + (empty-nullable-prefixes "$arithmetic unary" "$boolean factor" "$pattern source" + "$list value constructor" "$list literal") + (initial-shifts "UNWIND" "WITH" "RETURN")) + (fixtures "representative and negative fixture conformance") + (accepted-many "native identifier and numeric forms" + '("MATCH (`a``b`) RETURN `a``b`\n" "RETURN 1 AS one\n" "RETURN $1 AS value\n" + "MERGE (n:L) ON CREATE SET n.x = 1 ON MATCH SET n.x = 2 RETURN n\n" + "MERGE (a)-[r:R]-(b) RETURN r\n" "RETURN 1 AS n UNION ALL RETURN 2 AS n\n" + "RETURN false = true IS NULL AS value\n" + "UNWIND [0xFF, 0o77, .5, 1.25e+2, 3.0F] AS n RETURN n\n")) + (rejected-many "foreign numeric forms remain rejected" + '("UNWIND [0b101] AS n RETURN n\n" "UNWIND [1.] AS n RETURN n\n")) + (lr-receipt "300 boolean items complete without speculative paths" opencypher-parser + (string-append "RETURN [" (string-join (make-list 300 "true") ", ") "]") 1 + (rest '()) (branchesExplored 0) (speculativeBranchesExplored 0) + (maxSpeculativeDepth 0) (successfulCompletions 1))) diff --git a/languages/cypher/parser.ss b/languages/cypher/parser.ss new file mode 100644 index 00000000..d65a4f5c --- /dev/null +++ b/languages/cypher/parser.ss @@ -0,0 +1,37 @@ +;;; -*- Gerbil -*- +;;; Canonical public parser entry for openCypher 2024.1. + +(import (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-entry-ref language-metadata-ref) + ./grammar) +(export +opencypher-version+ +opencypher-syntax-contract+ opencypher-language-grammar (import: ./grammar) + opencypher-language + parse-opencypher) + +(export +opencypher-commit+ +opencypher-bnf-digest+ +opencypher-representative-query+) + +(def +opencypher-commit+ + "30b451d3b7c94ee5a84a0fdc223947a442dd9493") +(def +opencypher-bnf-digest+ + "sha256:c0b5454f001b59b401756158bf88e27847c8ace71f1abc8df1e05f8b710b9f50") +(def +opencypher-representative-query+ "MATCH (n) RETURN n\n") + +(deflanguage-parser-loader opencypher-language + (grammar opencypher-syntax) + (parse parse-opencypher) + (slots + source-schema: `((schema . "gerbil-parser.iso-bnf-rule-overlay.v1") + (namespace . opencypher) + (sourceVersion . "2024.1") + (upstreamCommit . ,+opencypher-commit+)) + metadata: `((language . "opencypher") + (version . "2024.1") + (contract . "opencypher-2024.1-syntax.v1") + (grammar-format . iso-bnf) + (reference-commit . ,+opencypher-commit+) + (source-digest . ,+opencypher-bnf-digest+)))) + +(def opencypher-language-grammar (language-parser-entry-ref opencypher-language 'descriptor)) + +(def +opencypher-version+ (language-metadata-ref (language-parser-entry-ref opencypher-language 'metadata) 'version)) + +(def +opencypher-syntax-contract+ (language-metadata-ref (language-parser-entry-ref opencypher-language 'metadata) 'contract)) diff --git a/languages/fhirpath/v2.0.0/corpus/au-core-patient/SOURCE.org b/languages/fhirpath/corpus/au-core-patient/SOURCE.org similarity index 100% rename from languages/fhirpath/v2.0.0/corpus/au-core-patient/SOURCE.org rename to languages/fhirpath/corpus/au-core-patient/SOURCE.org diff --git a/languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-01.fhirpath b/languages/fhirpath/corpus/au-core-patient/au-core-pat-01.fhirpath similarity index 100% rename from languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-01.fhirpath rename to languages/fhirpath/corpus/au-core-patient/au-core-pat-01.fhirpath diff --git a/languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-02.fhirpath b/languages/fhirpath/corpus/au-core-patient/au-core-pat-02.fhirpath similarity index 100% rename from languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-02.fhirpath rename to languages/fhirpath/corpus/au-core-patient/au-core-pat-02.fhirpath diff --git a/languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-03.fhirpath b/languages/fhirpath/corpus/au-core-patient/au-core-pat-03.fhirpath similarity index 100% rename from languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-03.fhirpath rename to languages/fhirpath/corpus/au-core-patient/au-core-pat-03.fhirpath diff --git a/languages/fhirpath/v2.0.0/grammar-source/SOURCE.org b/languages/fhirpath/grammar-source/SOURCE.org similarity index 100% rename from languages/fhirpath/v2.0.0/grammar-source/SOURCE.org rename to languages/fhirpath/grammar-source/SOURCE.org diff --git a/languages/fhirpath/v2.0.0/grammar-source/fhirpath.g4 b/languages/fhirpath/grammar-source/fhirpath.g4 similarity index 100% rename from languages/fhirpath/v2.0.0/grammar-source/fhirpath.g4 rename to languages/fhirpath/grammar-source/fhirpath.g4 diff --git a/languages/fhirpath/grammar.ss b/languages/fhirpath/grammar.ss new file mode 100644 index 00000000..937e87b1 --- /dev/null +++ b/languages/fhirpath/grammar.ss @@ -0,0 +1,241 @@ +;;; -*- Gerbil -*- +;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors +;;; +;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later + +;;; Native, lossless projection of the pinned normative ANTLR grammar. This +;;; module owns syntax only; FHIR model navigation and function semantics are +;;; deliberately outside the parser authority. +(import (only-in :gerbil-parser/language-support/grammar deflanguage deftext-profile) + (only-in :gerbil-parser/language-support/grammar-source defsyntax-antlr4-source)) +(export fhirpath-normative-antlr4-source fhirpath-syntax fhirpath-grammar fhirpath-bound-grammar-ir fhirpath-parser-ir fhirpath-parser) + +(defsyntax-antlr4-source fhirpath-normative-antlr4-source + (identity "fhirpath" "2.0.0" + "https://hl7.org/fhirpath/N1/fhirpath.g4") + (digest + "sha256:cf2a7cf29475e29b1a9188fcabea77782db59c9309b200059b3ef3f781eaae13") + (source "grammar-source/fhirpath.g4")) + +;;; Shared text-profile fragments expand into closed declaration data. +(deftext-profile fhirpath-date-format + (seq (run (numeric) 4 4) + (if-next (characters "-") + (seq (literal "-") (run (numeric) 2 2) + (if-next (characters "-") + (seq (literal "-") (run (numeric) 2 2))))))) + +(deftext-profile fhirpath-time-format + (seq (run (numeric) 2 2) + (if-next (characters ":") + (seq (literal ":") (run (numeric) 2 2) + (if-next (characters ":") + (seq (literal ":") (run (numeric) 2 2) + (optional (seq (literal ".") (run (numeric) 1 #f))))))))) + +(deftext-profile fhirpath-timezone-format + (if-next (characters "Z") (literal "Z") + (if-next (characters "+-") + (seq (run (characters "+-") 1 1) (run (numeric) 2 2) + (literal ":") (run (numeric) 2 2))))) + +(deflanguage fhirpath + (syntax + (lexical + (root expression) + (lex + (whitespace WhitespaceTrivia (whitespace+)) + (comment CommentTrivia (choice (line-comment "//") (block-comment "/*" "*/"))) + (datetime DateTimeToken + (text-profile + (seq (literal "@") (ref fhirpath-date-format) (literal "T") + (if-next (numeric) + (seq (ref fhirpath-time-format) (ref fhirpath-timezone-format)))))) + (time TimeToken + (text-profile (seq (literal "@T") (ref fhirpath-time-format)))) + (date DateToken + (text-profile + (seq (literal "@") (ref fhirpath-date-format) (not-next (characters "T"))))) + (delimited-identifier DelimitedIdentifierToken + (quoted-string-profile "`" "`'\\/fnrt" 4)) + (string StringToken (quoted-string-profile "'" "`'\\/fnrt" 4)) + (number NumberToken + (text-profile + (seq (run (numeric) 1 #f) (optional (seq (literal ".") (run (numeric) 1 #f)))))) + (identifier IdentifierToken + (text-profile + (seq (run (union (ascii-letter) (characters "_")) 1 1) + (run (union (ascii-letter) (characters "_") (numeric)) 0 #f)))) + (punctuation PunctuationToken + (literals "<=" ">=" "!=" "!~" "$this" "$index" "$total" + "." "[" "]" "+" "-" "*" "/" "&" "|" "<" ">" + "=" "~" "%" "(" ")" "{" "}" ","))) + (extras whitespace comment) + (keywords) + (recoveries (expression "GERBIL-PARSER-FHIRPATH-N2" preserve-source)) + (conflicts reject) + (case-insensitive #f))) + (rules + (expression + (node Expression (field operand (reference implies-expression)))) + ;; The normative ANTLR rule is directly left recursive. Its alternative + ;; order is projected into an explicit precedence ladder so the native LR + ;; machine has one CST for contextual keywords and qualified type names. + (implies-expression + (node Expression + (seq (field left (reference or-expression)) + (optional + (seq (field operator (literal "implies")) + (field right (reference implies-expression))))))) + (or-expression + (node Expression + (seq (field left (reference and-expression)) + (repeat + (seq (field operator (choice (literal "or") (literal "xor"))) + (field right (reference and-expression))))))) + (and-expression + (node Expression + (seq (field left (reference membership-expression)) + (repeat + (seq (field operator (literal "and")) + (field right (reference membership-expression))))))) + (membership-expression + (node Expression + (seq (field left (reference equality-expression)) + (repeat + (seq + (field operator (choice (literal "in") (literal "contains"))) + (field right (reference equality-expression))))))) + (equality-expression + (node Expression + (seq (field left (reference inequality-expression)) + (repeat + (seq + (field operator + (choice (literal "=") (literal "~") + (literal "!=") (literal "!~"))) + (field right (reference inequality-expression))))))) + (inequality-expression + (node Expression + (seq (field left (reference union-expression)) + (repeat + (seq + (field operator + (choice (literal "<=") (literal "<") + (literal ">") (literal ">="))) + (field right (reference union-expression))))))) + (union-expression + (node Expression + (seq (field left (reference type-expression)) + (repeat + (seq (field operator (literal "|")) + (field right (reference type-expression))))))) + (type-expression + (node Expression + (seq (field left (reference additive-expression)) + (repeat + (seq (field operator (choice (literal "is") (literal "as"))) + (field right (reference type-specifier))))))) + (additive-expression + (node Expression + (seq (field left (reference multiplicative-expression)) + (repeat + (seq + (field operator + (choice (literal "+") (literal "-") (literal "&"))) + (field right (reference multiplicative-expression))))))) + (multiplicative-expression + (node Expression + (seq (field left (reference polarity-expression)) + (repeat + (seq + (field operator + (choice (literal "*") (literal "/") + (literal "div") (literal "mod"))) + (field right (reference polarity-expression))))))) + (polarity-expression + (node Expression + (choice + (seq (field operator (choice (literal "+") (literal "-"))) + (field operand (reference polarity-expression))) + (field operand (reference postfix-expression))))) + (postfix-expression + (node Expression + (seq + (field operand (reference term)) + (repeat + (choice + (seq (literal ".") + (field invocation (reference invocation))) + (seq (literal "[") (field index (reference expression)) + (literal "]"))))))) + (term + (node Term + (field value + (choice (reference invocation) (reference literal) + (reference external-constant) + (seq (literal "(") (reference expression) (literal ")")))))) + (literal + (node Literal + (field value + (choice (seq (literal "{") (literal "}")) + (literal "true") (literal "false") + (token string) (token number) (token date) + (token datetime) (token time) (reference quantity))))) + (external-constant + (node ExternalConstant + (seq (literal "%") + (field name (choice (reference identifier) (token string)))))) + (invocation + (node Invocation + (field value + (choice (reference function) (reference identifier) + (literal "$this") (literal "$index") (literal "$total"))))) + (function + (node Function + (seq (field name (reference identifier)) (literal "(") + (optional (field parameter (reference param-list))) (literal ")")))) + (param-list + (node ParamList + (seq (field parameter (reference expression)) + (repeat + (seq (literal ",") + (field parameter (reference expression))))))) + (quantity + (node Quantity + (seq (field value (token number)) + ;; The normative grammar writes NUMBER unit?, but its preceding + ;; NUMBER literal already owns the empty-unit branch. Requiring the + ;; unit here preserves the accepted language while removing the two + ;; distinct CSTs for every ordinary number. + (field unit (reference unit))))) + (unit + (node Unit + (field value + (choice (reference date-time-precision) + (reference plural-date-time-precision) (token string))))) + (date-time-precision + (node DateTimePrecision + (field value + (choice (literal "year") (literal "month") (literal "week") + (literal "day") (literal "hour") (literal "minute") + (literal "second") (literal "millisecond"))))) + (plural-date-time-precision + (node PluralDateTimePrecision + (field value + (choice (literal "years") (literal "months") (literal "weeks") + (literal "days") (literal "hours") (literal "minutes") + (literal "seconds") (literal "milliseconds"))))) + (type-specifier + (node TypeSpecifier (field name (reference qualified-identifier)))) + (qualified-identifier + (node QualifiedIdentifier + (seq (field name (reference identifier)) + (repeat + (seq (literal ".") (field name (reference identifier))))))) + (identifier + (node Identifier + (field value + (choice (token identifier) (token delimited-identifier) + (literal "as") (literal "contains") + (literal "in") (literal "is"))))))) diff --git a/languages/fhirpath/parser-test.ss b/languages/fhirpath/parser-test.ss new file mode 100644 index 00000000..2d2bfbca --- /dev/null +++ b/languages/fhirpath/parser-test.ss @@ -0,0 +1,48 @@ +;;; -*- Gerbil -*- +;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors +;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later +(import :gerbil-parser/language-test-support + (only-in :std/misc/ports read-all-as-string) + (only-in :gerbil-parser/src/runtime/identity sha256-text) + ./parser) +(import (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/fixture defsyntax-corpus) + (only-in :gerbil-parser/language-support/development deflanguage-development-loader LanguageDevelopmentLoader.)) +(export fhirpath-fixtures fhirpath-test-language) + +(defsyntax-corpus fhirpath-fixtures + (identity "fhirpath" "2.0.0" +fhirpath-syntax-contract+) + (accepted + ("fhirpath/au-core/1" fhirpath-au-core-1 "corpus/au-core-patient/au-core-pat-01.fhirpath" Expression ()) + ("fhirpath/au-core/2" fhirpath-au-core-2 "corpus/au-core-patient/au-core-pat-02.fhirpath" Expression ()) + ("fhirpath/au-core/3" fhirpath-au-core-3 "corpus/au-core-patient/au-core-pat-03.fhirpath" Expression ())) + (rejected + ("fhirpath/incomplete" fhirpath-incomplete (text "Patient.name.where(")))) + +(deflanguage-development-loader fhirpath-test-language + (grammar fhirpath-language-grammar) + (parse parse-fhirpath-test) + (slots metadata: `((grammar-format . concise-dsl) (source-digest . ,+fhirpath-antlr4-digest+)) + fixtures: fhirpath-fixtures)) + +(deflanguage-parser-tests fhirpath-parser-test "FHIRPath 2.0.0 normative syntax" + (loader fhirpath-test-language) + (property "official ANTLR grammar identity and bytes" + (bindings) + (equal +fhirpath-standard-reference+ "HL7 Cross-Paradigm Specification: FHIRPath, Release 1") + (equal +fhirpath-standard-version+ "2.0.0") + (equal +fhirpath-source-uri+ "https://hl7.org/fhirpath/N1/fhirpath.g4") + (equal (sha256-text (call-with-input-file "languages/fhirpath/grammar-source/fhirpath.g4" read-all-as-string)) + +fhirpath-antlr4-digest+) + (equal +fhirpath-syntax-contract+ "fhirpath-normative-2.0.0-syntax.v1")) + (fixtures "AU Core Patient loader fixtures parse losslessly") + (accepted-many "normative literal and invocation forms" + '("Patient.name.where(use = 'official').given" + "%resource.birthDate >= @2000-01-01" + "@2015-02-04T14:34:28+09:00 < now()" "@T14:34:28.123" + "4.5 'mg'" "text.`div`.exists()" "value is FHIR.string")) + (accepted-many "strict string and delimited identifier profiles" + '("'α'" "'\\u0041'" "`a\\`b`")) + (rejected-many "malformed string and identifier escapes" + '("'\\q'" "'\\u123'" "`a\\x`")) + (rejected "incomplete invocation fails closed" "Patient.name.where(")) diff --git a/languages/fhirpath/parser.ss b/languages/fhirpath/parser.ss new file mode 100644 index 00000000..14b5921a --- /dev/null +++ b/languages/fhirpath/parser.ss @@ -0,0 +1,35 @@ +;;; -*- Gerbil -*- +;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors +;;; +;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later + +(import (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-entry-ref language-metadata-ref) + ./grammar) +(export +fhirpath-syntax-contract+ fhirpath-language-grammar (import: ./grammar) + fhirpath-language + parse-fhirpath) + +(export +fhirpath-standard-reference+ +fhirpath-standard-version+ +fhirpath-source-uri+ +fhirpath-antlr4-digest+) + +(def +fhirpath-standard-reference+ + "HL7 Cross-Paradigm Specification: FHIRPath, Release 1") + +(def +fhirpath-standard-version+ "2.0.0") + +(def +fhirpath-source-uri+ "https://hl7.org/fhirpath/N1/fhirpath.g4") + +(def +fhirpath-antlr4-digest+ + "sha256:cf2a7cf29475e29b1a9188fcabea77782db59c9309b200059b3ef3f781eaae13") + +(deflanguage-parser-loader fhirpath-language + (grammar fhirpath-syntax) + (parse parse-fhirpath) + (slots metadata: `((language . "fhirpath") + (version . "2.0.0") + (contract . "fhirpath-normative-2.0.0-syntax.v1") + (grammar-format . concise-dsl) + (source-digest . ,+fhirpath-antlr4-digest+)))) + +(def fhirpath-language-grammar (language-parser-entry-ref fhirpath-language 'descriptor)) + +(def +fhirpath-syntax-contract+ (language-metadata-ref (language-parser-entry-ref fhirpath-language 'metadata) 'contract)) diff --git a/languages/fhirpath/v2.0.0/grammar.ss b/languages/fhirpath/v2.0.0/grammar.ss deleted file mode 100644 index 395156a1..00000000 --- a/languages/fhirpath/v2.0.0/grammar.ss +++ /dev/null @@ -1,246 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -;;; Native, lossless projection of the pinned normative ANTLR grammar. This -;;; module owns syntax only; FHIR model navigation and function semantics are -;;; deliberately outside the parser authority. -(import :gerbil-parser/src/language/grammar - ./scanner - ./source) -(export (import: ./source) - +fhirpath-syntax-contract+ - fhirpath-v2-language-grammar - fhirpath-v2-grammar - fhirpath-v2-bound-grammar-ir - fhirpath-v2-parser-ir - fhirpath-v2-parser) - -(def +fhirpath-syntax-contract+ "fhirpath-normative-2.0.0-syntax.v1") - -(deflanguage-grammar fhirpath-v2 - (identity "fhirpath" "2.0.0" "fhirpath-normative-2.0.0-syntax.v1") - (syntax-kinds - (Expression node (left operator right operand invocation index)) - (Term node (value)) - (Literal node (value)) - (ExternalConstant node (name)) - (Invocation node (value)) - (Function node (name parameter)) - (ParamList node (parameter)) - (Quantity node (value unit)) - (Unit node (value)) - (DateTimePrecision node (value)) - (PluralDateTimePrecision node (value)) - (TypeSpecifier node (name)) - (QualifiedIdentifier node (name)) - (Identifier node (value)) - (IdentifierToken token (text)) - (DelimitedIdentifierToken token (text)) - (StringToken token (text)) - (NumberToken token (text)) - (DateToken token (text)) - (DateTimeToken token (text)) - (TimeToken token (text)) - (WhitespaceTrivia token (text)) - (CommentTrivia token (text)) - (PunctuationToken token (text))) - (terminals - (identifier IdentifierToken) - (delimited-identifier DelimitedIdentifierToken) - (string StringToken) - (number NumberToken) - (date DateToken) - (datetime DateTimeToken) - (time TimeToken) - (whitespace WhitespaceTrivia) - (comment CommentTrivia) - (punctuation PunctuationToken)) - (lexical-rules - (whitespace (whitespace+)) - (comment (choice (line-comment "//") (block-comment "/*" "*/"))) - (datetime (external fhirpath-datetime-v2 scan-fhirpath-datetime)) - (time (external fhirpath-time-v2 scan-fhirpath-time)) - (date (external fhirpath-date-v2 scan-fhirpath-date)) - (delimited-identifier - (external fhirpath-delimited-identifier-v2 - scan-fhirpath-delimited-identifier)) - (string (external fhirpath-string-v2 scan-fhirpath-string)) - (number (external fhirpath-number-v2 scan-fhirpath-number)) - (identifier (external fhirpath-identifier-v2 scan-fhirpath-identifier)) - (punctuation - (literals "<=" ">=" "!=" "!~" "$this" "$index" "$total" - "." "[" "]" "+" "-" "*" "/" "&" "|" "<" ">" - "=" "~" "%" "(" ")" "{" "}" ",")) ) - (rules - (expression - (alias Expression (field operand (reference implies-expression)))) - ;; The normative ANTLR rule is directly left recursive. Its alternative - ;; order is projected into an explicit precedence ladder so the native LR - ;; machine has one CST for contextual keywords and qualified type names. - (implies-expression - (alias Expression - (seq (field left (reference or-expression)) - (optional - (seq (field operator (literal "implies")) - (field right (reference implies-expression))))))) - (or-expression - (alias Expression - (seq (field left (reference and-expression)) - (repeat - (seq (field operator (choice (literal "or") (literal "xor"))) - (field right (reference and-expression))))))) - (and-expression - (alias Expression - (seq (field left (reference membership-expression)) - (repeat - (seq (field operator (literal "and")) - (field right (reference membership-expression))))))) - (membership-expression - (alias Expression - (seq (field left (reference equality-expression)) - (repeat - (seq - (field operator (choice (literal "in") (literal "contains"))) - (field right (reference equality-expression))))))) - (equality-expression - (alias Expression - (seq (field left (reference inequality-expression)) - (repeat - (seq - (field operator - (choice (literal "=") (literal "~") - (literal "!=") (literal "!~"))) - (field right (reference inequality-expression))))))) - (inequality-expression - (alias Expression - (seq (field left (reference union-expression)) - (repeat - (seq - (field operator - (choice (literal "<=") (literal "<") - (literal ">") (literal ">="))) - (field right (reference union-expression))))))) - (union-expression - (alias Expression - (seq (field left (reference type-expression)) - (repeat - (seq (field operator (literal "|")) - (field right (reference type-expression))))))) - (type-expression - (alias Expression - (seq (field left (reference additive-expression)) - (repeat - (seq (field operator (choice (literal "is") (literal "as"))) - (field right (reference type-specifier))))))) - (additive-expression - (alias Expression - (seq (field left (reference multiplicative-expression)) - (repeat - (seq - (field operator - (choice (literal "+") (literal "-") (literal "&"))) - (field right (reference multiplicative-expression))))))) - (multiplicative-expression - (alias Expression - (seq (field left (reference polarity-expression)) - (repeat - (seq - (field operator - (choice (literal "*") (literal "/") - (literal "div") (literal "mod"))) - (field right (reference polarity-expression))))))) - (polarity-expression - (alias Expression - (choice - (seq (field operator (choice (literal "+") (literal "-"))) - (field operand (reference polarity-expression))) - (field operand (reference postfix-expression))))) - (postfix-expression - (alias Expression - (seq - (field operand (reference term)) - (repeat - (choice - (seq (literal ".") - (field invocation (reference invocation))) - (seq (literal "[") (field index (reference expression)) - (literal "]"))))))) - (term - (alias Term - (field value - (choice (reference invocation) (reference literal) - (reference external-constant) - (seq (literal "(") (reference expression) (literal ")")))))) - (literal - (alias Literal - (field value - (choice (seq (literal "{") (literal "}")) - (literal "true") (literal "false") - (token string) (token number) (token date) - (token datetime) (token time) (reference quantity))))) - (external-constant - (alias ExternalConstant - (seq (literal "%") - (field name (choice (reference identifier) (token string)))))) - (invocation - (alias Invocation - (field value - (choice (reference function) (reference identifier) - (literal "$this") (literal "$index") (literal "$total"))))) - (function - (alias Function - (seq (field name (reference identifier)) (literal "(") - (optional (field parameter (reference param-list))) (literal ")")))) - (param-list - (alias ParamList - (seq (field parameter (reference expression)) - (repeat - (seq (literal ",") - (field parameter (reference expression))))))) - (quantity - (alias Quantity - (seq (field value (token number)) - ;; The normative grammar writes NUMBER unit?, but its preceding - ;; NUMBER literal already owns the empty-unit branch. Requiring the - ;; unit here preserves the accepted language while removing the two - ;; distinct CSTs for every ordinary number. - (field unit (reference unit))))) - (unit - (alias Unit - (field value - (choice (reference date-time-precision) - (reference plural-date-time-precision) (token string))))) - (date-time-precision - (alias DateTimePrecision - (field value - (choice (literal "year") (literal "month") (literal "week") - (literal "day") (literal "hour") (literal "minute") - (literal "second") (literal "millisecond"))))) - (plural-date-time-precision - (alias PluralDateTimePrecision - (field value - (choice (literal "years") (literal "months") (literal "weeks") - (literal "days") (literal "hours") (literal "minutes") - (literal "seconds") (literal "milliseconds"))))) - (type-specifier - (alias TypeSpecifier (field name (reference qualified-identifier)))) - (qualified-identifier - (alias QualifiedIdentifier - (seq (field name (reference identifier)) - (repeat - (seq (literal ".") (field name (reference identifier))))))) - (identifier - (alias Identifier - (field value - (choice (token identifier) (token delimited-identifier) - (literal "as") (literal "contains") - (literal "in") (literal "is")))))) - (extras whitespace comment) - (keywords) - (parser-entrypoints (expression parse pure)) - (recoveries (expression "GERBIL-PARSER-FHIRPATH-N2" preserve-source)) - (conflicts reject) - (case-insensitive #f) - (flow (source lexical) (lexical fhirpath-expression) (fhirpath-expression cst))) diff --git a/languages/fhirpath/v2.0.0/parser-test.ss b/languages/fhirpath/v2.0.0/parser-test.ss deleted file mode 100644 index 4e8422ca..00000000 --- a/languages/fhirpath/v2.0.0/parser-test.ss +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(import :std/test - (only-in :std/misc/ports read-all-as-string) - (only-in :gerbil-parser/src/runtime/artifact - parse-artifact-ref parse-artifact-roundtrip - parse-artifact-success? parse-artifact-valid?) - (only-in :gerbil-parser/src/runtime/identity sha256-text) - ./parser) - -(def grammar-source-path - "languages/fhirpath/v2.0.0/grammar-source/fhirpath.g4") - -(def au-core-fixture-paths - '("languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-01.fhirpath" - "languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-02.fhirpath" - "languages/fhirpath/v2.0.0/corpus/au-core-patient/au-core-pat-03.fhirpath")) - -(def (read-source path) - (call-with-input-file path read-all-as-string)) - -(def fhirpath-v2-parser-test - (test-suite "FHIRPath 2.0.0 normative syntax" - (test-case "the official ANTLR grammar identity and bytes are fixed" - (check +fhirpath-standard-reference+ - => "HL7 Cross-Paradigm Specification: FHIRPath, Release 1") - (check +fhirpath-standard-version+ => "2.0.0") - (check +fhirpath-source-uri+ - => "https://hl7.org/fhirpath/N1/fhirpath.g4") - (check (sha256-text (read-source grammar-source-path)) - => +fhirpath-antlr4-digest+) - (check +fhirpath-syntax-contract+ - => "fhirpath-normative-2.0.0-syntax.v1")) - (test-case "AU Core Patient invariant expressions parse losslessly" - (for-each - (lambda (path) - (let* ((source (read-source path)) - (artifact (parse-fhirpath-v2 source))) - (check (list path (parse-artifact-success? artifact)) - => (list path #t)) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source) - (check (parse-artifact-ref artifact 'sourceDigest) - => (sha256-text source)))) - au-core-fixture-paths)) - (test-case "normative literal and invocation forms stay lossless" - (for-each - (lambda (source) - (let (artifact (parse-fhirpath-v2 source)) - (check (list source (parse-artifact-success? artifact)) - => (list source #t)) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source))) - '("Patient.name.where(use = 'official').given" - "%resource.birthDate >= @2000-01-01" - "@2015-02-04T14:34:28+09:00 < now()" - "@T14:34:28.123" - "4.5 'mg'" - "text.`div`.exists()" - "value is FHIR.string"))) - (test-case "an incomplete invocation fails closed" - (let (artifact (parse-fhirpath-v2 "Patient.name.where(")) - (check (parse-artifact-success? artifact) => #f))))) - -(export fhirpath-v2-parser-test) diff --git a/languages/fhirpath/v2.0.0/parser.ss b/languages/fhirpath/v2.0.0/parser.ss deleted file mode 100644 index 63322176..00000000 --- a/languages/fhirpath/v2.0.0/parser.ss +++ /dev/null @@ -1,14 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(import :gerbil-parser/src/language/entry - ./grammar) -(export (import: ./grammar) - fhirpath-v2-language - parse-fhirpath-v2) - -(deflanguage-parser fhirpath-v2-language - (grammar fhirpath-v2-language-grammar) - (parse parse-fhirpath-v2)) diff --git a/languages/fhirpath/v2.0.0/scanner.ss b/languages/fhirpath/v2.0.0/scanner.ss deleted file mode 100644 index dd602a60..00000000 --- a/languages/fhirpath/v2.0.0/scanner.ss +++ /dev/null @@ -1,169 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(export scan-fhirpath-identifier - scan-fhirpath-delimited-identifier - scan-fhirpath-string - scan-fhirpath-number - scan-fhirpath-date - scan-fhirpath-datetime - scan-fhirpath-time) - -(def (scan-while source start predicate) - (let (length (string-length source)) - (let loop ((offset start)) - (if (and (< offset length) (predicate (string-ref source offset))) - (loop (+ offset 1)) - offset)))) - -(def (ascii-letter? character) - (or (and (char>=? character #\A) (char<=? character #\Z)) - (and (char>=? character #\a) (char<=? character #\z)))) - -(def (identifier-start? character) - (or (ascii-letter? character) (char=? character #\_))) - -(def (identifier-continue? character) - (or (identifier-start? character) (char-numeric? character))) - -(def (scan-fhirpath-identifier source start) - (and (< start (string-length source)) - (identifier-start? (string-ref source start)) - (scan-while source (+ start 1) identifier-continue?))) - -(def (hex-character? character) - (or (char-numeric? character) - (and (char>=? character #\a) (char<=? character #\f)) - (and (char>=? character #\A) (char<=? character #\F)))) - -(def (escaped-end source slash-offset) - (let* ((length (string-length source)) - (escape-offset (+ slash-offset 1))) - (and (< escape-offset length) - (let (escape (string-ref source escape-offset)) - (cond - ((memv escape '(#\` #\' #\\ #\/ #\f #\n #\r #\t)) - (+ escape-offset 1)) - ((and (char=? escape #\u) - (<= (+ escape-offset 5) length) - (let loop ((offset (+ escape-offset 1))) - (or (= offset (+ escape-offset 5)) - (and (hex-character? (string-ref source offset)) - (loop (+ offset 1)))))) - (+ escape-offset 5)) - (else #f)))))) - -(def (scan-quoted source start quote) - (let (length (string-length source)) - (and (< start length) - (char=? (string-ref source start) quote) - (let loop ((offset (+ start 1))) - (and (< offset length) - (let (character (string-ref source offset)) - (cond - ((char=? character quote) (+ offset 1)) - ((char=? character #\\) - (alet (end (escaped-end source offset)) - (loop end))) - (else (loop (+ offset 1)))))))))) - -(def (scan-fhirpath-delimited-identifier source start) - (scan-quoted source start #\`)) - -(def (scan-fhirpath-string source start) - (scan-quoted source start #\')) - -(def (scan-digits source start count) - (let (end (+ start count)) - (and (<= end (string-length source)) - (let loop ((offset start)) - (or (= offset end) - (and (char-numeric? (string-ref source offset)) - (loop (+ offset 1))))) - end))) - -(def (scan-fhirpath-number source start) - (and (< start (string-length source)) - (char-numeric? (string-ref source start)) - (let* ((integer-end (scan-while source start char-numeric?)) - (length (string-length source))) - (if (and (< integer-end length) - (char=? (string-ref source integer-end) #\.) - (< (+ integer-end 1) length) - (char-numeric? (string-ref source (+ integer-end 1)))) - (scan-while source (+ integer-end 1) char-numeric?) - integer-end)))) - -(def (scan-date-format source start) - (alet (year-end (scan-digits source start 4)) - (let (length (string-length source)) - (if (and (< year-end length) - (char=? (string-ref source year-end) #\-)) - (alet (month-end (scan-digits source (+ year-end 1) 2)) - (if (and (< month-end length) - (char=? (string-ref source month-end) #\-)) - (scan-digits source (+ month-end 1) 2) - month-end)) - year-end)))) - -(def (scan-time-format source start) - (alet (hour-end (scan-digits source start 2)) - (let (length (string-length source)) - (if (and (< hour-end length) - (char=? (string-ref source hour-end) #\:)) - (alet (minute-end (scan-digits source (+ hour-end 1) 2)) - (if (and (< minute-end length) - (char=? (string-ref source minute-end) #\:)) - (alet (second-end (scan-digits source (+ minute-end 1) 2)) - (if (and (< second-end length) - (char=? (string-ref source second-end) #\.) - (< (+ second-end 1) length) - (char-numeric? (string-ref source (+ second-end 1)))) - (scan-while source (+ second-end 1) char-numeric?) - second-end)) - minute-end)) - hour-end)))) - -(def (scan-timezone source start) - (let (length (string-length source)) - (cond - ((and (< start length) (char=? (string-ref source start) #\Z)) - (+ start 1)) - ((and (< start length) - (memv (string-ref source start) '(#\+ #\-))) - (alet (hour-end (scan-digits source (+ start 1) 2)) - (and (< hour-end length) - (char=? (string-ref source hour-end) #\:) - (scan-digits source (+ hour-end 1) 2)))) - (else start)))) - -(def (scan-fhirpath-date source start) - (and (< start (string-length source)) - (char=? (string-ref source start) #\@) - (alet (end (scan-date-format source (+ start 1))) - (and (or (= end (string-length source)) - (not (char=? (string-ref source end) #\T))) - end)))) - -(def (scan-fhirpath-datetime source start) - (and (< start (string-length source)) - (char=? (string-ref source start) #\@) - (alet (date-end (scan-date-format source (+ start 1))) - (and (< date-end (string-length source)) - (char=? (string-ref source date-end) #\T) - (let ((time-start (+ date-end 1)) - (length (string-length source))) - (if (and (< time-start length) - (char-numeric? (string-ref source time-start))) - (alet (time-end (scan-time-format source time-start)) - (scan-timezone source time-end)) - time-start)))))) - -(def (scan-fhirpath-time source start) - (let (length (string-length source)) - (and (< (+ start 2) length) - (char=? (string-ref source start) #\@) - (char=? (string-ref source (+ start 1)) #\T) - (scan-time-format source (+ start 2))))) diff --git a/languages/fhirpath/v2.0.0/source.ss b/languages/fhirpath/v2.0.0/source.ss deleted file mode 100644 index 92f15195..00000000 --- a/languages/fhirpath/v2.0.0/source.ss +++ /dev/null @@ -1,26 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(import (only-in :gerbil-parser/src/language-support/grammar-source - defsyntax-antlr4-source)) -(export +fhirpath-standard-reference+ - +fhirpath-standard-version+ - +fhirpath-source-uri+ - +fhirpath-antlr4-digest+ - fhirpath-normative-antlr4-source) - -(def +fhirpath-standard-reference+ - "HL7 Cross-Paradigm Specification: FHIRPath, Release 1") -(def +fhirpath-standard-version+ "2.0.0") -(def +fhirpath-source-uri+ "https://hl7.org/fhirpath/N1/fhirpath.g4") -(def +fhirpath-antlr4-digest+ - "sha256:cf2a7cf29475e29b1a9188fcabea77782db59c9309b200059b3ef3f781eaae13") - -(defsyntax-antlr4-source fhirpath-normative-antlr4-source - (identity "fhirpath" "2.0.0" - "https://hl7.org/fhirpath/N1/fhirpath.g4") - (digest - "sha256:cf2a7cf29475e29b1a9188fcabea77782db59c9309b200059b3ef3f781eaae13") - (source "grammar-source/fhirpath.g4")) diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_closed_graph_from_graph_type_double_colon.gql b/languages/gql/corpus/reference/create_closed_graph_from_graph_type_double_colon.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_closed_graph_from_graph_type_double_colon.gql rename to languages/gql/corpus/reference/create_closed_graph_from_graph_type_double_colon.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_closed_graph_from_graph_type_lexical.gql b/languages/gql/corpus/reference/create_closed_graph_from_graph_type_lexical.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_closed_graph_from_graph_type_lexical.gql rename to languages/gql/corpus/reference/create_closed_graph_from_graph_type_lexical.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_closed_graph_from_nested_graph_type_double_colon.gql b/languages/gql/corpus/reference/create_closed_graph_from_nested_graph_type_double_colon.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_closed_graph_from_nested_graph_type_double_colon.gql rename to languages/gql/corpus/reference/create_closed_graph_from_nested_graph_type_double_colon.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_graph.gql b/languages/gql/corpus/reference/create_graph.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_graph.gql rename to languages/gql/corpus/reference/create_graph.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_schema.gql b/languages/gql/corpus/reference/create_schema.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/create_schema.gql rename to languages/gql/corpus/reference/create_schema.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/insert_statement.gql b/languages/gql/corpus/reference/insert_statement.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/insert_statement.gql rename to languages/gql/corpus/reference/insert_statement.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/match_and_insert_example.gql b/languages/gql/corpus/reference/match_and_insert_example.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/match_and_insert_example.gql rename to languages/gql/corpus/reference/match_and_insert_example.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/match_with_exists_predicate_braces.gql b/languages/gql/corpus/reference/match_with_exists_predicate_braces.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/match_with_exists_predicate_braces.gql rename to languages/gql/corpus/reference/match_with_exists_predicate_braces.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/match_with_exists_predicate_nested_match.gql b/languages/gql/corpus/reference/match_with_exists_predicate_nested_match.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/match_with_exists_predicate_nested_match.gql rename to languages/gql/corpus/reference/match_with_exists_predicate_nested_match.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/match_with_exists_predicate_parentheses.gql b/languages/gql/corpus/reference/match_with_exists_predicate_parentheses.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/match_with_exists_predicate_parentheses.gql rename to languages/gql/corpus/reference/match_with_exists_predicate_parentheses.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/session_set_graph_to_current_graph.gql b/languages/gql/corpus/reference/session_set_graph_to_current_graph.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/session_set_graph_to_current_graph.gql rename to languages/gql/corpus/reference/session_set_graph_to_current_graph.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/session_set_graph_to_current_property_graph.gql b/languages/gql/corpus/reference/session_set_graph_to_current_property_graph.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/session_set_graph_to_current_property_graph.gql rename to languages/gql/corpus/reference/session_set_graph_to_current_property_graph.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/session_set_property_as_value.gql b/languages/gql/corpus/reference/session_set_property_as_value.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/session_set_property_as_value.gql rename to languages/gql/corpus/reference/session_set_property_as_value.gql diff --git a/languages/gql/iso-39075-2024/corpus/opengql-1.9.0/session_set_time_zone.gql b/languages/gql/corpus/reference/session_set_time_zone.gql similarity index 100% rename from languages/gql/iso-39075-2024/corpus/opengql-1.9.0/session_set_time_zone.gql rename to languages/gql/corpus/reference/session_set_time_zone.gql diff --git a/languages/gql/iso-39075-2024/grammar-source/GQL.g4 b/languages/gql/grammar-source/GQL.g4 similarity index 100% rename from languages/gql/iso-39075-2024/grammar-source/GQL.g4 rename to languages/gql/grammar-source/GQL.g4 diff --git a/languages/gql/grammar.ss b/languages/gql/grammar.ss new file mode 100644 index 00000000..461ffc9c --- /dev/null +++ b/languages/gql/grammar.ss @@ -0,0 +1,59 @@ +;;; -*- Gerbil -*- +;;; Grammar declaration and admitted compiler products for ISO/IEC 39075:2024 GQL generated from the pinned OpenGQL 1.9.0 grammar. + +(import (only-in :gerbil-parser/language-support/grammar deflanguage) + (only-in :gerbil-parser/language-support/antlr4-language + antlr4)) +(export gql-antlr4-token-bindings gql-antlr4-source gql-syntax gql-grammar gql-bound-grammar-ir gql-parser-ir gql-parser) + +(deflanguage gql + (syntax (antlr4 + (reference "1.9.0" + "16ea71bd320ad07fd2c46a3066afbaef7d226922") + (digest + "sha256:e1b4a24c6b88dedddc0a1fff97df0fc30bf118cea51539e26d71c717cb737bbf") + (source "grammar-source/GQL.g4") + (entrypoint gqlProgram) + (conflicts selective-glr) + (case-insensitive #t) + (token-bindings + ("PARAMETER_NAME" token identifier) + ("SINGLE_QUOTED_CHARACTER_SEQUENCE" token string) + ("DOUBLE_QUOTED_CHARACTER_SEQUENCE" token string) + ("ACCENT_QUOTED_CHARACTER_SEQUENCE" token string) + ("UNBROKEN_SINGLE_QUOTED_CHARACTER_SEQUENCE" token string) + ("UNBROKEN_DOUBLE_QUOTED_CHARACTER_SEQUENCE" token string) + ("UNBROKEN_ACCENT_QUOTED_CHARACTER_SEQUENCE" token string) + ("BYTE_STRING_LITERAL" token string) + ("UNSIGNED_DECIMAL_IN_SCIENTIFIC_NOTATION_WITH_EXACT_NUMBER_SUFFIX" token number) + ("UNSIGNED_DECIMAL_IN_SCIENTIFIC_NOTATION_WITHOUT_SUFFIX" token number) + ("UNSIGNED_DECIMAL_IN_SCIENTIFIC_NOTATION_WITH_APPROXIMATE_NUMBER_SUFFIX" token number) + ("UNSIGNED_DECIMAL_IN_COMMON_NOTATION_WITH_EXACT_NUMBER_SUFFIX" token number) + ("UNSIGNED_DECIMAL_IN_COMMON_NOTATION_WITHOUT_SUFFIX" token number) + ("UNSIGNED_DECIMAL_IN_COMMON_NOTATION_WITH_APPROXIMATE_NUMBER_SUFFIX" token number) + ("UNSIGNED_DECIMAL_INTEGER_WITH_EXACT_NUMBER_SUFFIX" token number) + ("UNSIGNED_DECIMAL_INTEGER_WITH_APPROXIMATE_NUMBER_SUFFIX" token number) + ("UNSIGNED_DECIMAL_INTEGER" token number) + ("UNSIGNED_DECIMAL_IN_SCIENTIFIC_NOTATION" token number) + ("UNSIGNED_DECIMAL_IN_COMMON_NOTATION" token number) + ("UNSIGNED_HEXADECIMAL_INTEGER" token number) + ("UNSIGNED_OCTAL_INTEGER" token number) + ("UNSIGNED_BINARY_INTEGER" token number) + ("SEPARATED_IDENTIFIER" token identifier) + ("REGULAR_IDENTIFIER" token identifier) + ("EXTENDED_IDENTIFIER" token identifier) + ("DELIMITED_IDENTIFIER" token identifier) + ("IDENTIFIER_START" token identifier) + ("IDENTIFIER_EXTEND" token identifier)) + (lex + (whitespace WhitespaceTrivia (whitespace+)) + (comment CommentTrivia (choice (line-comment "//") (block-comment "/*" "*/"))) + (string StringLiteralToken (quoted-string "\"" "'" "`")) + (number NumericLiteralToken (number-literal ("0x" "0o" "0b") "_" + ("M" "m" "F" "f" "D" "d") #t #t)) + (identifier LexicalIdentifier (identifier)) + (punctuation PunctuationToken (source-literals)) + (unknown UnknownToken (fallback))) + (extras whitespace comment))) + (rules)) + diff --git a/languages/gql/iso-39075-2024/fixtures.ss b/languages/gql/iso-39075-2024/fixtures.ss deleted file mode 100644 index 11f9476f..00000000 --- a/languages/gql/iso-39075-2024/fixtures.ss +++ /dev/null @@ -1,67 +0,0 @@ -;;; -*- Gerbil -*- -;;; openGQL 1.9.0 official examples, embedded as immutable syntax fixtures. - -(import (only-in :gerbil-parser/language-support defsyntax-corpus) - (only-in ./grammar +gql-standard-edition+ +gql-syntax-contract+)) -(export gql-iso-official-fixtures) - -(defsyntax-corpus gql-iso-official-fixtures - (identity "gql" +gql-standard-edition+ +gql-syntax-contract+) - (accepted - ("opengql/create-closed-graph-double-colon" - gql-create-closed-graph-double-colon - "corpus/opengql-1.9.0/create_closed_graph_from_graph_type_double_colon.gql" - GqlProgram (CreateGraphStatement)) - ("opengql/create-closed-graph-lexical" - gql-create-closed-graph-lexical - "corpus/opengql-1.9.0/create_closed_graph_from_graph_type_lexical.gql" - GqlProgram (CreateGraphStatement)) - ("opengql/create-closed-nested-graph-double-colon" - gql-create-closed-nested-graph-double-colon - "corpus/opengql-1.9.0/create_closed_graph_from_nested_graph_type_double_colon.gql" - GqlProgram (CreateGraphStatement)) - ("opengql/create-graph" - gql-create-graph - "corpus/opengql-1.9.0/create_graph.gql" - GqlProgram (CreateGraphStatement)) - ("opengql/create-schema" - gql-create-schema - "corpus/opengql-1.9.0/create_schema.gql" - GqlProgram (CreateSchemaStatement)) - ("opengql/insert-statement" - gql-insert-statement - "corpus/opengql-1.9.0/insert_statement.gql" - GqlProgram (InsertStatement)) - ("opengql/match-and-insert" - gql-match-and-insert - "corpus/opengql-1.9.0/match_and_insert_example.gql" - GqlProgram (MatchStatement InsertStatement)) - ("opengql/exists-braces" - gql-exists-braces - "corpus/opengql-1.9.0/match_with_exists_predicate_braces.gql" - GqlProgram (MatchStatement ExistsPredicate ReturnStatement)) - ("opengql/exists-parentheses" - gql-exists-parentheses - "corpus/opengql-1.9.0/match_with_exists_predicate_parentheses.gql" - GqlProgram (MatchStatement ExistsPredicate ReturnStatement)) - ("opengql/exists-nested-match" - gql-exists-nested-match - "corpus/opengql-1.9.0/match_with_exists_predicate_nested_match.gql" - GqlProgram (MatchStatement ExistsPredicate ReturnStatement)) - ("opengql/session-current-graph" - gql-session-current-graph - "corpus/opengql-1.9.0/session_set_graph_to_current_graph.gql" - GqlProgram (SessionSetCommand)) - ("opengql/session-current-property-graph" - gql-session-current-property-graph - "corpus/opengql-1.9.0/session_set_graph_to_current_property_graph.gql" - GqlProgram (SessionSetCommand)) - ("opengql/session-property-value" - gql-session-property-value - "corpus/opengql-1.9.0/session_set_property_as_value.gql" - GqlProgram (SessionSetCommand)) - ("opengql/session-time-zone" - gql-session-time-zone - "corpus/opengql-1.9.0/session_set_time_zone.gql" - GqlProgram (SessionSetCommand))) - (rejected)) diff --git a/languages/gql/iso-39075-2024/grammar.ss b/languages/gql/iso-39075-2024/grammar.ss deleted file mode 100644 index 84146bc1..00000000 --- a/languages/gql/iso-39075-2024/grammar.ss +++ /dev/null @@ -1,43 +0,0 @@ -;;; -*- Gerbil -*- -;;; ISO/IEC 39075:2024 GQL generated from the pinned OpenGQL 1.9.0 grammar. - -(import (only-in :gerbil-parser/language-support - deflanguage-antlr4-grammar)) -(export +gql-standard-reference+ - +gql-standard-edition+ - +gql-opengql-reference-version+ - +gql-opengql-reference-commit+ - +gql-antlr4-digest+ - +gql-syntax-contract+ - +gql-representative-query+ - gql-iso-language-grammar - gql-iso-grammar - gql-iso-bound-grammar-ir - gql-iso-parser-ir - gql-iso-parser) - -(def +gql-standard-reference+ "ISO/IEC 39075:2024") -(def +gql-standard-edition+ "edition-1-2024-04") -(def +gql-opengql-reference-version+ "1.9.0") -(def +gql-opengql-reference-commit+ - "16ea71bd320ad07fd2c46a3066afbaef7d226922") -(def +gql-antlr4-digest+ - "sha256:e1b4a24c6b88dedddc0a1fff97df0fc30bf118cea51539e26d71c717cb737bbf") -(def +gql-syntax-contract+ "iso-iec-39075-2024.opengql-1.9.0-syntax.v1") -(def +gql-representative-query+ - (string-append - "MATCH (person:Person {name: \"Ada\"}) " - "OPTIONAL MATCH (person)-[:KNOWS]->(friend:Person) " - "RETURN person.name AS source, friend.name AS target\n")) - -(deflanguage-antlr4-grammar gql-iso - (identity "gql" "edition-1-2024-04" - "iso-iec-39075-2024.opengql-1.9.0-syntax.v1") - (reference "1.9.0" - "16ea71bd320ad07fd2c46a3066afbaef7d226922") - (digest - "sha256:e1b4a24c6b88dedddc0a1fff97df0fc30bf118cea51539e26d71c717cb737bbf") - (source "grammar-source/GQL.g4") - (entrypoint gqlProgram) - (conflicts selective-glr) - (case-insensitive #t)) diff --git a/languages/gql/iso-39075-2024/parser-test.ss b/languages/gql/iso-39075-2024/parser-test.ss deleted file mode 100644 index 736e0994..00000000 --- a/languages/gql/iso-39075-2024/parser-test.ss +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- -;;; Acceptance owner for immutable OpenGQL identity, complete fixture parsing, -;;; lossless CST projection, and typed malformed-input terminals. - -(import (only-in :std/test check test-case test-suite) - :gerbil-parser/languages/gql/iso-39075-2024/parser - :gerbil-parser/src/compiler/bound-ir - :gerbil-parser/src/runtime/artifact - :gerbil-parser/src/runtime/cst - (only-in :gerbil-parser/src/runtime/token token? token-lexeme) - (only-in :gerbil-parser/language-support - syntax-fixture-expected-status - syntax-fixture-id - syntax-fixture-required-kinds - syntax-fixture-root-kind - syntax-fixture-source - syntax-fixture-source-digest - syntax-fixture-version) - (only-in ./fixtures gql-iso-official-fixtures)) -(export gql-iso-39075-2024-parser-test) - -;;; Structural traversal is intentionally generic over CST nodes and fields so -;;; corpus assertions observe emitted kinds without depending on child layout. -;; : (-> CSTValue (List Symbol)) -(def (cst-node-kinds value) - (cond - ((syntax-node? value) - (cons (syntax-node-kind value) - (apply append (map cst-node-kinds (syntax-node-children value))))) - ((syntax-field? value) - (apply append (map cst-node-kinds (syntax-field-children value)))) - (else '()))) - -;; : (-> CSTValue (List SyntaxNode)) -(def (cst-direct-nodes value) - (cond - ((syntax-node? value) (list value)) - ((syntax-field? value) - (apply append (map cst-direct-nodes (syntax-field-children value)))) - (else '()))) - -;; : (-> CSTValue Symbol (List SyntaxNode)) -(def (cst-nodes-of-kind value kind) - (cond - ((syntax-node? value) - (let (nested - (apply append - (map (cut cst-nodes-of-kind <> kind) - (syntax-node-children value)))) - (if (eq? (syntax-node-kind value) kind) - (cons value nested) - nested))) - ((syntax-field? value) - (apply append - (map (cut cst-nodes-of-kind <> kind) - (syntax-field-children value)))) - (else '()))) - -;; : (-> CSTValue (List String)) -(def (cst-token-lexemes value) - (cond - ((token? value) (list (token-lexeme value))) - ((syntax-node? value) - (apply append (map cst-token-lexemes (syntax-node-children value)))) - ((syntax-field? value) - (apply append (map cst-token-lexemes (syntax-field-children value)))) - (else '()))) - -;; : (-> SyntaxNode Boolean) -(def (conjunction-preserves-comparison-operands? node) - (let* ((direct - (apply append - (map cst-direct-nodes (syntax-node-children node)))) - (operands - (filter (lambda (child) - (eq? (syntax-node-kind child) 'ValueExpression)) - direct))) - (and (member "AND" (cst-token-lexemes node)) - (= (length operands) 2) - (member ">" (cst-token-lexemes (car operands))) - (member "=" (cst-token-lexemes (cadr operands)))))) - -;; gql-parser-ir-ref -;; : (forall (a) (-> [(Pair Symbol a)] Symbol a)) -;; : (-> Alist Symbol Datum) -(def (gql-parser-ir-ref parser-ir key) - (alet (entry (assq key parser-ir)) - (cdr entry))) - -;;; Acceptance couples immutable standard identity, AOT Parser IR, and -;;; lossless runtime artifacts; no fixture is accepted by parse count alone. -;; : TestSuite -(def gql-iso-39075-2024-parser-test - (test-suite "ISO/IEC 39075:2024 OpenGQL 1.9.0 syntax" - (test-case "the GQL standard identity is immutable" - (check +gql-standard-reference+ => "ISO/IEC 39075:2024") - (check +gql-standard-edition+ => "edition-1-2024-04") - (check +gql-opengql-reference-version+ => "1.9.0") - (check +gql-opengql-reference-commit+ - => "16ea71bd320ad07fd2c46a3066afbaef7d226922") - (check +gql-syntax-contract+ - => "iso-iec-39075-2024.opengql-1.9.0-syntax.v1") - (check +gql-antlr4-digest+ - => "sha256:e1b4a24c6b88dedddc0a1fff97df0fc30bf118cea51539e26d71c717cb737bbf") - (check (length (gql-parser-ir-ref gql-iso-parser-ir 'rules)) => 574) - (check (gql-parser-ir-ref gql-iso-parser-ir 'materialization) - => 'aot-expansion) - (check (gql-parser-ir-ref gql-iso-parser-ir 'conflict-policy) - => 'selective-glr) - (check (gql-parser-ir-ref gql-iso-parser-ir 'case-insensitive?) => #t) - (check (bound-grammar-ir-ref gql-iso-bound-grammar-ir 'bindingCount) - => 1176) - (let* ((binding - (bound-grammar-ir-binding - gql-iso-bound-grammar-ir 'rule 'gqlProgram)) - (source (bound-grammar-ir-ref binding 'source))) - (check (bound-grammar-ir-ref binding 'bindingId) - => '((package . gerbil-parser) - (grammar . gql-iso-grammar) - (namespace . rule) - (name . gqlProgram))) - (check (gql-parser-ir-ref source 'path) => "grammar-source/GQL.g4") - (check (gql-parser-ir-ref source 'location) - => "grammar-source/GQL.g4#gqlProgram") - (check (map (lambda (reference) - (gql-parser-ir-ref reference 'name)) - (bound-grammar-ir-ref binding 'references)) - => '(programActivity sessionCloseCommand sessionCloseCommand))) - (check (length gql-iso-official-fixtures) => 14)) - (test-case "the generic machine parses multi-clause GQL losslessly" - (let* ((artifact (parse-gql-iso-39075-2024 +gql-representative-query+)) - (root (parse-artifact->cst artifact)) - (kinds (cst-node-kinds root))) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => +gql-representative-query+) - (check (syntax-node-kind root) => 'GqlProgram) - (check (= (length (filter (cut eq? <> 'MatchStatement) kinds)) 2) - => #t) - (check (member 'NodePattern kinds) ? values) - (check (member 'EdgePattern kinds) ? values) - (check (member 'ElementPropertySpecification kinds) ? values) - (check (member 'PropertyKeyValuePair kinds) ? values) - (check (member 'ReturnStatement kinds) ? values))) - (test-case "GQL keywords are case-insensitive without changing source" - (let* ((source "match (n) return n\n") - (artifact (parse-gql-iso-39075-2024 source))) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source))) - (test-case "comparison operands bind before boolean conjunction" - (let* ((source - "MATCH (n) WHERE n.score > 2 AND n.active = TRUE RETURN n") - (artifact (parse-gql-iso-39075-2024 source)) - (root (parse-artifact->cst artifact))) - (check (parse-artifact-success? artifact) => #t) - (check (find conjunction-preserves-comparison-operands? - (cst-nodes-of-kind root 'ValueExpression)) - ? values))) - (test-case "GQL delimited identifiers and numeric profile parse losslessly" - (for-each - (lambda (source) - (let (artifact (parse-gql-iso-39075-2024 source)) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source))) - '("MATCH (n) RETURN n AS \"a\"\"b\"\n" - "RETURN 0xFF, 0o77, 0b101, .5, 1., 2.0M\n"))) - (test-case "all openGQL 1.9.0 official examples parse losslessly" - (for-each - (lambda (fixture) - (let* ((source (syntax-fixture-source fixture)) - (artifact (parse-gql-iso-39075-2024 source)) - (accepted? (parse-artifact-success? artifact))) - (check (syntax-fixture-expected-status fixture) => 'accepted) - (check (list (syntax-fixture-id fixture) accepted?) - => (list (syntax-fixture-id fixture) #t)) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-ref artifact 'sourceDigest) - => (syntax-fixture-source-digest fixture)) - (check (parse-artifact-roundtrip artifact) => source) - (when accepted? - (let* ((root (parse-artifact->cst artifact)) - (kinds (cst-node-kinds root))) - (check (syntax-node-kind root) - => (syntax-fixture-root-kind fixture)) - (for-each - (lambda (required-kind) - (check (member required-kind kinds) ? values)) - (syntax-fixture-required-kinds fixture)))))) - gql-iso-official-fixtures)) - (test-case "malformed graph patterns reject as one typed artifact" - (for-each - (lambda (source) - (let (artifact (parse-gql-iso-39075-2024 source)) - (check (parse-artifact-success? artifact) => #f) - (check (parse-artifact-valid? artifact) => #t) - (check (length (parse-artifact-ref artifact 'diagnostics)) => 1))) - '("MATCH\n" "CREATE\n" "SESSION\n"))))) diff --git a/languages/gql/iso-39075-2024/parser.ss b/languages/gql/iso-39075-2024/parser.ss deleted file mode 100644 index b50568a2..00000000 --- a/languages/gql/iso-39075-2024/parser.ss +++ /dev/null @@ -1,12 +0,0 @@ -;;; -*- Gerbil -*- -;;; Canonical public parser entry for ISO/IEC 39075:2024 GQL. - -(import :gerbil-parser/src/language/entry - ./grammar) -(export (import: ./grammar) - gql-iso-39075-2024-language - parse-gql-iso-39075-2024) - -(deflanguage-parser gql-iso-39075-2024-language - (grammar gql-iso-language-grammar) - (parse parse-gql-iso-39075-2024)) diff --git a/languages/gql/iso-39075-2024/query-syntax-test.ss b/languages/gql/iso-39075-2024/query-syntax-test.ss deleted file mode 100644 index f0eae465..00000000 --- a/languages/gql/iso-39075-2024/query-syntax-test.ss +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- - -(import (only-in :clan/poo/object .o) - (only-in :std/test check test-case test-suite) - ./query-syntax) - -(export gql-query-syntax-test) - -(def Program - (.o (:: @ GqlQueryProgram.) - match: - (.o (:: @ GqlQueryPath.) - start: (.o (:: @ GqlQueryNode.) binding: 's label: 'Scenario) - next: - (.o (:: @ GqlQueryStep.) relation: 'HAS_CASE - target: (.o (:: @ GqlQueryNode.) binding: 'c label: 'Case))) - where: - (.o (:: @ GqlQueryEquals.) - left: (.o (:: @ GqlQueryProperty.) binding: 's property: 'identity) - right: - (.o (:: @ GqlQueryLiteral.) - literal-kind: 'string value: "patient's-case")) - project: - (.o (:: @ GqlQueryProjection.) - expression: - (.o (:: @ GqlQueryProperty.) binding: 'c property: 'id)))) - -(def gql-query-syntax-test - (test-suite - "ISO GQL POO query syntax graph" - (test-case "projects canonical source from the language-owned graph" - (check (gql-query-program? Program) => #t) - (check - (gql-query-program->source Program) - => - (string-append - "MATCH (s:Scenario)-[:HAS_CASE]->(c:Case)\n" - "WHERE s.identity = 'patient''s-case'\n" - "RETURN c.id\n"))))) diff --git a/languages/gql/iso-39075-2024/query-syntax.ss b/languages/gql/iso-39075-2024/query-syntax.ss deleted file mode 100644 index 5ec27eae..00000000 --- a/languages/gql/iso-39075-2024/query-syntax.ss +++ /dev/null @@ -1,164 +0,0 @@ -;;; -*- Gerbil -*- -;;; Canonical POO-native ISO GQL query syntax graph and source projection. -;;; -;;; The syntax graph is language-owned. Domain packages extend these -;;; prototypes and never reimplement GQL quoting or source rendering. - -(import (only-in :clan/poo/object .o .ref .slot? object?) - (only-in :gerbil/core string-join) - (only-in :std/list/list every) - (only-in :std/string/misc string-concatenate string-subst)) - -(export GqlQueryNode. - GqlQueryStep. - GqlQueryPath. - GqlQueryProperty. - GqlQueryLiteral. - GqlQueryEquals. - GqlQueryProjection. - GqlQueryProgram. - gql-query-program? - gql-query-program->source) - -(def (gql-has-slots? value slots) - (and (object? value) - (every (lambda (slot) (.slot? value slot)) slots))) - -(def GqlQueryNode. - (.o syntax-kind: 'gql.node binding: #f label: #f)) - -(def GqlQueryStep. - (.o syntax-kind: 'gql.step relation: #f target: #f next: #f)) - -(def GqlQueryPath. - (.o syntax-kind: 'gql.path start: #f next: #f)) - -(def GqlQueryProperty. - (.o syntax-kind: 'gql.property binding: #f property: #f)) - -(def GqlQueryLiteral. - (.o syntax-kind: 'gql.literal literal-kind: #f value: #f)) - -(def GqlQueryEquals. - (.o syntax-kind: 'gql.equals left: #f right: #f)) - -(def GqlQueryProjection. - (.o syntax-kind: 'gql.projection expression: #f next: #f)) - -(def GqlQueryProgram. - (.o syntax-kind: 'gql.program match: #f where: #f project: #f)) - -(def (gql-query-node? value) - (and (gql-has-slots? value '(syntax-kind binding label)) - (eq? (.ref value 'syntax-kind) 'gql.node) - (symbol? (.ref value 'binding)) - (symbol? (.ref value 'label)))) - -(def (gql-query-step? value) - (and (gql-has-slots? value '(syntax-kind relation target next)) - (eq? (.ref value 'syntax-kind) 'gql.step) - (symbol? (.ref value 'relation)) - (gql-query-node? (.ref value 'target)) - (or (not (.ref value 'next)) - (gql-query-step? (.ref value 'next))))) - -(def (gql-query-path? value) - (and (gql-has-slots? value '(syntax-kind start next)) - (eq? (.ref value 'syntax-kind) 'gql.path) - (gql-query-node? (.ref value 'start)) - (or (not (.ref value 'next)) - (gql-query-step? (.ref value 'next))))) - -(def (gql-query-property? value) - (and (gql-has-slots? value '(syntax-kind binding property)) - (eq? (.ref value 'syntax-kind) 'gql.property) - (symbol? (.ref value 'binding)) - (symbol? (.ref value 'property)))) - -(def (gql-query-literal? value) - (and (gql-has-slots? value '(syntax-kind literal-kind value)) - (eq? (.ref value 'syntax-kind) 'gql.literal) - (case (.ref value 'literal-kind) - ((string) (string? (.ref value 'value))) - ((symbol) (symbol? (.ref value 'value))) - ((integer) (exact-integer? (.ref value 'value))) - ((boolean) (boolean? (.ref value 'value))) - (else #f)))) - -(def (gql-query-equals? value) - (and (gql-has-slots? value '(syntax-kind left right)) - (eq? (.ref value 'syntax-kind) 'gql.equals) - (gql-query-property? (.ref value 'left)) - (gql-query-literal? (.ref value 'right)))) - -(def (gql-query-projection? value) - (and (gql-has-slots? value '(syntax-kind expression next)) - (eq? (.ref value 'syntax-kind) 'gql.projection) - (gql-query-property? (.ref value 'expression)) - (or (not (.ref value 'next)) - (gql-query-projection? (.ref value 'next))))) - -(def (gql-query-program? value) - (and (gql-has-slots? value '(syntax-kind match where project)) - (eq? (.ref value 'syntax-kind) 'gql.program) - (gql-query-path? (.ref value 'match)) - (or (not (.ref value 'where)) - (gql-query-equals? (.ref value 'where))) - (gql-query-projection? (.ref value 'project)))) - -(def (gql-name value) - (if (symbol? value) (symbol->string value) value)) - -(def (gql-string value) - (string-append "'" (string-subst value "'" "''") "'")) - -(def (gql-node-document node) - (string-append "(" (gql-name (.ref node 'binding)) ":" - (gql-name (.ref node 'label)) ")")) - -(def (gql-path-document path) - (let loop ((step (.ref path 'next)) - (documents (list (gql-node-document (.ref path 'start))))) - (if step - (loop (.ref step 'next) - (cons - (string-append "-[:" (gql-name (.ref step 'relation)) "]->" - (gql-node-document (.ref step 'target))) - documents)) - (string-concatenate (reverse documents))))) - -(def (gql-property-document property) - (string-append (gql-name (.ref property 'binding)) "." - (gql-name (.ref property 'property)))) - -(def (gql-literal-document literal) - (case (.ref literal 'literal-kind) - ((string) (gql-string (.ref literal 'value))) - ((symbol) (gql-string (symbol->string (.ref literal 'value)))) - ((integer) (number->string (.ref literal 'value))) - ((boolean) (if (.ref literal 'value) "TRUE" "FALSE")))) - -(def (gql-equals-document equals) - (string-append (gql-property-document (.ref equals 'left)) " = " - (gql-literal-document (.ref equals 'right)))) - -(def (gql-projection-document projection) - (let loop ((current projection) (documents '())) - (if current - (loop (.ref current 'next) - (cons (gql-property-document (.ref current 'expression)) - documents)) - (string-join (reverse documents) ", ")))) - -(def (gql-query-program->source program) - (unless (gql-query-program? program) - (error "invalid GQL query syntax graph" program)) - (string-concatenate - (list - (string-append "MATCH " (gql-path-document (.ref program 'match)) "\n") - (if (.ref program 'where) - (string-append "WHERE " - (gql-equals-document (.ref program 'where)) "\n") - "") - (string-append "RETURN " - (gql-projection-document (.ref program 'project)) "\n")))) diff --git a/languages/gql/iso-39075-2024/source.ss b/languages/gql/iso-39075-2024/source.ss deleted file mode 100644 index e96c32af..00000000 --- a/languages/gql/iso-39075-2024/source.ss +++ /dev/null @@ -1,16 +0,0 @@ -;;; -*- Gerbil -*- -;;; OpenGQL 1.9.0 immutable grammar-source admission for ISO GQL 2024. - -(import :gerbil-parser/language-support) -(export +gql-antlr4-digest+ - gql-iso-antlr4-source) - -(def +gql-antlr4-digest+ - "sha256:e1b4a24c6b88dedddc0a1fff97df0fc30bf118cea51539e26d71c717cb737bbf") - -(defsyntax-antlr4-source gql-iso-antlr4-source - (identity "gql" "1.9.0" - "16ea71bd320ad07fd2c46a3066afbaef7d226922") - (digest - "sha256:e1b4a24c6b88dedddc0a1fff97df0fc30bf118cea51539e26d71c717cb737bbf") - (source "grammar-source/GQL.g4")) diff --git a/languages/gql/parser-test.ss b/languages/gql/parser-test.ss new file mode 100644 index 00000000..b4d4f0bb --- /dev/null +++ b/languages/gql/parser-test.ss @@ -0,0 +1,137 @@ +;;; -*- Gerbil -*- +;;; Declarative standards, bound-source identity, lossless syntax and precedence. +(import :gerbil-parser/language-test-support + (only-in :gerbil-parser/src/language/descriptor + language-grammar-grammar language-grammar-ir language-grammar-machine) + (only-in :gerbil-parser/language-support/entry language-parser-entry-ref) + (only-in :gerbil-parser/language-support/fixture syntax-fixture-id syntax-fixture-source) + (only-in :clan/poo/object .ref) + (only-in :gerbil-parser/language-support/fixture defsyntax-corpus defsyntax-fixture) + (only-in :gerbil-parser/language-support/development + deflanguage-development-loader LanguageDevelopmentLoader. language-loader-fixtures) + ./parser) + +(defsyntax-corpus gql-official-fixtures + (identity "gql" +gql-standard-edition+ +gql-syntax-contract+) + (accepted + ("opengql/create-closed-graph-double-colon" + gql-create-closed-graph-double-colon + "corpus/reference/create_closed_graph_from_graph_type_double_colon.gql" + GqlProgram (CreateGraphStatement)) + ("opengql/create-closed-graph-lexical" + gql-create-closed-graph-lexical + "corpus/reference/create_closed_graph_from_graph_type_lexical.gql" + GqlProgram (CreateGraphStatement)) + ("opengql/create-closed-nested-graph-double-colon" + gql-create-closed-nested-graph-double-colon + "corpus/reference/create_closed_graph_from_nested_graph_type_double_colon.gql" + GqlProgram (CreateGraphStatement)) + ("opengql/create-graph" + gql-create-graph + "corpus/reference/create_graph.gql" + GqlProgram (CreateGraphStatement)) + ("opengql/create-schema" + gql-create-schema + "corpus/reference/create_schema.gql" + GqlProgram (CreateSchemaStatement)) + ("opengql/insert-statement" + gql-insert-statement + "corpus/reference/insert_statement.gql" + GqlProgram (InsertStatement)) + ("opengql/match-and-insert" + gql-match-and-insert + "corpus/reference/match_and_insert_example.gql" + GqlProgram (MatchStatement InsertStatement)) + ("opengql/exists-braces" + gql-exists-braces + "corpus/reference/match_with_exists_predicate_braces.gql" + GqlProgram (MatchStatement ExistsPredicate ReturnStatement)) + ("opengql/exists-parentheses" + gql-exists-parentheses + "corpus/reference/match_with_exists_predicate_parentheses.gql" + GqlProgram (MatchStatement ExistsPredicate ReturnStatement)) + ("opengql/exists-nested-match" + gql-exists-nested-match + "corpus/reference/match_with_exists_predicate_nested_match.gql" + GqlProgram (MatchStatement ExistsPredicate ReturnStatement)) + ("opengql/session-current-graph" + gql-session-current-graph + "corpus/reference/session_set_graph_to_current_graph.gql" + GqlProgram (SessionSetCommand)) + ("opengql/session-current-property-graph" + gql-session-current-property-graph + "corpus/reference/session_set_graph_to_current_property_graph.gql" + GqlProgram (SessionSetCommand)) + ("opengql/session-property-value" + gql-session-property-value + "corpus/reference/session_set_property_as_value.gql" + GqlProgram (SessionSetCommand)) + ("opengql/session-time-zone" + gql-session-time-zone + "corpus/reference/session_set_time_zone.gql" + GqlProgram (SessionSetCommand))) + (rejected)) + + +(defsyntax-fixture gql-incomplete-return-fixture + (identity "gql/incomplete-return" "gql" +gql-standard-edition+ +gql-syntax-contract+) + (text "RETURN (") + (expect rejected #f ())) +(deflanguage-development-loader gql-test-language + (grammar gql-language-grammar) + (parse parse-gql-test) + (slots fixtures: (append gql-official-fixtures (list gql-incomplete-return-fixture)))) + +(export gql-official-fixtures gql-incomplete-return-fixture gql-test-language) + +(deflanguage-parser-tests gql-parser-test "ISO/IEC 39075:2024 OpenGQL 1.9.0 syntax" + (loader gql-test-language) + (property "immutable standard and corpus identity" (bindings) + (equal +gql-standard-reference+ "ISO/IEC 39075:2024") + (equal +gql-standard-edition+ "edition-1-2024-04") + (equal +gql-opengql-reference-version+ "1.9.0") + (equal +gql-opengql-reference-commit+ "16ea71bd320ad07fd2c46a3066afbaef7d226922") + (equal +gql-syntax-contract+ "iso-iec-39075-2024.opengql-1.9.0-syntax.v1") + (equal +gql-antlr4-digest+ "sha256:e1b4a24c6b88dedddc0a1fff97df0fc30bf118cea51539e26d71c717cb737bbf")) + (fixture-catalog "official examples and rejection control" (total 15)) + (property "published products and checked entry share one declaration" (bindings) + (equal (eq? (language-grammar-grammar gql-language-grammar) gql-grammar) #t) + (equal (eq? (language-grammar-ir gql-language-grammar) gql-parser-ir) #t) + (equal (eq? (language-grammar-machine gql-language-grammar) gql-parser) #t) + (equal (language-parser-entry-ref gql-language 'language) "gql") + (equal (language-parser-entry-ref gql-language 'contract) + "iso-iec-39075-2024.opengql-1.9.0-syntax.v1") + (equal (map syntax-fixture-id (language-loader-fixtures gql-test-language 'accepted)) + (map syntax-fixture-id gql-official-fixtures)) + (equal (map syntax-fixture-source (language-loader-fixtures gql-test-language 'accepted)) + (map syntax-fixture-source gql-official-fixtures))) + (parser-ir "generated Parser IR identity" gql-parser-ir + (rule-count 574) (materialization 'aot-expansion) + (conflict-policy 'selective-glr) (case-insensitive? #t)) + (bound-ir "bound catalog identity" gql-bound-grammar-ir (binding-count 1176)) + (bound-rules "source-bound root rule and ordered references" gql-bound-grammar-ir + (gqlProgram + (binding-id '((package . gerbil-parser) (grammar . gql-grammar) + (namespace . rule) (name . gqlProgram))) + (source path "grammar-source/GQL.g4") + (source location "grammar-source/GQL.g4#gqlProgram") + (reference-names '(programActivity sessionCloseCommand sessionCloseCommand)))) + (accepted "multi-clause query retains public syntax" +gql-representative-query+ + (root GqlProgram) (counts (MatchStatement 2)) + (nodes NodePattern EdgePattern ElementPropertySpecification PropertyKeyValuePair ReturnStatement)) + (accepted "case-insensitive keywords retain source" "match (n) return n\n") + (accepted "comparison operands bind before conjunction" + "MATCH (n) WHERE n.score > 2 AND n.active = TRUE RETURN n" + (subtree (node ValueExpression (lexemes "AND") + (children (node ValueExpression (lexemes ">")) + (node ValueExpression (lexemes "="))))) + (without-subtree (node ValueExpression (lexemes "AND") + (children (node ValueExpression (lexemes "=")) + (node ValueExpression (lexemes ">")))))) + (accepted-many "delimited identifiers and numeric profile" + '("MATCH (n) RETURN n AS \"a\"\"b\"\n" + "RETURN 0xFF, 0o77, 0b101, .5, 1., 2.0M\n")) + (fixture-group "all official examples retain their required structure" accepted) + (fixture-group "declared rejection controls" rejected) + (rejected-many "malformed graph patterns yield one diagnostic" + '("MATCH\n" "CREATE\n" "SESSION\n") (diagnostics 1))) diff --git a/languages/gql/parser.ss b/languages/gql/parser.ss new file mode 100644 index 00000000..d1ac1cd9 --- /dev/null +++ b/languages/gql/parser.ss @@ -0,0 +1,43 @@ +;;; -*- Gerbil -*- +;;; Canonical public parser entry for ISO/IEC 39075:2024 GQL. + +(import (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-entry-ref language-metadata-ref) + ./grammar) +(export +gql-standard-edition+ +gql-syntax-contract+ gql-language-grammar (import: ./grammar) + gql-language + parse-gql) + +(import (only-in :gerbil-parser/language-support/antlr4-source + antlr4-source-version antlr4-source-commit antlr4-source-digest)) +(export +gql-standard-reference+ +gql-opengql-reference-version+ +gql-opengql-reference-commit+ +gql-antlr4-digest+ +gql-representative-query+) + +(def +gql-standard-reference+ "ISO/IEC 39075:2024") + +(def +gql-representative-query+ + (string-append + "MATCH (person:Person {name: \"Ada\"}) " + "OPTIONAL MATCH (person)-[:KNOWS]->(friend:Person) " + "RETURN person.name AS source, friend.name AS target\n")) + +(def +gql-opengql-reference-version+ (antlr4-source-version gql-antlr4-source)) + +(def +gql-opengql-reference-commit+ (antlr4-source-commit gql-antlr4-source)) + +(def +gql-antlr4-digest+ (antlr4-source-digest gql-antlr4-source)) + +(deflanguage-parser-loader gql-language + (grammar gql-syntax) + (parse parse-gql) + (slots metadata: `((language . "gql") + (version . "edition-1-2024-04") + (contract . "iso-iec-39075-2024.opengql-1.9.0-syntax.v1") + (grammar-format . antlr4) + (reference-version . ,+gql-opengql-reference-version+) + (reference-commit . ,+gql-opengql-reference-commit+) + (source-digest . ,+gql-antlr4-digest+)))) + +(def gql-language-grammar (language-parser-entry-ref gql-language 'descriptor)) + +(def +gql-standard-edition+ (language-metadata-ref (language-parser-entry-ref gql-language 'metadata) 'version)) + +(def +gql-syntax-contract+ (language-metadata-ref (language-parser-entry-ref gql-language 'metadata) 'contract)) diff --git a/languages/hcl/v2-24/corpus/invalid/malformed-tuple.hcl b/languages/hcl/corpus/invalid/malformed-tuple.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/invalid/malformed-tuple.hcl rename to languages/hcl/corpus/invalid/malformed-tuple.hcl diff --git a/languages/hcl/v2-24/corpus/invalid/missing-equals.hcl b/languages/hcl/corpus/invalid/missing-equals.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/invalid/missing-equals.hcl rename to languages/hcl/corpus/invalid/missing-equals.hcl diff --git a/languages/hcl/v2-24/corpus/invalid/unterminated-label.hcl b/languages/hcl/corpus/invalid/unterminated-label.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/invalid/unterminated-label.hcl rename to languages/hcl/corpus/invalid/unterminated-label.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/comments/hash_comment.hcl b/languages/hcl/corpus/reference/comments/hash_comment.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/comments/hash_comment.hcl rename to languages/hcl/corpus/reference/comments/hash_comment.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/comments/multiline_comment.hcl b/languages/hcl/corpus/reference/comments/multiline_comment.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/comments/multiline_comment.hcl rename to languages/hcl/corpus/reference/comments/multiline_comment.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/comments/slash_comment.hcl b/languages/hcl/corpus/reference/comments/slash_comment.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/comments/slash_comment.hcl rename to languages/hcl/corpus/reference/comments/slash_comment.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/empty.hcl b/languages/hcl/corpus/reference/empty.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/empty.hcl rename to languages/hcl/corpus/reference/empty.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/expressions/heredoc.hcl b/languages/hcl/corpus/reference/expressions/heredoc.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/expressions/heredoc.hcl rename to languages/hcl/corpus/reference/expressions/heredoc.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/expressions/operators.hcl b/languages/hcl/corpus/reference/expressions/operators.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/expressions/operators.hcl rename to languages/hcl/corpus/reference/expressions/operators.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/expressions/primitive_literals.hcl b/languages/hcl/corpus/reference/expressions/primitive_literals.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/expressions/primitive_literals.hcl rename to languages/hcl/corpus/reference/expressions/primitive_literals.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/invalid/attribute_singleline_bad.hcl b/languages/hcl/corpus/reference/invalid/attribute_singleline_bad.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/invalid/attribute_singleline_bad.hcl rename to languages/hcl/corpus/reference/invalid/attribute_singleline_bad.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/invalid/block_single_oneline_invalid.hcl b/languages/hcl/corpus/reference/invalid/block_single_oneline_invalid.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/invalid/block_single_oneline_invalid.hcl rename to languages/hcl/corpus/reference/invalid/block_single_oneline_invalid.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/invalid/block_single_unclosed.hcl b/languages/hcl/corpus/reference/invalid/block_single_unclosed.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/invalid/block_single_unclosed.hcl rename to languages/hcl/corpus/reference/invalid/block_single_unclosed.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/attributes_expected.hcl b/languages/hcl/corpus/reference/structure/attributes_expected.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/attributes_expected.hcl rename to languages/hcl/corpus/reference/structure/attributes_expected.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/attributes_unexpected.hcl b/languages/hcl/corpus/reference/structure/attributes_unexpected.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/attributes_unexpected.hcl rename to languages/hcl/corpus/reference/structure/attributes_unexpected.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/block_empty_multiline.hcl b/languages/hcl/corpus/reference/structure/block_empty_multiline.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/block_empty_multiline.hcl rename to languages/hcl/corpus/reference/structure/block_empty_multiline.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/block_empty_oneline.hcl b/languages/hcl/corpus/reference/structure/block_empty_oneline.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/block_empty_oneline.hcl rename to languages/hcl/corpus/reference/structure/block_empty_oneline.hcl diff --git a/languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/block_single_oneline.hcl b/languages/hcl/corpus/reference/structure/block_single_oneline.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/hashicorp-v2.24.0/structure/block_single_oneline.hcl rename to languages/hcl/corpus/reference/structure/block_single_oneline.hcl diff --git a/languages/hcl/v2-24/corpus/representative.hcl b/languages/hcl/corpus/representative.hcl similarity index 100% rename from languages/hcl/v2-24/corpus/representative.hcl rename to languages/hcl/corpus/representative.hcl diff --git a/languages/hcl/grammar.ss b/languages/hcl/grammar.ss new file mode 100644 index 00000000..6b16411f --- /dev/null +++ b/languages/hcl/grammar.ss @@ -0,0 +1,165 @@ +;;; -*- Gerbil -*- +;;; Version-pinned HCL native syntax grammar owner. + +(import (only-in :gerbil-parser/language-support/grammar deflanguage)) +(export hcl-syntax hcl-grammar hcl-parser-ir hcl-parser) + +(deflanguage hcl + (syntax + (lexical + (root config-file) + (lex + (horizontal-whitespace HorizontalWhitespace (horizontal-whitespace+)) + (newline Newline (newline+)) + (comment Comment (line-comment "#" "//")) + (block-comment-token Comment (block-comment "/*" "*/")) + (heredoc Heredoc (heredoc)) + (string String (quoted-string "\"")) + (number Number (number)) + (identifier Identifier (identifier)) + (punctuation Punctuation + (literals "==" "!=" "<=" ">=" "&&" "||" + "{" "}" "[" "]" "(" ")" "=" "," "." ":" + "+" "-" "*" "/" "%" "!" "<" ">" "?")) + (unknown Unknown (fallback))) + (extras horizontal-whitespace comment block-comment-token) + (keywords) + (recoveries + (config-file "GERBIL-PARSER-HCL-V2-24" preserve-source)) + (conflicts reject) + (case-insensitive #f) + (node-fields + (CallExpression callee argument) + (IndexExpression collection key)))) + (rules + (config-file + (node HclFile + (repeat (choice (token newline) (field item (reference body-item)))))) + (body + (node Body + (repeat (choice (token newline) (field item (reference body-item)))))) + (body-item + (choice (reference block) (reference attribute))) + (attribute + (node Attribute + (seq + (field name (token identifier)) + (literal "=") + (field value (reference expression))))) + (block + (node Block + (seq + (field type (token identifier)) + (repeat + (field label (choice (token string) (token identifier)))) + (literal "{") + (field body (reference body)) + (literal "}")))) + (expression (reference conditional-expression)) + (conditional-expression + (node ConditionalExpression + (seq + (field condition (reference binary-expression)) + (optional + (seq + (literal "?") + (field consequent (reference expression)) + (literal ":") + (field alternative (reference expression))))))) + (binary-expression + (node BinaryExpression + (seq + (field left (reference unary-expression)) + (repeat + (seq + (field operator (reference binary-operator)) + (field right (reference unary-expression))))))) + (binary-operator + (choice + (literal "==") (literal "!=") (literal "<=") (literal ">=") + (literal "&&") (literal "||") (literal "+") (literal "-") + (literal "*") (literal "/") (literal "%") (literal "<") + (literal ">"))) + (unary-expression + (node UnaryExpression + (seq + (repeat (field operator + (choice (literal "!") (literal "-") (literal "+")))) + (field operand (reference postfix-expression))))) + (postfix-expression + (node TraversalExpression + (seq + (field root (reference primary-expression)) + (repeat (field step (reference postfix-suffix)))))) + (postfix-suffix + (choice + (seq (literal ".") + (choice (token identifier) (token number) (literal "*"))) + (node IndexExpression + (seq (literal "[") + (field key (choice (literal "*") (reference expression))) + (literal "]"))) + (node CallExpression + (seq + (literal "(") + (optional + (seq + (field argument (reference expression)) + (repeat + (seq (literal ",") (field argument (reference expression)))))) + (literal ")"))))) + (primary-expression + (choice + (reference object-expression) + (reference tuple-expression) + (reference heredoc-expression) + (reference string-expression) + (reference number-expression) + (node LiteralExpression (field value (token identifier))) + (seq (literal "(") (reference expression) (literal ")")))) + (traversal-expression + (node TraversalExpression + (seq + (field root (token identifier)) + (repeat + (seq (literal ".") (field step (token identifier))))))) + (string-expression + (node StringExpression (field value (token string)))) + (heredoc-expression + (node HeredocExpression (field value (token heredoc)))) + (number-expression + (node NumberExpression (field value (token number)))) + (tuple-expression + (node TupleExpression + (seq + (literal "[") + (repeat (token newline)) + (optional (reference tuple-elements)) + (literal "]")))) + (tuple-elements + (seq + (field element (reference expression)) + (optional (reference tuple-tail)))) + (tuple-tail + (choice + (seq (literal ",") + (repeat (token newline)) + (optional (reference tuple-elements))) + (seq (repeat1 (token newline)) + (optional (reference tuple-elements))))) + (object-expression + (node ObjectExpression + (seq + (literal "{") + (repeat + (choice + (token newline) + (field entry (reference object-entry)))) + (literal "}")))) + (object-entry + (node ObjectEntry + (seq + (field key (choice (token identifier) (token string))) + (choice (literal "=") (literal ":")) + (field value (reference expression)) + (optional (literal ","))))))) diff --git a/languages/hcl/parser-test.ss b/languages/hcl/parser-test.ss new file mode 100644 index 00000000..abdf7fc2 --- /dev/null +++ b/languages/hcl/parser-test.ss @@ -0,0 +1,199 @@ +;;; -*- Gerbil -*- +;;; Declarative pinned corpus and generated native-route qualification. +(import :gerbil-parser/language-test-support ./parser) +(import (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/fixture defsyntax-corpus defsyntax-fixture) + (only-in :gerbil-parser/language-support/development deflanguage-development-loader LanguageDevelopmentLoader.) + (only-in :gerbil-parser/language-build-support declare-language-build-strategy make-fused-reduction-strategy make-rust-runtime-strategy) + (only-in :gerbil-parser/src/compiler/hcl-source direct-parse-hcl direct-lex-hcl direct-hcl-grammar-digest)) +(export hcl-representative-fixture hcl-official-fixtures hcl-test-language) + +(defsyntax-fixture hcl-representative-fixture + (identity "hcl/v2.24.0/representative" + "hcl" +hcl-native-syntax-version+ +hcl-syntax-contract+) + (source "corpus/representative.hcl") + (expect accepted HclFile + (Block Attribute TupleExpression ObjectExpression + TraversalExpression))) + +(defsyntax-corpus hcl-official-fixtures + (identity "hcl" +hcl-native-syntax-version+ +hcl-syntax-contract+) + (accepted + ("hcl/specsuite/comments/hash" hcl-spec-hash-comment + "corpus/reference/comments/hash_comment.hcl" HclFile ()) + ("hcl/specsuite/comments/multiline" hcl-spec-multiline-comment + "corpus/reference/comments/multiline_comment.hcl" HclFile ()) + ("hcl/specsuite/comments/slash" hcl-spec-slash-comment + "corpus/reference/comments/slash_comment.hcl" HclFile ()) + ("hcl/specsuite/empty" hcl-spec-empty + "corpus/reference/empty.hcl" HclFile ()) + ("hcl/specsuite/expressions/heredoc" hcl-spec-heredoc + "corpus/reference/expressions/heredoc.hcl" + HclFile (Attribute ObjectExpression HeredocExpression)) + ("hcl/specsuite/expressions/operators" hcl-spec-operators + "corpus/reference/expressions/operators.hcl" + HclFile (Block Attribute BinaryExpression ConditionalExpression)) + ("hcl/specsuite/expressions/primitive-literals" hcl-spec-primitives + "corpus/reference/expressions/primitive_literals.hcl" + HclFile (Attribute NumberExpression StringExpression LiteralExpression)) + ("hcl/specsuite/structure/attributes/expected" hcl-spec-attributes-expected + "corpus/reference/structure/attributes_expected.hcl" + HclFile (Attribute StringExpression)) + ("hcl/specsuite/structure/attributes/unexpected" hcl-spec-attributes-unexpected + "corpus/reference/structure/attributes_unexpected.hcl" + HclFile (Attribute StringExpression)) + ("hcl/specsuite/structure/blocks/empty-oneline" hcl-spec-block-empty-oneline + "corpus/reference/structure/block_empty_oneline.hcl" + HclFile (Block)) + ("hcl/specsuite/structure/blocks/empty-multiline" hcl-spec-block-empty-multiline + "corpus/reference/structure/block_empty_multiline.hcl" + HclFile (Block)) + ("hcl/specsuite/structure/blocks/single-oneline" hcl-spec-block-single-oneline + "corpus/reference/structure/block_single_oneline.hcl" + HclFile (Block Attribute StringExpression))) + (rejected + ("hcl/specsuite/structure/attributes/singleline-bad" + hcl-spec-attribute-singleline-bad + "corpus/reference/invalid/attribute_singleline_bad.hcl") + ("hcl/specsuite/structure/blocks/single-oneline-invalid" + hcl-spec-block-single-oneline-invalid + "corpus/reference/invalid/block_single_oneline_invalid.hcl") + ("hcl/specsuite/structure/blocks/single-unclosed" + hcl-spec-block-single-unclosed + "corpus/reference/invalid/block_single_unclosed.hcl"))) +(deflanguage-development-loader hcl-test-language + (grammar hcl-language-grammar) + (parse parse-hcl-test) + (slots metadata: `((grammar-format . concise-dsl) (reference-commit . ,+hcl-native-syntax-commit+)) + build-strategies: (list (declare-language-build-strategy 'fused-reductions + (make-fused-reduction-strategy hcl-language-grammar)) + (declare-language-build-strategy 'rust-runtime + (make-rust-runtime-strategy hcl-language-grammar))) + fixtures: hcl-official-fixtures + native-test-profile: (.o profile: 'recursive-source + digest: direct-hcl-grammar-digest + source: direct-parse-hcl lexer: direct-lex-hcl))) + +(deflanguage-parser-tests hcl-parser-test "HCL native syntax v2.24.0 official corpus" + (loader hcl-test-language) + (installed "HCL machine installs the generated fused step" step) + (installed "HCL machine installs the Grammar IR recursive source parser" source) + (strategy-parity "the closed ASCII lexer agrees with canonical tokens" + (append (sources "key0 = 1\n" "a-b = 123\r\n" "a = 1.2\n" + "a == 1\n" "a = \"text\"\n" "é = 1\n") + (seeded 1729 512 (repeat 0 32 (character "abcXYZ_eE0123= \t\r\n")))) + (routes) (lexical optional) (coverage 64 0)) + (strategy-parity "the 1024-line Basic source retains its exact artifact" + (indexed 1024 (concat "key" index " = 1\n")) + (routes entry source-default source-raw source-generic indexed) (accepted #t)) + (strategy-parity "ASCII quoted strings retain ranked tokens and artifacts" + (sources "name = \"value\"\n" + "name = \"a\\\"b\"\n" + "name = \"a\"\"b\"\n" + "name = \"// not a comment\"\n" + "name = \"a\nb\"\n") + (routes entry source-ranked source-raw indexed) (lexical fast) (accepted #t)) + (strategy-parity "unsupported quoted strings fall back to ranked lexing" + (sources "name = \"雪\"\n" "name = \"unterminated\n") + (routes source-default source-ranked) (lexical fallback)) + (strategy-parity "generated quoted-string tokens match ranked scanning" + (seeded 3857 128 (concat "key = \"" (repeat 0 12 (choice "a" "0" " " "\\\\" "\\\"" "//")) "\"\n")) + (routes source-default source-ranked) (lexical fast) (accepted #t)) + (strategy-parity "1024 ASCII string lines retain the indexed artifact" + (indexed 1024 (concat "key" index " = \"value\"\n")) + (routes entry source-ranked source-raw indexed) (accepted #t)) + (strategy-parity "ASCII comment forms retain ranked tokens and artifacts" + (sources "# heading\nkey = 1\n" + "key=1 // trailing\r\n" + "key = 1 /* block\n comment */\n" + "key = \"// value\" # comment\n" + "/* leading */\nkey = 1\n") + (routes entry source-ranked source-raw indexed) (lexical fast) (accepted #t)) + (strategy-parity "Unicode and unterminated comments use ranked fallback" + (sources "key = 1 # 雪\n" "key = 1 /* unclosed") + (routes source-default source-ranked) (lexical fallback)) + (strategy-parity "generated comment tokens match ranked scanning" + (seeded 7103 128 + (bind ((start (choice "#" "//" "/*"))) + (concat "key = 1 " start (repeat 0 16 (character "ab09 #/\t")) + (when-equal start "/*" "*/") "\r\n"))) + (routes source-default source-ranked) (lexical fast) (accepted #t)) + (strategy-parity "1024 commented lines retain the indexed artifact" + (indexed 1024 (concat "key" index " = 1 # note\n")) + (routes entry source-ranked source-raw indexed) (accepted #t)) + (strategy-parity "ASCII fractional and exponent numbers retain artifacts" + (sources "key = 0.5\n" "key=12e3\n" "key=1.25E-3\n" + "key=7e+1\n" "key=4.0 # comment\n") + (routes entry source-ranked source-raw indexed) (lexical fast) (accepted #t)) + (strategy-parity "numeric boundaries and seeded literals match ranked scanning" + (sources "key=1e\n" "key=1e+\n" "key=1.\n" + "key=1.2.3\n" "key=1e2foo\n" "key=雪2\n") + (routes source-default source-ranked) (lexical optional)) + (strategy-parity "seeded numeric literals retain fast tokens and accepted artifacts" + (seeded 7207 128 (concat "key = " (integer 1 999) "." (integer 0 1000) + (if-even "e+" "E-") (integer 0 30) "\n")) + (routes source-default source-ranked) (lexical fast) (accepted #t)) + (strategy-parity "1024 decimal lines retain the indexed artifact" + (indexed 1024 (concat "key" index " = 1.25\n")) + (routes entry source-ranked source-raw indexed) (accepted #t)) + (strategy-parity "simple attributes and late complex fallback retain artifacts" + (sources "" "\n\n" "name = \"雪\"\r\n" "flag = true\n" + "é = 2\n" "# comment\nfoo = 2\n" + "foo = 1 # comment\nbar = \"x\"\n" + "plain = 1\ncomplex = [1, 2]\n") + (routes entry source-default source-raw source-generic indexed) (accepted #t)) + (strategy-parity "mixed simple attribute corridor matches generic events" + (seeded 4919 128 + (repeat 1 8 (concat (choice "a" "foo" "é" "name_2") " = " + (choice "1" "23" "foo" "true" "\"x\"" "\"雪\"") + (choice "\n" "\r\n" " # comment\n")))) + (routes source-generic source-default) (accepted #t)) + (strategy-parity "compact HCL preserves canonical and rollback event paths" + (indexed 128 (concat "key" index "=1\n")) + (routes entry source-raw indexed) (accepted #t)) + (strategy-parity "closed flat blocks retain ranked tokens and full artifacts" + (sources "srv {}\n" + "srv { key = 1 }\n" + "srv \"web\" { key = 1 }\n" + "srv \"web\" internal { key = 1 }\n" + "srv { child { key = 1 } }\n" + "srv { child { grandchild { key = 1 } } }\n" + "srv { /* lead */ child { key = 1 } /* trail */ }\n" + "srv \"web\" { child \"db\" {\r\n key = 1\r\n }\r\n}\n" + "srv { child {}\n another { key = 2 } }\n" + "srv /* between */ \"web\" /* label */ internal {\n}\n" + "srv {\n key = 1\n port = 2\n}\n" + "# before\nsrv {\r\n name = \"x\" // after\r\n}\r\n" + "one {\n}\ntwo {\n x = 1.25e+3\n}\n") + (routes entry source-generic indexed) (lexical fast) (accepted #t)) + (strategy-parity "Unicode block label uses ranked lexer and exact byte offsets" + (sources "srv \"λ\" internal { child \"μ\" { key = 1 } }\n") + (routes entry source-generic indexed) (lexical fallback) (accepted #t)) + (strategy-parity "complex nested values and mixed roots keep generic fallback" + (sources "srv { child { key = [1, 2] } }\n" + "top = 1\nsrv { key = 2 }\n") + (routes entry source-generic indexed) (accepted #t)) + (strategy-parity "1024 flat-block lines retain the indexed artifact" + (append + (indexed 256 (concat "srv" index " {\nkey = 1\nport = 2\n}\n")) + (indexed 256 (concat "srv" index " \"web\" internal {\nkey = 1\nport = 2\n}\n"))) + (routes entry source-generic indexed) (lexical fast) (accepted #t)) + (strategy-parity "1024 nested-block lines retain the indexed artifact" + (indexed 256 (concat "outer" index " \"env\" {\nsrv" index + " \"web\" internal {\nkey = 1\n}}\n")) + (routes entry source-generic indexed) (lexical fast) (accepted #t)) + (strategy-parity "generated events preserve Unicode byte offsets and trivia" + (sources "名称 = \"λ中😀\"\n# 注释\n") + (routes entry indexed) (accepted #t)) + (property "the upstream syntax identity is immutable" (bindings) + (equal +hcl-native-syntax-version+ "v2.24.0") + (equal +hcl-native-syntax-commit+ "6b5068090eef06b1f127f61529db5ba0be7ed343") + (equal +hcl-syntax-contract+ "hcl-native-v2.24.0.v1")) + (fixture-catalog "official HCL fixture counts" (total 15) (accepted 12) (rejected 3)) + (identity "Loader retains the pinned grammar identity" + (schema "gerbil-parser.language-entry.v2") (language "hcl") + (version +hcl-native-syntax-version+) (contract +hcl-syntax-contract+)) + (fixture-parity "complete official HCL specsuite sources parse losslessly" accepted + (routes entry source-default source-raw indexed) (accepted #t)) + (fixture-parity "official invalid HCL sources fail as one typed artifact" rejected + (routes source-default) (accepted #f) (diagnostics 1))) diff --git a/languages/hcl/parser.ss b/languages/hcl/parser.ss new file mode 100644 index 00000000..91758194 --- /dev/null +++ b/languages/hcl/parser.ss @@ -0,0 +1,34 @@ +(import (only-in :gerbil-parser/src/compiler/hcl-reductions direct-step direct-event-step direct-grammar-digest) + (only-in :gerbil-parser/src/compiler/hcl-source direct-parse-hcl direct-hcl-grammar-digest)) +;;; -*- Gerbil -*- +;;; Canonical public parser entry for HCL native syntax v2.24.0. + +(import (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-entry-ref language-metadata-ref) + ./grammar) +(export +hcl-native-syntax-version+ +hcl-syntax-contract+ hcl-language-grammar (import: ./grammar) + hcl-language + parse-hcl) + +(export +hcl-native-syntax-commit+) + +(def +hcl-native-syntax-commit+ + "6b5068090eef06b1f127f61529db5ba0be7ed343") + +(deflanguage-parser-loader hcl-language + (grammar hcl-syntax) + (parse parse-hcl) + (slots metadata: `((language . "hcl") + (version . "v2.24.0") + (contract . "hcl-native-v2.24.0.v1") + (grammar-format . concise-dsl) + (reference-commit . ,+hcl-native-syntax-commit+))) + (backends hcl-parser + (step direct-grammar-digest direct-step) + (source direct-hcl-grammar-digest direct-parse-hcl) + (event-step direct-grammar-digest direct-event-step))) + +(def hcl-language-grammar (language-parser-entry-ref hcl-language 'descriptor)) + +(def +hcl-native-syntax-version+ (language-metadata-ref (language-parser-entry-ref hcl-language 'metadata) 'version)) + +(def +hcl-syntax-contract+ (language-metadata-ref (language-parser-entry-ref hcl-language 'metadata) 'contract)) diff --git a/languages/hcl/v2-24/fixtures.ss b/languages/hcl/v2-24/fixtures.ss deleted file mode 100644 index 23bb5552..00000000 --- a/languages/hcl/v2-24/fixtures.ss +++ /dev/null @@ -1,76 +0,0 @@ -;;; -*- Gerbil -*- -;;; HCL v2.24.0 official specsuite sources embedded at expansion time. - -(import :gerbil-parser/languages/hcl/v2-24/parser - (only-in :gerbil-parser/language-support - defsyntax-corpus - defsyntax-fixture - syntax-fixture-expected-status)) -(export hcl-v2-24-representative-fixture - hcl-v2-24-official-fixtures - hcl-v2-24-official-accepted-fixtures - hcl-v2-24-official-rejected-fixtures) - -(defsyntax-fixture hcl-v2-24-representative-fixture - (identity "hcl/v2.24.0/representative" - "hcl" +hcl-native-syntax-version+ +hcl-syntax-contract+) - (source "corpus/representative.hcl") - (expect accepted HclFile - (Block Attribute TupleExpression ObjectExpression - TraversalExpression))) - -(defsyntax-corpus hcl-v2-24-official-fixtures - (identity "hcl" +hcl-native-syntax-version+ +hcl-syntax-contract+) - (accepted - ("hcl/specsuite/comments/hash" hcl-spec-hash-comment - "corpus/hashicorp-v2.24.0/comments/hash_comment.hcl" HclFile ()) - ("hcl/specsuite/comments/multiline" hcl-spec-multiline-comment - "corpus/hashicorp-v2.24.0/comments/multiline_comment.hcl" HclFile ()) - ("hcl/specsuite/comments/slash" hcl-spec-slash-comment - "corpus/hashicorp-v2.24.0/comments/slash_comment.hcl" HclFile ()) - ("hcl/specsuite/empty" hcl-spec-empty - "corpus/hashicorp-v2.24.0/empty.hcl" HclFile ()) - ("hcl/specsuite/expressions/heredoc" hcl-spec-heredoc - "corpus/hashicorp-v2.24.0/expressions/heredoc.hcl" - HclFile (Attribute ObjectExpression HeredocExpression)) - ("hcl/specsuite/expressions/operators" hcl-spec-operators - "corpus/hashicorp-v2.24.0/expressions/operators.hcl" - HclFile (Block Attribute BinaryExpression ConditionalExpression)) - ("hcl/specsuite/expressions/primitive-literals" hcl-spec-primitives - "corpus/hashicorp-v2.24.0/expressions/primitive_literals.hcl" - HclFile (Attribute NumberExpression StringExpression LiteralExpression)) - ("hcl/specsuite/structure/attributes/expected" hcl-spec-attributes-expected - "corpus/hashicorp-v2.24.0/structure/attributes_expected.hcl" - HclFile (Attribute StringExpression)) - ("hcl/specsuite/structure/attributes/unexpected" hcl-spec-attributes-unexpected - "corpus/hashicorp-v2.24.0/structure/attributes_unexpected.hcl" - HclFile (Attribute StringExpression)) - ("hcl/specsuite/structure/blocks/empty-oneline" hcl-spec-block-empty-oneline - "corpus/hashicorp-v2.24.0/structure/block_empty_oneline.hcl" - HclFile (Block)) - ("hcl/specsuite/structure/blocks/empty-multiline" hcl-spec-block-empty-multiline - "corpus/hashicorp-v2.24.0/structure/block_empty_multiline.hcl" - HclFile (Block)) - ("hcl/specsuite/structure/blocks/single-oneline" hcl-spec-block-single-oneline - "corpus/hashicorp-v2.24.0/structure/block_single_oneline.hcl" - HclFile (Block Attribute StringExpression))) - (rejected - ("hcl/specsuite/structure/attributes/singleline-bad" - hcl-spec-attribute-singleline-bad - "corpus/hashicorp-v2.24.0/invalid/attribute_singleline_bad.hcl") - ("hcl/specsuite/structure/blocks/single-oneline-invalid" - hcl-spec-block-single-oneline-invalid - "corpus/hashicorp-v2.24.0/invalid/block_single_oneline_invalid.hcl") - ("hcl/specsuite/structure/blocks/single-unclosed" - hcl-spec-block-single-unclosed - "corpus/hashicorp-v2.24.0/invalid/block_single_unclosed.hcl"))) - -(def hcl-v2-24-official-accepted-fixtures - (filter (lambda (fixture) - (eq? (syntax-fixture-expected-status fixture) 'accepted)) - hcl-v2-24-official-fixtures)) - -(def hcl-v2-24-official-rejected-fixtures - (filter (lambda (fixture) - (eq? (syntax-fixture-expected-status fixture) 'rejected)) - hcl-v2-24-official-fixtures)) diff --git a/languages/hcl/v2-24/grammar.ss b/languages/hcl/v2-24/grammar.ss deleted file mode 100644 index 1fe66df8..00000000 --- a/languages/hcl/v2-24/grammar.ss +++ /dev/null @@ -1,213 +0,0 @@ -;;; -*- Gerbil -*- -;;; Version-pinned HCL native syntax grammar owner. - -(import (only-in :gerbil-parser/language-support deflanguage-grammar)) -(export +hcl-native-syntax-version+ - +hcl-native-syntax-commit+ - +hcl-syntax-contract+ - hcl-v2-24-language-grammar - hcl-v2-24-grammar - hcl-v2-24-parser-ir - hcl-v2-24-parser) - -(def +hcl-native-syntax-version+ "v2.24.0") -(def +hcl-native-syntax-commit+ - "6b5068090eef06b1f127f61529db5ba0be7ed343") -(def +hcl-syntax-contract+ "hcl-native-v2.24.0.v1") - -(deflanguage-grammar hcl-v2-24 - (identity "hcl" +hcl-native-syntax-version+ +hcl-syntax-contract+) - (syntax-kinds - (HclFile node (item)) - (Body node (item)) - (Attribute node (name value)) - (Block node (type label body)) - (TraversalExpression node (root step)) - (CallExpression node (callee argument)) - (BinaryExpression node (left operator right)) - (UnaryExpression node (operator operand)) - (ConditionalExpression node (condition consequent alternative)) - (IndexExpression node (collection key)) - (StringExpression node (value)) - (HeredocExpression node (value)) - (NumberExpression node (value)) - (LiteralExpression node (value)) - (TupleExpression node (element)) - (ObjectExpression node (entry)) - (ObjectEntry node (key value)) - (Identifier token (text)) - (Number token (text)) - (String token (text)) - (Heredoc token (text)) - (HorizontalWhitespace token (text)) - (Newline token (text)) - (Comment token (text)) - (Punctuation token (text)) - (Unknown token (text))) - (terminals - (identifier Identifier) - (number Number) - (string String) - (heredoc Heredoc) - (horizontal-whitespace HorizontalWhitespace) - (newline Newline) - (comment Comment) - (block-comment-token Comment) - (punctuation Punctuation) - (unknown Unknown)) - (lexical-rules - (horizontal-whitespace (horizontal-whitespace+)) - (newline (newline+)) - (comment (line-comment "#" "//")) - (block-comment-token (block-comment "/*" "*/")) - (heredoc (heredoc)) - (string (quoted-string "\"")) - (number (number)) - (identifier (identifier)) - (punctuation - (literals "==" "!=" "<=" ">=" "&&" "||" - "{" "}" "[" "]" "(" ")" "=" "," "." ":" - "+" "-" "*" "/" "%" "!" "<" ">" "?")) - (unknown (fallback))) - (rules - (config-file - (alias HclFile - (repeat (choice (token newline) (field item (reference body-item)))))) - (body - (alias Body - (repeat (choice (token newline) (field item (reference body-item)))))) - (body-item - (choice (reference block) (reference attribute))) - (attribute - (alias Attribute - (seq - (field name (token identifier)) - (literal "=") - (field value (reference expression))))) - (block - (alias Block - (seq - (field type (token identifier)) - (repeat - (field label (choice (token string) (token identifier)))) - (literal "{") - (field body (reference body)) - (literal "}")))) - (expression (reference conditional-expression)) - (conditional-expression - (alias ConditionalExpression - (seq - (field condition (reference binary-expression)) - (optional - (seq - (literal "?") - (field consequent (reference expression)) - (literal ":") - (field alternative (reference expression))))))) - (binary-expression - (alias BinaryExpression - (seq - (field left (reference unary-expression)) - (repeat - (seq - (field operator (reference binary-operator)) - (field right (reference unary-expression))))))) - (binary-operator - (choice - (literal "==") (literal "!=") (literal "<=") (literal ">=") - (literal "&&") (literal "||") (literal "+") (literal "-") - (literal "*") (literal "/") (literal "%") (literal "<") - (literal ">"))) - (unary-expression - (alias UnaryExpression - (seq - (repeat (field operator - (choice (literal "!") (literal "-") (literal "+")))) - (field operand (reference postfix-expression))))) - (postfix-expression - (alias TraversalExpression - (seq - (field root (reference primary-expression)) - (repeat (field step (reference postfix-suffix)))))) - (postfix-suffix - (choice - (seq (literal ".") - (choice (token identifier) (token number) (literal "*"))) - (alias IndexExpression - (seq (literal "[") - (field key (choice (literal "*") (reference expression))) - (literal "]"))) - (alias CallExpression - (seq - (literal "(") - (optional - (seq - (field argument (reference expression)) - (repeat - (seq (literal ",") (field argument (reference expression)))))) - (literal ")"))))) - (primary-expression - (choice - (reference object-expression) - (reference tuple-expression) - (reference heredoc-expression) - (reference string-expression) - (reference number-expression) - (alias LiteralExpression (field value (token identifier))) - (seq (literal "(") (reference expression) (literal ")")))) - (traversal-expression - (alias TraversalExpression - (seq - (field root (token identifier)) - (repeat - (seq (literal ".") (field step (token identifier))))))) - (string-expression - (alias StringExpression (field value (token string)))) - (heredoc-expression - (alias HeredocExpression (field value (token heredoc)))) - (number-expression - (alias NumberExpression (field value (token number)))) - (tuple-expression - (alias TupleExpression - (seq - (literal "[") - (repeat (token newline)) - (optional (reference tuple-elements)) - (literal "]")))) - (tuple-elements - (seq - (field element (reference expression)) - (optional (reference tuple-tail)))) - (tuple-tail - (choice - (seq (literal ",") - (repeat (token newline)) - (optional (reference tuple-elements))) - (seq (repeat1 (token newline)) - (optional (reference tuple-elements))))) - (object-expression - (alias ObjectExpression - (seq - (literal "{") - (repeat - (choice - (token newline) - (field entry (reference object-entry)))) - (literal "}")))) - (object-entry - (alias ObjectEntry - (seq - (field key (choice (token identifier) (token string))) - (choice (literal "=") (literal ":")) - (field value (reference expression)) - (optional (literal ",")))))) - (extras horizontal-whitespace comment block-comment-token) - (keywords) - (parser-entrypoints - (config-file parse pure)) - (recoveries - (config-file "GERBIL-PARSER-HCL-V2-24" preserve-source)) - (flow - (source lexical) - (lexical hcl-structural-core) - (hcl-structural-core cst))) diff --git a/languages/hcl/v2-24/parser-test.ss b/languages/hcl/v2-24/parser-test.ss deleted file mode 100644 index 7396a43a..00000000 --- a/languages/hcl/v2-24/parser-test.ss +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- -;;; Acceptance owner for the pinned HCL specsuite, descriptor identity, -;;; lossless accepted artifacts, and typed rejected artifacts. - -(import (only-in :std/test check test-case test-suite) - :gerbil-parser/languages/hcl/v2-24/parser - :gerbil-parser/src/language/entry - :gerbil-parser/src/runtime/artifact - :gerbil-parser/src/runtime/cst - (only-in :gerbil-parser/language-support - syntax-fixture-contract - syntax-fixture-expected-status - syntax-fixture-id - syntax-fixture-required-kinds - syntax-fixture-root-kind - syntax-fixture-source - syntax-fixture-source-digest - syntax-fixture-version) - (only-in ./fixtures - hcl-v2-24-official-accepted-fixtures - hcl-v2-24-official-fixtures - hcl-v2-24-official-rejected-fixtures)) -(export hcl-v2-24-parser-test) - -;;; Structural traversal stays independent of HCL production nesting so the -;;; official corpus checks semantic kinds while roundtrip checks retain bytes. -;; : (-> CSTValue (List Symbol)) -(def (cst-node-kinds value) - (cond - ((syntax-node? value) - (cons (syntax-node-kind value) - (apply append (map cst-node-kinds (syntax-node-children value))))) - ((syntax-field? value) - (apply append (map cst-node-kinds (syntax-field-children value)))) - (else '()))) - -;; : TestSuite -(def hcl-v2-24-parser-test - (test-suite "HCL native syntax v2.24.0 official corpus" - (test-case "the upstream syntax identity is immutable" - (check +hcl-native-syntax-version+ => "v2.24.0") - (check +hcl-native-syntax-commit+ - => "6b5068090eef06b1f127f61529db5ba0be7ed343") - (check +hcl-syntax-contract+ - => "hcl-native-v2.24.0.v1") - (check (language-parser-entry-ref hcl-v2-24-language 'schema) - => +language-parser-entry-schema+) - (check (language-parser-entry-ref hcl-v2-24-language 'language) - => "hcl") - (check (language-parser-entry-ref hcl-v2-24-language 'version) - => +hcl-native-syntax-version+) - (check (language-parser-entry-ref hcl-v2-24-language 'contract) - => +hcl-syntax-contract+) - (check (length hcl-v2-24-official-fixtures) => 15) - (check (length hcl-v2-24-official-accepted-fixtures) => 12) - (check (length hcl-v2-24-official-rejected-fixtures) => 3)) - (test-case "complete official HCL specsuite sources parse losslessly" - (for-each - (lambda (fixture) - (let* ((source (syntax-fixture-source fixture)) - (artifact (parse-hcl-v2-24 source)) - (accepted? (parse-artifact-success? artifact))) - (check (syntax-fixture-version fixture) => +hcl-native-syntax-version+) - (check (syntax-fixture-contract fixture) => +hcl-syntax-contract+) - (check (syntax-fixture-expected-status fixture) => 'accepted) - (check (list (syntax-fixture-id fixture) accepted?) - => (list (syntax-fixture-id fixture) #t)) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-ref artifact 'sourceDigest) - => (syntax-fixture-source-digest fixture)) - (check (parse-artifact-roundtrip artifact) => source) - (when accepted? - (let* ((root (parse-artifact->cst artifact)) - (kinds (cst-node-kinds root))) - (check (syntax-node-kind root) - => (syntax-fixture-root-kind fixture)) - (for-each - (lambda (required-kind) - (check (member required-kind kinds) ? values)) - (syntax-fixture-required-kinds fixture)))))) - hcl-v2-24-official-accepted-fixtures)) - (test-case "official invalid HCL sources fail as one typed artifact" - (for-each - (lambda (fixture) - (let* ((source (syntax-fixture-source fixture)) - (artifact (parse-hcl-v2-24 source))) - (check (syntax-fixture-version fixture) => +hcl-native-syntax-version+) - (check (syntax-fixture-contract fixture) => +hcl-syntax-contract+) - (check (syntax-fixture-expected-status fixture) => 'rejected) - (check (parse-artifact-success? artifact) => #f) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-ref artifact 'sourceDigest) - => (syntax-fixture-source-digest fixture)) - (check (parse-artifact-roundtrip artifact) => source) - (check (length (parse-artifact-ref artifact 'diagnostics)) => 1))) - hcl-v2-24-official-rejected-fixtures)))) diff --git a/languages/hcl/v2-24/parser.ss b/languages/hcl/v2-24/parser.ss deleted file mode 100644 index 937ed740..00000000 --- a/languages/hcl/v2-24/parser.ss +++ /dev/null @@ -1,12 +0,0 @@ -;;; -*- Gerbil -*- -;;; Canonical public parser entry for HCL native syntax v2.24.0. - -(import :gerbil-parser/src/language/entry - ./grammar) -(export (import: ./grammar) - hcl-v2-24-language - parse-hcl-v2-24) - -(deflanguage-parser hcl-v2-24-language - (grammar hcl-v2-24-language-grammar) - (parse parse-hcl-v2-24)) diff --git a/languages/hl7/v2-2.5.1/corpus/adt-a08-patient.hl7 b/languages/hl7/corpus/adt-a08-patient.hl7 similarity index 100% rename from languages/hl7/v2-2.5.1/corpus/adt-a08-patient.hl7 rename to languages/hl7/corpus/adt-a08-patient.hl7 diff --git a/languages/hl7/grammar.ss b/languages/hl7/grammar.ss new file mode 100644 index 00000000..ebff087a --- /dev/null +++ b/languages/hl7/grammar.ss @@ -0,0 +1,105 @@ +;;; -*- Gerbil -*- +;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors +;;; +;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later + +(import (only-in :gerbil-parser/language-support/grammar deflanguage)) +(export hl7-syntax + hl7-grammar + hl7-parser-ir + hl7-parser) + + +;;; Closed engine rules derive message-local delimiters from MSH-1/MSH-2. +(deflanguage hl7 + (syntax + (lexical + (root message) + (lex + (segment-id SegmentId + (text-profile + (seq (run (union (characters "ABCDEFGHIJKLMNOPQRSTUVWXYZ") (numeric)) 3 3) + (not-next (union (characters "ABCDEFGHIJKLMNOPQRSTUVWXYZ") (numeric)))))) + (data Data (header-data "MSH" 5 "\r\n")) + (segment-terminator SegmentTerminator + (text-profile + (if-next (characters "\r") + (seq (literal "\r") (optional (literal "\n"))) + (literal "\n")))) + (field-separator FieldSeparator + (header-delimiter "MSH" 5 0)) + (component-separator ComponentSeparator + (header-delimiter "MSH" 5 1)) + (repetition-separator RepetitionSeparator + (header-delimiter "MSH" 5 2)) + (escape-character EscapeCharacter + (header-delimiter "MSH" 5 3)) + (subcomponent-separator SubcomponentSeparator + (header-delimiter "MSH" 5 4))) + (extras) + (keywords) + (recoveries + (message "GERBIL-PARSER-HL7V2-ER7" preserve-source)) + (conflicts reject) + (case-insensitive #f))) + (rules + (message + (node Message + (seq + (field header (reference header-segment)) + (repeat (field segment (reference segment)))))) + (header-segment + (node HeaderSegment + (seq + (literal "MSH") + (token field-separator) + (token component-separator) + (token repetition-separator) + (token escape-character) + (token subcomponent-separator) + (repeat + (seq (token field-separator) + (optional (field field (reference field))))) + (token segment-terminator)))) + (segment + (node Segment + (seq + (field name (token segment-id)) + (repeat + (seq (token field-separator) + (optional (field field (reference field))))) + (token segment-terminator)))) + (field + (node Field + (seq + (field repetition (reference repetition)) + (repeat + (seq (token repetition-separator) + (field repetition (reference repetition))))))) + (repetition + (node Repetition + (seq + (field component (reference component)) + (repeat + (seq (token component-separator) + (optional (field component (reference component)))))))) + (component + (node Component + (seq + (field subcomponent (reference subcomponent)) + (repeat + (seq (token subcomponent-separator) + (optional + (field subcomponent (reference subcomponent)))))))) + (subcomponent + (node Subcomponent + (repeat1 + (field value + (choice (token data) (token segment-id) + (reference escape)))))) + (escape + (node Escape + (seq (token escape-character) + (optional + (field value (choice (token data) (token segment-id)))) + (token escape-character)))))) diff --git a/languages/hl7/parser-test.ss b/languages/hl7/parser-test.ss new file mode 100644 index 00000000..8bc8e6a4 --- /dev/null +++ b/languages/hl7/parser-test.ss @@ -0,0 +1,43 @@ +;;; -*- Gerbil -*- +;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors +;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later +(import :gerbil-parser/language-test-support + (only-in :std/misc/ports read-all-as-string) + (only-in :gerbil-parser/language-support parse-artifact-ref) + ./parser) +(import (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/fixture defsyntax-corpus) + (only-in :gerbil-parser/language-support/development deflanguage-development-loader LanguageDevelopmentLoader.)) +(export hl7-fixtures hl7-test-language) + +(defsyntax-corpus hl7-fixtures + (identity "hl7v2" +hl7-language-version+ +hl7-syntax-contract+) + (accepted + ("hl7v2/adt-a08" hl7-adt-a08 "corpus/adt-a08-patient.hl7" Message (HeaderSegment Segment Field))) + (rejected + ("hl7v2/missing-header" hl7-missing-header (text "PID|1\r")))) + +(deflanguage-development-loader hl7-test-language + (grammar hl7-language-grammar) + (parse parse-hl7-test) + (slots metadata: '((grammar-format . concise-dsl)) + fixtures: hl7-fixtures)) + +(deflanguage-parser-tests hl7-parser-test "HL7v2 ER7 2.5.1 parser" + (loader hl7-test-language) + (accepted "ADT A08 is lossless and projected" + (call-with-input-file "languages/hl7/corpus/adt-a08-patient.hl7" read-all-as-string) + (projection hl7-adt-a08-patient-projection + (sourceMessageType "ADT^A08") (identifierValue "8003608166690503") + (familyName "Nguyen") (givenNames '("Ava")))) + (property "projection source identity matches its artifact" + (bindings (artifact (parse-hl7 (call-with-input-file + "languages/hl7/corpus/adt-a08-patient.hl7" read-all-as-string))) + (projection (hl7-adt-a08-patient-projection artifact))) + (equal (cdr (assq 'sourceDigest projection)) + (parse-artifact-ref artifact 'sourceDigest))) + (rejected-many "missing delimiter headers retain valid lossless artifacts" + '("PID|1\r" "PID|α\r")) + (accepted "message-local delimiters" + "MSH*$%!?*LEGACY*AU*FHIR*AU*202609170900**ADT$A08*1*P*2.5.1\r") + (fixtures "loader fixture conformance")) diff --git a/languages/hl7/parser.ss b/languages/hl7/parser.ss new file mode 100644 index 00000000..11ba5b40 --- /dev/null +++ b/languages/hl7/parser.ss @@ -0,0 +1,38 @@ +;;; -*- Gerbil -*- +;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors +;;; +;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later + +(import (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-entry-ref language-metadata-ref) + (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/projection RecordProjection. deflanguage-projection) + ./grammar) +(export +hl7-language-version+ +hl7-syntax-contract+ hl7-language-grammar (import: ./grammar) + hl7-language + parse-hl7 hl7-adt-a08-patient-projection) + +(deflanguage-parser-loader hl7-language + (grammar hl7-syntax) + (parse parse-hl7) + (slots metadata: '((language . "hl7v2") + (version . "2.5.1") + (contract . "hl7v2-er7.v1") + (grammar-format . concise-dsl)))) + +(def hl7-language-grammar (language-parser-entry-ref hl7-language 'descriptor)) + +(def +hl7-language-version+ (language-metadata-ref (language-parser-entry-ref hl7-language 'metadata) 'version)) + +(def +hl7-syntax-contract+ (language-metadata-ref (language-parser-entry-ref hl7-language 'metadata) 'contract)) + +(deflanguage-projection hl7-adt-a08-patient-projection + (grammar hl7-syntax) + (strategy (.o (:: self RecordProjection.) prefix: "MSH" delimiter-count: 5 + expectations: '(((list (split (record "MSH" 8) 1 0) (split (record "MSH" 8) 1 1)) ("ADT" "A08")) + ((record "MSH" 11) "2.5.1")) + constants: '((schema . "gerbil-parser.hl7v2-adt-a08-patient.v1") + (sourceInterface . hl7) (sourceMessageType . "ADT^A08") (sourceVersion . "2.5.1")) + columns: '((identifierSystem concat (text "urn:oid:") (split (split (record "PID" 3) 1 3) 4 1)) + (identifierValue split (record "PID" 3) 1 0) + (familyName split (record "PID" 5) 1 0) + (givenNames list (split (record "PID" 5) 1 1)))))) diff --git a/languages/hl7/v2-2.5.1/grammar.ss b/languages/hl7/v2-2.5.1/grammar.ss deleted file mode 100644 index e5f956f5..00000000 --- a/languages/hl7/v2-2.5.1/grammar.ss +++ /dev/null @@ -1,135 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(import :gerbil-parser/src/language/grammar - ./scanner) -(export +hl7v2-language-version+ - +hl7v2-syntax-contract+ - hl7v2-language-grammar - hl7v2-grammar - hl7v2-parser-ir - hl7v2-parser) - -(def +hl7v2-language-version+ "2.5.1") -(def +hl7v2-syntax-contract+ "hl7v2-er7.v1") - -;;; Every delimiter scanner derives its character from MSH-1/MSH-2, preserving -;;; ER7's message-local delimiter contract without mutable parser state. -(deflanguage-grammar hl7v2 - (identity "hl7v2" +hl7v2-language-version+ +hl7v2-syntax-contract+) - (syntax-kinds - (Message node (header segment)) - (HeaderSegment node (field)) - (Segment node (name field)) - (Field node (repetition)) - (Repetition node (component)) - (Component node (subcomponent)) - (Subcomponent node (value)) - (Escape node (value)) - (SegmentId token (text)) - (Data token (text)) - (SegmentTerminator token (text)) - (FieldSeparator token (text)) - (ComponentSeparator token (text)) - (RepetitionSeparator token (text)) - (EscapeCharacter token (text)) - (SubcomponentSeparator token (text))) - (terminals - (segment-id SegmentId) - (data Data) - (segment-terminator SegmentTerminator) - (field-separator FieldSeparator) - (component-separator ComponentSeparator) - (repetition-separator RepetitionSeparator) - (escape-character EscapeCharacter) - (subcomponent-separator SubcomponentSeparator)) - (lexical-rules - (segment-id (external hl7v2-segment-id-v1 scan-hl7v2-segment-id)) - (data (external hl7v2-data-v1 scan-hl7v2-data)) - (segment-terminator - (external hl7v2-segment-terminator-v1 - scan-hl7v2-segment-terminator)) - (field-separator - (external hl7v2-field-separator-v1 scan-hl7v2-field-separator)) - (component-separator - (external hl7v2-component-separator-v1 scan-hl7v2-component-separator)) - (repetition-separator - (external hl7v2-repetition-separator-v1 scan-hl7v2-repetition-separator)) - (escape-character - (external hl7v2-escape-character-v1 scan-hl7v2-escape-character)) - (subcomponent-separator - (external hl7v2-subcomponent-separator-v1 - scan-hl7v2-subcomponent-separator))) - (rules - (message - (alias Message - (seq - (field header (reference header-segment)) - (repeat (field segment (reference segment)))))) - (header-segment - (alias HeaderSegment - (seq - (literal "MSH") - (token field-separator) - (token component-separator) - (token repetition-separator) - (token escape-character) - (token subcomponent-separator) - (repeat - (seq (token field-separator) - (optional (field field (reference field))))) - (token segment-terminator)))) - (segment - (alias Segment - (seq - (field name (token segment-id)) - (repeat - (seq (token field-separator) - (optional (field field (reference field))))) - (token segment-terminator)))) - (field - (alias Field - (seq - (field repetition (reference repetition)) - (repeat - (seq (token repetition-separator) - (field repetition (reference repetition))))))) - (repetition - (alias Repetition - (seq - (field component (reference component)) - (repeat - (seq (token component-separator) - (optional (field component (reference component)))))))) - (component - (alias Component - (seq - (field subcomponent (reference subcomponent)) - (repeat - (seq (token subcomponent-separator) - (optional - (field subcomponent (reference subcomponent)))))))) - (subcomponent - (alias Subcomponent - (repeat1 - (field value - (choice (token data) (token segment-id) - (reference escape)))))) - (escape - (alias Escape - (seq (token escape-character) - (optional - (field value (choice (token data) (token segment-id)))) - (token escape-character))))) - (extras) - (keywords) - (parser-entrypoints - (message parse pure)) - (recoveries - (message "GERBIL-PARSER-HL7V2-ER7" preserve-source)) - (flow - (source lexical) - (lexical hl7v2-er7-structure) - (hl7v2-er7-structure cst))) diff --git a/languages/hl7/v2-2.5.1/parser-test.ss b/languages/hl7/v2-2.5.1/parser-test.ss deleted file mode 100644 index aa85e841..00000000 --- a/languages/hl7/v2-2.5.1/parser-test.ss +++ /dev/null @@ -1,39 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(import :std/test - (only-in :std/misc/ports read-all-as-string) - (only-in :gerbil-parser/src/runtime/artifact - parse-artifact-ref parse-artifact-roundtrip - parse-artifact-success? parse-artifact-valid?) - ./parser - ./projection) - -(def fixture-path "languages/hl7/v2-2.5.1/corpus/adt-a08-patient.hl7") - -(def hl7v2-parser-test - (test-suite "HL7v2 ER7 2.5.1 parser" - (test-case "ADT A08 is accepted, lossless and projected with receipt identity" - (let* ((source (call-with-input-file fixture-path read-all-as-string)) - (artifact (parse-hl7v2 source)) - (projection (hl7v2-adt-a08-patient-projection artifact))) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source) - (check (cdr (assq 'sourceDigest projection)) - => (parse-artifact-ref artifact 'sourceDigest)) - (check (cdr (assq 'sourceMessageType projection)) => "ADT^A08") - (check (cdr (assq 'identifierValue projection)) => "8003608166690503") - (check (cdr (assq 'familyName projection)) => "Nguyen") - (check (cdr (assq 'givenNames projection)) => '("Ava")))) - (test-case "message-local delimiter declaration is parsed losslessly" - (let* ((source - "MSH*$%!?*LEGACY*AU*FHIR*AU*202609170900**ADT$A08*1*P*2.5.1\r") - (artifact (parse-hl7v2 source))) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source))))) - -(export hl7v2-parser-test) diff --git a/languages/hl7/v2-2.5.1/parser.ss b/languages/hl7/v2-2.5.1/parser.ss deleted file mode 100644 index c16dc291..00000000 --- a/languages/hl7/v2-2.5.1/parser.ss +++ /dev/null @@ -1,14 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(import :gerbil-parser/src/language/entry - ./grammar) -(export (import: ./grammar) - hl7v2-language - parse-hl7v2) - -(deflanguage-parser hl7v2-language - (grammar hl7v2-language-grammar) - (parse parse-hl7v2)) diff --git a/languages/hl7/v2-2.5.1/projection.ss b/languages/hl7/v2-2.5.1/projection.ss deleted file mode 100644 index aa440f0e..00000000 --- a/languages/hl7/v2-2.5.1/projection.ss +++ /dev/null @@ -1,89 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(import (only-in :gerbil-parser/src/runtime/artifact - parse-artifact-ref parse-artifact-roundtrip - parse-artifact-success? parse-artifact-valid?)) -(export hl7v2-adt-a08-patient-projection) - -(def (split-character value delimiter) - (let (length (string-length value)) - (let loop ((start 0) (offset 0) (parts '())) - (cond - ((= offset length) - (reverse (cons (substring value start offset) parts))) - ((char=? (string-ref value offset) delimiter) - (loop (+ offset 1) (+ offset 1) - (cons (substring value start offset) parts))) - (else (loop start (+ offset 1) parts)))))) - -(def (message-lines source) - (filter (lambda (line) (> (string-length line) 0)) - (split-character - (list->string - (map (lambda (character) - (if (char=? character #\newline) #\return character)) - (string->list source))) - #\return))) - -(def (required-index values index owner) - (if (< index (length values)) - (list-ref values index) - (error "missing required HL7v2 field" owner index))) - -(def (line-with-prefix lines prefix) - (let loop ((rest lines)) - (cond - ((null? rest) #f) - ((and (>= (string-length (car rest)) 4) - (string=? (substring (car rest) 0 4) prefix)) - (car rest)) - (else (loop (cdr rest)))))) - -(def (hl7v2-adt-a08-patient-projection artifact) - (unless (and (parse-artifact-valid? artifact) - (parse-artifact-success? artifact)) - (error "HL7v2 projection requires an accepted ParseArtifact")) - (let* ((source (parse-artifact-roundtrip artifact)) - (lines (message-lines source)) - (msh-line (required-index lines 0 'MSH)) - (field-separator (string-ref msh-line 3)) - (msh (split-character msh-line field-separator)) - (encoding-characters (required-index msh 1 'MSH-2)) - (component-separator (string-ref encoding-characters 0)) - (subcomponent-separator (string-ref encoding-characters 3)) - (pid-line - (line-with-prefix lines - (string-append "PID" (string field-separator))))) - (unless (and (string=? (required-index msh 0 'MSH) "MSH") - (= (string-length encoding-characters) 4) - (equal? (split-character - (required-index msh 8 'MSH-9) - component-separator) - '("ADT" "A08")) - (string=? (required-index msh 11 'MSH-12) "2.5.1") - pid-line) - (error "HL7v2 artifact is not an ADT^A08 2.5.1 patient message")) - (let* ((pid (split-character pid-line field-separator)) - (cx (split-character (required-index pid 3 'PID-3) - component-separator)) - (assigning-authority - (split-character (required-index cx 3 'PID-3.4) - subcomponent-separator)) - (name (split-character (required-index pid 5 'PID-5) - component-separator))) - (list - (cons 'schema "gerbil-parser.hl7v2-adt-a08-patient.v1") - (cons 'sourceDigest (parse-artifact-ref artifact 'sourceDigest)) - (cons 'grammarDigest (parse-artifact-ref artifact 'grammarDigest)) - (cons 'sourceInterface 'hl7v2) - (cons 'sourceMessageType "ADT^A08") - (cons 'sourceVersion "2.5.1") - (cons 'identifierSystem - (string-append "urn:oid:" - (required-index assigning-authority 1 'PID-3.4.2))) - (cons 'identifierValue (required-index cx 0 'PID-3.1)) - (cons 'familyName (required-index name 0 'PID-5.1)) - (cons 'givenNames (list (required-index name 1 'PID-5.2))))))) diff --git a/languages/hl7/v2-2.5.1/scanner.ss b/languages/hl7/v2-2.5.1/scanner.ss deleted file mode 100644 index 4976451b..00000000 --- a/languages/hl7/v2-2.5.1/scanner.ss +++ /dev/null @@ -1,88 +0,0 @@ -;;; -*- Gerbil -*- -;;; SPDX-FileCopyrightText: 2026 tao3k team and Contributors -;;; -;;; SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -(import (only-in :std/list/list delete-duplicates/hash)) - -(export scan-hl7v2-segment-id - scan-hl7v2-data - scan-hl7v2-segment-terminator - scan-hl7v2-field-separator - scan-hl7v2-component-separator - scan-hl7v2-repetition-separator - scan-hl7v2-escape-character - scan-hl7v2-subcomponent-separator) - -(def (scan-while source start predicate) - (let (length (string-length source)) - (let loop ((offset start)) - (if (and (< offset length) (predicate (string-ref source offset))) - (loop (+ offset 1)) - (and (> offset start) offset))))) - -(def (hl7v2-segment-id-character? character) - (or (and (char>=? character #\A) (char<=? character #\Z)) - (char-numeric? character))) - -(def (scan-hl7v2-segment-id source start) - (let (end (scan-while source start hl7v2-segment-id-character?)) - (and end (= (- end start) 3) end))) - -(def (hl7v2-delimiters source) - (and (>= (string-length source) 8) - (string=? (substring source 0 3) "MSH") - (let (delimiters - (list (string-ref source 3) (string-ref source 4) - (string-ref source 5) (string-ref source 6) - (string-ref source 7))) - (and (= (length (delete-duplicates/hash delimiters)) 5) - (not (ormap (lambda (character) - (or (char-alphabetic? character) - (char-numeric? character))) - delimiters)) - delimiters)))) - -(def (scan-hl7v2-delimiter source start index) - (let (delimiters (hl7v2-delimiters source)) - (and delimiters - (< start (string-length source)) - (char=? (string-ref source start) (list-ref delimiters index)) - (+ start 1)))) - -(def (scan-hl7v2-field-separator source start) - (scan-hl7v2-delimiter source start 0)) - -(def (scan-hl7v2-component-separator source start) - (scan-hl7v2-delimiter source start 1)) - -(def (scan-hl7v2-repetition-separator source start) - (scan-hl7v2-delimiter source start 2)) - -(def (scan-hl7v2-escape-character source start) - (scan-hl7v2-delimiter source start 3)) - -(def (scan-hl7v2-subcomponent-separator source start) - (scan-hl7v2-delimiter source start 4)) - -(def (scan-hl7v2-data source start) - (let (delimiters (hl7v2-delimiters source)) - (and delimiters - (scan-while - source start - (lambda (character) - (and (not (memv character delimiters)) - (not (char=? character #\return)) - (not (char=? character #\newline)))))))) - -(def (scan-hl7v2-segment-terminator source start) - (let (length (string-length source)) - (and (< start length) - (case (string-ref source start) - ((#\return) - (if (and (< (+ start 1) length) - (char=? (string-ref source (+ start 1)) #\newline)) - (+ start 2) - (+ start 1))) - ((#\newline) (+ start 1)) - (else #f))))) diff --git a/languages/tla-plus/v1/corpus/core/Counter.tla b/languages/tla-plus/corpus/core/Counter.tla similarity index 100% rename from languages/tla-plus/v1/corpus/core/Counter.tla rename to languages/tla-plus/corpus/core/Counter.tla diff --git a/languages/tla-plus/v1/corpus/core/MalformedIf.tla b/languages/tla-plus/corpus/core/MalformedIf.tla similarity index 100% rename from languages/tla-plus/v1/corpus/core/MalformedIf.tla rename to languages/tla-plus/corpus/core/MalformedIf.tla diff --git a/languages/tla-plus/v1/corpus/core/ModuleForms.tla b/languages/tla-plus/corpus/core/ModuleForms.tla similarity index 100% rename from languages/tla-plus/v1/corpus/core/ModuleForms.tla rename to languages/tla-plus/corpus/core/ModuleForms.tla diff --git a/languages/tla-plus/v1/corpus/core/NestedComment.tla b/languages/tla-plus/corpus/core/NestedComment.tla similarity index 100% rename from languages/tla-plus/v1/corpus/core/NestedComment.tla rename to languages/tla-plus/corpus/core/NestedComment.tla diff --git a/languages/tla-plus/v1/corpus/core/StructuredExpressions.tla b/languages/tla-plus/corpus/core/StructuredExpressions.tla similarity index 100% rename from languages/tla-plus/v1/corpus/core/StructuredExpressions.tla rename to languages/tla-plus/corpus/core/StructuredExpressions.tla diff --git a/languages/tla-plus/v1/corpus/tlaplus-examples/HourClock.tla b/languages/tla-plus/corpus/tlaplus-examples/HourClock.tla similarity index 100% rename from languages/tla-plus/v1/corpus/tlaplus-examples/HourClock.tla rename to languages/tla-plus/corpus/tlaplus-examples/HourClock.tla diff --git a/languages/tla-plus/v1/grammar-source/tla+.jj b/languages/tla-plus/grammar-source/tla+.jj similarity index 100% rename from languages/tla-plus/v1/grammar-source/tla+.jj rename to languages/tla-plus/grammar-source/tla+.jj diff --git a/languages/tla-plus/v1/grammar-source/tla+.jj.license b/languages/tla-plus/grammar-source/tla+.jj.license similarity index 100% rename from languages/tla-plus/v1/grammar-source/tla+.jj.license rename to languages/tla-plus/grammar-source/tla+.jj.license diff --git a/languages/tla-plus/grammar.ss b/languages/tla-plus/grammar.ss new file mode 100644 index 00000000..aff86445 --- /dev/null +++ b/languages/tla-plus/grammar.ss @@ -0,0 +1,5 @@ +;;; -*- Gerbil -*- +;;; Public syntax interface; private modules contain declarative recognition data. +(import ./syntax/core ./syntax/layout ./syntax/proof ./syntax/lexical) +(export tla-plus-core-syntax tla-plus-layout-syntax tla-plus-sany-candidate-syntax + tla-proof-name tla-proof-start tla-proof-reference tla-identifier) diff --git a/languages/tla-plus/parser-test.ss b/languages/tla-plus/parser-test.ss new file mode 100644 index 00000000..f1e98fe7 --- /dev/null +++ b/languages/tla-plus/parser-test.ss @@ -0,0 +1,220 @@ +;;; -*- Gerbil -*- +;;; Declarative layout fields, concurrency and engine receipts. +(import :gerbil-parser/language-test-support ./parser) +(import (only-in :gerbil-parser/src/language/descriptor language-grammar-language language-grammar-version language-grammar-contract) + (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/entry language-metadata-ref language-parser-entry-ref) + (only-in :gerbil-parser/language-support/fixture defsyntax-corpus) + (only-in :gerbil-parser/language-support/development deflanguage-development-loader LanguageDevelopmentLoader.)) +(export tla-plus-core-fixtures tla-plus-layout-fixtures tla-plus-sany-candidate-fixtures tla-plus-core-test-language tla-plus-layout-test-language tla-plus-sany-candidate-test-language) + +(defsyntax-corpus tla-plus-core-fixtures + (identity (language-grammar-language tla-plus-core-language-grammar) + (language-grammar-version tla-plus-core-language-grammar) + (language-grammar-contract tla-plus-core-language-grammar)) + (accepted + ("tla-plus/examples/hour-clock" tla-plus-hour-clock + "corpus/tlaplus-examples/HourClock.tla" SourceFile + (Module ExtendsDeclaration VariableDeclaration OperatorDefinition + TheoremDeclaration)) + ("tla-plus/core/counter" tla-plus-counter + "corpus/core/Counter.tla" SourceFile + (Module ExtendsDeclaration ConstantDeclaration VariableDeclaration + OperatorDefinition TheoremDeclaration)) + ("tla-plus/core/nested-comment" tla-plus-nested-comment + "corpus/core/NestedComment.tla" SourceFile + (Module VariableDeclaration OperatorDefinition)) + ("tla-plus/core/structured-expressions" tla-plus-structured-expressions + "corpus/core/StructuredExpressions.tla" SourceFile + (Module OperatorDefinition Expression IfExpression ChooseExpression + QuantifiedExpression TupleExpression SetExpression + FunctionConstructor OperatorApplication LetExpression + CaseExpression RecordExpression SetFilterExpression)) + ("tla-plus/core/module-forms" tla-plus-module-forms + "corpus/core/ModuleForms.tla" SourceFile + (Module RecursiveDeclaration InstanceDeclaration AssumptionDeclaration + TheoremDeclaration OperatorDefinition))) + (rejected + ("tla-plus/core/malformed-if" tla-plus-malformed-if + "corpus/core/MalformedIf.tla"))) + +(defsyntax-corpus tla-plus-layout-fixtures + (identity "tla-plus" "v2" "tla-plus.native-layout.v2") + (accepted + ("tla-plus/layout/aligned" tla-layout-aligned (text "---- MODULE J ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n") SourceFile (JunctionExpression))) + (rejected + ("tla-plus/layout/incomplete" tla-layout-incomplete (text "---- MODULE J ----\nInit ==\n /\\\n====\n")))) + +(defsyntax-corpus tla-plus-sany-candidate-fixtures + (identity "tla-plus" "p4-draft" "tla-plus.native-sany-candidate.p4") + (accepted + ("tla-plus/candidate/proof" tla-candidate-proof (text "---- MODULE P ----\nTHEOREM TRUE\nBY TRUE\n====\n") SourceFile (TerminalProof))) + (rejected + ("tla-plus/candidate/missing-qed" tla-candidate-missing-qed (text "---- MODULE P ----\nTHEOREM TRUE\n<1>1. TRUE OBVIOUS\n====\n")))) + +(deflanguage-development-loader tla-plus-core-test-language + (grammar tla-plus-core-language-grammar) + (parse parse-tla-plus-core-test) + (slots metadata: '((grammar-format . concise-dsl)) + fixtures: tla-plus-core-fixtures)) + +(deflanguage-development-loader tla-plus-layout-test-language + (grammar tla-plus-layout-language-grammar) + (parse parse-tla-plus-layout-test) + (slots metadata: '((grammar-format . concise-dsl)) + fixtures: tla-plus-layout-fixtures)) + +(deflanguage-development-loader tla-plus-sany-candidate-test-language + (grammar tla-plus-sany-candidate-language-grammar) + (parse parse-tla-plus-sany-candidate-test) + (slots metadata: '((grammar-format . concise-dsl)) + fixtures: tla-plus-sany-candidate-fixtures)) + +(deflanguage-parser-tests tla-plus-layout-parser-test "TLA+ column layout grammar" + (loader tla-plus-layout-test-language) + (accepted "aligned markers form one list" "---- MODULE J ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" + (field-counts JunctionExpression body (2))) + (accepted "nested quantifier closes at outer reference" "---- MODULE J ----\nInit ==\n /\\ \\E x \\in S :\n /\\ x = 1\n /\\ x = 2\n /\\ TRUE\n====\n" + (field-counts JunctionExpression body (2 2))) + (accepted "different columns remain distinct" "---- MODULE J ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" + (field-counts JunctionExpression body (1))) + (accepted "CRLF and tabs retain alignment" "---- MODULE J ----\r\nInit ==\r\n\t/\\ TRUE\r\n\t/\\ FALSE\r\n====\r\n" + (field-counts JunctionExpression body (2))) + (parallel "nine requests isolate layout frames and columns" 9 + (list "---- MODULE A ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" "---- MODULE B ----\nInit ==\n \\/ TRUE\n \\/ FALSE\n====\n" "---- MODULE C ----\nInit ==\n /\\ \\E x \\in S :\n /\\ x = 1\n /\\ x = 2\n /\\ TRUE\n====\n")) + (incremental "edits retain fresh layout semantics" "---- MODULE J ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" + (replace "FALSE" "TRUE") (freshFallback? #t)) + (rejected "dangling item remains rejected" "---- MODULE J ----\nInit ==\n /\\\n====\n") + (recovery-rejected "recovery retains layout rejection" "---- MODULE J ----\nInit ==\n /\\\n====\n" (outcome 'disabled))) + +(deflanguage-parser-tests tla-plus-core-parser-test "TLA+ core grammar" + (loader tla-plus-core-test-language) + (accepted "modular multiline action contracts keep native source bytes" + "---- MODULE Modular ----\nVARIABLES active,\n queued\nWork == INSTANCE WorkLifecycle\nInit ==\n /\\ Work!Init\n /\\ active = TRUE\nNext == Work!Step \\/\n (active' = FALSE /\\\n UNCHANGED <>)\n====\n" + (nodes InstanceExpression QualifiedNameExpression JunctionExpression)) + (accepted-many "set union and intersection keep official ASCII spellings" + (list "---- MODULE SetOperator ----\nLeft == {\"a\"} \\cup {\"b\"}\nRight == Left \\in {\"a\", \"b\"}\n====\n" "---- MODULE SetOperator ----\nLeft == {\"a\"} \\union {\"b\"}\nRight == Left \\in {\"a\", \"b\"}\n====\n" "---- MODULE SetOperator ----\nLeft == {\"a\"} \\cap {\"b\"}\nRight == Left \\in {\"a\", \"b\"}\n====\n" "---- MODULE SetOperator ----\nLeft == {\"a\"} \\intersect {\"b\"}\nRight == Left \\in {\"a\", \"b\"}\n====\n") (nodes Expression)) + (rejected "set operators require a right operand" + "---- MODULE BrokenSet ----\nLeft == {\"a\"} \\cup\n====\n") + (accepted "Temporal set maps, products, difference, and subset retain bytes" + "---- MODULE TemporalSyntax ----\nIds == {o[1] : o \\in Observations}\nRemaining == Ids \\ {id}\nPairs == Ids \\X Cuts\nWithin == Remaining \\subseteq Ids\n====\n" + (nodes SetMapExpression)) + (rejected-many "incomplete Temporal operators remain syntax errors" + (list "---- MODULE BrokenTemporal ----\nIds == {o[1] : o \\in}\n====\n" "---- MODULE BrokenTemporal ----\nPairs == Ids \\X\n====\n" "---- MODULE BrokenTemporal ----\nRemaining == Ids \\\n====\n" "---- MODULE BrokenTemporal ----\nWithin == Ids \\subseteq\n====\n")) + (rejected-many "incomplete modular forms remain syntax errors" + (list "---- MODULE Broken ----\nWork == INSTANCE\n====\n" "---- MODULE Broken ----\nValue == Work!\n====\n" "---- MODULE Broken ----\nVARIABLES active,\n\n====\n" "---- MODULE Broken ----\nInit ==\n /\\\n====\n" "---- MODULE Broken ----\nNext == TRUE /\\\n\n====\n")) + (property "native syntax and corpus identities are immutable" (bindings) + (equal (language-metadata-ref (language-parser-entry-ref tla-plus-core-language 'metadata) 'syntax-source) "Specifying Systems, Chapter 15: TLAPlusGrammar") + (equal (language-metadata-ref (language-parser-entry-ref tla-plus-core-language 'metadata) 'sany-release) "v1.7.4") + (equal (language-metadata-ref (language-parser-entry-ref tla-plus-core-language 'metadata) 'sany-commit) "5a47802b5c391f59ecdd44117981f4ff8c0656ba") + (equal (language-metadata-ref (language-parser-entry-ref tla-plus-core-language 'metadata) 'sany-grammar-blob) "bf9e7acb5337f4b6c2a4d6a973a1a65c95e72f56") + (equal (language-metadata-ref (language-parser-entry-ref tla-plus-core-language 'metadata) 'sany-grammar-digest) "sha256:15edd079cf16cf91556ba66b496c9ffc16f26ab30856753ed58e60b0d54f2d07") + (equal (language-metadata-ref (language-parser-entry-ref tla-plus-core-language 'metadata) 'examples-commit) "ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10")) + (fixture-catalog "core fixture counts and first source digest" (total 6) (accepted 5) (rejected 1) + (first-digest "sha256:985903176db4725f9cf25df84ad84dcd53ba94be80d26298b87ec86fbc9b08b3")) + (fixture-group "all admitted modules publish lossless structural CSTs" accepted (root SourceFile)) + (accepted-fixture "nested block comments remain one lossless trivia token" 2 (token-counts (comment 1))) + (accepted "unary negation is admitted by the native lexical contract" + "---- MODULE Negation ----\nVARIABLES enabled, ready\nDisabled == ~enabled /\\ ready\nNext == enabled' = FALSE /\\ ready' = TRUE\n====\n") + (model-receipt "one qualification API composes parser and TLC receipts" "Qualified" + "---- MODULE Qualified ----\nVARIABLE enabled\nInit == enabled = FALSE\nNext == enabled' = TRUE\n====\n" "INIT Init\nNEXT Next\n" + (stdout "TLC2 Version fixture\nProgress(3) at 00:00:00: 7 states generated, 7 distinct states found, 5 states left on queue.\nModel checking completed. No error has been found.\n4 states generated, 2 distinct states found, 0 states left on queue.\nThe depth of the complete state graph search is 1.\n" 0) 1 + (schema "gerbil-parser.tla-plus-model-qualification.v1") (admitted #t) (syntax-accepted #t) (roundtrip #t) + (states-generated 4) (distinct-states 2) (states-left 0) (graph-depth 1)) + (model-receipt "qualification fails closed on misleading TLC completion" "Rejected" + "---- MODULE Rejected ----\nVARIABLE enabled\nInit == enabled = FALSE\nNext == enabled' = TRUE\n====\n" "INIT Init\nNEXT Next\n" + (stdout "TLC2 Version fixture\nModel checking completed. No error has been found.\n4 states generated, 2 distinct states found, 0 states left on queue.\nThe depth of the complete state graph search is 1.\n" 7) 1 + (admitted #f) (nonzero exit-status)) + (model-receipt "progress-only output never establishes completed state counts" "Rejected" + "---- MODULE Rejected ----\nVARIABLE enabled\nInit == enabled = FALSE\nNext == enabled' = TRUE\n====\n" "INIT Init\nNEXT Next\n" + (stdout "TLC2 Version fixture\nProgress(3) at 00:00:00: 7 states generated, 7 distinct states found, 5 states left on queue.\nModel checking completed. No error has been found.\n" 0) 1 + (admitted #f) (states-generated #f)) + (model-receipt "syntax rejection stops before resolving TLC" "Malformed" + "---- MODULE Malformed ----\nVARIABLE x\nBroken == IF x = 0 THEN ELSE x\n====\n" "INIT Broken\n" (unresolved) 1 + (admitted #f) (syntax-accepted #f) (tool-path #f) (exit-status #f)) + (rejected "unterminated nested comments fail as one typed artifact" + "---- MODULE Broken ----\n(* outer (* nested *)\nVARIABLE x\n====\n" + (diagnostics 1)) + (fixture-group "recognized but malformed expressions fail closed" rejected (diagnostics 1))) + +(deflanguage-parser-tests tla-plus-sany-candidate-parser-test "TLA+ SANY candidate grammar" + (loader tla-plus-sany-candidate-test-language) + (distinct-contract "candidate identity is distinct from the published layout contract" tla-plus-layout-language) + (identity "candidate version remains pinned" (version "p4-draft")) + (syntax-kind "TerminalProof fields derive from its transparent fact-list rule" TerminalProof node (item)) + (accepted "TerminalProof metadata admits the fields emitted by its fact list" + "---- MODULE P ----\nTHEOREM TRUE\nBY TRUE\n====\n" + (fields TerminalProof item)) + (accepted "aligned markers make one list" + "---- MODULE J ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" + (field-counts JunctionExpression body (2))) + (accepted "an explicit action closer ends an aligned list on the same line" + "---- MODULE J ----\nCONSTANT S\nVARIABLE x\nQ == [][ /\\ (\\A i \\in S : i = i)\n /\\ (\\A j \\in S : j = j) ]_<>\n====\n" + (field-counts JunctionExpression body (2))) + (accepted "nested quantifier lists close at the outer reference" + "---- MODULE J ----\nInit ==\n /\\ \\E x \\in S :\n /\\ x = 1\n /\\ x = 2\n /\\ TRUE\n====\n" + (field-counts JunctionExpression body (2 2))) + (accepted "different marker columns do not join one list" + "---- MODULE J ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" + (field-counts JunctionExpression body (1))) + (accepted "CRLF and tab stops retain marker alignment" + "---- MODULE J ----\r\nInit ==\r\n\t/\\ TRUE\r\n\t/\\ FALSE\r\n====\r\n" + (field-counts JunctionExpression body (2))) + (accepted "variable-width borders and function-set syntax" + "-------- MODULE J --------\n----------------\nF == [S -> T]\n==========\n" + (counts (FunctionSetExpression 1))) + (accepted "operator constants, EXCEPT updates, and field selection" + "---- MODULE J ----\nCONSTANTS Send(_, _), T\nF == [pc EXCEPT ![self] = \"b1\"]\nG == participant[i].decision\n====\n" + (counts (ExceptExpression 1) (RecordFieldExpression 1))) + (accepted "instantiated module operator qualification" + "---- MODULE J ----\nSpec == Inner(q)!ISpec\n====\n" + (counts (InstanceQualifiedExpression 1))) + (accepted "infix operator definitions and qualified symbolic operators" + "---- MODULE N ----\na \\div b == R!\\div(a, b)\na % b == a - b * (a \\div b)\na < b == (a \\leq b) /\\ (a # b)\n====\n" + (counts (OperatorDefinition 3))) + (accepted "record sets keep colon fields distinct from record values" + "---- MODULE R ----\nType == [val : Data, rdy : {0, 1}]\nValue == [val |-> x, rdy |-> TRUE]\n====\n" + (counts (RecordSetExpression 1) (RecordExpression 1))) + (accepted "temporal leads-to is one lossless infix operator" + "---- MODULE Leads ----\nLeads == A ~> B\nGrouped == (A /\\ B) ~> C\n====\n" + (counts (Expression 3))) + (rejected "action subscripts reject a missing closing delimiter" + "---- MODULE Bad ----\nP == [A]x\n====\n") + (accepted "action subscripts require SANY's closing delimiter" + "---- MODULE Tuple ----\nP == [A]_<>\nQ == [][A]_(<>)\nR == <><>_x\n====\n" + (counts (TemporalSubscriptExpression 2) (AngleActionExpression 1))) + (accepted "function definitions retain domain bindings in LET" + "---- MODULE Functions ----\nF[x \\in S, y \\in T] == <>\nL == LET G[x \\in S] == x H[y \\in T] == y IN <>\n====\n" + (counts (FunctionDefinition 1) (LocalFunctionDefinition 2) (DomainBinding 4))) + (parallel "concurrent requests keep layout columns and frames isolated" 9 (list "---- MODULE A ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" "---- MODULE B ----\nInit ==\n \\/ TRUE\n \\/ FALSE\n====\n" "---- MODULE C ----\nInit ==\n /\\ \\E x \\in S :\n /\\ x = 1\n /\\ x = 2\n /\\ TRUE\n====\n")) + (incremental "edits reparse with the same layout semantics" "---- MODULE J ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" (replace "FALSE" "TRUE") (freshFallback? #t)) + (rejected "dangling list item remains rejected" + "---- MODULE J ----\nInit ==\n /\\\n====\n") + (recovery-rejected "recovery keeps layout rejection unchanged" "---- MODULE J ----\nInit ==\n /\\\n====\n" (outcome 'disabled))) + +(deflanguage-parser-tests sany-closure-test "SANY corpus syntax families" + (loader tla-plus-sany-candidate-test-language) + (accepted "document text and nested and concatenated modules retain nodes" + "notes\n---- MODULE 2Outer ----\n---- MODULE Inner ----\nP == TRUE\n====\nQ == \"==== ---- MODULE Fake ----\"\n====\ntrailing notes\n---- MODULE Peer ----\nR == FALSE\n====\nend notes\n" + (diagnostics 0) (counts (Module 3))) + (accepted "higher order formals lambdas labels and operator declarations" + "---- MODULE Higher ----\nApply(Op(_,_), x, y) == Op(x,y)\nf ** g == f\n-. x == x\nx ^+ == x\nP == Apply(**, a, b)\nQ == Apply(LAMBDA x,y: x, a, b)\nR == tag(x):: x = x\nS == LET IsGroup(G, _+_) == G IN IsGroup(S,+)\n====\n" + (diagnostics 0) (counts (LambdaExpression 1) (LabelExpression 1) (FormalParameter 9))) + (accepted "CASE application and selection stay in the final arm" + "---- MODULE Cases ----\nP == CASE A -> B [] OTHER -> F[x]\nQ == CASE A -> B [] OTHER -> F.x\nR == CASE A -> B [] OTHER -> x+y\n====\n" + (diagnostics 0) (counts (CaseExpression 3) (FunctionApplication 1) (RecordFieldExpression 1))) + (accepted "outdent restores a precedence suppressed reduction" + "---- MODULE Outdent ----\nP ==\n /\\ \\A p \\in S :\n /\\ IF s # top THEN s.ctxt = p ELSE FALSE\n => pc[p] = s\n /\\ TRUE\n====\n" + (diagnostics 0)) + (accepted "nested IF lists can return through multiple enclosing references" + "---- MODULE NestedIF ----\ne2(self) == /\\ pc[self] = s\n /\\ IF unchecked[self] # {}\n THEN /\\ \\E i \\in unchecked[self]:\n /\\ unchecked = unchecked\n /\\ IF num[i] > max[self]\n THEN /\\ max = max\n ELSE /\\ TRUE\n /\\ max = max\n /\\ pc = pc\n ELSE /\\ pc = pc\n /\\ UNCHANGED <>\n /\\ UNCHANGED <>\n====\n" + (diagnostics 0)) + (accepted "proof levels qualifiers assumptions and commands retain structure" + "---- MODULE Proofs ----\nLEMMA P == ASSUME NEW x PROVE TRUE\n<1>1. SUFFICES ASSUME TRUE PROVE TRUE\n OBVIOUS\n<1>2. TRUE\n <2>1. CASE TRUE\n BY P!(x)\n <2>. QED OBVIOUS\n<1>. QED BY <1>1, <1>2\n====\n" + (diagnostics 0) (counts (ProofStep 5) (AssumeProve 2) (SelectorArguments 1))) + (native-entry "public loader descriptor and native entry: accepted" "---- MODULE Entry ----\nTHEOREM TRUE\nOBVIOUS\n====\n" accepted) + (native-entry "public loader descriptor and native entry: rejected" "---- MODULE Entry ----\nTHEOREM TRUE\n<1>1. TRUE OBVIOUS\n====\n" rejected) + (portable-rejected "build ABI rejects public candidate policy rather than emitting recognition only" "languages/tla-plus/parser.ss#tla-plus-sany-candidate-language-grammar" + "language parser policy is unsupported by standalone Rust AOT") + (rejected-many "missing QED inconsistent levels and proof of HAVE reject" + (list "---- MODULE Bad ----\nTHEOREM TRUE\n<1>1. TRUE OBVIOUS\n====\n" "---- MODULE Bad ----\nTHEOREM TRUE\n<1>1. TRUE\n<2>1. TRUE OBVIOUS\n<3>2. TRUE OBVIOUS\n<2>. QED OBVIOUS\n<1>. QED OBVIOUS\n====\n" "---- MODULE Bad ----\nTHEOREM TRUE\n<1>1. HAVE TRUE\n<2>. QED OBVIOUS\n<1>. QED OBVIOUS\n====\n"))) diff --git a/languages/tla-plus/parser.ss b/languages/tla-plus/parser.ss new file mode 100644 index 00000000..cf879723 --- /dev/null +++ b/languages/tla-plus/parser.ss @@ -0,0 +1,61 @@ +;;; -*- Gerbil -*- +;;; Parser names describe syntax scope; parser metadata owns version identity. + +(import (only-in :clan/poo/object .o) + (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-entry-ref) + (only-in :gerbil-parser/language-support/grammar-source defsyntax-javacc-source) + (only-in :gerbil-parser/src/language/descriptor language-grammar-with-parser-policy) + :gerbil-parser/language-support/structured + ./grammar + :gerbil-parser/src/language/tlc) +(export tla-plus-layout-language-grammar tla-plus-core-language-grammar (import: ./grammar) + tla-plus-sany-candidate-language tla-plus-sany-candidate-language-grammar parse-tla-plus-sany-candidate + +tla-plus-model-qualification-schema+ tla-plus-model-receipt? + tla-plus-model-receipt-admitted tla-plus-model-receipt-output tla-plus-model-receipt->alist + qualify-tla-plus-model qualify-tla-plus-core-model + tla-plus-core-language tla-plus-layout-language + parse-tla-plus-core parse-tla-plus-layout parse-tla-plus) +;;; Parser metadata owns release and pinned-source identity. +(defsyntax-javacc-source tla-plus-sany-source + (identity "tla-plus" "v1.7.4" "5a47802b5c391f59ecdd44117981f4ff8c0656ba") + (digest "sha256:15edd079cf16cf91556ba66b496c9ffc16f26ab30856753ed58e60b0d54f2d07") + (source "grammar-source/tla+.jj")) +(def tla-plus-source-metadata + (.o language: "tla-plus" grammar-format: 'concise-dsl + syntax-source: "Specifying Systems, Chapter 15: TLAPlusGrammar" + examples-commit: "ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10" + sany-release: "v1.7.4" + sany-commit: "5a47802b5c391f59ecdd44117981f4ff8c0656ba" + sany-grammar-blob: "bf9e7acb5337f4b6c2a4d6a973a1a65c95e72f56" + sany-grammar-digest: "sha256:15edd079cf16cf91556ba66b496c9ffc16f26ab30856753ed58e60b0d54f2d07" + source-catalog: tla-plus-sany-source)) + +(deflanguage-parser-loader tla-plus-core-language + (grammar tla-plus-core-syntax) + (parse parse-tla-plus-core) + (slots metadata: (.o (:: @ tla-plus-source-metadata) version: "v1" contract: "tla-plus.native-core.v1"))) + +(def tla-plus-core-language-grammar (language-parser-entry-ref tla-plus-core-language 'descriptor)) + +(deflanguage-parser-loader tla-plus-layout-language + (grammar tla-plus-layout-syntax) + (parse parse-tla-plus-layout) + (slots metadata: (.o (:: @ tla-plus-source-metadata) version: "v2" contract: "tla-plus.native-layout.v2"))) + +(def tla-plus-layout-language-grammar (language-parser-entry-ref tla-plus-layout-language 'descriptor)) + +(def parse-tla-plus parse-tla-plus-layout) + +(deflanguage-model-entry qualify-tla-plus-model (grammar tla-plus-layout-language-grammar)) +(deflanguage-model-entry qualify-tla-plus-core-model (grammar tla-plus-core-language-grammar)) + +(def proof-syntax + (language-grammar-with-parser-policy + tla-plus-sany-candidate-syntax "tla-plus.sany-proof-policy.v1" 4096 (bind-structured-proof-policy (.o (:: self StructuredProofPolicy.) code: "GERBIL-PARSER-TLA-PLUS-SANY-PROOF")))) + +(deflanguage-parser-loader tla-plus-sany-candidate-language + (grammar proof-syntax) + (parse parse-tla-plus-sany-candidate) + (slots metadata: (.o (:: @ tla-plus-source-metadata) version: "p4-draft" contract: "tla-plus.native-sany-candidate.p4"))) + +(def tla-plus-sany-candidate-language-grammar (language-parser-entry-ref tla-plus-sany-candidate-language 'descriptor)) diff --git a/languages/tla-plus/syntax/core.ss b/languages/tla-plus/syntax/core.ss new file mode 100644 index 00000000..860dc70f --- /dev/null +++ b/languages/tla-plus/syntax/core.ss @@ -0,0 +1,183 @@ +;;; -*- Gerbil -*- +;;; TLA+ recognition declarations; kinds, fields and references are DSL-derived. +(import (only-in :gerbil-parser/language-support/grammar deflanguage) ./lexical) +(export tla-plus-core-syntax) +(deflanguage tla-plus-core + (syntax + (lexical + (root source-file) + (lex (horizontal-whitespace HorizontalWhitespace (horizontal-whitespace+)) + (newline Newline (newline+)) + (comment Comment (choice (line-comment "\\*") (nested-block-comment "(*" "*)"))) + (string String (quoted-string "\"")) + (module-end ModuleEnd + (literals "==============================================================" "====")) + (separator-line SeparatorLine + (literals "--------------------------------------------------------------")) + (module-border ModuleBorder (literals "----------------------" "----")) (number Number (number)) + (identifier Identifier (identifier)) + (punctuation Punctuation + (literals "<=>" "|->" "[]" "<>" "=>" "==" "/\\" "\\/" "\\AA" "\\EE" "\\A" "\\E" "\\notin" + "\\intersect" "\\union" "\\cap" "\\cup" "\\subseteq" "\\X" "\\in" "\\" ".." "<<" ">>" "<=" + ">=" "->" "<-" "#" "=" "<" ">" "+" "-" "*" "/" "^" "'" "~" "(" ")" "[" "]" "{" "}" "," ":" + "!" "_" "."))) + (extras horizontal-whitespace newline comment) + (keywords (module "MODULE") (extends "EXTENDS") (constant "CONSTANT") (constants "CONSTANTS") + (variable "VARIABLE") (variables "VARIABLES") (theorem "THEOREM") (if "IF") (then "THEN") + (else "ELSE") (choose "CHOOSE") (let "LET") (in "IN") (case "CASE") (other "OTHER") + (enabled "ENABLED") (unchanged "UNCHANGED") (subset "SUBSET") (union "UNION") (domain "DOMAIN") + (local "LOCAL") (recursive "RECURSIVE") (instance "INSTANCE") (with "WITH") (assume "ASSUME") + (assumption "ASSUMPTION") (axiom "AXIOM") (proposition "PROPOSITION") (use "USE") (hide "HIDE")) + (recoveries (module "GERBIL-PARSER-TLA-PLUS-V1" preserve-source)) + (conflicts selective-glr) + (case-insensitive #f))) + (rules (source-file (node SourceFile (field module module))) + (module + (node Module module-border "MODULE" (field name identifier) module-border + (repeat (field item module-item)) module-end)) + (module-item + (choice extends-declaration constant-declaration variable-declaration operator-definition + recursive-declaration instance-declaration assumption-declaration theorem-declaration + use-hide-declaration separator)) + (extends-declaration + (node ExtendsDeclaration "EXTENDS" (field module identifier) + (repeat (seq "," (field module identifier))))) + (constant-declaration + (node ConstantDeclaration (choice "CONSTANT" "CONSTANTS") (field name identifier) + (repeat (seq "," (field name identifier))))) + (variable-declaration + (node VariableDeclaration (choice "VARIABLE" "VARIABLES") (field name identifier) + (repeat (seq "," (field name identifier))))) + (operator-definition + (node OperatorDefinition (optional "LOCAL") (field name identifier) + (optional + (seq "(" (field parameter identifier) (repeat (seq "," (field parameter identifier))) ")")) + "==" (field body (choice instance-expression junction-expression expression)))) + (recursive-declaration + (node RecursiveDeclaration "RECURSIVE" (field name identifier) + (optional + (seq "(" (field parameter identifier) (repeat (seq "," (field parameter identifier))) ")")) + (repeat (seq "," (field name identifier))))) + (instance-declaration + (node InstanceDeclaration (optional "LOCAL") "INSTANCE" (field module identifier) + (optional + (seq "WITH" (field substitution substitution) + (repeat (seq "," (field substitution substitution))))))) + (substitution (node Substitution (field name identifier) "<-" (field value expression))) + (instance-expression + (node InstanceExpression "INSTANCE" (field module identifier) + (optional + (seq "WITH" (field substitution substitution) + (repeat (seq "," (field substitution substitution))))))) + (junction-expression + (node JunctionExpression (field operator (choice "/\\" "\\/")) (field body expression))) + (assumption-declaration + (node AssumptionDeclaration (choice "ASSUME" "ASSUMPTION" "AXIOM") + (optional (seq (field name identifier) "==")) (field body expression))) + (theorem-declaration + (node TheoremDeclaration (choice "THEOREM" "PROPOSITION") + (optional (seq (field name identifier) "==")) (field body expression))) + (use-hide-declaration + (node UseHideDeclaration (choice "USE" "HIDE") (field item expression) + (repeat (seq "," (field item expression))))) (separator (node Separator separator-line)) + (expression + (choice + (prec right 10 + (node Expression (field left expression) (field operator (choice "<=>" "=>")) + (field right expression))) + (prec left 20 + (node Expression (field left expression) (field operator "\\/") (field right expression))) + (prec left 30 + (node Expression (field left expression) (field operator "/\\") (field right expression))) + (prec none 40 + (node Expression (field left expression) + (field operator (choice "=" "#" "<" ">" "<=" ">=" "\\in" "\\notin" "\\subseteq")) + (field right expression))) + (prec left 45 + (node Expression (field left expression) + (field operator (choice "\\cup" "\\union" "\\cap" "\\intersect" "\\")) + (field right expression))) + (prec left 50 + (node Expression (field left expression) (field operator "..") (field right expression))) + (prec left 60 + (node Expression (field left expression) (field operator (choice "+" "-")) + (field right expression))) + (prec left 70 + (node Expression (field left expression) (field operator (choice "*" "/" "\\X")) + (field right expression))) + (prec right 80 + (node Expression (field left expression) (field operator "^") (field right expression))) + (prec right 90 prefix-expression) (prec left 100 postfix-expression) + (prec left 110 function-application) (prec left 110 operator-application) if-expression + (prec right 1 choose-expression) (prec right 1 quantified-expression) + (prec right 1 let-expression) case-expression function-constructor record-expression + set-filter-expression set-map-expression temporal-subscript-expression grouped-expression + tuple-expression set-expression name-expression qualified-name-expression number-expression + string-expression)) + (if-expression + (node IfExpression "IF" (field condition expression) "THEN" (field consequent expression) + "ELSE" (field alternative expression))) + (choose-expression + (prec right 1 + (node ChooseExpression "CHOOSE" (field name identifier) + (optional (seq "\\in" (field domain expression))) ":" (field predicate expression)))) + (quantified-expression + (prec right 1 + (node QuantifiedExpression (field quantifier (choice "\\A" "\\E" "\\AA" "\\EE")) + (field name identifier) (optional (seq "\\in" (field domain expression))) ":" + (field predicate expression)))) + (let-expression + (prec right 1 + (node LetExpression "LET" (field definition local-definition) "IN" (field body expression)))) + (local-definition + (node LocalDefinition (field name identifier) + (optional + (seq "(" (field parameter identifier) (repeat (seq "," (field parameter identifier))) ")")) + "==" (field body expression))) + (case-expression + (node CaseExpression "CASE" (field arm case-arm) (repeat (seq "[]" (field arm case-arm))) + (optional (seq "[]" "OTHER" "->" (field other expression))))) + (case-arm (node CaseArm (field condition expression) "->" (field result expression))) + (prefix-expression + (prec right 90 + (node PrefixExpression + (field operator (choice "~" "-" "[]" "<>" "ENABLED" "UNCHANGED" "SUBSET" "UNION" "DOMAIN")) + (field operand expression)))) + (postfix-expression + (prec left 100 (node PostfixExpression (field operand expression) (field operator "'")))) + (function-application + (prec left 110 + (node FunctionApplication (field function expression) "[" (field argument expression) + (repeat (seq "," (field argument expression))) "]"))) + (operator-application + (prec left 110 + (node OperatorApplication (field operator expression) "(" + (optional (field argument expression)) (repeat (seq "," (field argument expression))) ")"))) + (function-constructor + (node FunctionConstructor "[" (field name identifier) "\\in" (field domain expression) "|->" + (field body expression) "]")) + (record-expression + (node RecordExpression "[" (field name identifier) "|->" (field value expression) + (repeat (seq "," (field name identifier) "|->" (field value expression))) "]")) + (set-filter-expression + (node SetFilterExpression "{" (field name identifier) "\\in" (field domain expression) ":" + (field predicate expression) "}")) + (set-map-expression + (node SetMapExpression "{" (field body expression) ":" (field name identifier) "\\in" + (field domain expression) "}")) + (temporal-subscript-expression + (node TemporalSubscriptExpression "[" (field action expression) "]" + (field subscript identifier))) + (grouped-expression (node GroupedExpression "(" (field expression expression) ")")) + (tuple-expression + (node TupleExpression "<<" (optional (field item expression)) + (repeat (seq "," (field item expression))) ">>")) + (set-expression + (node SetExpression "{" (optional (field item expression)) + (repeat (seq "," (field item expression))) "}")) + (name-expression (node NameExpression (field name identifier))) + (qualified-name-expression + (node QualifiedNameExpression (field module identifier) + (repeat1 (seq "!" (field name identifier))))) + (number-expression (node NumberExpression (field value number))) + (string-expression (node StringExpression (field value string))))) diff --git a/languages/tla-plus/syntax/layout.ss b/languages/tla-plus/syntax/layout.ss new file mode 100644 index 00000000..f307a226 --- /dev/null +++ b/languages/tla-plus/syntax/layout.ss @@ -0,0 +1,190 @@ +;;; -*- Gerbil -*- +;;; TLA+ recognition declarations; kinds, fields and references are DSL-derived. +(import (only-in :gerbil-parser/language-support/grammar deflanguage) ./lexical) +(export tla-plus-layout-syntax) +(deflanguage tla-plus-layout + (syntax + (lexical + (root source-file) + (lex (horizontal-whitespace HorizontalWhitespace (horizontal-whitespace+)) + (newline Newline (newline+)) + (comment Comment (choice (line-comment "\\*") (nested-block-comment "(*" "*)"))) + (string String (quoted-string "\"")) + (module-end ModuleEnd + (literals "==============================================================" "====")) + (separator-line SeparatorLine + (literals "--------------------------------------------------------------")) + (module-border ModuleBorder (literals "----------------------" "----")) (number Number (number)) + (identifier Identifier (identifier)) + (punctuation Punctuation + (literals "<=>" "|->" "[]" "<>" "=>" "==" "/\\" "\\/" "\\AA" "\\EE" "\\A" "\\E" "\\notin" + "\\intersect" "\\union" "\\cap" "\\cup" "\\subseteq" "\\X" "\\in" "\\" ".." "<<" ">>" "<=" + ">=" "->" "<-" "#" "=" "<" ">" "+" "-" "*" "/" "^" "'" "~" "(" ")" "[" "]" "{" "}" "," ":" + "!" "_" "."))) + (extras horizontal-whitespace newline comment) + (keywords (module "MODULE") (extends "EXTENDS") (constant "CONSTANT") (constants "CONSTANTS") + (variable "VARIABLE") (variables "VARIABLES") (theorem "THEOREM") (if "IF") (then "THEN") + (else "ELSE") (choose "CHOOSE") (let "LET") (in "IN") (case "CASE") (other "OTHER") + (enabled "ENABLED") (unchanged "UNCHANGED") (subset "SUBSET") (union "UNION") (domain "DOMAIN") + (local "LOCAL") (recursive "RECURSIVE") (instance "INSTANCE") (with "WITH") (assume "ASSUME") + (assumption "ASSUMPTION") (axiom "AXIOM") (proposition "PROPOSITION") (use "USE") (hide "HIDE")) + (recoveries (module "GERBIL-PARSER-TLA-PLUS-V1" preserve-source)) + (conflicts selective-glr) + (case-insensitive #f))) + (rules (source-file (node SourceFile (field module module))) + (module + (node Module module-border "MODULE" (field name identifier) module-border + (repeat (field item module-item)) module-end)) + (module-item + (choice extends-declaration constant-declaration variable-declaration operator-definition + recursive-declaration instance-declaration assumption-declaration theorem-declaration + use-hide-declaration separator)) + (extends-declaration + (node ExtendsDeclaration "EXTENDS" (field module identifier) + (repeat (seq "," (field module identifier))))) + (constant-declaration + (node ConstantDeclaration (choice "CONSTANT" "CONSTANTS") (field name identifier) + (repeat (seq "," (field name identifier))))) + (variable-declaration + (node VariableDeclaration (choice "VARIABLE" "VARIABLES") (field name identifier) + (repeat (seq "," (field name identifier))))) + (operator-definition + (node OperatorDefinition (optional "LOCAL") (field name identifier) + (optional + (seq "(" (field parameter identifier) (repeat (seq "," (field parameter identifier))) ")")) + "==" (field body (choice instance-expression expression)))) + (recursive-declaration + (node RecursiveDeclaration "RECURSIVE" (field name identifier) + (optional + (seq "(" (field parameter identifier) (repeat (seq "," (field parameter identifier))) ")")) + (repeat (seq "," (field name identifier))))) + (instance-declaration + (node InstanceDeclaration (optional "LOCAL") "INSTANCE" (field module identifier) + (optional + (seq "WITH" (field substitution substitution) + (repeat (seq "," (field substitution substitution))))))) + (substitution (node Substitution (field name identifier) "<-" (field value expression))) + (instance-expression + (node InstanceExpression "INSTANCE" (field module identifier) + (optional + (seq "WITH" (field substitution substitution) + (repeat (seq "," (field substitution substitution))))))) + (junction-expression + (choice + (node JunctionExpression (field operator (layout-start "/\\")) (field body expression) + (repeat (seq (field operator (layout-next "/\\")) (field body expression))) (layout-end)) + (node JunctionExpression (field operator (layout-start "\\/")) (field body expression) + (repeat (seq (field operator (layout-next "\\/")) (field body expression))) (layout-end)))) + (assumption-declaration + (node AssumptionDeclaration (choice "ASSUME" "ASSUMPTION" "AXIOM") + (optional (seq (field name identifier) "==")) (field body expression))) + (theorem-declaration + (node TheoremDeclaration (choice "THEOREM" "PROPOSITION") + (optional (seq (field name identifier) "==")) (field body expression))) + (use-hide-declaration + (node UseHideDeclaration (choice "USE" "HIDE") (field item expression) + (repeat (seq "," (field item expression))))) (separator (node Separator separator-line)) + (expression + (choice + (prec right 10 + (node Expression (field left expression) (field operator (choice "<=>" "=>")) + (field right expression))) + (prec left 20 + (node Expression (field left expression) (field operator "\\/") (field right expression))) + (prec left 30 + (node Expression (field left expression) (field operator "/\\") (field right expression))) + (prec none 40 + (node Expression (field left expression) + (field operator (choice "=" "#" "<" ">" "<=" ">=" "\\in" "\\notin" "\\subseteq")) + (field right expression))) + (prec left 45 + (node Expression (field left expression) + (field operator (choice "\\cup" "\\union" "\\cap" "\\intersect" "\\")) + (field right expression))) + (prec left 50 + (node Expression (field left expression) (field operator "..") (field right expression))) + (prec left 60 + (node Expression (field left expression) (field operator (choice "+" "-")) + (field right expression))) + (prec left 70 + (node Expression (field left expression) (field operator (choice "*" "/" "\\X")) + (field right expression))) + (prec right 80 + (node Expression (field left expression) (field operator "^") (field right expression))) + (prec right 90 prefix-expression) (prec left 100 postfix-expression) + (prec left 110 function-application) (prec left 110 operator-application) if-expression + (prec right 1 choose-expression) (prec right 1 quantified-expression) + (prec right 1 let-expression) case-expression function-constructor record-expression + set-filter-expression set-map-expression temporal-subscript-expression grouped-expression + tuple-expression set-expression name-expression junction-expression + qualified-name-expression number-expression string-expression)) + (if-expression + (prec right 1 + (node IfExpression "IF" (field condition expression) "THEN" (field consequent expression) + "ELSE" (field alternative expression)))) + (choose-expression + (prec right 1 + (node ChooseExpression "CHOOSE" (field name identifier) + (optional (seq "\\in" (field domain expression))) ":" (field predicate expression)))) + (quantified-expression + (prec right 1 + (node QuantifiedExpression (field quantifier (choice "\\A" "\\E" "\\AA" "\\EE")) + (field name identifier) (optional (seq "\\in" (field domain expression))) + (repeat + (seq "," (field name identifier) (optional (seq "\\in" (field domain expression))))) + ":" (field predicate expression)))) + (let-expression + (prec right 1 + (node LetExpression "LET" (field definition local-definition) "IN" (field body expression)))) + (local-definition + (node LocalDefinition (field name identifier) + (optional + (seq "(" (field parameter identifier) (repeat (seq "," (field parameter identifier))) ")")) + "==" (field body expression))) + (case-expression + (node CaseExpression "CASE" (field arm case-arm) (repeat (seq "[]" (field arm case-arm))) + (optional (seq "[]" "OTHER" "->" (field other expression))))) + (case-arm (node CaseArm (field condition expression) "->" (field result expression))) + (prefix-expression + (prec right 90 + (node PrefixExpression + (field operator (choice "~" "-" "[]" "<>" "ENABLED" "UNCHANGED" "SUBSET" "UNION" "DOMAIN")) + (field operand expression)))) + (postfix-expression + (prec left 100 (node PostfixExpression (field operand expression) (field operator "'")))) + (function-application + (prec left 110 + (node FunctionApplication (field function expression) "[" (field argument expression) + (repeat (seq "," (field argument expression))) "]"))) + (operator-application + (prec left 110 + (node OperatorApplication (field operator (choice identifier qualified-name-expression)) "(" + (optional (field argument expression)) (repeat (seq "," (field argument expression))) ")"))) + (function-constructor + (node FunctionConstructor "[" (field name identifier) "\\in" (field domain expression) "|->" + (field body expression) "]")) + (record-expression + (node RecordExpression "[" (field name identifier) "|->" (field value expression) + (repeat (seq "," (field name identifier) "|->" (field value expression))) "]")) + (set-filter-expression + (node SetFilterExpression "{" (field name identifier) "\\in" (field domain expression) ":" + (field predicate expression) "}")) + (set-map-expression + (node SetMapExpression "{" (field body expression) ":" (field name identifier) "\\in" + (field domain expression) "}")) + (temporal-subscript-expression + (node TemporalSubscriptExpression "[" (field action expression) "]" + (field subscript identifier))) + (grouped-expression (node GroupedExpression "(" (field expression expression) ")")) + (tuple-expression + (node TupleExpression "<<" (optional (field item expression)) + (repeat (seq "," (field item expression))) ">>")) + (set-expression + (node SetExpression "{" (optional (field item expression)) + (repeat (seq "," (field item expression))) "}")) + (name-expression (node NameExpression (field name identifier))) + (qualified-name-expression + (node QualifiedNameExpression (field module identifier) + (repeat1 (seq "!" (field name identifier))))) + (number-expression (node NumberExpression (field value number))) + (string-expression (node StringExpression (field value string))))) diff --git a/languages/tla-plus/syntax/lexical.ss b/languages/tla-plus/syntax/lexical.ss new file mode 100644 index 00000000..16d5ecb8 --- /dev/null +++ b/languages/tla-plus/syntax/lexical.ss @@ -0,0 +1,17 @@ +;;; -*- Gerbil -*- +;;; Shared TLA+ lexical expression data; no version or source metadata. +(import (only-in :gerbil-parser/language-support/grammar deftext-profile)) +(export tla-proof-name tla-proof-start tla-proof-reference tla-identifier) +(deftext-profile tla-proof-name + (seq (literal "<") (if-next (characters "+*") (run (characters "+*") 1 1) (run (numeric) 1 #f)) + (literal ">") + (if-next (union (alphabetic) (numeric) (characters "_")) + (run (union (alphabetic) (numeric) (characters "_")) 1 #f) + (optional (run (characters "*-") 1 1))) (run (characters ".") 0 #f))) +(deftext-profile tla-proof-start + (ends-not-in (union (alphabetic) (numeric) (characters "_")) (ref tla-proof-name))) +(deftext-profile tla-proof-reference + (ends-in (union (alphabetic) (numeric) (characters "_")) (ref tla-proof-name))) +(deftext-profile tla-identifier + (run-containing (union (alphabetic) (numeric) (characters "_")) + (union (alphabetic) (characters "_")) 1 #f)) diff --git a/languages/tla-plus/syntax/proof.ss b/languages/tla-plus/syntax/proof.ss new file mode 100644 index 00000000..494bf51e --- /dev/null +++ b/languages/tla-plus/syntax/proof.ss @@ -0,0 +1,410 @@ +;;; -*- Gerbil -*- +;;; TLA+ recognition declarations; kinds, fields and references are DSL-derived. +(import (only-in :gerbil-parser/language-support/grammar deflanguage) ./lexical) +(export tla-plus-sany-candidate-syntax) +(deflanguage tla-plus-sany-candidate + (syntax + (lexical + (root source-file) + (lex (module-text ModuleText (module-text "----" "MODULE" "====" "(*" "*)" "\\*" "_")) + (proof-step-name ProofStepName (text-profile (ref tla-proof-start))) + (proof-reference ProofReference (text-profile (ref tla-proof-reference))) + (horizontal-whitespace HorizontalWhitespace (horizontal-whitespace+)) + (newline Newline (newline+)) + (comment Comment (choice (line-comment "\\*") (nested-block-comment "(*" "*)"))) + (string String (quoted-string "\"")) (module-end ModuleEnd (character-run "=" 4)) + (module-border ModuleBorder (character-run "-" 4)) (number Number (number)) + (identifier Identifier (text-profile (ref tla-identifier))) + (punctuation Punctuation + (literals "\\land" "\\lor" "\\cdot" "\\equiv" "-+->" "\\times" "..." "|" "||" "&&" "&" "$$" + "$" "??" "%%" "##" "++" "--" "**" "//" "^^" "!!" "|-" "|=" "-|" "=|" "<:" ":=" "::=" + "\\oplus" "\\ominus" "\\odot" "\\oslash" "\\otimes" "\\uplus" "\\sqcap" "\\sqcup" "\\wr" + "\\star" "\\bigcirc" "\\bullet" "\\prec" "\\succ" "\\preceq" "\\succeq" "\\sim" "\\simeq" + "\\ll" "\\gg" "\\asymp" "\\subset" "\\supset" "\\supseteq" "\\approx" "\\cong" "\\sqsubset" + "\\sqsubseteq" "\\sqsupset" "\\sqsupseteq" "\\doteq" "\\propto" "\\mod" "(+)" "(-)" "(.)" + "(/)" "(\\X)" "-." "^+" "^*" "^#" "\\lnot" "\\neg" "<=>" "|->" "[]" "<>" "=>" "~>" "==" + "/\\" "\\/" "\\AA" "\\EE" "\\A" "\\E" "\\notin" "\\intersect" "\\union" "\\cap" "\\cup" + "\\subseteq" "\\X" "\\in" "\\" "\\div" "\\leq" "\\geq" "\\o" "\\circ" "@@" ":>" ".." "<<" + ">>" "]_" ">>_" "<=" ">=" "=<" "/=" "->" "<-" "#" "=" "<" ">" "+" "-" "*" "/" "%" "^" "'" + "~" "@" "[]ASSUME" "[]PROVE" "::" "(" ")" "[" "]" "{" "}" "," ":" "!" "_" "."))) + (extras horizontal-whitespace newline comment) + (keywords (module "MODULE") (extends "EXTENDS") (constant "CONSTANT") (constants "CONSTANTS") + (variable "VARIABLE") (variables "VARIABLES") (theorem "THEOREM") (if "IF") (then "THEN") + (else "ELSE") (choose "CHOOSE") (let "LET") (in "IN") (case "CASE") (other "OTHER") + (enabled "ENABLED") (unchanged "UNCHANGED") (weak-fairness "WF_") (strong-fairness "SF_") + (subset "SUBSET") (union "UNION") (domain "DOMAIN") (local "LOCAL") (recursive "RECURSIVE") + (instance "INSTANCE") (with "WITH") (except "EXCEPT") (assume "ASSUME") + (assumption "ASSUMPTION") (axiom "AXIOM") (proof "PROOF") (by "BY") (only "ONLY") + (obvious "OBVIOUS") (omitted "OMITTED") (qed "QED") (suffices "SUFFICES") (have "HAVE") + (take "TAKE") (witness "WITNESS") (pick "PICK") (define "DEFINE") (def "DEF") (defs "DEFS") + (prove "PROVE") (new "NEW") (state "STATE") (action "ACTION") (temporal "TEMPORAL") + (lambda "LAMBDA") (lemma "LEMMA") (corollary "COROLLARY") (proposition "PROPOSITION") + (use "USE") (hide "HIDE")) + (recoveries (module "GERBIL-PARSER-TLA-PLUS-SANY-CANDIDATE" preserve-source)) + (conflicts selective-glr) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile (optional (field text module-text)) (field module module) + (repeat (seq (optional (field text module-text)) (field module module))) + (optional (field text module-text)))) + (module + (node Module module-border "MODULE" (field name identifier) module-border + (repeat (field item module-item)) module-end)) + (module-item + (choice extends-declaration constant-declaration variable-declaration operator-definition + function-definition infix-operator-definition prefix-operator-definition + postfix-operator-definition recursive-declaration instance-declaration + assumption-declaration theorem-declaration use-hide-declaration separator module)) + (extends-declaration + (node ExtendsDeclaration "EXTENDS" (field module identifier) + (repeat (seq "," (field module identifier))))) + (constant-declaration + (node ConstantDeclaration (choice "CONSTANT" "CONSTANTS") (field name identifier) + (optional (seq "(" (field parameter "_") (repeat (seq "," (field parameter "_"))) ")")) + (repeat + (seq "," (field name identifier) + (optional (seq "(" (field parameter "_") (repeat (seq "," (field parameter "_"))) ")")))))) + (variable-declaration + (node VariableDeclaration (choice "VARIABLE" "VARIABLES") (field name identifier) + (repeat (seq "," (field name identifier))))) + (operator-definition + (node OperatorDefinition (optional "LOCAL") (field name identifier) + (optional + (seq "(" (field parameter formal-parameter) + (repeat (seq "," (field parameter formal-parameter))) ")")) "==" + (field body (choice instance-expression expression)))) + (formal-parameter + (choice + (node FormalParameter (field name (choice identifier "_")) + (optional (seq "(" (field parameter "_") (repeat (seq "," (field parameter "_"))) ")"))) + (node FormalParameter "_" (field name infix-symbol) "_"))) + (lambda-expression + (prec right 1 + (node LambdaExpression "LAMBDA" (field name identifier) + (repeat (seq "," (field name identifier))) ":" (field body expression)))) + (label-expression + (prec right 1 (node LabelExpression (field name expression) "::" (field body expression)))) + (function-definition + (node FunctionDefinition (optional "LOCAL") (field name identifier) "[" + (field binding domain-binding) (repeat (seq "," (field binding domain-binding))) "]" "==" + (field body expression))) + (domain-binding + (choice + (node DomainBinding (field name identifier) (repeat (seq "," (field name identifier))) + "\\in" (field domain expression)) tuple-domain-binding)) + (tuple-domain-binding + (node TupleBinding "<<" (field name identifier) (repeat (seq "," (field name identifier))) + ">>" "\\in" (field domain expression))) + (single-domain-binding + (choice (node DomainBinding (field name identifier) "\\in" (field domain expression)) + tuple-domain-binding)) + (infix-operator-definition + (node OperatorDefinition (optional "LOCAL") (field parameter formal-parameter) + (field name infix-symbol) (field parameter formal-parameter) "==" (field body expression))) + (infix-symbol + (choice "^" "/" "*" "-" "+" "=" "\\land" "\\lor" "~>" "=>" "\\cdot" "\\equiv" "-+->" "/=" + "\\subseteq" "\\in" "<" "\\leq" ">" "\\geq" "\\times" "\\" "\\intersect" "\\union" "..." + ".." "|" "||" "&&" "&" "$$" "$" "??" "%%" "%" "##" "++" "--" "**" "//" "^^" "@@" "!!" "|-" + "|=" "-|" "=|" "<:" ":>" ":=" "::=" "\\oplus" "\\ominus" "\\odot" "\\oslash" "\\otimes" + "\\uplus" "\\sqcap" "\\sqcup" "\\div" "\\wr" "\\star" "\\o" "\\bigcirc" "\\bullet" "\\prec" + "\\succ" "\\preceq" "\\succeq" "\\sim" "\\simeq" "\\ll" "\\gg" "\\asymp" "\\subset" + "\\supset" "\\supseteq" "\\approx" "\\cong" "\\sqsubset" "\\sqsubseteq" "\\sqsupset" + "\\sqsupseteq" "\\doteq" "\\propto" "/\\" "\\/" "<=>" "#" "<=" "=<" ">=" "\\X" "\\cap" + "\\cup" "\\mod" "(+)" "(-)" "(.)" "(/)" "(\\X)" "\\circ")) + (prefix-operator-definition + (node OperatorDefinition (optional "LOCAL") (field name "-.") + (field parameter formal-parameter) "==" (field body expression))) + (postfix-operator-definition + (node OperatorDefinition (optional "LOCAL") (field parameter formal-parameter) + (field name (choice "^+" "^*" "^#")) "==" (field body expression))) + (recursive-declaration + (node RecursiveDeclaration "RECURSIVE" (field name identifier) + (optional + (seq "(" (field parameter formal-parameter) + (repeat (seq "," (field parameter formal-parameter))) ")")) + (repeat (seq "," (field name identifier))))) + (instance-declaration + (node InstanceDeclaration (optional "LOCAL") "INSTANCE" (field module identifier) + (optional + (seq "WITH" (field substitution substitution) + (repeat (seq "," (field substitution substitution))))))) + (substitution (node Substitution (field name identifier) "<-" (field value expression))) + (instance-expression + (node InstanceExpression "INSTANCE" (field module identifier) + (optional + (seq "WITH" (field substitution substitution) + (repeat (seq "," (field substitution substitution))))))) + (junction-expression + (choice + (node JunctionExpression (field operator (layout-start "/\\")) (field body expression) + (repeat (seq (field operator (layout-next "/\\")) (field body expression))) + (layout-end ")" "]" "}" ">>" "]_" ">>_" "," ":" "->" "|->" "THEN" "ELSE" "IN")) + (node JunctionExpression (field operator (layout-start "\\/")) (field body expression) + (repeat (seq (field operator (layout-next "\\/")) (field body expression))) + (layout-end ")" "]" "}" ">>" "]_" ">>_" "," ":" "->" "|->" "THEN" "ELSE" "IN")))) + (assumption-declaration + (node AssumptionDeclaration (choice "ASSUME" "ASSUMPTION" "AXIOM") + (optional (seq (field name identifier) "==")) (field body expression))) + (theorem-declaration + (node TheoremDeclaration (choice "THEOREM" "PROPOSITION" "LEMMA" "COROLLARY") + (optional (seq (field name identifier) "==")) (field body (choice expression assume-prove)) + (optional (field proof proof)))) + (fact-list (seq (field item proof-fact) (repeat (seq "," (field item proof-fact))))) + (proof-fact (choice expression infix-symbol (seq "MODULE" identifier))) + (use-hide-declaration + (node UseHideDeclaration (choice "USE" "HIDE") (optional "ONLY") (optional fact-list) + (optional (seq (choice "DEF" "DEFS") fact-list)))) + (terminal-proof + (node TerminalProof (optional "PROOF") + (choice "OBVIOUS" "OMITTED" + (seq "BY" (optional "ONLY") (optional fact-list) + (optional (seq (choice "DEF" "DEFS") fact-list)))))) + (proof (choice terminal-proof (node Proof (optional "PROOF") (repeat1 (field step proof-step))))) + (proof-step + (node ProofStep (field name (choice proof-step-name proof-reference)) + (field body + (choice "QED" use-hide-declaration instance-declaration + (seq (optional "DEFINE") (repeat1 let-definition)) proof-command + (seq (optional "SUFFICES") (choice assume-prove expression)))) + (optional (field proof terminal-proof)))) + (proof-command + (node ProofCommand + (choice (seq (field operator (choice "HAVE" "CASE")) (field item expression)) + (seq (field operator "WITNESS") fact-list) (seq (field operator "TAKE") proof-bindings) + (seq (field operator "PICK") proof-bindings ":" (field item expression))))) + (proof-bindings + (choice (seq domain-binding (repeat (seq "," domain-binding))) + (seq identifier (repeat (seq "," identifier))))) + (assume-prove + (node AssumeProve (choice "ASSUME" "[]ASSUME") (field assumption proof-assumption) + (repeat (seq "," (field assumption proof-assumption))) (choice "PROVE" "[]PROVE") + (field conclusion expression))) + (proof-assumption (choice expression assume-prove new-symbol)) + (new-symbol + (node NewSymbol + (choice (seq "NEW" (optional "CONSTANT")) "CONSTANT" + (seq (optional "NEW") (choice "VARIABLE" "STATE" "ACTION" "TEMPORAL"))) + (field name formal-parameter) (optional (seq "\\in" (field domain expression))))) + (separator (node Separator module-border)) + (expression + (choice + (prec left 30 + (node Expression (field left expression) (field operator (choice "\\land" "\\lor")) + (field right expression))) + (prec left 40 + (node Expression (field left expression) (field operator (choice "\\cdot")) + (field right expression))) + (prec none 15 + (node Expression (field left expression) (field operator (choice "\\equiv" "-+->")) + (field right expression))) + (prec left 60 + (node Expression (field left expression) + (field operator (choice "\\times" "|" "||" "%%" "++" "\\oplus" "(+)")) + (field right expression))) + (prec none 50 + (node Expression (field left expression) (field operator (choice "..." "!!" "\\wr")) + (field right expression))) + (prec left 70 + (node Expression (field left expression) + (field operator + (choice "&&" "&" "**" "\\odot" "\\otimes" "\\star" "\\bigcirc" "\\bullet" "(.)" + "(\\X)")) (field right expression))) + (prec left 50 + (node Expression (field left expression) + (field operator (choice "$$" "$" "??" "##" "\\uplus" "\\sqcap" "\\sqcup")) + (field right expression))) + (prec left 65 + (node Expression (field left expression) (field operator (choice "--" "\\ominus" "(-)")) + (field right expression))) + (prec none 70 + (node Expression (field left expression) (field operator (choice "//" "\\oslash" "(/)")) + (field right expression))) + (prec none 80 + (node Expression (field left expression) (field operator (choice "^^")) + (field right expression))) + (prec none 40 + (node Expression (field left expression) + (field operator + (choice "|-" "|=" "-|" "=|" ":=" "::=" "\\prec" "\\succ" "\\preceq" "\\succeq" "\\sim" + "\\simeq" "\\ll" "\\gg" "\\asymp" "\\subset" "\\supset" "\\supseteq" "\\approx" + "\\cong" "\\sqsubset" "\\sqsubseteq" "\\sqsupset" "\\sqsupseteq" "\\doteq" + "\\propto")) (field right expression))) + (prec none 43 + (node Expression (field left expression) (field operator (choice "<:")) + (field right expression))) + (prec none 60 + (node Expression (field left expression) (field operator (choice "\\mod")) + (field right expression))) + (prec right 10 + (node Expression (field left expression) (field operator (choice "<=>" "=>")) + (field right expression))) + (prec none 15 + (node Expression (field left expression) (field operator "~>") (field right expression))) + (prec left 20 + (node Expression (field left expression) (field operator "\\/") (field right expression))) + (prec left 30 + (node Expression (field left expression) (field operator "/\\") (field right expression))) + (prec none 40 + (node Expression (field left expression) + (field operator + (choice "=" "#" "/=" "<" ">" "<=" "=<" ">=" "\\leq" "\\geq" "\\in" "\\notin" + "\\subseteq")) (field right expression))) + (prec left 42 + (node Expression (field left expression) (field operator "@@") (field right expression))) + (prec none 43 + (node Expression (field left expression) (field operator ":>") (field right expression))) + (prec left 45 + (node Expression (field left expression) + (field operator (choice "\\cup" "\\union" "\\cap" "\\intersect" "\\")) + (field right expression))) + (prec left 50 + (node Expression (field left expression) (field operator "..") (field right expression))) + (prec left 60 + (node Expression (field left expression) (field operator "+") (field right expression))) + (prec left 65 + (node Expression (field left expression) (field operator "-") (field right expression))) + (prec left 70 + (node Expression (field left expression) + (field operator (choice "*" "\\o" "\\circ" "%" "\\X")) (field right expression))) + (prec none 70 + (node Expression (field left expression) (field operator (choice "/" "\\div")) + (field right expression))) + (prec none 80 + (node Expression (field left expression) (field operator "^") (field right expression))) + (prec right 90 prefix-expression) (prec left 100 postfix-expression) + (prec left 110 function-application) (prec left 110 operator-application) + (prec left 110 record-field-expression) if-expression (prec right 1 choose-expression) + (prec right 1 quantified-expression) (prec right 1 let-expression) + (prec right 1 case-expression) function-constructor function-set-expression + except-expression record-expression set-filter-expression record-set-expression + set-map-expression temporal-subscript-expression angle-action-expression fairness-expression + grouped-expression tuple-expression set-expression name-expression at-expression + (prec right 1 label-expression) junction-expression qualified-name-expression + (prec left 120 instance-qualified-expression) number-expression string-expression + (prec right 1 lambda-expression))) + (if-expression + (prec right 1 + (node IfExpression "IF" (field condition expression) "THEN" (field consequent expression) + "ELSE" (field alternative expression)))) + (choose-expression + (prec right 1 + (node ChooseExpression "CHOOSE" (field name identifier) + (optional (seq "\\in" (field domain expression))) ":" (field predicate expression)))) + (quantified-expression + (choice + (prec right 1 + (node QuantifiedExpression (field quantifier (choice "\\A" "\\E")) + (field binding domain-binding) (repeat (seq "," (field binding domain-binding))) ":" + (field predicate expression))) + (prec right 1 + (node QuantifiedExpression (field quantifier (choice "\\A" "\\E" "\\AA" "\\EE")) + (field name identifier) (repeat (seq "," (field name identifier))) ":" + (field predicate expression))))) + (let-expression + (prec right 1 + (node LetExpression "LET" (field definition let-definition) + (repeat (field definition let-definition)) "IN" (field body expression)))) + (let-definition + (choice local-definition local-function-definition infix-operator-definition + prefix-operator-definition postfix-operator-definition recursive-declaration + instance-declaration)) + (local-function-definition + (node LocalFunctionDefinition (field name identifier) "[" (field binding domain-binding) + (repeat (seq "," (field binding domain-binding))) "]" "==" (field body expression))) + (local-definition + (node LocalDefinition (field name identifier) + (optional + (seq "(" (field parameter formal-parameter) + (repeat (seq "," (field parameter formal-parameter))) ")")) "==" + (field body (choice expression instance-expression)))) + (case-expression + (prec right 1 + (node CaseExpression "CASE" (field arm case-arm) (repeat (seq "[]" (field arm case-arm))) + (optional (prec right 1 (seq "[]" "OTHER" "->" (field other expression))))))) + (case-arm + (prec right 1 (node CaseArm (field condition expression) "->" (field result expression)))) + (prefix-expression + (prec right 90 + (node PrefixExpression + (field operator + (choice "~" "\\lnot" "\\neg" "-." "-" "[]" "<>" "ENABLED" "UNCHANGED" "SUBSET" "UNION" + "DOMAIN")) (field operand expression)))) + (postfix-expression + (prec left 100 + (node PostfixExpression (field operand expression) + (field operator (choice "'" "^+" "^*" "^#"))))) + (function-application + (prec left 110 + (node FunctionApplication (field function expression) "[" (field argument expression) + (repeat (seq "," (field argument expression))) "]"))) + (operator-application + (prec left 110 + (node OperatorApplication (field operator (choice identifier qualified-name-expression)) "(" + (optional (field argument (choice expression infix-symbol))) + (repeat (seq "," (field argument (choice expression infix-symbol)))) ")"))) + (record-field-expression + (prec left 110 + (node RecordFieldExpression (field record expression) "." (field field identifier)))) + (function-constructor + (node FunctionConstructor "[" (field binding domain-binding) + (repeat (seq "," (field binding domain-binding))) "|->" (field body expression) "]")) + (function-set-expression + (node FunctionSetExpression "[" (field domain expression) "->" (field codomain expression) "]")) + (except-expression + (node ExceptExpression "[" (field base expression) "EXCEPT" (field update except-update) + (repeat (seq "," (field update except-update))) "]")) + (except-update + (node ExceptUpdate "!" (field path except-path) (repeat (field path except-path)) "=" + (field value expression))) + (except-path + (choice + (node ExceptIndex "[" (field index expression) (repeat (seq "," (field index expression))) + "]") (node ExceptField "." (field name identifier)))) + (at-expression (node AtExpression "@")) + (record-expression + (node RecordExpression "[" (field name identifier) "|->" (field value expression) + (repeat (seq "," (field name identifier) "|->" (field value expression))) "]")) + (record-set-expression + (node RecordSetExpression "[" (field name identifier) ":" (field domain expression) + (repeat (seq "," (field name identifier) ":" (field domain expression))) "]")) + (set-filter-expression + (prec dynamic 1 + (node SetFilterExpression "{" (field binding single-domain-binding) ":" + (field predicate expression) "}"))) + (set-map-expression + (node SetMapExpression "{" (field body expression) ":" (field binding domain-binding) + (repeat (seq "," (field binding domain-binding))) "}")) + (temporal-subscript-expression + (node TemporalSubscriptExpression "[" (field action expression) "]_" + (field subscript reduced-subscript-expression))) + (angle-action-expression + (node AngleActionExpression "<<" (field action expression) ">>_" + (field subscript reduced-subscript-expression))) + (reduced-subscript-expression + (choice name-expression qualified-name-expression instance-qualified-expression + tuple-expression grouped-expression)) + (fairness-expression + (node FairnessExpression (field operator (choice "WF_" "SF_")) + (field subscript reduced-subscript-expression) "(" (field action expression) ")")) + (grouped-expression (node GroupedExpression "(" (field expression expression) ")")) + (tuple-expression + (node TupleExpression "<<" (optional (field item expression)) + (repeat (seq "," (field item expression))) ">>")) + (set-expression + (node SetExpression "{" (optional (field item expression)) + (repeat (seq "," (field item expression))) "}")) + (name-expression (node NameExpression (field name (choice identifier proof-reference)))) + (selector + (choice identifier number infix-symbol ":" "@" "<<" ">>" + (node SelectorArguments "(" (field argument expression) + (repeat (seq "," (field argument expression))) ")"))) + (qualified-name-expression + (node QualifiedNameExpression (field module identifier) + (repeat1 (seq "!" (field name selector))))) + (instance-qualified-expression + (prec left 120 + (node InstanceQualifiedExpression (field instance operator-application) "!" + (field name selector) (repeat (seq "!" (field name selector)))))) + (number-expression (node NumberExpression (field value number))) + (string-expression (node StringExpression (field value string))))) diff --git a/languages/tla-plus/v1/fixtures.ss b/languages/tla-plus/v1/fixtures.ss deleted file mode 100644 index adbafbe9..00000000 --- a/languages/tla-plus/v1/fixtures.ss +++ /dev/null @@ -1,46 +0,0 @@ -;;; -*- Gerbil -*- -;;; Versioned TLA+ sources embedded at expansion time. - -(import :gerbil-parser/languages/tla-plus/v1/parser - (only-in :gerbil-parser/language-support - defsyntax-corpus syntax-fixture-expected-status)) -(export tla-plus-v1-fixtures tla-plus-v1-accepted-fixtures - tla-plus-v1-rejected-fixtures) - -(defsyntax-corpus tla-plus-v1-fixtures - (identity "tla-plus" +tla-plus-contract-version+ +tla-plus-syntax-contract+) - (accepted - ("tla-plus/examples/hour-clock" tla-plus-hour-clock - "corpus/tlaplus-examples/HourClock.tla" SourceFile - (Module ExtendsDeclaration VariableDeclaration OperatorDefinition - TheoremDeclaration)) - ("tla-plus/core/counter" tla-plus-counter - "corpus/core/Counter.tla" SourceFile - (Module ExtendsDeclaration ConstantDeclaration VariableDeclaration - OperatorDefinition TheoremDeclaration)) - ("tla-plus/core/nested-comment" tla-plus-nested-comment - "corpus/core/NestedComment.tla" SourceFile - (Module VariableDeclaration OperatorDefinition)) - ("tla-plus/core/structured-expressions" tla-plus-structured-expressions - "corpus/core/StructuredExpressions.tla" SourceFile - (Module OperatorDefinition Expression IfExpression ChooseExpression - QuantifiedExpression TupleExpression SetExpression - FunctionConstructor OperatorApplication LetExpression - CaseExpression RecordExpression SetFilterExpression)) - ("tla-plus/core/module-forms" tla-plus-module-forms - "corpus/core/ModuleForms.tla" SourceFile - (Module RecursiveDeclaration InstanceDeclaration AssumptionDeclaration - TheoremDeclaration OperatorDefinition))) - (rejected - ("tla-plus/core/malformed-if" tla-plus-malformed-if - "corpus/core/MalformedIf.tla"))) - -(def tla-plus-v1-accepted-fixtures - (filter (lambda (fixture) - (eq? (syntax-fixture-expected-status fixture) 'accepted)) - tla-plus-v1-fixtures)) - -(def tla-plus-v1-rejected-fixtures - (filter (lambda (fixture) - (eq? (syntax-fixture-expected-status fixture) 'rejected)) - tla-plus-v1-fixtures)) diff --git a/languages/tla-plus/v1/grammar.ss b/languages/tla-plus/v1/grammar.ss deleted file mode 100644 index a0197260..00000000 --- a/languages/tla-plus/v1/grammar.ss +++ /dev/null @@ -1,376 +0,0 @@ -;;; -*- Gerbil -*- -;;; Native TLA+ core syntax compiled by the language-neutral engine. - -(import (only-in :gerbil-parser/language-support deflanguage-grammar) - ./source) -(export (import: ./source) - +tla-plus-contract-version+ +tla-plus-syntax-contract+ - +tla-plus-examples-commit+ tla-plus-v1-language-grammar - tla-plus-v1-grammar tla-plus-v1-parser-ir tla-plus-v1-parser) - -(def +tla-plus-contract-version+ "v1") -(def +tla-plus-syntax-contract+ "tla-plus.native-core.v1") -(def +tla-plus-examples-commit+ "ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10") - -(deflanguage-grammar tla-plus-v1 - (identity "tla-plus" +tla-plus-contract-version+ +tla-plus-syntax-contract+) - (syntax-kinds - (SourceFile node (module)) (Module node (name item)) - (ExtendsDeclaration node (module)) (ConstantDeclaration node (name)) - (VariableDeclaration node (name)) - (OperatorDefinition node (name parameter body)) - (RecursiveDeclaration node (name parameter)) - (InstanceDeclaration node (module substitution)) - (InstanceExpression node (module substitution)) - (QualifiedNameExpression node (module name)) - (JunctionExpression node (operator body)) - (Substitution node (name value)) - (AssumptionDeclaration node (name body)) - (TheoremDeclaration node (name body)) - (UseHideDeclaration node (item)) - (Separator node ()) (EmptyLine node ()) - (Expression node (left operator right)) - (IfExpression node (condition consequent alternative)) - (ChooseExpression node (name domain predicate)) - (QuantifiedExpression node (quantifier name domain predicate)) - (LetExpression node (definition body)) - (LocalDefinition node (name parameter body)) - (CaseExpression node (arm other)) (CaseArm node (condition result)) - (PrefixExpression node (operator operand)) - (PostfixExpression node (operand operator)) - (FunctionApplication node (function argument)) - (OperatorApplication node (operator argument)) - (FunctionConstructor node (name domain body)) - (RecordExpression node (name value)) - (SetFilterExpression node (name domain predicate)) - (TemporalSubscriptExpression node (action subscript)) - (GroupedExpression node (expression)) - (TupleExpression node (item)) (SetExpression node (item)) - (NameExpression node (name)) (NumberExpression node (value)) - (StringExpression node (value)) (Identifier token (text)) - (Number token (text)) (String token (text)) - (HorizontalWhitespace token (text)) (Newline token (text)) - (Comment token (text)) (ModuleBorder token (text)) - (ModuleEnd token (text)) (SeparatorLine token (text)) - (Punctuation token (text))) - (terminals - (identifier Identifier) (number Number) (string String) - (horizontal-whitespace HorizontalWhitespace) (newline Newline) - (comment Comment) (module-border ModuleBorder) (module-end ModuleEnd) - (separator-line SeparatorLine) (punctuation Punctuation)) - (lexical-rules - (horizontal-whitespace (horizontal-whitespace+)) (newline (newline+)) - (comment - (choice (line-comment "\\*") (nested-block-comment "(*" "*)"))) - (string (quoted-string "\"")) - (module-end (literals "==============================================================" "====")) - (separator-line (literals "--------------------------------------------------------------")) - (module-border (literals "----------------------" "----")) - (number (number)) (identifier (identifier)) - (punctuation - (literals "<=>" "|->" "[]" "<>" "=>" "==" "/\\" "\\/" - "\\AA" "\\EE" "\\A" "\\E" "\\notin" "\\in" - ".." "<<" ">>" "<=" ">=" "->" "<-" - "#" "=" "<" ">" "+" "-" "*" "/" "^" "'" "~" - "(" ")" "[" "]" "{" "}" "," ":" "!" "_" ".")) - ) - (rules - (source-file (alias SourceFile (field module (reference module)))) - (module - (alias Module - (seq (token module-border) (literal "MODULE") - (field name (token identifier)) (token module-border) - (repeat (field item (reference module-item))) - (token module-end)))) - (module-item - (choice (reference extends-declaration) (reference constant-declaration) - (reference variable-declaration) (reference operator-definition) - (reference recursive-declaration) (reference instance-declaration) - (reference assumption-declaration) (reference theorem-declaration) - (reference use-hide-declaration) (reference separator))) - (extends-declaration - (alias ExtendsDeclaration - (seq (literal "EXTENDS") (field module (token identifier)) - (repeat (seq (literal ",") (field module (token identifier))))))) - (constant-declaration - (alias ConstantDeclaration - (seq (choice (literal "CONSTANT") (literal "CONSTANTS")) - (field name (token identifier)) - (repeat (seq (literal ",") (field name (token identifier))))))) - (variable-declaration - (alias VariableDeclaration - (seq (choice (literal "VARIABLE") (literal "VARIABLES")) - (field name (token identifier)) - (repeat (seq (literal ",") (field name (token identifier))))))) - (operator-definition - (alias OperatorDefinition - (seq (optional (literal "LOCAL")) (field name (token identifier)) - (optional - (seq (literal "(") (field parameter (token identifier)) - (repeat (seq (literal ",") (field parameter (token identifier)))) - (literal ")"))) - (literal "==") - (field body (choice (reference instance-expression) - (reference junction-expression) - (reference expression)))))) - (recursive-declaration - (alias RecursiveDeclaration - (seq (literal "RECURSIVE") - (field name (token identifier)) - (optional - (seq (literal "(") (field parameter (token identifier)) - (repeat (seq (literal ",") - (field parameter (token identifier)))) - (literal ")"))) - (repeat (seq (literal ",") (field name (token identifier))))))) - (instance-declaration - (alias InstanceDeclaration - (seq (optional (literal "LOCAL")) (literal "INSTANCE") - (field module (token identifier)) - (optional - (seq (literal "WITH") - (field substitution (reference substitution)) - (repeat - (seq (literal ",") - (field substitution (reference substitution))))))))) - (substitution - (alias Substitution - (seq (field name (token identifier)) (literal "<-") - (field value (reference expression))))) - (instance-expression - (alias InstanceExpression - (seq (literal "INSTANCE") (field module (token identifier)) - (optional - (seq (literal "WITH") - (field substitution (reference substitution)) - (repeat (seq (literal ",") - (field substitution (reference substitution))))))))) - ;; Flat junction lists retain their leading marker in the native CST. - ;; Nested indentation-sensitive junction lists are not admitted here. - (junction-expression - (alias JunctionExpression - (seq (field operator (choice (literal "/\\") (literal "\\/"))) - (field body (reference expression))))) - (assumption-declaration - (alias AssumptionDeclaration - (seq (choice (literal "ASSUME") (literal "ASSUMPTION") - (literal "AXIOM")) - (optional - (seq (field name (token identifier)) (literal "=="))) - (field body (reference expression))))) - (theorem-declaration - (alias TheoremDeclaration - (seq (choice (literal "THEOREM") (literal "PROPOSITION")) - (optional - (seq (field name (token identifier)) (literal "=="))) - (field body (reference expression))))) - (use-hide-declaration - (alias UseHideDeclaration - (seq (choice (literal "USE") (literal "HIDE")) - (field item (reference expression)) - (repeat (seq (literal ",") (field item (reference expression))))))) - (separator (alias Separator (token separator-line))) - (expression - (choice - (prec right 10 - (alias Expression - (seq (field left (reference expression)) - (field operator (choice (literal "<=>") (literal "=>"))) - (field right (reference expression))))) - (prec left 20 - (alias Expression - (seq (field left (reference expression)) - (field operator (literal "\\/")) - (field right (reference expression))))) - (prec left 30 - (alias Expression - (seq (field left (reference expression)) - (field operator (literal "/\\")) - (field right (reference expression))))) - (prec none 40 - (alias Expression - (seq (field left (reference expression)) - (field operator - (choice (literal "=") (literal "#") (literal "<") (literal ">") - (literal "<=") (literal ">=") (literal "\\in") - (literal "\\notin"))) - (field right (reference expression))))) - (prec left 50 - (alias Expression - (seq (field left (reference expression)) (field operator (literal "..")) - (field right (reference expression))))) - (prec left 60 - (alias Expression - (seq (field left (reference expression)) - (field operator (choice (literal "+") (literal "-"))) - (field right (reference expression))))) - (prec left 70 - (alias Expression - (seq (field left (reference expression)) - (field operator (choice (literal "*") (literal "/"))) - (field right (reference expression))))) - (prec right 80 - (alias Expression - (seq (field left (reference expression)) (field operator (literal "^")) - (field right (reference expression))))) - (prec right 90 (reference prefix-expression)) - (prec left 100 (reference postfix-expression)) - (prec left 110 (reference function-application)) - (prec left 110 (reference operator-application)) - (reference if-expression) - ;; TLA+ binders and LET extend as far right as possible. Their low - ;; precedence lets comparison and application shifts remain inside the - ;; predicate/body instead of also completing an outer expression tree. - (prec right 1 (reference choose-expression)) - (prec right 1 (reference quantified-expression)) - (prec right 1 (reference let-expression)) - (reference case-expression) (reference function-constructor) - (reference record-expression) (reference set-filter-expression) - (reference temporal-subscript-expression) - (reference grouped-expression) (reference tuple-expression) - (reference set-expression) (reference name-expression) - (reference qualified-name-expression) - (reference number-expression) (reference string-expression))) - (if-expression - (alias IfExpression - (seq (literal "IF") (field condition (reference expression)) - (literal "THEN") (field consequent (reference expression)) - (literal "ELSE") (field alternative (reference expression))))) - (choose-expression - (prec right 1 - (alias ChooseExpression - (seq (literal "CHOOSE") (field name (token identifier)) - (optional (seq (literal "\\in") - (field domain (reference expression)))) - (literal ":") (field predicate (reference expression)))))) - (quantified-expression - (prec right 1 - (alias QuantifiedExpression - (seq (field quantifier - (choice (literal "\\A") (literal "\\E") - (literal "\\AA") (literal "\\EE"))) - (field name (token identifier)) - (optional (seq (literal "\\in") - (field domain (reference expression)))) - (literal ":") (field predicate (reference expression)))))) - (let-expression - (prec right 1 - (alias LetExpression - (seq (literal "LET") (field definition (reference local-definition)) - (literal "IN") (field body (reference expression)))))) - (local-definition - (alias LocalDefinition - (seq (field name (token identifier)) - (optional - (seq (literal "(") (field parameter (token identifier)) - (repeat (seq (literal ",") - (field parameter (token identifier)))) - (literal ")"))) - (literal "==") (field body (reference expression))))) - (case-expression - (alias CaseExpression - (seq (literal "CASE") (field arm (reference case-arm)) - (repeat (seq (literal "[]") (field arm (reference case-arm)))) - (optional - (seq (literal "[]") (literal "OTHER") (literal "->") - (field other (reference expression))))))) - (case-arm - (alias CaseArm - (seq (field condition (reference expression)) (literal "->") - (field result (reference expression))))) - (prefix-expression - (prec right 90 - (alias PrefixExpression - (seq (field operator - (choice (literal "~") (literal "-") (literal "[]") - (literal "<>") (literal "ENABLED") - (literal "UNCHANGED") (literal "SUBSET") - (literal "UNION") (literal "DOMAIN"))) - (field operand (reference expression)))))) - (postfix-expression - (prec left 100 - (alias PostfixExpression - (seq (field operand (reference expression)) - (field operator (literal "'")))))) - (function-application - (prec left 110 - (alias FunctionApplication - (seq (field function (reference expression)) (literal "[") - (field argument (reference expression)) - (repeat (seq (literal ",") - (field argument (reference expression)))) - (literal "]"))))) - (operator-application - (prec left 110 - (alias OperatorApplication - (seq (field operator (reference expression)) (literal "(") - (optional (field argument (reference expression))) - (repeat (seq (literal ",") - (field argument (reference expression)))) - (literal ")"))))) - (function-constructor - (alias FunctionConstructor - (seq (literal "[") (field name (token identifier)) (literal "\\in") - (field domain (reference expression)) (literal "|->") - (field body (reference expression)) (literal "]")))) - (record-expression - (alias RecordExpression - (seq (literal "[") (field name (token identifier)) (literal "|->") - (field value (reference expression)) - (repeat - (seq (literal ",") (field name (token identifier)) - (literal "|->") (field value (reference expression)))) - (literal "]")))) - (set-filter-expression - (alias SetFilterExpression - (seq (literal "{") (field name (token identifier)) (literal "\\in") - (field domain (reference expression)) (literal ":") - (field predicate (reference expression)) (literal "}")))) - (temporal-subscript-expression - (alias TemporalSubscriptExpression - (seq (literal "[") (field action (reference expression)) (literal "]") - ;; SANY lexes the lossless `_name` spelling as one identifier. - (field subscript (token identifier))))) - (grouped-expression - (alias GroupedExpression - (seq (literal "(") (field expression (reference expression)) - (literal ")")))) - (tuple-expression - (alias TupleExpression - (seq (literal "<<") (optional (field item (reference expression))) - (repeat (seq (literal ",") (field item (reference expression)))) - (literal ">>")))) - (set-expression - (alias SetExpression - (seq (literal "{") (optional (field item (reference expression))) - (repeat (seq (literal ",") (field item (reference expression)))) - (literal "}")))) - (name-expression - (alias NameExpression (field name (token identifier)))) - (qualified-name-expression - (alias QualifiedNameExpression - (seq (field module (token identifier)) - (repeat1 (seq (literal "!") (field name (token identifier))))))) - (number-expression - (alias NumberExpression (field value (token number)))) - (string-expression - (alias StringExpression (field value (token string))))) - ;; Native TLA+ declarations are delimited by grammar, not physical lines. - ;; Newlines remain lossless trivia; no downstream source normalization. - (extras horizontal-whitespace newline comment) - (keywords (module "MODULE") (extends "EXTENDS") (constant "CONSTANT") - (constants "CONSTANTS") (variable "VARIABLE") - (variables "VARIABLES") (theorem "THEOREM") - (if "IF") (then "THEN") (else "ELSE") (choose "CHOOSE") - (let "LET") (in "IN") (case "CASE") (other "OTHER") - (enabled "ENABLED") (unchanged "UNCHANGED") - (subset "SUBSET") (union "UNION") (domain "DOMAIN") - (local "LOCAL") (recursive "RECURSIVE") - (instance "INSTANCE") (with "WITH") - (assume "ASSUME") (assumption "ASSUMPTION") (axiom "AXIOM") - (proposition "PROPOSITION") (use "USE") (hide "HIDE")) - (parser-entrypoints (source-file parse pure)) - (recoveries (module "GERBIL-PARSER-TLA-PLUS-V1" preserve-source)) - (conflicts selective-glr) - (case-insensitive #f) - (flow (source lexical) (lexical tla-plus-module) (tla-plus-module cst))) diff --git a/languages/tla-plus/v1/parser-test.ss b/languages/tla-plus/v1/parser-test.ss deleted file mode 100644 index a5fd860d..00000000 --- a/languages/tla-plus/v1/parser-test.ss +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- -;;; Acceptance owner for pinned TLA+ sources, structural CST obligations, -;;; nested-comment losslessness, and typed unterminated-comment failure. - -(import (only-in :std/test check test-case test-suite) - (only-in :std/misc/process run-process) - (only-in :std/string/misc string-trim-eol) - :gerbil-parser/languages/tla-plus/v1/parser - (only-in :gerbil-parser/languages/tla-plus/v1/qualification - +tla-plus-model-qualification-schema+ - qualify-tla-plus-model - tla-plus-model-receipt-admitted - tla-plus-model-receipt->alist) - :gerbil-parser/src/runtime/artifact - :gerbil-parser/src/runtime/cst - (only-in :gerbil-parser/language-support - syntax-fixture-required-kinds - syntax-fixture-source - syntax-fixture-source-digest - syntax-fixture-expected-status) - (only-in ./fixtures tla-plus-v1-fixtures - tla-plus-v1-accepted-fixtures - tla-plus-v1-rejected-fixtures)) -(export tla-plus-v1-parser-test) - -(def (receipt-ref receipt key) - (let (entry (assq key (tla-plus-model-receipt->alist receipt))) - (and entry (cdr entry)))) - -(def (call-with-qualification-fixture procedure) - (let* ((template - (path-expand "gerbil-parser-tlc-test.XXXXXX" - (getenv "TMPDIR" "/tmp"))) - (directory - (string-trim-eol (run-process ["mktemp" "-d" template])))) - (unwind-protect - (procedure directory) - (when (file-exists? directory) - (delete-file-or-directory directory #t))))) - -;;; CST traversal intentionally treats fields as transparent containers; the -;;; observable contract is the ordered set of emitted syntax-node kinds. -;; : (-> CSTValue (List Symbol)) -(def (cst-node-kinds value) - (cond - ((syntax-node? value) - (cons (syntax-node-kind value) - (apply append (map cst-node-kinds (syntax-node-children value))))) - ((syntax-field? value) - (apply append (map cst-node-kinds (syntax-field-children value)))) - (else '()))) - -;; : (-> ParseArtifact Symbol Integer) -(def (artifact-token-count artifact kind) - (length - (filter (lambda (event) - (and (token-event? event) - (eq? (token-event-token-kind event) kind))) - (parse-artifact-events artifact)))) - -;; : TestSuite -(def tla-plus-v1-parser-test - (test-suite "TLA+ v1 versioned language pack" - (test-case "modular multiline action contracts keep native source bytes" - (let* ((source - "---- MODULE Modular ----\nVARIABLES active,\n queued\nWork == INSTANCE WorkLifecycle\nInit ==\n /\\ Work!Init\n /\\ active = TRUE\nNext == Work!Step \\/\n (active' = FALSE /\\\n UNCHANGED <>)\n====\n") - (artifact (parse-tla-plus-v1 source)) - (kinds (and (parse-artifact-success? artifact) - (cst-node-kinds (parse-artifact->cst artifact))))) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source) - (for-each - (lambda (kind) (check (member kind kinds) ? values)) - '(InstanceExpression QualifiedNameExpression JunctionExpression)))) - (test-case "incomplete modular forms remain syntax errors" - (for-each - (lambda (body) - (let* ((source (string-append "---- MODULE Broken ----\n" body "\n====\n")) - (artifact (parse-tla-plus-v1 source))) - (check (parse-artifact-success? artifact) => #f) - (check (parse-artifact-roundtrip artifact) => source))) - '("Work == INSTANCE" "Value == Work!" "VARIABLES active,\n" - "Init ==\n /\\" "Next == TRUE /\\\n"))) - (test-case "native syntax and corpus identities are immutable" - (check +tla-plus-contract-version+ => "v1") - (check +tla-plus-syntax-contract+ => "tla-plus.native-core.v1") - (check +tla-plus-syntax-source+ - => "Specifying Systems, Chapter 15: TLAPlusGrammar") - (check +tla-plus-sany-release+ => "v1.7.4") - (check +tla-plus-sany-commit+ - => "5a47802b5c391f59ecdd44117981f4ff8c0656ba") - (check +tla-plus-sany-grammar-blob+ - => "bf9e7acb5337f4b6c2a4d6a973a1a65c95e72f56") - (check +tla-plus-sany-grammar-digest+ - => "sha256:15edd079cf16cf91556ba66b496c9ffc16f26ab30856753ed58e60b0d54f2d07") - (check +tla-plus-examples-commit+ - => "ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10") - (check (length tla-plus-v1-fixtures) => 6) - (check (length tla-plus-v1-accepted-fixtures) => 5) - (check (length tla-plus-v1-rejected-fixtures) => 1) - (check (syntax-fixture-source-digest (car tla-plus-v1-fixtures)) - => "sha256:985903176db4725f9cf25df84ad84dcd53ba94be80d26298b87ec86fbc9b08b3")) - (test-case "all admitted modules publish lossless structural CSTs" - (for-each - (lambda (fixture) - (let* ((source (syntax-fixture-source fixture)) - (artifact (parse-tla-plus-v1 source)) - (root (and (parse-artifact-success? artifact) - (parse-artifact->cst artifact))) - (kinds (and root (cst-node-kinds root)))) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source) - (check (syntax-node-kind root) => 'SourceFile) - (for-each - (lambda (kind) (check (member kind kinds) ? values)) - (syntax-fixture-required-kinds fixture)))) - tla-plus-v1-accepted-fixtures)) - (test-case "nested block comments remain one lossless trivia token" - (let* ((fixture (caddr tla-plus-v1-accepted-fixtures)) - (artifact (parse-tla-plus-v1 - (syntax-fixture-source fixture)))) - (check (parse-artifact-success? artifact) => #t) - (check (artifact-token-count artifact 'comment) => 1))) - (test-case "unary negation is admitted by the native lexical contract" - (let* ((source - "---- MODULE Negation ----\nVARIABLES enabled, ready\nDisabled == ~enabled /\\ ready\nNext == enabled' = FALSE /\\ ready' = TRUE\n====\n") - (artifact (parse-tla-plus-v1 source))) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source))) - (test-case "one qualification API composes parser and TLC receipts" - (call-with-qualification-fixture - (lambda (directory) - (let ((spec (path-expand "Qualified.tla" directory)) - (config (path-expand "Qualified.cfg" directory)) - (tlc (path-expand "tlc-fixture" directory))) - (call-with-output-file - spec - (lambda (port) - (write-string - "---- MODULE Qualified ----\nVARIABLE enabled\nInit == enabled = FALSE\nNext == enabled' = TRUE\n====\n" - port))) - (call-with-output-file - config - (lambda (port) (write-string "INIT Init\nNEXT Next\n" port))) - (call-with-output-file - tlc - (lambda (port) - (write-string - "#!/bin/sh\ncat <<'EOF'\nTLC2 Version fixture\nModel checking completed. No error has been found.\n4 states generated, 2 distinct states found, 0 states left on queue.\nThe depth of the complete state graph search is 1.\nEOF\n" - port))) - (run-process ["chmod" "+x" tlc]) - (let (receipt - (qualify-tla-plus-model spec config tlc: tlc workers: 1)) - (check +tla-plus-model-qualification-schema+ - => "gerbil-parser.tla-plus-model-qualification.v1") - (check (tla-plus-model-receipt-admitted receipt) => #t) - (check (receipt-ref receipt 'syntax-accepted) => #t) - (check (receipt-ref receipt 'roundtrip) => #t) - (check (receipt-ref receipt 'states-generated) => 4) - (check (receipt-ref receipt 'distinct-states) => 2) - (check (receipt-ref receipt 'states-left) => 0) - (check (receipt-ref receipt 'graph-depth) => 1)))))) - (test-case "qualification fails closed on misleading TLC completion" - (call-with-qualification-fixture - (lambda (directory) - (let ((spec (path-expand "Rejected.tla" directory)) - (config (path-expand "Rejected.cfg" directory)) - (tlc (path-expand "tlc-fixture" directory))) - (call-with-output-file - spec - (lambda (port) - (write-string - "---- MODULE Rejected ----\nVARIABLE enabled\nInit == enabled = FALSE\nNext == enabled' = TRUE\n====\n" - port))) - (call-with-output-file - config - (lambda (port) (write-string "INIT Init\nNEXT Next\n" port))) - (call-with-output-file - tlc - (lambda (port) - (write-string - "#!/bin/sh\necho 'TLC2 Version fixture'\necho 'Model checking completed. No error has been found.'\necho '4 states generated, 2 distinct states found, 0 states left on queue.'\necho 'The depth of the complete state graph search is 1.'\nexit 7\n" - port))) - (run-process ["chmod" "+x" tlc]) - (let (receipt - (qualify-tla-plus-model spec config tlc: tlc workers: 1)) - (check (tla-plus-model-receipt-admitted receipt) => #f) - (check (zero? (receipt-ref receipt 'exit-status)) => #f)) - (call-with-output-file - tlc - (lambda (port) - (write-string - "#!/bin/sh\necho 'TLC2 Version fixture'\necho 'Model checking completed. No error has been found.'\n" - port))) - (let (receipt - (qualify-tla-plus-model spec config tlc: tlc workers: 1)) - (check (tla-plus-model-receipt-admitted receipt) => #f) - (check (receipt-ref receipt 'states-generated) => #f)))))) - (test-case "syntax rejection stops before resolving TLC" - (call-with-qualification-fixture - (lambda (directory) - (let ((spec (path-expand "Malformed.tla" directory)) - (config (path-expand "Malformed.cfg" directory))) - (call-with-output-file - spec - (lambda (port) - (write-string - "---- MODULE Malformed ----\nVARIABLE x\nBroken == IF x = 0 THEN ELSE x\n====\n" - port))) - (call-with-output-file - config - (lambda (port) (write-string "INIT Broken\n" port))) - (let (receipt - (qualify-tla-plus-model - spec config tlc: "this-tlc-must-not-be-resolved")) - (check (tla-plus-model-receipt-admitted receipt) => #f) - (check (receipt-ref receipt 'syntax-accepted) => #f) - (check (receipt-ref receipt 'tool-path) => #f) - (check (receipt-ref receipt 'exit-status) => #f)))))) - (test-case "unterminated nested comments fail as one typed artifact" - (let (artifact - (parse-tla-plus-v1 - "---- MODULE Broken ----\n(* outer (* nested *)\nVARIABLE x\n====\n")) - (check (parse-artifact-success? artifact) => #f) - (check (parse-artifact-valid? artifact) => #t) - (check (length (parse-artifact-ref artifact 'diagnostics)) => 1))) - (test-case "recognized but malformed expressions fail closed" - (for-each - (lambda (fixture) - (let* ((source (syntax-fixture-source fixture)) - (artifact (parse-tla-plus-v1 source))) - (check (syntax-fixture-expected-status fixture) => 'rejected) - (check (parse-artifact-success? artifact) => #f) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => source) - (check (length (parse-artifact-ref artifact 'diagnostics)) => 1))) - tla-plus-v1-rejected-fixtures)))) diff --git a/languages/tla-plus/v1/parser.ss b/languages/tla-plus/v1/parser.ss deleted file mode 100644 index 0c74a9cb..00000000 --- a/languages/tla-plus/v1/parser.ss +++ /dev/null @@ -1,8 +0,0 @@ -;;; -*- Gerbil -*- -;;; Boundary: the versioned package publishes one descriptor-backed parser; -;;; grammar compilation remains owned by grammar.ss at expansion time. -(import :gerbil-parser/src/language/entry ./grammar) -(export (import: ./grammar) tla-plus-v1-language parse-tla-plus-v1) -(deflanguage-parser tla-plus-v1-language - (grammar tla-plus-v1-language-grammar) - (parse parse-tla-plus-v1)) diff --git a/languages/tla-plus/v1/source.ss b/languages/tla-plus/v1/source.ss deleted file mode 100644 index 87e3a13c..00000000 --- a/languages/tla-plus/v1/source.ss +++ /dev/null @@ -1,24 +0,0 @@ -;;; -*- Gerbil -*- -;;; Official SANY JavaCC grammar admitted only as native syntax evidence. - -(import :gerbil-parser/language-support) -(export +tla-plus-syntax-source+ - +tla-plus-sany-release+ - +tla-plus-sany-commit+ - +tla-plus-sany-grammar-blob+ - +tla-plus-sany-grammar-digest+ - tla-plus-sany-source) - -(def +tla-plus-syntax-source+ "Specifying Systems, Chapter 15: TLAPlusGrammar") -(def +tla-plus-sany-release+ "v1.7.4") -(def +tla-plus-sany-commit+ "5a47802b5c391f59ecdd44117981f4ff8c0656ba") -(def +tla-plus-sany-grammar-blob+ "bf9e7acb5337f4b6c2a4d6a973a1a65c95e72f56") -(def +tla-plus-sany-grammar-digest+ - "sha256:15edd079cf16cf91556ba66b496c9ffc16f26ab30856753ed58e60b0d54f2d07") - -(defsyntax-javacc-source tla-plus-sany-source - (identity "tla-plus" "v1.7.4" - "5a47802b5c391f59ecdd44117981f4ff8c0656ba") - (digest - "sha256:15edd079cf16cf91556ba66b496c9ffc16f26ab30856753ed58e60b0d54f2d07") - (source "grammar-source/tla+.jj")) diff --git a/line-structure-support.ss b/line-structure-support.ss index fdc6dc22..90e79452 100644 --- a/line-structure-support.ss +++ b/line-structure-support.ss @@ -1,6 +1,6 @@ ;;; -*- Gerbil -*- ;;; Public AOT support for Scheme-authored contextual line grammars. -(import (only-in ./src/compiler/line-structure-rowan - generate-line-structure-rowan-module)) -(export generate-line-structure-rowan-module) +(import (only-in ./src/compiler/line-structure-runtime + generate-line-structure-runtime-module)) +(export generate-line-structure-runtime-module) diff --git a/parser-actor-support.ss b/parser-actor-support.ss new file mode 100644 index 00000000..6d5b3dad --- /dev/null +++ b/parser-actor-support.ss @@ -0,0 +1,5 @@ +;;; -*- Gerbil -*- +;;; Optional public facade for application-owned local parser actors. +(import (only-in ./src/runtime/actor + spawn-parser-actor parser-actor-parse parser-actor-stop!)) +(export spawn-parser-actor parser-actor-parse parser-actor-stop!) diff --git a/qualify.ss b/qualify.ss new file mode 100644 index 00000000..e1b25a9b --- /dev/null +++ b/qualify.ss @@ -0,0 +1,84 @@ +#!/usr/bin/env gxi +;;; Native qualification entry; declarations own process contracts. +(displayln "QUALIFICATION-START") (force-output) +(import :gerbil-parser/t/support/qualification/process) +;;; Load compiler/benchmark services only for the selected source profile. +(def (profile-value module name) + (eval (list 'import module)) + (eval name)) +(def (run-process-cli arguments) + (let loop ((arguments arguments) (budget 60) (idle #f) (log #f) (receipts '())) + (unless (pair? arguments) (error "missing qualification command")) + (let (flag (car arguments)) + (cond + ((equal? flag "--") + (unless log (error "missing --log")) + (let (result (run-observed-process (cdr arguments) log timeout: budget idle-timeout: idle)) + (let (missing (filter (lambda (pattern) + (not (receipt-present? pattern (utf8->string (process-result-output result))))) receipts)) + (displayln "PROCESS-RESULT elapsed=" (process-result-elapsed result) " exit=" (process-result-status result) + " reason=" (process-result-reason result) " missing=" missing) (force-output) + (exit (cond ((member (process-result-reason result) '(total-timeout idle-timeout)) 124) + ((or (process-result-reason result) (pair? missing) (not (zero? (process-result-status result)))) 1) + (else 0)))))) + ((member flag '("--timeout" "--idle-timeout" "--log" "--require")) + (unless (pair? (cdr arguments)) (error "missing option value" flag)) + (let (value (cadr arguments)) + (cond ((equal? flag "--timeout") (loop (cddr arguments) (string->number value) idle log receipts)) + ((equal? flag "--idle-timeout") (loop (cddr arguments) budget (string->number value) log receipts)) + ((equal? flag "--log") (loop (cddr arguments) budget idle value receipts)) + (else (loop (cddr arguments) budget idle log (cons value receipts)))))) + ((equal? flag "--full-output") (loop (cdr arguments) budget idle log receipts)) + (else (error "unknown process contract option" flag)))))) + +(def (local-cli arguments) + (let loop ((rest arguments) (suites '()) (prepared? #f) (root ".gerbil") (logs ".data/local-native") (jobs (min 4 (##core-count)))) + (if (null? rest) + ((profile-value ':gerbil-parser/t/support/qualification/plans 'qualify-local!) (if (null? suites) '("rust" "gql" "ffi" "aot") (reverse suites)) prepared? root logs jobs) + (cond + ((equal? (car rest) "--prepared-diagnostics") (loop (cdr rest) suites #t root logs jobs)) + ((member (car rest) '("--suite" "--gerbil-path" "--log-dir" "--jobs")) + (unless (pair? (cdr rest)) (error "missing local option value" (car rest))) + (let (value (cadr rest)) + (cond ((equal? (car rest) "--suite") (loop (cddr rest) (if (member value suites) suites (cons value suites)) prepared? root logs jobs)) + ((equal? (car rest) "--gerbil-path") (loop (cddr rest) suites prepared? value logs jobs)) + ((equal? (car rest) "--log-dir") (loop (cddr rest) suites prepared? root value jobs)) + (else (loop (cddr rest) suites prepared? root logs (string->number value)))))) + (else (error "unknown local option" (car rest))))))) + +(def (performance-cli arguments) + (let loop ((rest arguments) (selected '()) (budget 120) (logs ".data/performance-source") (continue? #f) (list? #f)) + (if (null? rest) + (let* ((catalog (profile-value ':gerbil-parser/t/support/qualification/performance 'performance-suites)) + (names (if (null? selected) (map car catalog) (reverse selected)))) + (if list? (for-each (lambda (name) (displayln name ": " (cdr (or (assoc name catalog) (error "unknown suite" name))))) names) + ((profile-value ':gerbil-parser/t/support/qualification/performance 'qualify-performance!) names budget logs continue?))) + (cond + ((equal? (car rest) "--continue-on-failure") (loop (cdr rest) selected budget logs #t list?)) + ((equal? (car rest) "--list") (loop (cdr rest) selected budget logs continue? #t)) + ((member (car rest) '("--suite" "--timeout" "--log-dir")) + (unless (pair? (cdr rest)) (error "missing performance option value" (car rest))) + (let (value (cadr rest)) + (cond ((equal? (car rest) "--suite") (loop (cddr rest) (cons value selected) budget logs continue? list?)) + ((equal? (car rest) "--timeout") (loop (cddr rest) selected (string->number value) logs continue? list?)) + (else (loop (cddr rest) selected budget value continue? list?))))) + (else (error "unknown performance option" (car rest))))))) + +(def (main mode . arguments) + (cond + ((equal? mode "run") (run-process-cli arguments)) + ((equal? mode "compiled-event-fold") (apply (profile-value ':gerbil-parser/t/support/qualification/plans 'qualify-event-fold!) arguments)) + ((equal? mode "performance") (performance-cli arguments)) + ((equal? mode "local") (local-cli arguments)) + ((equal? mode "host") (unless (null? arguments) (error "host takes no arguments")) ((profile-value ':gerbil-parser/t/support/qualification/products 'qualify-runtime-host!))) + ((equal? mode "shared-library") + (let loop ((rest arguments) (output #f) (modules '())) + (if (null? rest) + (begin (unless output (error "missing --output")) ((profile-value ':gerbil-parser/t/support/qualification/products 'build-language-library!) output (reverse modules))) + (begin + (unless (pair? (cdr rest)) (error "missing shared-library option value")) + (cond ((equal? (car rest) "--output") (loop (cddr rest) (cadr rest) modules)) + ((equal? (car rest) "--language-module") (loop (cddr rest) output (cons (cadr rest) modules))) + (else (error "unknown shared-library option" (car rest)))))))) + (else (error "unknown qualification profile" mode))) + (exit 0)) diff --git a/rust-rowan-event-support.ss b/rust-rowan-event-support.ss deleted file mode 100644 index 8638abb9..00000000 --- a/rust-rowan-event-support.ss +++ /dev/null @@ -1,7 +0,0 @@ -;;; -*- Gerbil -*- -;;; Public Scheme-owned event algorithm boundary for Rust/Rowan AOT. - -(import (only-in ./src/compiler/event-fold-aot - define-event-fold-parser run-event-fold - event-fold-ir-json)) -(export define-event-fold-parser run-event-fold event-fold-ir-json) diff --git a/rust-rowan-grammar-support.ss b/rust-rowan-grammar-support.ss deleted file mode 100644 index ce15ef93..00000000 --- a/rust-rowan-grammar-support.ss +++ /dev/null @@ -1,5 +0,0 @@ -;;; -*- Gerbil -*- -;;; Minimal public grammar-authoring facade for the Rust/Rowan AOT product. - -(import (only-in ./src/language/grammar deflanguage)) -(export deflanguage) diff --git a/rust-rowan-support.ss b/rust-rowan-support.ss deleted file mode 100644 index d0f0f2a1..00000000 --- a/rust-rowan-support.ss +++ /dev/null @@ -1,6 +0,0 @@ -;;; -*- Gerbil -*- -;;; Stable public AOT boundary from one language descriptor to one Rust module. - -(import (only-in ./src/compiler/rust-rowan - generate-language-rust-rowan-module)) -(export generate-language-rust-rowan-module) diff --git a/rust-runtime-event-support.ss b/rust-runtime-event-support.ss new file mode 100644 index 00000000..95c9da9d --- /dev/null +++ b/rust-runtime-event-support.ss @@ -0,0 +1,29 @@ +;;; -*- Gerbil -*- +;;; Public Scheme-owned event algorithm boundary for Rust AOT. + +(import (only-in ./src/compiler/event-fold-aot + define-event-fold-parser run-event-fold + event-fold-ir-json) + (only-in ./src/modules/parser/source-pattern-funs + source-offset-after source-pattern-end source-pattern-at? + source-ascii-ci-pattern-at?) + (only-in ./src/modules/parser/source-fragment-objects + make-source-delimited-fragment make-source-first-split + make-source-reference-scan) + (only-in ./src/modules/parser/source-fragment-types + source-delimited-fragment? source-first-split? + source-reference-scan?) + (only-in ./src/modules/parser/source-fragment-funs + source-delimited-fragment-initial + source-delimited-fragment-forms + source-first-split-initial source-first-split-forms + source-reference-scan-initial source-reference-scan-forms)) +(export define-event-fold-parser run-event-fold event-fold-ir-json + source-offset-after source-pattern-end source-pattern-at? + source-ascii-ci-pattern-at? + make-source-delimited-fragment source-delimited-fragment? + source-delimited-fragment-initial source-delimited-fragment-forms + make-source-first-split source-first-split? + source-first-split-initial source-first-split-forms + make-source-reference-scan source-reference-scan? + source-reference-scan-initial source-reference-scan-forms) diff --git a/rust-runtime-grammar-support.ss b/rust-runtime-grammar-support.ss new file mode 100644 index 00000000..b51e7866 --- /dev/null +++ b/rust-runtime-grammar-support.ss @@ -0,0 +1,7 @@ +;;; -*- Gerbil -*- +;;; Minimal public grammar-authoring facade for the Rust AOT product. + +(import (only-in ./src/grammar/lexical-algebra deftext-profile) + (only-in ./src/language/grammar deflanguage)) +(export deftext-profile + deflanguage) diff --git a/rust-runtime-support.ss b/rust-runtime-support.ss new file mode 100644 index 00000000..278431b9 --- /dev/null +++ b/rust-runtime-support.ss @@ -0,0 +1,7 @@ +;;; -*- Gerbil -*- +;;; Stable public AOT boundary from one language descriptor to one Rust module. + +(import (only-in ./src/compiler/rust-runtime + generate-language-rust-runtime-module)) +(import (only-in ./src/compiler/rust-scanner generate-command-source-rust-module generate-rust-scanner-module generate-contextual-language-rust-runtime-module)) +(export generate-language-rust-runtime-module generate-command-source-rust-module generate-rust-scanner-module generate-contextual-language-rust-runtime-module) diff --git a/rust/gerbil-parser-rowan-arithmetic/Cargo.toml b/rust/gerbil-parser-artifact/Cargo.toml similarity index 50% rename from rust/gerbil-parser-rowan-arithmetic/Cargo.toml rename to rust/gerbil-parser-artifact/Cargo.toml index 5ef1fc0a..3e5d7f88 100644 --- a/rust/gerbil-parser-rowan-arithmetic/Cargo.toml +++ b/rust/gerbil-parser-artifact/Cargo.toml @@ -1,15 +1,18 @@ [package] -name = "gerbil-parser-rowan-arithmetic" +name = "gerbil-parser-artifact" version.workspace = true -description = "Generated arithmetic reference language for gerbil-parser-rowan" edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true -publish = false +description = "Validated borrowed access to Scheme parser artifacts" + +[features] +benchmark-controls = [] [dependencies] -gerbil-parser-rowan.workspace = true +sha2.workspace = true +serde_json = "1.0" [lints] workspace = true diff --git a/rust/gerbil-parser-artifact/README.org b/rust/gerbil-parser-artifact/README.org new file mode 100644 index 00000000..eb5536cc --- /dev/null +++ b/rust/gerbil-parser-artifact/README.org @@ -0,0 +1,67 @@ +#+title: Scheme parser artifacts in Rust + +* Authority and ownership + +=gerbil-parser-artifact= admits the Scheme-owned GPA1 result and the inert +native descriptor from the same language handle. It contains no language +recognizer and depends on no execution backend or third-party tree representation. +=gerbil-parser-ffi= obtains and caches this catalog through its handle; +callers use =parsed.view()= without passing a generated parser specification. + +#+begin_src rust +let parsed = language.parse(source)?; +let view = parsed.view()?; +if let Some(root) = view.root() { + for field in root.field("body") { + for child in field.children() { + // Node and token kinds, spans and source text come from Scheme. + } + } +} +#+end_src + +The descriptor uses the existing bounded =(object ...)= / =(list ...)= Scheme +metadata wire. Its reader accepts inert values only, rejects duplicate object +keys and trailing forms, and enforces the producer's depth limit. It does not +evaluate Scheme. =rootKind= is published by the parser product, rather than +inferred from catalog order. Node fields are checked against their owner's +published declaration. + +* Borrowed AST contract + +The view borrows both the original C result bytes and source. A flat index +records matching scope boundaries and node parents. Navigation preserves +producer order, token trivia, UTF-8 byte spans, named field scopes, repeated +fields and zero-width fields. Node and token text are slices of the original source. +Field scopes remain associations; they are not converted to anonymous nodes. +Rejected syntax exposes =view.tokens()= but no accepted root. The same borrowed +token API is available for accepted input: =id()= is the scanner ordinal, +=class()= is the original terminal class, and =kind()= is its published syntax +kind (or the class for unmapped source tokens). These identities remain distinct; +for example, an =item-token= scanner class can publish an =Identifier= syntax +kind. Iteration preserves all trivia and returns slices of the original source, +with no token-text allocation or recognition replay. Accepted tokens expose their +owning node; rejected tokens have no invented node parent. + +Admission checks grammar and source digests, complete source coverage, catalog +indices, node/token identities, nesting, field ownership and UTF-8 boundaries +before navigation. The native wrapper's lifetimes retain the result allocation, +language handle and owner-thread runtime until borrowed access ends. + +* Independent Rust execution + +The =syntax= module stores an owned source allocation plus a validated flat +preorder index for the independent Rust AOT backend. It is not used to convert +native Scheme artifacts. AOT execution remains in =gerbil-parser-runtime=; +its generated tables and semantic controls remain independent of native FFI. +Both access paths avoid per-token string storage and additional tree materialization. + +* Evidence boundary + +Canonical and contextual Scheme fixtures are regenerated by +=t/fixtures/shared-scanner/generate-artifact.ss=. Native AST shape, token +order, spans and source borrowing are compared with independent AOT results. +Truncation, header/record mutation, product mismatch, rejection, UTF-8 and field +identity controls remain executable. Field traversal additionally checks actual +published field names and values. This separation establishes ownership and +representation contracts; it makes no end-to-end performance claim. diff --git a/rust/gerbil-parser-artifact/src/admission.rs b/rust/gerbil-parser-artifact/src/admission.rs new file mode 100644 index 00000000..d73c3091 --- /dev/null +++ b/rust/gerbil-parser-artifact/src/admission.rs @@ -0,0 +1,151 @@ +//! Validate identities, canonical records, nesting, and exact UTF-8 coverage. +use super::wire::{self, NativeArtifactError, NativeEventKind as Kind}; +use crate::view::{Index, Navigation}; +use crate::{KindCategory, NativeCatalog}; +use sha2::{Digest, Sha256}; + +pub(super) fn admit( + payload: &[u8], + source: &str, + language: &NativeCatalog, +) -> Result { + let fail = |reason| wire::error(reason, None); + if source.len() > 67_108_864 + || payload.len() < 80 + || &payload[..4] != b"GPA1" + || wire::u32_at(payload, 4) != 1 + || wire::u32_at(payload, 8) > 1 + { + return Err(fail("header")); + } + let count = wire::u32_at(payload, 12) as usize; + if count.checked_mul(24).and_then(|n| n.checked_add(80)) != Some(payload.len()) { + return Err(fail("length")); + } + let digest = language + .grammar_digest + .as_str() + .strip_prefix("sha256:") + .ok_or_else(|| fail("grammar-digest"))?; + if digest.len() != 64 + || !digest.is_ascii() + || !(0..32).all(|i| { + u8::from_str_radix(&digest[i * 2..i * 2 + 2], 16).ok() == Some(payload[16 + i]) + }) + { + return Err(fail("grammar-digest")); + } + if payload[48..80] != Sha256::digest(source.as_bytes())[..] { + return Err(fail("source-digest")); + } + let accepted = wire::u32_at(payload, 8) == 0; + let index = validate_events(payload, source, language, language.fields.len(), accepted)?; + Ok(Navigation { accepted, index }) +} + +fn validate_events( + payload: &[u8], + source: &str, + language: &NativeCatalog, + fields: usize, + accepted: bool, +) -> Result, NativeArtifactError> { + let (mut offset, mut nodes, mut tokens, mut roots) = (0, 0, 0, 0); + let mut stack = Vec::new(); + // Do not reserve an untrusted declared event count before row admission. + // Larger valid inputs grow with admitted records, not the wire header. + let mut navigation = Vec::with_capacity(((payload.len() - 80) / 24).min(1024)); + let mut owner: Option<(usize, u32)> = None; + for (index, row) in payload[80..].as_chunks::<24>().0.iter().enumerate() { + let fail = |reason| wire::error(reason, Some(index)); + if !(1..=5).contains(&wire::u32_at(row, 0)) { + return Err(fail("event-tag")); + } + let event = wire::event(row); + navigation.push(Index { + end: 0, + parent: owner.map(|(at, _)| at), + }); + match event.kind { + Kind::StartNode => { + if !accepted + || event.id != nodes + || event.start != offset + || event.end != 0 + || (stack.is_empty() + && (roots != 0 || event.symbol as usize != language.root_kind)) + || language + .kinds + .get(event.symbol as usize) + .is_none_or(|kind| kind.category != KindCategory::Node) + { + return Err(fail("start-node")); + } + if stack.is_empty() { + roots += 1; + } + nodes += 1; + stack.push((Kind::StartNode, event.id, event.symbol, index, owner)); + owner = Some((index, event.symbol)); + } + Kind::FinishNode => { + let frame = stack.pop().ok_or_else(|| fail("finish-node"))?; + if event.start != 0 + || event.end != offset + || (frame.0, frame.1, frame.2) != (Kind::StartNode, event.id, event.symbol) + { + return Err(fail("finish-node")); + } + navigation[frame.3].end = index; + owner = frame.4; + } + Kind::StartField => { + if stack.is_empty() + || event.id != 0 + || event.start != offset + || event.end != 0 + || event.symbol as usize >= fields + { + return Err(fail("start-field")); + } + let owner_kind = owner.ok_or_else(|| fail("field-owner"))?.1; + if !language.kinds[owner_kind as usize] + .fields + .contains(&language.fields[event.symbol as usize]) + { + return Err(fail("field-owner")); + } + stack.push((Kind::StartField, 0, event.symbol, index, owner)); + } + Kind::FinishField => { + let frame = stack.pop().ok_or_else(|| fail("finish-field"))?; + if event.id != 0 + || event.start != 0 + || event.end != offset + || (frame.0, frame.1, frame.2) != (Kind::StartField, 0, event.symbol) + { + return Err(fail("finish-field")); + } + navigation[frame.3].end = index; + } + Kind::Token => { + if event.id != tokens + || event.start != offset + || event.end <= offset + || source.get(event.start..event.end).is_none() + || (accepted && stack.is_empty()) + || event.symbol as usize >= language.terminals.len() + { + return Err(fail("token")); + } + navigation[index].end = index; + tokens += 1; + offset = event.end; + } + } + } + if !stack.is_empty() || offset != source.len() || roots != usize::from(accepted) { + return Err(wire::error("coverage", None)); + } + Ok(navigation) +} diff --git a/rust/gerbil-parser-artifact/src/catalog.rs b/rust/gerbil-parser-artifact/src/catalog.rs new file mode 100644 index 00000000..b5d536a5 --- /dev/null +++ b/rust/gerbil-parser-artifact/src/catalog.rs @@ -0,0 +1,185 @@ +//! An owned vocabulary published by the Scheme parser, independent of LR tables. +use crate::NativeArtifactError; +use serde_json::Value; + +/// Category of a published syntax kind. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum KindCategory { + Node, + Token, +} +/// A named published kind with its allowed fields. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NativeKind { + pub name: String, + pub category: KindCategory, + pub fields: Vec, +} +/// A named token class emitted by the scanner. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NativeTerminal { + pub name: String, + /// Syntax token kind, when published by a grammar catalog. + pub syntax_kind: Option, +} +/// The immutable catalog and digest obtained from the same native language handle. +#[derive(Clone, Debug)] +pub struct NativeCatalog { + pub(crate) language: String, + pub(crate) grammar_digest: String, + pub(crate) root_kind: usize, + pub(crate) kinds: Vec, + pub(crate) terminals: Vec, + pub(crate) fields: Vec, +} +fn invalid() -> NativeArtifactError { + crate::wire::error("descriptor", None) +} +fn text(value: &Value) -> Result { + value + .as_str() + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .ok_or_else(invalid) +} +fn texts(value: &Value) -> Result, NativeArtifactError> { + value + .as_array() + .ok_or_else(invalid)? + .iter() + .map(text) + .collect() +} +fn distinct<'a>(values: impl IntoIterator) -> bool { + let mut seen = std::collections::HashSet::new(); + values.into_iter().all(|v| seen.insert(v)) +} +impl NativeCatalog { + /// Decode the native descriptor; never import a generated parser specification. + /// # Errors + /// Rejects malformed schemas, catalogs, identities or undeclared fields/roots. + pub fn from_descriptor(bytes: &[u8]) -> Result { + let value = crate::datum::read(bytes)?; + if value["schema"] != "gerbil-parser.native-descriptor.v1" { + return Err(invalid()); + } + let language = text(&value["language"])?; + let grammar_digest = text(&value["grammarDigest"])?; + let hex = grammar_digest.strip_prefix("sha256:").ok_or_else(invalid)?; + if hex.len() != 64 + || !hex + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(invalid()); + } + let fields = texts(&value["fields"])?; + let kinds = read_kinds(&value["syntaxKinds"], &fields)?; + let terminals = read_terminals(&value["terminals"], &kinds)?; + let root_name = text(&value["rootKind"])?; + let root_kind = kinds + .iter() + .position(|k| k.name == root_name && k.category == KindCategory::Node) + .ok_or_else(invalid)?; + if !distinct(kinds.iter().map(|k| k.name.as_str())) + || !distinct(terminals.iter().map(|t| t.name.as_str())) + || !distinct(fields.iter().map(String::as_str)) + { + return Err(invalid()); + } + Ok(Self { + language, + grammar_digest, + root_kind, + kinds, + terminals, + fields, + }) + } + /// Language identity published by the handle. + #[must_use] + pub fn language(&self) -> &str { + &self.language + } + /// Exact parser product digest, including contextual identity when present. + #[must_use] + pub fn grammar_digest(&self) -> &str { + &self.grammar_digest + } + /// Declared root kind name. + #[must_use] + pub fn root_kind(&self) -> &str { + &self.kinds[self.root_kind].name + } + /// Published node kinds and their field declarations. + #[must_use] + pub fn kinds(&self) -> &[NativeKind] { + &self.kinds + } + /// Published field names in wire-index order. + #[must_use] + pub fn fields(&self) -> &[String] { + &self.fields + } + /// Published terminal classes in wire-index order. + #[must_use] + pub fn terminals(&self) -> &[NativeTerminal] { + &self.terminals + } +} + +fn read_kinds(value: &Value, fields: &[String]) -> Result, NativeArtifactError> { + let declared: std::collections::HashSet<_> = fields.iter().collect(); + let mut kinds = Vec::new(); + for row in value.as_array().ok_or_else(invalid)? { + let row = row + .as_array() + .filter(|r| r.len() == 3) + .ok_or_else(invalid)?; + let category = match row[1].as_str() { + Some("result" | "node") => KindCategory::Node, + Some("token") => KindCategory::Token, + _ => return Err(invalid()), + }; + let names = texts(&row[2])?; + if !distinct(names.iter().map(String::as_str)) + || names.iter().any(|n| !declared.contains(n)) + { + return Err(invalid()); + } + kinds.push(NativeKind { + name: text(&row[0])?, + category, + fields: names, + }); + } + Ok(kinds) +} +fn read_terminals( + value: &Value, + kinds: &[NativeKind], +) -> Result, NativeArtifactError> { + let token_kinds: std::collections::HashMap<_, _> = kinds + .iter() + .enumerate() + .filter(|(_, kind)| kind.category == KindCategory::Token) + .map(|(index, kind)| (kind.name.as_str(), index)) + .collect(); + let mut terminals = Vec::new(); + for row in value.as_array().ok_or_else(invalid)? { + let row = row + .as_array() + .filter(|r| r.len() == 2) + .ok_or_else(invalid)?; + let token_kind = text(&row[1])?; + let syntax_kind = token_kinds.get(token_kind.as_str()).copied(); + if syntax_kind.is_none() && token_kind != "token" { + return Err(invalid()); + } + terminals.push(NativeTerminal { + name: text(&row[0])?, + syntax_kind, + }); + } + Ok(terminals) +} diff --git a/rust/gerbil-parser-artifact/src/datum.rs b/rust/gerbil-parser-artifact/src/datum.rs new file mode 100644 index 00000000..efa51e77 --- /dev/null +++ b/rust/gerbil-parser-artifact/src/datum.rs @@ -0,0 +1,145 @@ +//! Inert reader for the bounded `(object ...)` / `(list ...)` native metadata wire. +use crate::{NativeArtifactError, wire}; +use serde_json::{Map, Value}; +fn invalid() -> NativeArtifactError { + wire::error("descriptor-datum", None) +} +pub(crate) fn read(bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes).map_err(|_| invalid())?; + let mut reader = Reader { text, at: 0 }; + let value = reader.value(0)?; + reader.space(); + if reader.at != text.len() { + return Err(invalid()); + } + Ok(value) +} +struct Reader<'a> { + text: &'a str, + at: usize, +} +impl Reader<'_> { + fn peek(&self) -> Option { + self.text[self.at..].chars().next() + } + fn take(&mut self) -> Result { + let ch = self.peek().ok_or_else(invalid)?; + self.at += ch.len_utf8(); + Ok(ch) + } + fn space(&mut self) { + while self.peek().is_some_and(char::is_whitespace) { + self.take().expect("peeked character"); + } + } + fn expect(&mut self, ch: char) -> Result<(), NativeArtifactError> { + self.space(); + if self.take()? != ch { + return Err(invalid()); + } + Ok(()) + } + fn atom(&mut self) -> &str { + let start = self.at; + while self + .peek() + .is_some_and(|ch| !ch.is_whitespace() && ch != '(' && ch != ')') + { + self.take().expect("peeked character"); + } + &self.text[start..self.at] + } + fn string(&mut self) -> Result { + self.expect('"')?; + let mut value = String::new(); + loop { + match self.take()? { + '"' => return Ok(value), + '\\' => value.push(self.escape()?), + ch => value.push(ch), + } + } + } + fn escape(&mut self) -> Result { + match self.take()? { + '"' => Ok('"'), + '\\' => Ok('\\'), + 'n' => Ok('\n'), + 'r' => Ok('\r'), + 't' => Ok('\t'), + 'a' => Ok('\u{7}'), + 'b' => Ok('\u{8}'), + 'v' => Ok('\u{b}'), + 'f' => Ok('\u{c}'), + 'x' => self.hex_character(), + _ => Err(invalid()), + } + } + fn hex_character(&mut self) -> Result { + let start = self.at; + while self.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + self.take()?; + } + let hex = &self.text[start..self.at]; + if hex.is_empty() || hex.len() > 6 || self.take()? != ';' { + return Err(invalid()); + } + char::from_u32(u32::from_str_radix(hex, 16).map_err(|_| invalid())?).ok_or_else(invalid) + } + fn value(&mut self, depth: usize) -> Result { + if depth > 64 { + return Err(invalid()); + } + self.space(); + match self.peek().ok_or_else(invalid)? { + '"' => Ok(Value::String(self.string()?)), + '(' => { + self.take()?; + self.space(); + match self.atom() { + "list" => { + let mut values = Vec::new(); + loop { + self.space(); + if self.peek() == Some(')') { + self.take()?; + break; + } + values.push(self.value(depth + 1)?); + } + Ok(Value::Array(values)) + } + "object" => { + let mut values = Map::new(); + loop { + self.space(); + if self.peek() == Some(')') { + self.take()?; + break; + } + self.expect('(')?; + self.space(); + let key = self.string()?; + let value = self.value(depth + 1)?; + self.expect(')')?; + if values.insert(key, value).is_some() { + return Err(invalid()); + } + } + Ok(Value::Object(values)) + } + _ => Err(invalid()), + } + } + _ => match self.atom() { + "#t" => Ok(Value::Bool(true)), + "#f" => Ok(Value::Bool(false)), + number => Ok(Value::Number(number.parse().map_err(|_| invalid())?)), + }, + } + } +} + +#[cfg(test)] +#[path = "../tests/unit/datum.rs"] +mod tests; diff --git a/rust/gerbil-parser-artifact/src/lib.rs b/rust/gerbil-parser-artifact/src/lib.rs new file mode 100644 index 00000000..9f0a4cf0 --- /dev/null +++ b/rust/gerbil-parser-artifact/src/lib.rs @@ -0,0 +1,20 @@ +//! Rust access to Scheme-owned parser results, with no parser or tree dependency. +#![forbid(unsafe_code)] +mod admission; +mod catalog; +mod datum; +mod storage; +mod view; +pub use storage::{NativeArtifactBytes, NativeArtifactStorage}; +mod wire; +pub use catalog::{KindCategory, NativeCatalog, NativeKind, NativeTerminal}; +pub use view::{ + NativeArtifactView, NativeChildren, NativeElement, NativeField, NativeFields, NativeNode, + NativeToken, SourceRange, +}; +pub use wire::{NativeArtifactError, NativeEvent, NativeEventKind}; +pub mod syntax; + +#[cfg(test)] +#[path = "../tests/unit/language_alignment.rs"] +mod language_alignment_tests; diff --git a/rust/gerbil-parser-artifact/src/storage.rs b/rust/gerbil-parser-artifact/src/storage.rs new file mode 100644 index 00000000..8370b2ec --- /dev/null +++ b/rust/gerbil-parser-artifact/src/storage.rs @@ -0,0 +1,94 @@ +//! An immutable payload owner admits and indexes its result once. +use crate::view::Navigation; +use crate::{NativeArtifactError, NativeArtifactView, NativeCatalog}; +use std::cell::OnceCell; + +/// Length-bearing bytes owned for the lifetime of a parse result. +/// +/// Implementations must return the same immutable bytes throughout their lifetime. +/// Ownership may refer to a foreign allocation; reads must preserve its bounds +/// and release must wait until all borrows end. This contract has no parser API. +pub trait NativeArtifactBytes { + /// Borrow the original immutable allocation. + /// # Errors + /// Rejects an invalid allocation or unavailable bytes. + fn bytes(&self) -> Result<&[u8], NativeArtifactError>; +} + +struct Admitted<'catalog> { + catalog: &'catalog NativeCatalog, + navigation: Navigation, + payload_identity: (usize, usize), +} + +/// Own a result while borrowing its exact source and handle catalog. +/// +/// The cached state contains only external catalog references and owned indexes; +/// it contains no references into `owner`, so moving this storage is safe. +pub struct NativeArtifactStorage<'catalog, 'source, P> { + owner: P, + source: &'source str, + admitted: OnceCell>, +} +impl<'catalog, 'source, P: NativeArtifactBytes> NativeArtifactStorage<'catalog, 'source, P> { + /// Keep the payload alive; admission occurs on the first view request. + #[must_use] + pub const fn new(owner: P, source: &'source str) -> Self { + Self { + owner, + source, + admitted: OnceCell::new(), + } + } + + /// Borrow the immutable payload owner without exposing mutable access. + #[must_use] + pub const fn owner(&self) -> &P { + &self.owner + } + + /// Admit once and reuse the same navigation indexes for this handle catalog. + /// # Errors + /// Rejects invalid bytes/source/catalog or a different catalog after admission. + pub fn view( + &self, + catalog: &'catalog NativeCatalog, + ) -> Result, NativeArtifactError> { + let payload = self.owner.bytes()?; + if self.admitted.get().is_none() { + let navigation = Navigation::decode(payload, self.source, catalog)?; + self.admitted + .set(Admitted { + catalog, + navigation, + payload_identity: (payload.as_ptr() as usize, payload.len()), + }) + .map_err(|_| NativeArtifactError { + reason: "native-view-cache", + event: None, + })?; + } + let admitted = self.admitted.get().ok_or(NativeArtifactError { + reason: "native-view-cache", + event: None, + })?; + if admitted.payload_identity != (payload.as_ptr() as usize, payload.len()) { + return Err(NativeArtifactError { + reason: "native-view-payload", + event: None, + }); + } + if !std::ptr::eq(catalog, admitted.catalog) { + return Err(NativeArtifactError { + reason: "native-view-catalog", + event: None, + }); + } + Ok(NativeArtifactView::from_navigation( + payload, + self.source, + admitted.catalog, + &admitted.navigation, + )) + } +} diff --git a/rust/gerbil-parser-artifact/src/syntax/mod.rs b/rust/gerbil-parser-artifact/src/syntax/mod.rs new file mode 100644 index 00000000..4ba54df2 --- /dev/null +++ b/rust/gerbil-parser-artifact/src/syntax/mod.rs @@ -0,0 +1,8 @@ +//! Source-backed navigation for independently executing AOT products. +//! FFI clients use borrowed native views and never materialize this storage. +mod navigation; +mod storage; +mod walk; +pub use navigation::{SyntaxElement, SyntaxNode, SyntaxToken}; +pub use storage::{IndexedEntry, SyntaxKind, SyntaxTree, TextRange, TextSize}; +pub use walk::WalkEvent; diff --git a/rust/gerbil-parser-artifact/src/syntax/navigation.rs b/rust/gerbil-parser-artifact/src/syntax/navigation.rs new file mode 100644 index 00000000..40cb86fa --- /dev/null +++ b/rust/gerbil-parser-artifact/src/syntax/navigation.rs @@ -0,0 +1,274 @@ +//! Node and token access over an existing flat source index. +use super::storage::{Storage, SyntaxKind, TextRange, TextSize}; +use super::walk::WalkEvent; +use std::{fmt, sync::Arc}; +/// An owned-source node view, sharing the producer's navigation index. +#[derive(Clone)] +pub struct SyntaxNode { + pub(super) storage: Arc, + pub(super) at: usize, +} +/// An owned-source token view, borrowing its text from the shared source. +#[derive(Clone)] +pub struct SyntaxToken { + pub(super) storage: Arc, + pub(super) at: usize, +} +/// Node or token child in producer order. +#[derive(Clone, Debug)] +pub enum SyntaxElement { + Node(SyntaxNode), + Token(SyntaxToken), +} +impl SyntaxElement { + /// Keep a token child, if this element is a token. + #[must_use] + pub fn into_token(self) -> Option { + match self { + Self::Token(t) => Some(t), + Self::Node(_) => None, + } + } + /// Keep a node child, if this element is a node. + #[must_use] + pub fn into_node(self) -> Option { + match self { + Self::Node(n) => Some(n), + Self::Token(_) => None, + } + } + /// Shared published kind. + #[must_use] + pub fn kind(&self) -> SyntaxKind { + match self { + Self::Node(n) => n.kind(), + Self::Token(t) => t.kind(), + } + } + /// Shared original source range. + #[must_use] + // Construction rejects sources beyond u32 and spans beyond source length. + #[allow(clippy::cast_possible_truncation)] + pub fn text_range(&self) -> TextRange { + match self { + Self::Node(n) => n.text_range(), + Self::Token(t) => t.text_range(), + } + } +} +fn element(storage: &Arc, at: usize) -> SyntaxElement { + if storage.entries[at].token { + SyntaxElement::Token(SyntaxToken { + storage: Arc::clone(storage), + at, + }) + } else { + SyntaxElement::Node(SyntaxNode { + storage: Arc::clone(storage), + at, + }) + } +} +impl SyntaxNode { + /// Published node kind. + #[must_use] + pub fn kind(&self) -> SyntaxKind { + SyntaxKind(self.storage.entries[self.at].kind) + } + /// Original source span. + #[must_use] + // Construction rejects sources beyond u32 and spans beyond source length. + #[allow(clippy::cast_possible_truncation)] + pub fn text_range(&self) -> TextRange { + let r = &self.storage.entries[self.at]; + TextRange::new(TextSize(r.start as u32), TextSize(r.end as u32)) + } + /// Borrow the source slice directly, including trivia. + #[must_use] + pub fn text(&self) -> &str { + let r = &self.storage.entries[self.at]; + &self.storage.source[r.start..r.end] + } + /// Existing parent index. + #[must_use] + pub fn parent(&self) -> Option { + self.storage.entries[self.at].parent.map(|at| Self { + storage: Arc::clone(&self.storage), + at, + }) + } + /// Direct children, skipping each subtree using its admitted interval. + pub fn children_with_tokens(&self) -> impl Iterator + use<> { + let storage = Arc::clone(&self.storage); + let end = storage.entries[self.at].subtree_end; + let mut next = self.at + 1; + std::iter::from_fn(move || { + if next >= end { + return None; + } + let at = next; + next = storage.entries[at].subtree_end; + Some(element(&storage, at)) + }) + } + /// Direct node children. + pub fn children(&self) -> impl Iterator + use<> { + self.children_with_tokens() + .filter_map(SyntaxElement::into_node) + } + /// First direct node child. + #[must_use] + pub fn first_child(&self) -> Option { + self.children().next() + } + /// Preorder nodes including this node, without recursively traversing Rust stacks. + pub fn descendants(&self) -> impl Iterator + use<> { + self.descendants_with_tokens() + .filter_map(SyntaxElement::into_node) + } + /// Preorder elements including this node. + pub fn descendants_with_tokens(&self) -> impl Iterator + use<> { + let storage = Arc::clone(&self.storage); + let range = self.at..storage.entries[self.at].subtree_end; + range.map(move |at| element(&storage, at)) + } + /// Enter/leave traversal over existing indexes. + pub fn preorder_with_tokens(&self) -> impl Iterator> + use<> { + Preorder::new(Arc::clone(&self.storage), self.at) + } +} +impl SyntaxToken { + /// Published token kind. + #[must_use] + pub fn kind(&self) -> SyntaxKind { + SyntaxKind(self.storage.entries[self.at].kind) + } + /// Exact source slice, without allocating token strings. + #[must_use] + pub fn text(&self) -> &str { + let r = &self.storage.entries[self.at]; + &self.storage.source[r.start..r.end] + } + /// Original byte range. + #[must_use] + // Construction rejects sources beyond u32 and spans beyond source length. + #[allow(clippy::cast_possible_truncation)] + pub fn text_range(&self) -> TextRange { + let r = &self.storage.entries[self.at]; + TextRange::new(TextSize(r.start as u32), TextSize(r.end as u32)) + } + /// Existing owning node. + #[must_use] + pub fn parent(&self) -> Option { + self.storage.entries[self.at].parent.map(|at| SyntaxNode { + storage: Arc::clone(&self.storage), + at, + }) + } +} +impl fmt::Display for SyntaxNode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.text()) + } +} +impl fmt::Display for SyntaxToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.text()) + } +} +impl fmt::Debug for SyntaxNode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SyntaxNode") + .field("kind", &self.kind()) + .field("range", &self.text_range()) + .field( + "entries", + &&self.storage.entries[self.at..self.storage.entries[self.at].subtree_end], + ) + .finish() + } +} +impl fmt::Debug for SyntaxToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SyntaxToken") + .field("kind", &self.kind()) + .field("range", &self.text_range()) + .field("text", &self.text()) + .finish() + } +} +impl PartialEq for SyntaxNode { + fn eq(&self, other: &Self) -> bool { + self.at == other.at && Arc::ptr_eq(&self.storage, &other.storage) + } +} +impl Eq for SyntaxNode {} +pub(super) struct Preorder { + storage: Arc, + next: usize, + end: usize, + stack: Vec, + pending_token: Option, +} +impl Preorder { + pub(super) fn new(storage: Arc, at: usize) -> Self { + let end = storage.entries[at].subtree_end; + Self { + storage, + next: at, + end, + stack: Vec::new(), + pending_token: None, + } + } +} +impl Iterator for Preorder { + type Item = WalkEvent; + fn next(&mut self) -> Option { + if let Some(at) = self.pending_token.take() { + return Some(WalkEvent::Leave(SyntaxElement::Token(SyntaxToken { + storage: Arc::clone(&self.storage), + at, + }))); + } + if let Some(&at) = self.stack.last() + && self.next == self.storage.entries[at].subtree_end + { + self.stack.pop(); + return Some(WalkEvent::Leave(SyntaxElement::Node(SyntaxNode { + storage: Arc::clone(&self.storage), + at, + }))); + } + if self.next >= self.end { + return None; + } + let at = self.next; + self.next += 1; + let value = if self.storage.entries[at].token { + self.pending_token = Some(at); + SyntaxElement::Token(SyntaxToken { + storage: Arc::clone(&self.storage), + at, + }) + } else { + self.stack.push(at); + SyntaxElement::Node(SyntaxNode { + storage: Arc::clone(&self.storage), + at, + }) + }; + Some(WalkEvent::Enter(value)) + } +} + +impl super::storage::SyntaxTree { + /// Access the existing root index; no additional tree is built. + #[must_use] + pub fn root(&self) -> SyntaxNode { + SyntaxNode { + storage: Arc::clone(&self.0), + at: 0, + } + } +} diff --git a/rust/gerbil-parser-artifact/src/syntax/storage.rs b/rust/gerbil-parser-artifact/src/syntax/storage.rs new file mode 100644 index 00000000..933b899f --- /dev/null +++ b/rust/gerbil-parser-artifact/src/syntax/storage.rs @@ -0,0 +1,156 @@ +//! Flat source ranges and navigation indexes; token strings are not duplicated. +use std::sync::Arc; +/// Published syntax kind index. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub struct SyntaxKind(pub u16); +/// UTF-8 byte offset within an owned syntax source. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub struct TextSize(pub u32); +impl From for TextSize { + fn from(n: u32) -> Self { + Self(n) + } +} +impl From for u32 { + fn from(n: TextSize) -> Self { + n.0 + } +} +impl From for usize { + fn from(n: TextSize) -> Self { + n.0 as Self + } +} +/// Half-open source range used by indexed AST access and graph projections. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct TextRange { + start: TextSize, + end: TextSize, +} +impl TextRange { + /// Construct an ordered source span. + #[must_use] + /// # Panics + /// Panics when the end precedes the start. + pub fn new(start: TextSize, end: TextSize) -> Self { + assert!(start <= end); + Self { start, end } + } + /// Construct an empty span at an offset. + #[must_use] + pub fn empty(at: TextSize) -> Self { + Self::new(at, at) + } + /// First byte offset. + #[must_use] + pub fn start(self) -> TextSize { + self.start + } + /// Exclusive final byte offset. + #[must_use] + pub fn end(self) -> TextSize { + self.end + } +} +/// One producer-owned node or token range and its structural links. +#[derive(Clone, Debug)] +pub struct IndexedEntry { + pub kind: u16, + pub token: bool, + pub start: usize, + pub end: usize, + pub parent: Option, + pub subtree_end: usize, +} +#[derive(Debug)] +pub(super) struct Storage { + pub source: Arc, + pub entries: Vec, +} +/// One immutable source allocation with a flat navigation index. +#[derive(Clone, Debug)] +pub struct SyntaxTree(pub(super) Arc); +impl SyntaxTree { + /// Admit the producer's ordered flat index without copying any token text. + /// # Errors + /// Rejects malformed ranges, parent references, subtree intervals or coverage. + pub fn from_index(source: &str, entries: Vec) -> Result { + if source.len() > u32::MAX as usize + || entries.is_empty() + || entries[0].token + || entries[0].parent.is_some() + || entries[0].start != 0 + || entries[0].end != source.len() + || entries[0].subtree_end != entries.len() + { + return Err("invalid indexed syntax root".into()); + } + let mut offset = 0; + let mut ancestors: Vec = Vec::new(); + for (i, row) in entries.iter().enumerate() { + while ancestors + .last() + .is_some_and(|&owner| entries[owner].subtree_end == i) + { + ancestors.pop(); + } + if row.parent != ancestors.last().copied() || row.start != offset { + return Err("invalid indexed syntax preorder".into()); + } + if !row.token { + ancestors.push(i); + } + if row.start > row.end + || source.get(row.start..row.end).is_none() + || row.subtree_end <= i + || row.subtree_end > entries.len() + { + return Err("invalid indexed syntax span".into()); + } + if let Some(parent) = row.parent { + let Some(owner) = entries.get(parent) else { + return Err("invalid indexed syntax parent".into()); + }; + if parent >= i + || owner.token + || owner.subtree_end <= i + || owner.subtree_end < row.subtree_end + || owner.start > row.start + || owner.end < row.end + { + return Err("invalid indexed syntax ancestry".into()); + } + } else if i != 0 { + return Err("extra indexed syntax root".into()); + } + if row.subtree_end < entries.len() && entries[row.subtree_end].start != row.end { + return Err("invalid indexed syntax subtree coverage".into()); + } + if row.token { + if row.start != offset || row.end <= row.start || row.subtree_end != i + 1 { + return Err("invalid indexed syntax token coverage".into()); + } + offset = row.end; + } + } + if offset != source.len() { + return Err("incomplete indexed syntax coverage".into()); + } + Ok(Self(Arc::new(Storage { + source: Arc::from(source), + entries, + }))) + } + /// Complete original source length. + #[must_use] + // Construction admits only source lengths representable by TextSize. + #[allow(clippy::cast_possible_truncation)] + pub fn text_len(&self) -> TextSize { + TextSize(self.0.source.len() as u32) + } +} +impl std::fmt::Display for SyntaxTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0.source) + } +} diff --git a/rust/gerbil-parser-artifact/src/syntax/walk.rs b/rust/gerbil-parser-artifact/src/syntax/walk.rs new file mode 100644 index 00000000..b88ed6d8 --- /dev/null +++ b/rust/gerbil-parser-artifact/src/syntax/walk.rs @@ -0,0 +1,7 @@ +//! Generic traversal direction for indexed syntax access. +/// Entering or leaving an existing syntax element. +#[derive(Clone, Debug)] +pub enum WalkEvent { + Enter(T), + Leave(T), +} diff --git a/rust/gerbil-parser-artifact/src/view.rs b/rust/gerbil-parser-artifact/src/view.rs new file mode 100644 index 00000000..eee3e86a --- /dev/null +++ b/rust/gerbil-parser-artifact/src/view.rs @@ -0,0 +1,355 @@ +//! Navigation indexes point into original GPA1 records and source; no CST copy. +use crate::{NativeArtifactError, NativeCatalog, NativeEvent, NativeEventKind as Kind}; +use crate::{admission, wire}; +use std::borrow::Cow; + +/// UTF-8 byte span in the source admitted with this result. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SourceRange { + pub start: usize, + pub end: usize, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct Index { + pub(super) end: usize, + pub(super) parent: Option, +} +/// Validated native bytes and source borrowed from the same parse result. +#[derive(Debug)] +pub struct NativeArtifactView<'a> { + payload: &'a [u8], + source: &'a str, + catalog: &'a NativeCatalog, + accepted: bool, + index: Cow<'a, [Index]>, +} +impl<'a> NativeArtifactView<'a> { + /// Validate identities, vocabulary, event nesting and full UTF-8 coverage. + /// # Errors + /// Rejects malformed or mismatched artifacts before exposing navigation. + pub fn decode( + payload: &'a [u8], + source: &'a str, + catalog: &'a NativeCatalog, + ) -> Result { + let navigation = Navigation::decode(payload, source, catalog)?; + Ok(Self { + payload, + source, + catalog, + accepted: navigation.accepted, + index: Cow::Owned(navigation.index), + }) + } + pub(crate) fn from_navigation( + payload: &'a [u8], + source: &'a str, + catalog: &'a NativeCatalog, + navigation: &'a Navigation, + ) -> Self { + Self { + payload, + source, + catalog, + accepted: navigation.accepted, + index: Cow::Borrowed(&navigation.index), + } + } + /// Frozen two-pass decoder for complete-request benchmark controls. + /// # Errors + /// Rejects invalid artifacts under the same former admission contract. + #[cfg(feature = "benchmark-controls")] + pub fn decode_reference( + payload: &'a [u8], + source: &'a str, + catalog: &'a NativeCatalog, + ) -> Result { + let navigation = reference_navigation::decode(payload, source, catalog)?; + Ok(Self { + payload, + source, + catalog, + accepted: navigation.accepted, + index: Cow::Owned(navigation.index), + }) + } + /// Whether Scheme accepted this source without repair. + #[must_use] + pub fn accepted(&self) -> bool { + self.accepted + } + /// Catalog from the same native language handle. + #[must_use] + pub fn catalog(&self) -> &NativeCatalog { + self.catalog + } + /// Borrow wire records without copying payloads or token text. + pub fn events(&self) -> impl ExactSizeIterator + '_ { + self.payload[80..] + .as_chunks::<24>() + .0 + .iter() + .map(wire::event) + } + /// Borrow all scanner tokens in producer order, including syntax rejection. + pub fn tokens(&self) -> impl Iterator> + '_ { + (0..self.index.len()).filter_map(|at| { + (self.event(at).kind == Kind::Token).then_some(NativeToken { view: self, at }) + }) + } + /// Return a token source slice. Callers cannot forge an out-of-source range. + #[must_use] + pub fn token_text(&self, event: NativeEvent) -> Option<&'a str> { + (event.kind == Kind::Token) + .then(|| self.source.get(event.start..event.end)) + .flatten() + } + fn event(&self, at: usize) -> NativeEvent { + wire::event( + self.payload[80 + at * 24..80 + (at + 1) * 24] + .try_into() + .expect("admitted width"), + ) + } + /// Borrow the accepted root; rejected input has tokens but no accepted AST. + #[must_use] + pub fn root(&self) -> Option> { + self.accepted.then_some(NativeNode { view: self, at: 0 }) + } + fn range(&self, at: usize) -> SourceRange { + SourceRange { + start: self.event(at).start, + end: self.event(self.index[at].end).end, + } + } +} +/// Borrowed AST node; ids and fields remain those emitted by Scheme. +#[derive(Clone, Copy, Debug)] +pub struct NativeNode<'v, 'a> { + view: &'v NativeArtifactView<'a>, + at: usize, +} +impl<'v, 'a> NativeNode<'v, 'a> { + /// Canonical producer node id. + #[must_use] + pub fn id(&self) -> u64 { + self.view.event(self.at).id + } + /// Published node kind name. + #[must_use] + pub fn kind(&self) -> &'a str { + &self.view.catalog.kinds[self.view.event(self.at).symbol as usize].name + } + /// Exact source range of this node. + #[must_use] + pub fn range(&self) -> SourceRange { + self.view.range(self.at) + } + /// Borrow the original source substring, including trivia. + #[must_use] + pub fn text(&self) -> &'a str { + let r = self.range(); + &self.view.source[r.start..r.end] + } + /// Parent node, treating field scopes as associations rather than nodes. + #[must_use] + pub fn parent(&self) -> Option { + self.view.index[self.at].parent.map(|at| Self { + view: self.view, + at, + }) + } + /// Direct node/token children, preserving order across field scopes. + #[must_use] + pub fn children(&self) -> NativeChildren<'v, 'a> { + NativeChildren { + view: self.view, + next: self.at + 1, + end: self.view.index[self.at].end, + } + } + /// Direct named field scopes; repeated fields remain distinct. + #[must_use] + pub fn fields(&self) -> NativeFields<'v, 'a> { + NativeFields { + view: self.view, + next: self.at + 1, + end: self.view.index[self.at].end, + } + } + /// All occurrences of a declared field in source order. + pub fn field(&self, name: &str) -> impl Iterator> { + self.fields().filter(move |f| f.name() == name) + } +} +/// Borrowed field scope, including an empty or repeated field value. +#[derive(Clone, Copy, Debug)] +pub struct NativeField<'v, 'a> { + view: &'v NativeArtifactView<'a>, + at: usize, +} +impl<'v, 'a> NativeField<'v, 'a> { + /// Published field name. + #[must_use] + pub fn name(&self) -> &'a str { + &self.view.catalog.fields[self.view.event(self.at).symbol as usize] + } + /// Exact source span of this field. + #[must_use] + pub fn range(&self) -> SourceRange { + self.view.range(self.at) + } + /// Source text of this field, without copying. + #[must_use] + pub fn text(&self) -> &'a str { + let r = self.range(); + &self.view.source[r.start..r.end] + } + /// Direct field values in producer order. + #[must_use] + pub fn children(&self) -> NativeChildren<'v, 'a> { + NativeChildren { + view: self.view, + next: self.at + 1, + end: self.view.index[self.at].end, + } + } +} +/// Borrowed source token from the native scanner's terminal catalog. +#[derive(Clone, Copy, Debug)] +pub struct NativeToken<'v, 'a> { + view: &'v NativeArtifactView<'a>, + at: usize, +} +impl<'v, 'a> NativeToken<'v, 'a> { + /// Scanner token ordinal, independent of syntax node ids. + #[must_use] + pub fn id(&self) -> u64 { + self.view.event(self.at).id + } + /// Original scanner class, before its mapping to a syntax token kind. + #[must_use] + pub fn class(&self) -> &'a str { + &self.view.catalog.terminals[self.view.event(self.at).symbol as usize].name + } + + /// Published syntax kind, falling back to the scanner class when unmapped. + #[must_use] + pub fn kind(&self) -> &'a str { + let terminal = &self.view.catalog.terminals[self.view.event(self.at).symbol as usize]; + terminal.syntax_kind.map_or(terminal.name.as_str(), |kind| { + self.view.catalog.kinds[kind].name.as_str() + }) + } + /// Exact UTF-8 byte span. + #[must_use] + pub fn range(&self) -> SourceRange { + self.view.range(self.at) + } + /// Borrow the original token text. + #[must_use] + pub fn text(&self) -> &'a str { + let r = self.range(); + &self.view.source[r.start..r.end] + } + /// Owning node, if this is accepted syntax. + #[must_use] + pub fn parent(&self) -> Option> { + self.view.index[self.at].parent.map(|at| NativeNode { + view: self.view, + at, + }) + } +} +/// A direct native AST child. +#[derive(Clone, Copy, Debug)] +pub enum NativeElement<'v, 'a> { + Node(NativeNode<'v, 'a>), + Token(NativeToken<'v, 'a>), +} +/// Iterator over direct children, skipping each child subtree in constant time. +pub struct NativeChildren<'v, 'a> { + view: &'v NativeArtifactView<'a>, + next: usize, + end: usize, +} +impl<'v, 'a> Iterator for NativeChildren<'v, 'a> { + type Item = NativeElement<'v, 'a>; + fn next(&mut self) -> Option { + while self.next < self.end { + let at = self.next; + self.next += 1; + match self.view.event(at).kind { + Kind::StartNode => { + self.next = self.view.index[at].end + 1; + return Some(NativeElement::Node(NativeNode { + view: self.view, + at, + })); + } + Kind::Token => { + return Some(NativeElement::Token(NativeToken { + view: self.view, + at, + })); + } + _ => {} + } + } + None + } +} +/// Iterator over direct fields, preserving repeated and empty scopes. +pub struct NativeFields<'v, 'a> { + view: &'v NativeArtifactView<'a>, + next: usize, + end: usize, +} +impl<'v, 'a> Iterator for NativeFields<'v, 'a> { + type Item = NativeField<'v, 'a>; + fn next(&mut self) -> Option { + while self.next < self.end { + let at = self.next; + self.next += 1; + match self.view.event(at).kind { + Kind::StartField => { + self.next = self.view.index[at].end + 1; + return Some(NativeField { + view: self.view, + at, + }); + } + Kind::StartNode => self.next = self.view.index[at].end + 1, + _ => {} + } + } + None + } +} + +/// Admission and navigation contain no references into their payload owner. +#[derive(Debug)] +pub(crate) struct Navigation { + pub(super) accepted: bool, + pub(super) index: Vec, +} +impl Navigation { + pub(crate) fn decode( + payload: &[u8], + source: &str, + catalog: &NativeCatalog, + ) -> Result { + admission::admit(payload, source, catalog) + } +} + +#[cfg(test)] +#[path = "../tests/unit/storage.rs"] +mod storage_tests; + +#[cfg(test)] +#[path = "../tests/unit/admission_fusion.rs"] +mod admission_fusion_tests; +#[cfg(any(test, feature = "benchmark-controls"))] +#[path = "../tests/support/reference_navigation.rs"] +mod reference_navigation; diff --git a/rust/gerbil-parser-artifact/src/wire.rs b/rust/gerbil-parser-artifact/src/wire.rs new file mode 100644 index 00000000..10c0a46e --- /dev/null +++ b/rust/gerbil-parser-artifact/src/wire.rs @@ -0,0 +1,66 @@ +//! Fixed-width little-endian GPA1 records; admission precedes public iteration. +use std::fmt; + +/// Canonical node, field and token record tags from the Scheme producer. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NativeEventKind { + StartNode, + FinishNode, + StartField, + FinishField, + Token, +} + +/// Symbol indexes refer to the bound descriptor's kinds, fields, or terminals. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NativeEvent { + pub kind: NativeEventKind, + pub symbol: u32, + pub id: u64, + pub start: usize, + pub end: usize, +} + +/// A rejected wire invariant, with its event index when applicable. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NativeArtifactError { + pub reason: &'static str, + pub event: Option, +} +impl fmt::Display for NativeArtifactError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "native artifact {} at event {:?}", + self.reason, self.event + ) + } +} +impl std::error::Error for NativeArtifactError {} + +pub(super) fn error(reason: &'static str, event: Option) -> NativeArtifactError { + NativeArtifactError { reason, event } +} +pub(super) fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes( + bytes[offset..offset + 4] + .try_into() + .expect("admitted record width"), + ) +} +pub(super) fn event(row: &[u8; 24]) -> NativeEvent { + NativeEvent { + kind: match u32_at(row, 0) { + 1 => NativeEventKind::StartNode, + 2 => NativeEventKind::FinishNode, + 3 => NativeEventKind::StartField, + 4 => NativeEventKind::FinishField, + 5 => NativeEventKind::Token, + _ => unreachable!("admitted event tag"), + }, + symbol: u32_at(row, 4), + id: u64::from_le_bytes(row[8..16].try_into().expect("admitted record width")), + start: u32_at(row, 16) as usize, + end: u32_at(row, 20) as usize, + } +} diff --git a/rust/gerbil-parser-artifact/tests/fixtures/languages.bin b/rust/gerbil-parser-artifact/tests/fixtures/languages.bin new file mode 100644 index 00000000..af565c90 Binary files /dev/null and b/rust/gerbil-parser-artifact/tests/fixtures/languages.bin differ diff --git a/rust/gerbil-parser-artifact/tests/support/reference_admission.rs b/rust/gerbil-parser-artifact/tests/support/reference_admission.rs new file mode 100644 index 00000000..6de8be26 --- /dev/null +++ b/rust/gerbil-parser-artifact/tests/support/reference_admission.rs @@ -0,0 +1,140 @@ +//! Frozen pre-fusion admission for matched complete-request qualification. +//! Validate identities, canonical records, nesting, and exact UTF-8 coverage. +use crate::wire::{self, NativeArtifactError, NativeEventKind as Kind}; +use crate::{KindCategory, NativeCatalog}; +use sha2::{Digest, Sha256}; + +pub(crate) fn admit( + payload: &[u8], + source: &str, + language: &NativeCatalog, +) -> Result { + let fail = |reason| wire::error(reason, None); + if source.len() > 67_108_864 + || payload.len() < 80 + || &payload[..4] != b"GPA1" + || wire::u32_at(payload, 4) != 1 + || wire::u32_at(payload, 8) > 1 + { + return Err(fail("header")); + } + let count = wire::u32_at(payload, 12) as usize; + if count.checked_mul(24).and_then(|n| n.checked_add(80)) != Some(payload.len()) { + return Err(fail("length")); + } + let digest = language + .grammar_digest + .as_str() + .strip_prefix("sha256:") + .ok_or_else(|| fail("grammar-digest"))?; + if digest.len() != 64 + || !digest.is_ascii() + || !(0..32).all(|i| { + u8::from_str_radix(&digest[i * 2..i * 2 + 2], 16).ok() == Some(payload[16 + i]) + }) + { + return Err(fail("grammar-digest")); + } + if payload[48..80] != Sha256::digest(source.as_bytes())[..] { + return Err(fail("source-digest")); + } + let accepted = wire::u32_at(payload, 8) == 0; + validate_events(payload, source, language, language.fields.len(), accepted)?; + Ok(accepted) +} + +fn validate_events( + payload: &[u8], + source: &str, + language: &NativeCatalog, + fields: usize, + accepted: bool, +) -> Result<(), NativeArtifactError> { + let (mut offset, mut nodes, mut tokens, mut roots) = (0, 0, 0, 0); + let mut stack = Vec::new(); + for (index, row) in payload[80..].as_chunks::<24>().0.iter().enumerate() { + let fail = |reason| wire::error(reason, Some(index)); + if !(1..=5).contains(&wire::u32_at(row, 0)) { + return Err(fail("event-tag")); + } + let event = wire::event(row); + match event.kind { + Kind::StartNode => { + if !accepted + || event.id != nodes + || event.start != offset + || event.end != 0 + || (stack.is_empty() + && (roots != 0 || event.symbol as usize != language.root_kind)) + || language + .kinds + .get(event.symbol as usize) + .is_none_or(|kind| kind.category != KindCategory::Node) + { + return Err(fail("start-node")); + } + if stack.is_empty() { + roots += 1; + } + nodes += 1; + stack.push((Kind::StartNode, event.id, event.symbol)); + } + Kind::FinishNode => { + if event.start != 0 + || event.end != offset + || stack.pop() != Some((Kind::StartNode, event.id, event.symbol)) + { + return Err(fail("finish-node")); + } + } + Kind::StartField => { + if stack.is_empty() + || event.id != 0 + || event.start != offset + || event.end != 0 + || event.symbol as usize >= fields + { + return Err(fail("start-field")); + } + let owner = stack + .iter() + .rev() + .find(|(kind, _, _)| *kind == Kind::StartNode) + .ok_or_else(|| fail("field-owner"))?; + if !language.kinds[owner.2 as usize] + .fields + .contains(&language.fields[event.symbol as usize]) + { + return Err(fail("field-owner")); + } + stack.push((Kind::StartField, 0, event.symbol)); + } + Kind::FinishField => { + if event.id != 0 + || event.start != 0 + || event.end != offset + || stack.pop() != Some((Kind::StartField, 0, event.symbol)) + { + return Err(fail("finish-field")); + } + } + Kind::Token => { + if event.id != tokens + || event.start != offset + || event.end <= offset + || source.get(event.start..event.end).is_none() + || (accepted && stack.is_empty()) + || event.symbol as usize >= language.terminals.len() + { + return Err(fail("token")); + } + tokens += 1; + offset = event.end; + } + } + } + if !stack.is_empty() || offset != source.len() || roots != usize::from(accepted) { + return Err(wire::error("coverage", None)); + } + Ok(()) +} diff --git a/rust/gerbil-parser-artifact/tests/support/reference_navigation.rs b/rust/gerbil-parser-artifact/tests/support/reference_navigation.rs new file mode 100644 index 00000000..740f85e4 --- /dev/null +++ b/rust/gerbil-parser-artifact/tests/support/reference_navigation.rs @@ -0,0 +1,47 @@ +//! Frozen second traversal used only by the complete-request benchmark control. +use super::{Index, Navigation}; +use crate::{NativeArtifactError, NativeCatalog, NativeEventKind as Kind, wire}; +#[path = "reference_admission.rs"] +mod reference_admission; +pub(super) fn decode( + payload: &[u8], + source: &str, + catalog: &NativeCatalog, +) -> Result { + let accepted = reference_admission::admit(payload, source, catalog)?; + let count = (payload.len() - 80) / 24; + let mut index = vec![ + Index { + end: 0, + parent: None + }; + count + ]; + let mut stack = Vec::new(); + let mut nodes = Vec::new(); + for (at, row) in payload[80..].as_chunks::<24>().0.iter().enumerate() { + let event = wire::event(row); + index[at].parent = nodes.last().copied(); + match event.kind { + Kind::StartNode | Kind::StartField => { + stack.push(at); + if event.kind == Kind::StartNode { + nodes.push(at); + } + } + Kind::FinishNode | Kind::FinishField => { + let start = stack + .pop() + .ok_or_else(|| wire::error("index-nesting", Some(at)))?; + index[start].end = at; + if event.kind == Kind::FinishNode { + nodes.pop(); + } + } + Kind::Token => { + index[at].end = at; + } + } + } + Ok(Navigation { accepted, index }) +} diff --git a/rust/gerbil-parser-artifact/tests/unit/admission_fusion.rs b/rust/gerbil-parser-artifact/tests/unit/admission_fusion.rs new file mode 100644 index 00000000..6b86fb06 --- /dev/null +++ b/rust/gerbil-parser-artifact/tests/unit/admission_fusion.rs @@ -0,0 +1,48 @@ +//! Preserve complete admission decisions and every navigation entry. +use super::{Navigation, reference_navigation}; +use crate::NativeCatalog; +fn take<'a>(input: &mut &'a [u8]) -> &'a [u8] { + let (length, rest) = input.split_at(4); + let length = u32::from_le_bytes(length.try_into().unwrap()) as usize; + let (value, rest) = rest.split_at(length); + *input = rest; + value +} +fn parity(payload: &[u8], source: &str, catalog: &NativeCatalog) { + let candidate = Navigation::decode(payload, source, catalog); + let control = reference_navigation::decode(payload, source, catalog); + match (candidate, control) { + (Ok(candidate), Ok(control)) => { + assert_eq!(candidate.accepted, control.accepted); + assert_eq!(candidate.index, control.index); + } + (Err(candidate), Err(control)) => assert_eq!(candidate, control), + (candidate, control) => panic!("admission mismatch: {candidate:?} / {control:?}"), + } +} +#[test] +fn fused_admission_matches_former_decoder_for_all_native_languages_and_corrupt_rows() { + let mut input = include_bytes!("../fixtures/languages.bin").as_slice(); + let mut languages = 0; + while !input.is_empty() { + let _name = take(&mut input); + let descriptor = take(&mut input); + let source = std::str::from_utf8(take(&mut input)).unwrap(); + let payload = take(&mut input); + input = &input[1..]; + let catalog = NativeCatalog::from_descriptor(descriptor).unwrap(); + parity(payload, source, &catalog); + for at in 0..payload.len() { + let mut damaged = payload.to_vec(); + damaged[at] ^= 0x80; + parity(&damaged, source, &catalog); + } + // Extra/truncated records and source identity remain rejected equally. + for length in [0, 4, 79, payload.len() - 1] { + parity(&payload[..length], source, &catalog); + } + parity(payload, "wrong source", &catalog); + languages += 1; + } + assert_eq!(languages, 20); +} diff --git a/rust/gerbil-parser-artifact/tests/unit/datum.rs b/rust/gerbil-parser-artifact/tests/unit/datum.rs new file mode 100644 index 00000000..21e5c7c4 --- /dev/null +++ b/rust/gerbil-parser-artifact/tests/unit/datum.rs @@ -0,0 +1,37 @@ +use super::read; +#[test] +fn metadata_reader_is_inert_and_rejects_duplicate_keys_trailing_and_deep_forms() { + for source in [ + "(eval 1)", + "(object (\"x\" 1) (\"x\" 2))", + "(list 1) 2", + "(list #.(evil))", + "(list", + "\"\\q\"", + ] { + assert!(read(source.as_bytes()).is_err(), "{source}"); + } + let deep = format!("{}0{}", "(list ".repeat(66), ")".repeat(66)); + assert!(read(deep.as_bytes()).is_err()); +} +#[test] +fn metadata_reader_preserves_scheme_strings_lists_numbers_and_booleans() { + let value = read(br#"(object ("s" "a\n\"\x3b1;\\") ("v" (list #t #f 42)))"#).unwrap(); + assert_eq!(value["s"], "a\n\"α\\"); + assert_eq!(value["v"], serde_json::json!([true, false, 42])); +} + +#[test] +fn native_gambit_writer_control_character_spellings_are_preserved() { + for (wire, expected) in [ + (r#""\x0;""#, '\0'), + (r#""\a""#, '\u{7}'), + (r#""\b""#, '\u{8}'), + (r#""\v""#, '\u{b}'), + (r#""\f""#, '\u{c}'), + (r#""\x1b;""#, '\u{1b}'), + (r#""\x7f;""#, '\u{7f}'), + ] { + assert_eq!(read(wire.as_bytes()).unwrap(), expected.to_string()); + } +} diff --git a/rust/gerbil-parser-artifact/tests/unit/language_alignment.rs b/rust/gerbil-parser-artifact/tests/unit/language_alignment.rs new file mode 100644 index 00000000..7aff459c --- /dev/null +++ b/rust/gerbil-parser-artifact/tests/unit/language_alignment.rs @@ -0,0 +1,58 @@ +//! Real configured Scheme products admitted without generated Rust parser tables. +use crate::{NativeArtifactView, NativeCatalog}; +// Fixture records are four little-endian length-prefixed byte slices and a status. +const CASES: &[u8] = include_bytes!("../fixtures/languages.bin"); + +fn take_bytes<'a>(input: &mut &'a [u8]) -> &'a [u8] { + let (length, rest) = input.split_at(4); + let length = u32::from_le_bytes(length.try_into().unwrap()) as usize; + let (value, rest) = rest.split_at(length); + *input = rest; + value +} + +#[test] +fn all_production_entries_publish_accepted_and_rejected_native_results() { + let mut input = CASES; + let mut controls = 0; + let mut entries = std::collections::HashMap::new(); + while !input.is_empty() { + let name = std::str::from_utf8(take_bytes(&mut input)).unwrap(); + let descriptor = take_bytes(&mut input); + let source = take_bytes(&mut input); + let payload = take_bytes(&mut input); + let accepted = match input[0] { + 0 => false, + 1 => true, + status => panic!("invalid fixture status {status}"), + }; + input = &input[1..]; + controls += 1; + let statuses = entries.entry(name).or_insert([false; 2]); + assert!(!statuses[usize::from(accepted)], "duplicate control {name}"); + statuses[usize::from(accepted)] = true; + let catalog = NativeCatalog::from_descriptor(descriptor).expect(name); + let source = std::str::from_utf8(source).expect(name); + let view = NativeArtifactView::decode(payload, source, &catalog).expect(name); + assert_eq!(view.accepted(), accepted, "{name}"); + if let Some(root) = view.root() { + assert_eq!(root.kind(), catalog.root_kind(), "{name}"); + assert_eq!(root.text(), source, "{name}"); + } else { + assert!(!accepted, "{name}"); + } + let mut offset = 0; + for (ordinal, token) in view.tokens().enumerate() { + assert_eq!(token.id(), u64::try_from(ordinal).unwrap(), "{name}"); + assert_eq!(token.range().start, offset, "{name}"); + assert_eq!(token.text(), &source[offset..token.range().end], "{name}"); + assert_eq!(token.text().as_ptr(), source[offset..].as_ptr(), "{name}"); + assert_eq!(token.parent().is_some(), accepted, "{name}"); + offset = token.range().end; + } + assert_eq!(offset, source.len(), "{name}"); + } + assert_eq!(controls, 20); + assert_eq!(entries.len(), 10); + assert!(entries.values().all(|statuses| *statuses == [true; 2])); +} diff --git a/rust/gerbil-parser-artifact/tests/unit/storage.rs b/rust/gerbil-parser-artifact/tests/unit/storage.rs new file mode 100644 index 00000000..5ab8006e --- /dev/null +++ b/rust/gerbil-parser-artifact/tests/unit/storage.rs @@ -0,0 +1,100 @@ +//! Cache reuse retains admission, exact catalog binding and original source slices. +use super::NativeArtifactView; +use crate::{NativeArtifactBytes, NativeArtifactError, NativeArtifactStorage, NativeCatalog}; + +struct Payload<'a>(&'a [u8]); +impl NativeArtifactBytes for Payload<'_> { + fn bytes(&self) -> Result<&[u8], NativeArtifactError> { + Ok(self.0) + } +} +fn take<'a>(input: &mut &'a [u8]) -> &'a [u8] { + let (length, rest) = input.split_at(4); + let length = u32::from_le_bytes(length.try_into().unwrap()) as usize; + let (value, rest) = rest.split_at(length); + *input = rest; + value +} +#[test] +fn cached_views_share_navigation_for_all_twenty_real_native_controls() { + let mut input = include_bytes!("../fixtures/languages.bin").as_slice(); + let mut count = 0; + while !input.is_empty() { + let _name = take(&mut input); + let descriptor = take(&mut input); + let source = std::str::from_utf8(take(&mut input)).unwrap(); + let payload = take(&mut input); + let accepted = input[0] == 1; + input = &input[1..]; + let catalog = NativeCatalog::from_descriptor(descriptor).unwrap(); + let storage = NativeArtifactStorage::new(Payload(payload), source); + let first = storage.view(&catalog).unwrap(); + let second = storage.view(&catalog).unwrap(); + assert_eq!(first.index.as_ptr(), second.index.as_ptr()); + assert_eq!(first.accepted(), accepted); + assert_eq!( + second.events().collect::>(), + first.events().collect::>() + ); + for token in second.tokens() { + assert_eq!( + token.text().as_ptr(), + source[token.range().start..].as_ptr() + ); + } + drop(first); + drop(second); + let moved = storage; + assert_eq!(moved.view(&catalog).unwrap().accepted(), accepted); + let different_catalog = NativeCatalog::from_descriptor(descriptor).unwrap(); + assert_eq!( + moved.view(&different_catalog).unwrap_err().reason, + "native-view-catalog" + ); + count += 1; + } + assert_eq!(count, 20); +} +#[test] +fn cached_storage_preserves_source_and_wire_rejection() { + let mut input = include_bytes!("../fixtures/languages.bin").as_slice(); + let _name = take(&mut input); + let catalog = NativeCatalog::from_descriptor(take(&mut input)).unwrap(); + let source = std::str::from_utf8(take(&mut input)).unwrap(); + let payload = take(&mut input); + let wrong_source = NativeArtifactStorage::new(Payload(payload), "different source"); + assert_eq!( + wrong_source.view(&catalog).unwrap_err().reason, + NativeArtifactView::decode(payload, "different source", &catalog) + .unwrap_err() + .reason + ); + let mut corrupt = payload.to_vec(); + corrupt[0] = 0; + let corrupt_storage = NativeArtifactStorage::new(Payload(&corrupt), source); + assert!(corrupt_storage.view(&catalog).is_err()); + assert!(corrupt_storage.view(&catalog).is_err()); +} + +#[test] +fn owner_cannot_rebind_cached_navigation_to_another_allocation() { + struct Rebinding<'a>(std::cell::Cell<&'a [u8]>); + impl NativeArtifactBytes for Rebinding<'_> { + fn bytes(&self) -> Result<&[u8], NativeArtifactError> { + Ok(self.0.get()) + } + } + let mut input = include_bytes!("../fixtures/languages.bin").as_slice(); + let _name = take(&mut input); + let catalog = NativeCatalog::from_descriptor(take(&mut input)).unwrap(); + let source = std::str::from_utf8(take(&mut input)).unwrap(); + let payload = take(&mut input); + let replacement = payload.to_vec(); + let storage = NativeArtifactStorage::new(Rebinding(std::cell::Cell::new(payload)), source); + assert!(storage.view(&catalog).is_ok()); + storage.owner().0.set(&replacement); + assert_eq!( + storage.view(&catalog).unwrap_err().reason, + "native-view-payload" + ); +} diff --git a/rust/gerbil-parser-ffi/Cargo.toml b/rust/gerbil-parser-ffi/Cargo.toml new file mode 100644 index 00000000..b341a5a8 --- /dev/null +++ b/rust/gerbil-parser-ffi/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "gerbil-parser-ffi" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Thread-bound Rust ownership for the Gerbil parser language C ABI" +readme = "README.org" + +[features] +standalone = [] +benchmark-controls = ["gerbil-parser-artifact/benchmark-controls"] +tokio = ["standalone", "dep:tokio"] + +[dependencies] +gerbil-parser-artifact.workspace = true +tokio = { version = "1.53.1", features = ["sync"], optional = true } + +[dev-dependencies] +tokio = { version = "1.53.1", features = ["rt", "macros", "sync"] } + +# Unsafe code is confined to this FFI crate; the AOT runtime remains forbidden. +[lints.rust] +unsafe_op_in_unsafe_fn = "deny" +[lints.clippy] +all = "warn" +pedantic = "warn" diff --git a/rust/gerbil-parser-ffi/README.org b/rust/gerbil-parser-ffi/README.org new file mode 100644 index 00000000..4d1d0c5d --- /dev/null +++ b/rust/gerbil-parser-ffi/README.org @@ -0,0 +1,157 @@ +#+title: Rust FFI language ownership + +This crate calls the language-independent C ABI from an already initialized +Gerbil embedding host. It owns language handles and C result buffers. The +host must keep the VM and loaded modules alive until all sessions are dropped. +The optional =standalone= feature provides owned VM initialization and shutdown; +see the standalone host contract below. + +A language author generates the Scheme/C entry with +=generate-language-pack= and links that pack through the normal Gerbil +builder. Rust declares only the generated factory; the generic ABI functions +are supplied by =LanguageApi::linked=: + +#+begin_src rust +use gerbil_parser_ffi::{LanguageApi, NativeSession}; +unsafe extern "C" { + fn my_language_create() -> u64; +} +// SAFETY: this host has initialized the pack on this OS thread and will keep +// the VM and linked libraries alive until this scope and its borrowers end. +let session = unsafe { NativeSession::attach(LanguageApi::linked(my_language_create)) }?; +let language = session.language()?; +let parsed = language.parse(source)?; +let view = parsed.view()?; +let root = view.root(); // None for syntax rejection. +#+end_src + +=view= obtains its vocabulary and product digest from the same native handle. +Canonical and contextual packs use this same API; no generated Rust parser +specification or field-count argument is required. The immutable descriptor is +admitted once per handle. =root=, =children= and =field(name)= borrow the original +source and retain ordered nodes, tokens, repeated fields and byte spans. +A syntax rejection is a successful transport with =view.accepted() == false=. +=for token in view.tokens()= still exposes the original scanner classes, syntax +kinds, ids, byte spans and borrowed text for diagnostics. This does not fabricate +an accepted AST or recovery result. Accepted and rejected paths use the same +handle catalog and retain the same lifetime restrictions. +Handles, results and views are tied together by Rust borrows. Sessions cannot +cross threads. Dropping a result releases its C allocation; dropping a language +releases its registered handle. The wrapper does not copy source or GPA1 bytes, +though the C/Scheme boundary still copies input and publishes a C-owned payload. + +For Source descriptors, each language handle retains its own Scheme scanner +history across calls. Complete recognition and GPA1 publication still run. +History commits after successful C result allocation and copying; failed +transport leaves the previous history intact. The cache retains lexical state +without retaining the previous AST/artifact. Releasing the handle releases +that history. See the [[file:../../docs/research/scheme-source-edit-sessions.org][Source edit-session research contract]]. + +Verification uses a real Rust static consumer in +=t/fixtures/rust-language-abi=, linked into the compiled Gerbil probe. +Run the repository's =qualify.ss local --suite ffi= or its Scheme +Ubuntu CI. Compile-fail doctests verify the thread and borrow restrictions. + +* Rust ABI qualification build + +=build-rust-ffi-tests.ss= uses the official =std/build-script= / =std/make= +owner to compile and link the disposable Rust/C transport probe. Its +=optimize: #f= setting avoids whole-library optimizer analysis for this test +entry; Gambit still produces compiled machine code. Language packs and parser runtime +objects retain their own optimization settings. =verbose: 9= reports real +compiler work, and the fixture preloader traces imports throughout the build. + +Both local and CI entrypoints use the native Cargo compiler progress proxy. +The Rust consumer build/link gates remain 90 seconds with 30 seconds of output +inactivity; execution remains 90 seconds with 5 seconds of output inactivity. +Qualification requires actual native handle/result ownership and 100-call +markers, not only a successful static Rust build. + +* Standalone C and Cargo hosts + +Build the parser package and language pack with their normal Gerbil build owner, +then compile the language module into one native runtime library: + +#+begin_src shell +gerbil env gxi build-qualification.ss compile +gerbil env gxi qualify.ss shared-library --language-module my-language/abi \ + --output /absolute/path/libmy_language.so +#+end_src + +Use =.dylib= on macOS. Repeat =--language-module= for additional packs. The +builder delegates Scheme generation, dependency closure and static compilation +to the public Gerbil compiler API. The installed SDK's GCC compiler, +optimization flags and ABI macros are retained; ambient =CC= does not replace +the Scheme compiler. Its temporary root emits no unused dynamic +modules, and all of its intermediate files are removed. Do not mix an +unoptimized module with previously generated optimizer interfaces or consumers; +rebuild the producer and consumers through their normal build owner. + +The C ABI headers and lifecycle implementation live together in +=include/gerbil-parser/=; the bundle builder compiles =runtime.c= once with the +SDK compiler. C hosts include =gerbil-parser/runtime.h=, initialize once, use +generated pack factories and the generic language ABI, release all handles and results, and +shut down on the initializing OS thread. Status =-2= retains a live runtime +while C resources exist. Successful shutdown uses normal Gambit cleanup and +permanently closes admission. Loading another Gambit VM or restarting this +runtime in the same process is unsupported. + +Cargo hosts enable this crate's =standalone= feature, link that library in their +usual =build.rs=, and declare only their generated factory: + +#+begin_src rust +use gerbil_parser_ffi::{NativeRuntime, RuntimeApi}; +unsafe extern "C" { fn my_language_create() -> u64; } +// SAFETY: this is the only Gambit VM, and its library remains loaded. +let mut runtime = unsafe { NativeRuntime::start(RuntimeApi::linked(my_language_create)) }?; +{ + let language = runtime.language()?; + let parsed = language.parse(source)?; + let view = parsed.view()?; + let root = view.root(); // None for syntax rejection. +} +runtime.close()?; +#+end_src + +Language, result and view borrows prevent closing their runtime. =Drop= closes +it after its borrowers end; =close= permits reporting cleanup errors explicitly. +Independently retained C resources must also be released before dropping it. +The default feature set remains suitable for hosts that already own the VM. + +=qualify.ss host= builds independent C and Rust process hosts and +requires normal cleanup, live-resource barriers, foreign-thread rejection and +closed-runtime rejection. Builds retain 90/30 second total/inactivity limits; +execution retains 90/5 seconds. Build progress reports actual preprocessed, IR, +assembly and object file writes. No forced child exit qualifies this path. + +* Tokio callers + +Enable =features = ["tokio"]= for async callers of a standalone bundle. Link the +same generated language library and use =TokioParser=: + +#+begin_src rust +use gerbil_parser_ffi::{RuntimeApi, TokioParser}; +unsafe extern "C" { fn my_language_create() -> u64; } +// SAFETY: the process owns exactly one VM; the bundle stays loaded. +let parser = unsafe { + TokioParser::start(RuntimeApi::linked(my_language_create), 32).await? +}; +let parsed = parser.parse(source.to_owned()).await?; +assert!(parsed.view()?.accepted()); +parser.shutdown().await?; +// The independent Rust snapshot remains valid after native cleanup. +let view = parsed.view()?; +#+end_src + +Clients may be cloned and sent to Tokio tasks. The dedicated owner thread +retains the VM and language; each reply owns its source and a GPA1 byte snapshot. +The synchronous =NativeRuntime= interface continues to offer borrowed C payloads. +Async transfer copies each completed payload once before freeing the C allocation. +Queue capacity bounds admitted pending requests. Caller-held sources and retained +results have independent lifetimes. A shutdown request stops every clone; later +requests return =OwnerStopped=. Dropping all clients drains pending requests and +performs cleanup. The VM is one-shot and cannot be restarted in the same process. + +=qualify.ss host= exercises 32 concurrent complete 64-record requests, Unicode, +syntax rejection and views retained after shutdown. Async ownership support +alone does not establish lower parser CPU or parallel Scheme execution. diff --git a/rust/gerbil-parser-ffi/src/abi.rs b/rust/gerbil-parser-ffi/src/abi.rs new file mode 100644 index 00000000..66a4f8d9 --- /dev/null +++ b/rust/gerbil-parser-ffi/src/abi.rs @@ -0,0 +1,108 @@ +//! Exact public C representations and the only raw payload access. +use crate::NativeError; + +/// Layout of `gerbil_parser_result` from `language.h`. +/// Fields are public for foreign declarations; safe consumers use `NativePayload`. +#[repr(C)] +#[derive(Debug)] +pub struct RawResult { + pub status: i32, + pub payload: *mut u8, + pub length: usize, +} + +/// Functions from one loaded language ABI runtime and an initialized language pack. +/// The generated pack contributes only `create`; every other function is generic. +#[derive(Clone, Copy)] +pub struct LanguageApi { + pub version: unsafe extern "C" fn() -> u32, + pub is_owner_thread: unsafe extern "C" fn() -> i32, + pub create: unsafe extern "C" fn() -> u64, + pub descriptor: unsafe extern "C" fn(u64, *mut RawResult) -> i32, + pub parse: unsafe extern "C" fn(u64, *const u8, usize, *mut RawResult) -> i32, + pub release: unsafe extern "C" fn(u64) -> i32, + pub result_init: unsafe extern "C" fn(*mut RawResult), + pub result_release: unsafe extern "C" fn(*mut RawResult), +} + +impl RawResult { + pub(crate) fn empty() -> Self { + Self { + status: 0, + payload: std::ptr::null_mut(), + length: 0, + } + } + + // The caller owns this result and keeps its language/session alive. The API + // contract supplies a live C allocation, never an arbitrary foreign pointer. + pub(crate) fn bytes(&self) -> Result<&[u8], NativeError> { + if self.length == 0 { + return Ok(&[]); + } + if self.payload.is_null() || self.length > isize::MAX as usize { + return Err(NativeError::InvalidBuffer); + } + // SAFETY: attach requires ABI-conforming allocations. The result owns + // this allocation; &self prevents its release while the slice is live. + Ok(unsafe { std::slice::from_raw_parts(self.payload, self.length) }) + } +} + +unsafe extern "C" { + #[cfg(feature = "standalone")] + fn gerbil_parser_runtime_init() -> i32; + #[cfg(feature = "standalone")] + fn gerbil_parser_runtime_shutdown() -> i32; + fn gerbil_parser_language_abi_version() -> u32; + fn gerbil_parser_language_is_owner_thread() -> i32; + fn gerbil_parser_language_descriptor(handle: u64, result: *mut RawResult) -> i32; + fn gerbil_parser_language_parse( + handle: u64, + source: *const u8, + length: usize, + result: *mut RawResult, + ) -> i32; + fn gerbil_parser_language_release(handle: u64) -> i32; + fn gerbil_parser_result_init(result: *mut RawResult); + fn gerbil_parser_result_release(result: *mut RawResult); +} + +impl LanguageApi { + /// Bind the generic linked C ABI to one generated pack's factory. + /// This only constructs the function table; `NativeSession::attach` admits it. + #[must_use] + pub const fn linked(create: unsafe extern "C" fn() -> u64) -> Self { + Self { + version: gerbil_parser_language_abi_version, + is_owner_thread: gerbil_parser_language_is_owner_thread, + create, + descriptor: gerbil_parser_language_descriptor, + parse: gerbil_parser_language_parse, + release: gerbil_parser_language_release, + result_init: gerbil_parser_result_init, + result_release: gerbil_parser_result_release, + } + } +} + +/// Functions from one standalone runtime bundle and its selected language pack. +#[cfg(feature = "standalone")] +#[derive(Clone, Copy)] +pub struct RuntimeApi { + pub init: unsafe extern "C" fn() -> i32, + pub shutdown: unsafe extern "C" fn() -> i32, + pub language: LanguageApi, +} +#[cfg(feature = "standalone")] +impl RuntimeApi { + /// Bind a compiled standalone bundle to its selected generated factory. + #[must_use] + pub const fn linked(create: unsafe extern "C" fn() -> u64) -> Self { + Self { + init: gerbil_parser_runtime_init, + shutdown: gerbil_parser_runtime_shutdown, + language: LanguageApi::linked(create), + } + } +} diff --git a/rust/gerbil-parser-ffi/src/language.rs b/rust/gerbil-parser-ffi/src/language.rs new file mode 100644 index 00000000..3f238d87 --- /dev/null +++ b/rust/gerbil-parser-ffi/src/language.rs @@ -0,0 +1,180 @@ +//! Handle/result ownership with borrowed, validated GPA1 views. +use crate::{NativeError, NativeSession, RawResult}; +use gerbil_parser_artifact::{ + NativeArtifactBytes, NativeArtifactError, NativeArtifactStorage, NativeArtifactView, + NativeCatalog, +}; +use std::cell::OnceCell; + +/// A unique registered language, released once when its last borrow ends. +pub struct NativeLanguage<'runtime> { + pub(crate) session: &'runtime NativeSession, + pub(crate) handle: u64, + pub(crate) catalog: OnceCell, +} + +/// A C-owned result. Its bytes cannot outlive the result or replace its allocation. +pub struct NativePayload<'language, 'runtime> { + language: &'language NativeLanguage<'runtime>, + raw: RawResult, +} + +/// A result bound to the exact source used by the native parser. +/// +/// A live result prevents releasing its handle: +/// ```compile_fail +/// fn release(language: gerbil_parser_ffi::NativeLanguage<'_>) { +/// let parsed = language.parse("a=1\\n").unwrap(); +/// drop(language); +/// parsed.bytes().unwrap(); +/// } +/// ``` +/// A live view prevents freeing its C payload: +/// ```compile_fail +/// fn free(language: &gerbil_parser_ffi::NativeLanguage<'_>) { +/// let parsed = language.parse("a=1\\n").unwrap(); +/// let view = parsed.view().unwrap(); +/// drop(parsed); +/// view.accepted(); +/// } +/// ``` +pub struct NativeParsed<'language, 'runtime, 'source> { + storage: NativeArtifactStorage<'language, 'source, NativePayload<'language, 'runtime>>, +} + +impl<'runtime> NativeLanguage<'runtime> { + /// Read the length-bearing descriptor JSON bytes without copying them. + /// # Errors + /// Returns a native transport or invalid-buffer error. + pub fn descriptor(&self) -> Result, NativeError> { + let mut result = self.result(); + // SAFETY: result is initialized, exclusive, and belongs to this session. + let status = unsafe { (self.session.api.descriptor)(self.handle, &raw mut result.raw) }; + result.admit(status)?; + Ok(result) + } + + /// Read and admit the Scheme result vocabulary once for this handle. + /// # Errors + /// Rejects transport failures or malformed native descriptors. + pub fn catalog(&self) -> Result<&NativeCatalog, NativeArtifactError> { + if let Some(catalog) = self.catalog.get() { + return Ok(catalog); + } + let payload = self.descriptor().map_err(|_| NativeArtifactError { + reason: "native-descriptor", + event: None, + })?; + let bytes = payload.bytes().map_err(|_| NativeArtifactError { + reason: "native-buffer", + event: None, + })?; + let catalog = NativeCatalog::from_descriptor(bytes)?; + self.catalog.set(catalog).map_err(|_| NativeArtifactError { + reason: "native-catalog", + event: None, + })?; + self.catalog.get().ok_or(NativeArtifactError { + reason: "native-catalog", + event: None, + }) + } + + /// Parse UTF-8 source, including embedded NUL, with no Rust payload copy. + /// # Errors + /// Rejects source beyond the C ABI limit or native transport failure. + pub fn parse<'language, 'source>( + &'language self, + source: &'source str, + ) -> Result, NativeError> { + if source.len() > 64 * 1024 * 1024 { + return Err(NativeError::SourceTooLarge); + } + let mut payload = self.result(); + // SAFETY: source is readable for len bytes, the C ABI copies it during + // the call, and payload is an initialized exclusive result. + let status = unsafe { + (self.session.api.parse)( + self.handle, + source.as_ptr(), + source.len(), + &raw mut payload.raw, + ) + }; + payload.admit(status)?; + Ok(NativeParsed { + storage: NativeArtifactStorage::new(payload, source), + }) + } + + fn result(&self) -> NativePayload<'_, 'runtime> { + let mut raw = RawResult::empty(); + // SAFETY: a fresh result is initialized exactly once before use. + unsafe { (self.session.api.result_init)(&raw mut raw) }; + NativePayload { + language: self, + raw, + } + } +} + +impl Drop for NativeLanguage<'_> { + fn drop(&mut self) { + // SAFETY: all result borrows ended, the runtime is alive, and the owned + // handle has never been exposed for an external release. + unsafe { (self.session.api.release)(self.handle) }; + } +} + +impl NativePayload<'_, '_> { + fn admit(&self, status: i32) -> Result<(), NativeError> { + if status != 0 { + return Err(NativeError::Transport(status)); + } + if self.raw.status != 0 { + return Err(NativeError::Transport(self.raw.status)); + } + self.bytes().map(|_| ()) + } + + /// Borrow exactly the C allocation. Zero length never dereferences NULL. + /// # Errors + /// Rejects a NULL nonempty or unrepresentably large allocation. + pub fn bytes(&self) -> Result<&[u8], NativeError> { + self.raw.bytes() + } +} + +impl Drop for NativePayload<'_, '_> { + fn drop(&mut self) { + // SAFETY: this result is owned, initialized, and no byte borrow survives + // its drop. Its language and runtime are still alive on the owner thread. + unsafe { (self.language.session.api.result_release)(&raw mut self.raw) }; + } +} + +impl NativeParsed<'_, '_, '_> { + /// Borrow GPA1 bytes for external consumers; views should use `view`. + /// # Errors + /// Rejects an invalid C allocation. + pub fn bytes(&self) -> Result<&[u8], NativeError> { + self.storage.owner().bytes() + } + + /// Borrow AST/CST access using the descriptor from this exact native handle. + /// # Errors + /// Rejects incompatible product/source identities, catalogs or records. + pub fn view(&self) -> Result, NativeArtifactError> { + let catalog = self.storage.owner().language.catalog()?; + self.storage.view(catalog) + } +} + +impl NativeArtifactBytes for NativePayload<'_, '_> { + fn bytes(&self) -> Result<&[u8], NativeArtifactError> { + NativePayload::bytes(self).map_err(|_| NativeArtifactError { + reason: "native-buffer", + event: None, + }) + } +} diff --git a/rust/gerbil-parser-ffi/src/lib.rs b/rust/gerbil-parser-ffi/src/lib.rs new file mode 100644 index 00000000..59b8befe --- /dev/null +++ b/rust/gerbil-parser-ffi/src/lib.rs @@ -0,0 +1,28 @@ +//! Rust ownership for an already initialized, owner-thread Gerbil language pack. +//! +//! Use `NativeRuntime` with a standalone bundle, or `NativeSession` with an +//! embedding host that already initialized the VM. Libraries stay loaded until +//! the runtime/session and all their borrowers are dropped. +mod abi; +mod language; +#[cfg(feature = "standalone")] +mod runtime; +mod session; + +#[cfg(feature = "standalone")] +pub use abi::RuntimeApi; +pub use abi::{LanguageApi, RawResult}; +pub use language::{NativeLanguage, NativeParsed, NativePayload}; +#[cfg(feature = "standalone")] +pub use runtime::NativeRuntime; +pub use session::{NativeError, NativeSession}; + +pub use gerbil_parser_artifact::{ + NativeArtifactView, NativeCatalog, NativeElement, NativeField, NativeNode, NativeToken, + SourceRange, +}; + +#[cfg(feature = "tokio")] +mod tokio_owner; +#[cfg(feature = "tokio")] +pub use tokio_owner::{OwnedParsed, TokioParser, TokioParserError}; diff --git a/rust/gerbil-parser-ffi/src/runtime.rs b/rust/gerbil-parser-ffi/src/runtime.rs new file mode 100644 index 00000000..676d13fa --- /dev/null +++ b/rust/gerbil-parser-ffi/src/runtime.rs @@ -0,0 +1,86 @@ +//! Unique ownership of a standalone VM, extending the session borrow chain. +use crate::{NativeError, NativeLanguage, NativeSession, RuntimeApi}; + +/// Owner of the bundle's process-wide, one-shot SDK lifecycle. +/// Language/result borrows prevent closing or dropping their runtime. +/// ```compile_fail +/// fn close(runtime: &mut gerbil_parser_ffi::NativeRuntime) { +/// let language = runtime.language().unwrap(); +/// let parsed = language.parse("a=1\n").unwrap(); +/// runtime.close().unwrap(); +/// parsed.bytes().unwrap(); +/// } +/// ``` +pub struct NativeRuntime { + session: NativeSession, + shutdown: unsafe extern "C" fn() -> i32, + closed: bool, +} +impl NativeRuntime { + /// Initialize and own a standalone bundle on this OS thread. + /// # Safety + /// `api` must come from the bundle built by the native library builder (or + /// implement the same `runtime.h` and `language.h` contracts). No other + /// Gambit VM may be initialized in this process. The bundle and its functions + /// must remain loaded until this owner is dropped. The caller must not close + /// the VM or release resources borrowed from this owner independently. + /// Independently created C resources must be released before dropping it. + /// # Errors + /// Returns a lifecycle failure, incompatible ABI, or owner admission error. + pub unsafe fn start(api: RuntimeApi) -> Result { + // SAFETY: the caller supplies the unique live bundle and SDK lifecycle. + let status = unsafe { (api.init)() }; + if status != 0 { + return Err(NativeError::Runtime(status)); + } + // SAFETY: successful setup initialized the same linked language API. + let session = match unsafe { NativeSession::attach(api.language) } { + Ok(session) => session, + Err(error) => { + // SAFETY: initialization succeeded, but no handles were created. + unsafe { (api.shutdown)() }; + return Err(error); + } + }; + Ok(Self { + session, + shutdown: api.shutdown, + closed: false, + }) + } + + /// Register a language in this live runtime. + /// # Errors + /// Rejects a closed runtime or language registration failure. + pub fn language(&self) -> Result, NativeError> { + if self.closed { + return Err(NativeError::RuntimeClosed); + } + self.session.language() + } + + /// Disable VM admission and perform normal SDK cleanup once. + /// Mutable admission excludes all live language/result/view borrows. + /// # Errors + /// Returns the bundle's status if independently retained C resources prevent + /// shutdown. The runtime remains live, and this operation can be retried. + pub fn close(&mut self) -> Result<(), NativeError> { + if self.closed { + return Ok(()); + } + // SAFETY: borrowers have ended, this owner cannot cross OS threads, and + // the bundle enforces the remaining C resource barrier before cleanup. + let status = unsafe { (self.shutdown)() }; + if status != 0 { + return Err(NativeError::Runtime(status)); + } + self.closed = true; + Ok(()) + } +} +impl Drop for NativeRuntime { + fn drop(&mut self) { + // C shutdown retains the VM on error rather than invalidating live data. + let _ = self.close(); + } +} diff --git a/rust/gerbil-parser-ffi/src/session.rs b/rust/gerbil-parser-ffi/src/session.rs new file mode 100644 index 00000000..b0872c07 --- /dev/null +++ b/rust/gerbil-parser-ffi/src/session.rs @@ -0,0 +1,87 @@ +//! Owner-thread admission and the host's VM lifetime boundary. +use crate::{LanguageApi, NativeLanguage}; +use std::{marker::PhantomData, rc::Rc}; + +/// Transport/admission failure. Syntax rejection is represented in the artifact. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NativeError { + AbiVersion, + OwnerThread, + Create, + Transport(i32), + InvalidBuffer, + SourceTooLarge, + Runtime(i32), + RuntimeClosed, +} + +/// Borrowed host runtime admission, deliberately neither `Send` nor `Sync`. +/// Handles borrow this session, and results borrow their handle. +/// +/// ```compile_fail +/// fn send(session: gerbil_parser_ffi::NativeSession) { +/// std::thread::spawn(move || drop(session)); +/// } +/// ``` +pub struct NativeSession { + pub(crate) api: LanguageApi, + _owner: PhantomData>, +} + +impl NativeSession { + /// Admit an initialized runtime on its original OS thread. + /// # Safety + /// Every function must implement `language.h` and belong to the same live + /// initialized runtime. `create` must belong to an initialized pack in that + /// runtime. Functions must not unwind across C. The embedding host must keep + /// the VM, modules and function pointers alive until this session and all its + /// borrowers are dropped, and must not release their handles/results itself. + /// # Errors + /// Returns an incompatible ABI or owner-thread error before creating handles. + pub unsafe fn attach(api: LanguageApi) -> Result { + // SAFETY: the caller guarantees initialized, live ABI function pointers. + if unsafe { (api.version)() } != 2 { + return Err(NativeError::AbiVersion); + } + // SAFETY: this C guard does not enter Scheme on a non-owner OS thread. + if unsafe { (api.is_owner_thread)() } == 0 { + return Err(NativeError::OwnerThread); + } + Ok(Self { + api, + _owner: PhantomData, + }) + } + + /// Create a distinct owned handle using the attached pack's factory. + /// # Errors + /// Returns `Create` if the pack cannot register a language. + pub fn language(&self) -> Result, NativeError> { + // SAFETY: attach admitted this thread; the session cannot cross threads. + let handle = unsafe { (self.api.create)() }; + if handle == 0 { + return Err(NativeError::Create); + } + Ok(NativeLanguage { + session: self, + handle, + catalog: std::cell::OnceCell::new(), + }) + } +} + +impl std::fmt::Display for NativeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::AbiVersion => f.write_str("native language ABI version must be 2"), + Self::OwnerThread => f.write_str("native runtime requires its owner OS thread"), + Self::Create => f.write_str("native language registration failed"), + Self::Transport(status) => write!(f, "native language transport failed: {status}"), + Self::InvalidBuffer => f.write_str("native result has an invalid allocation"), + Self::Runtime(status) => write!(f, "native runtime lifecycle failed: {status}"), + Self::RuntimeClosed => f.write_str("native runtime has been closed"), + Self::SourceTooLarge => f.write_str("native source exceeds 64 MiB"), + } + } +} +impl std::error::Error for NativeError {} diff --git a/rust/gerbil-parser-ffi/src/tokio_owner.rs b/rust/gerbil-parser-ffi/src/tokio_owner.rs new file mode 100644 index 00000000..94ea6e16 --- /dev/null +++ b/rust/gerbil-parser-ffi/src/tokio_owner.rs @@ -0,0 +1,189 @@ +//! Bounded async admission to a single native owner OS thread. +use crate::{NativeArtifactView, NativeCatalog, NativeError, NativeRuntime, RuntimeApi}; +use gerbil_parser_artifact::NativeArtifactError; +use std::sync::Arc; +use tokio::sync::{mpsc, oneshot}; + +/// Independent Rust snapshot; no Scheme handle or C allocation crosses threads. +pub struct OwnedParsed { + source: String, + payload: Vec, + catalog: Arc, +} +impl OwnedParsed { + /// The exact source submitted to this request. + #[must_use] + pub fn source(&self) -> &str { + &self.source + } + /// Owned GPA1 bytes, still usable after the Scheme runtime closes. + #[must_use] + pub fn bytes(&self) -> &[u8] { + &self.payload + } + /// Admit the snapshot against its source and native language catalog. + /// # Errors + /// Returns an artifact validation error for incompatible or malformed bytes. + pub fn view(&self) -> Result, NativeArtifactError> { + NativeArtifactView::decode(&self.payload, &self.source, &self.catalog) + } + /// Frozen artifact decoder for complete-request benchmark qualification. + /// # Errors + /// Rejects artifacts under the independent former admission contract. + #[cfg(feature = "benchmark-controls")] + pub fn reference_view(&self) -> Result, NativeArtifactError> { + NativeArtifactView::decode_reference(&self.payload, &self.source, &self.catalog) + } +} + +/// Owner startup, transport, or shutdown failure. +#[derive(Debug)] +pub enum TokioParserError { + Native(NativeError), + Catalog(NativeArtifactError), + OwnerStopped, + InvalidCapacity, + Thread(std::io::Error), +} +impl std::fmt::Display for TokioParserError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} +impl std::error::Error for TokioParserError {} + +enum Request { + Parse( + String, + oneshot::Sender>, + ), + Shutdown(oneshot::Sender>), +} + +/// Cloneable async client; all Scheme execution stays on one dedicated thread. +/// The queue bounds waiting requests, not the caller's source allocation. +#[derive(Clone)] +pub struct TokioParser { + requests: mpsc::Sender, +} +impl TokioParser { + /// Initialize the standalone VM and language on their permanent owner thread. + /// # Safety + /// `api` must satisfy `NativeRuntime::start` and remain loaded until shutdown. + /// No other VM may be initialized in this process, including after shutdown. + /// # Errors + /// Rejects invalid capacity, thread creation, VM startup, or catalog admission. + pub async unsafe fn start(api: RuntimeApi, capacity: usize) -> Result { + if capacity == 0 || capacity > tokio::sync::Semaphore::MAX_PERMITS { + return Err(TokioParserError::InvalidCapacity); + } + let (requests, incoming) = mpsc::channel(capacity); + let (ready, started) = oneshot::channel(); + std::thread::Builder::new() + .name("gerbil-parser-owner".into()) + .spawn(move || { + // SAFETY: the caller admits this unique VM and live function table. + let mut runtime = match unsafe { NativeRuntime::start(api) } { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready.send(Err(TokioParserError::Native(error))); + return; + } + }; + let language = match runtime.language() { + Ok(language) => language, + Err(error) => { + let _ = ready.send(Err(TokioParserError::Native(error))); + return; + } + }; + let catalog = match language.catalog() { + Ok(catalog) => Arc::new(catalog.clone()), + Err(error) => { + let _ = ready.send(Err(TokioParserError::Catalog(error))); + return; + } + }; + if ready.send(Ok(())).is_err() { + return; + } + let shutdown = serve(incoming, |source| { + let payload = { + let parsed = language.parse(&source).map_err(TokioParserError::Native)?; + parsed.bytes().map_err(TokioParserError::Native)?.to_vec() + }; // Release the C allocation on the owner before publishing. + Ok(OwnedParsed { + source, + payload, + catalog: Arc::clone(&catalog), + }) + }); + drop(language); + let result = runtime.close().map_err(TokioParserError::Native); + if let Some(reply) = shutdown { + let _ = reply.send(result); + } + }) + .map_err(TokioParserError::Thread)?; + started + .await + .map_err(|_| TokioParserError::OwnerStopped)??; + Ok(Self { requests }) + } + + /// Parse through bounded admission; cancelling a queued request skips it. + /// Once execution starts, its C resources are released even if cancelled. + /// # Errors + /// Reports native transport failure or a stopped owner. + pub async fn parse(&self, source: String) -> Result { + if source.len() > 64 * 1024 * 1024 { + return Err(TokioParserError::Native(NativeError::SourceTooLarge)); + } + let permit = self + .requests + .reserve() + .await + .map_err(|_| TokioParserError::OwnerStopped)?; + let (reply, result) = oneshot::channel(); + permit.send(Request::Parse(source, reply)); + result.await.map_err(|_| TokioParserError::OwnerStopped)? + } + + /// Close admission for every client, release the language, and clean up the VM. + /// Requests ahead of this message finish; requests behind it are cancelled. + /// Dropping every client also drains pending requests and closes the VM. + /// # Errors + /// Reports cleanup failure or an owner that has already stopped. + pub async fn shutdown(&self) -> Result<(), TokioParserError> { + let (reply, result) = oneshot::channel(); + self.requests + .send(Request::Shutdown(reply)) + .await + .map_err(|_| TokioParserError::OwnerStopped)?; + result.await.map_err(|_| TokioParserError::OwnerStopped)? + } +} + +fn serve( + mut incoming: mpsc::Receiver, + mut parse: impl FnMut(String) -> Result, +) -> Option>> { + while let Some(request) = incoming.blocking_recv() { + match request { + Request::Parse(source, reply) => { + if !reply.is_closed() { + let _ = reply.send(parse(source)); + } + } + Request::Shutdown(reply) => { + incoming.close(); + return Some(reply); + } + } + } + None +} + +#[cfg(test)] +#[path = "../tests/unit/tokio_owner.rs"] +mod tests; diff --git a/rust/gerbil-parser-ffi/tests/unit/tokio_owner.rs b/rust/gerbil-parser-ffi/tests/unit/tokio_owner.rs new file mode 100644 index 00000000..c6bd1bf1 --- /dev/null +++ b/rust/gerbil-parser-ffi/tests/unit/tokio_owner.rs @@ -0,0 +1,69 @@ +use super::{OwnedParsed, Request, TokioParser, TokioParserError, serve}; +use crate::NativeError; +use tokio::sync::{mpsc, oneshot}; +#[tokio::test] +async fn cancelled_queue_entry_skips_execution_and_owner_drains_on_close() { + let (requests, incoming) = mpsc::channel(2); + let (cancelled, receiver) = oneshot::channel(); + drop(receiver); + requests + .send(Request::Parse("cancelled".into(), cancelled)) + .await + .unwrap(); + let (reply, result) = oneshot::channel(); + requests + .send(Request::Parse("a=1\n".repeat(64), reply)) + .await + .unwrap(); + assert!(requests.try_reserve().is_err(), "bounded backpressure"); + drop(requests); + let owner = std::thread::spawn(move || { + let mut calls = 0; + assert!( + serve(incoming, |source| { + calls += 1; + assert_eq!(source.lines().count(), 64); + Err(TokioParserError::Native(NativeError::Transport(17))) + }) + .is_none() + ); + calls + }); + assert!(matches!( + result.await.unwrap(), + Err(TokioParserError::Native(NativeError::Transport(17))) + )); + assert_eq!(owner.join().unwrap(), 1); +} +#[tokio::test] +async fn shutdown_cancels_later_requests() { + let (requests, incoming) = mpsc::channel(2); + let (shutdown, completion) = oneshot::channel(); + requests.send(Request::Shutdown(shutdown)).await.unwrap(); + let (reply, result) = oneshot::channel(); + requests + .send(Request::Parse(String::new(), reply)) + .await + .unwrap(); + let owner = std::thread::spawn(move || { + serve(incoming, |_| panic!("request after shutdown executed")) + .unwrap() + .send(Ok(())) + .unwrap(); + }); + completion.await.unwrap().unwrap(); + assert!(result.await.is_err()); + assert!( + requests + .send(Request::Shutdown(oneshot::channel().0)) + .await + .is_err() + ); + owner.join().unwrap(); +} +#[test] +fn snapshots_and_clients_are_send_and_sync() { + fn check() {} + check::(); + check::(); +} diff --git a/rust/gerbil-parser-rowan-arithmetic/src/lib.rs b/rust/gerbil-parser-rowan-arithmetic/src/lib.rs deleted file mode 100644 index 90d9b980..00000000 --- a/rust/gerbil-parser-rowan-arithmetic/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Generated arithmetic reference language for `gerbil-parser-rowan`. - -mod generated; -mod parser; - -pub use generated::LANGUAGE; -pub use parser::parse; diff --git a/rust/gerbil-parser-rowan-arithmetic/src/parser.rs b/rust/gerbil-parser-rowan-arithmetic/src/parser.rs deleted file mode 100644 index aa9265ab..00000000 --- a/rust/gerbil-parser-rowan-arithmetic/src/parser.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Arithmetic language parsing entrypoint. - -use crate::LANGUAGE; - -/// Parse arithmetic source with the generated language product. -/// -/// # Errors -/// -/// Returns a typed parser error when the source is rejected or the generated -/// language tables violate the runtime contract. -pub fn parse(source: &str) -> Result { - gerbil_parser_rowan::parse(&LANGUAGE, source) -} diff --git a/rust/gerbil-parser-rowan-scenarios/Cargo.toml b/rust/gerbil-parser-rowan-scenarios/Cargo.toml deleted file mode 100644 index 6de8749a..00000000 --- a/rust/gerbil-parser-rowan-scenarios/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "gerbil-parser-rowan-scenarios" -version.workspace = true -description = "Integration and performance scenarios for generated Rowan languages" -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true -publish = false - -[dependencies] -gerbil-parser-rowan.workspace = true -gerbil-parser-rowan-arithmetic.workspace = true - -[dev-dependencies] -gerbil-parser-build-support.workspace = true - -[lints] -workspace = true diff --git a/rust/gerbil-parser-rowan-scenarios/src/lib.rs b/rust/gerbil-parser-rowan-scenarios/src/lib.rs deleted file mode 100644 index 3db0c862..00000000 --- a/rust/gerbil-parser-rowan-scenarios/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -//! Executable integration and performance scenarios for generated languages. diff --git a/rust/gerbil-parser-rowan-scenarios/tests/integration_test.rs b/rust/gerbil-parser-rowan-scenarios/tests/integration_test.rs deleted file mode 100644 index 8e29a416..00000000 --- a/rust/gerbil-parser-rowan-scenarios/tests/integration_test.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[path = "integration/arithmetic_v1.rs"] -mod arithmetic_v1; diff --git a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/expected/README.md b/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/expected/README.md deleted file mode 100644 index 533c4660..00000000 --- a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/expected/README.md +++ /dev/null @@ -1 +0,0 @@ -Every generated-table engine execution must produce a complete lossless Rowan CST without external processes. diff --git a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/inputs/README.md b/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/inputs/README.md deleted file mode 100644 index e1edbe29..00000000 --- a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_deterministic_hot_path_v1/inputs/README.md +++ /dev/null @@ -1 +0,0 @@ -The engine workload is the fixed arithmetic source executed through the AOT-generated Rust tables and generic Rowan engine. diff --git a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/scenario.toml b/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/scenario.toml deleted file mode 100644 index d5baad3c..00000000 --- a/rust/gerbil-parser-rowan-scenarios/tests/performance/scenarios/rowan_engine_event_tree_hot_path_v1/scenario.toml +++ /dev/null @@ -1,11 +0,0 @@ -id = "rowan-engine-event-tree-hot-path-v1" -title = "Scheme-AOT structural events stay inside the Rowan latency red zone" -policy_ids = ["RUST-AGENT-ROWAN-ENGINE-PERFORMANCE-001"] -agent_goal = "Preserve bounded lossless Rowan tree construction from deeply nested generated parser events." -reference_repositories = ["rust-analyzer/rowan", "tree-sitter/tree-sitter"] -reference_patterns = [ - "Structural parser decisions remain downstream-owned", - "The generic sink validates exact source coverage without copying source text", -] -inputs = "inputs" -expected = "expected" diff --git a/rust/gerbil-parser-rowan-scenarios/tests/performance_test.rs b/rust/gerbil-parser-rowan-scenarios/tests/performance_test.rs deleted file mode 100644 index c25f9b2a..00000000 --- a/rust/gerbil-parser-rowan-scenarios/tests/performance_test.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[path = "performance/rowan_parse.rs"] -mod rowan_parse; diff --git a/rust/gerbil-parser-rowan/src/engine/mod.rs b/rust/gerbil-parser-rowan/src/engine/mod.rs deleted file mode 100644 index af0ae86a..00000000 --- a/rust/gerbil-parser-rowan/src/engine/mod.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! Generated-table execution owners for the Rowan runtime. - -mod event_tree; -mod generated_events; -mod graph_projection; -mod lexer; -mod model; -mod parser; -mod rowan_tree; -mod structural_block_opening; -mod structural_key_line; -mod structural_lines; -mod structural_list; -mod structural_table; -mod validation; - -pub use event_tree::build_rowan_events; -pub use generated_events::parse_generated_events; -pub use graph_projection::{ - GraphFieldMode, GraphFieldRule, GraphFieldValue, GraphNodeRule, GraphProjectionSpec, - GraphRecord, project_syntax_graph, -}; -pub use model::{ - ActionEntry, BlockContents, BlockHeaderRule, BlockLineRule, BlockOpeningMode, Diagnostic, - GerbilLanguage, GotoEntry, HeadingFieldsRule, HeadingLineRule, InlineLinkRule, KeyLineContext, - KeyLineMode, KeyLineRule, KeyValueLineRule, KindCategory, KindSpec, LanguageSpec, LexicalExpr, - LexicalRule, LineStructureSpec, ListLineRule, Operand, OperandAction, Parse, ParseError, - ParseReceipt, ParserAction, Production, Reduction, ScannedToken, SelectiveGlrReceipt, Symbol, - SyntaxKind, SyntaxNode, SyntaxToken, TableLineRule, Terminal, TerminalSpec, TreeEvent, - UnclosedBlockPolicy, -}; -pub use parser::{parse, parse_scanned}; -pub use structural_lines::parse_structural_lines; - -#[cfg(test)] -#[path = "../../tests/unit/lexical.rs"] -mod lexical_tests; - -#[cfg(test)] -#[path = "../../tests/unit/selective_glr.rs"] -mod selective_glr_tests; - -#[cfg(test)] -#[path = "../../tests/unit/event_tree.rs"] -mod event_tree_tests; - -#[cfg(test)] -#[path = "../../tests/unit/structural_lines.rs"] -mod structural_lines_tests; - -#[cfg(test)] -#[path = "../../tests/unit/pure_function_aot.rs"] -mod pure_function_aot_tests; - -#[cfg(test)] -#[path = "../../tests/unit/event_strategy_aot.rs"] -mod event_strategy_aot_tests; - -#[cfg(test)] -#[path = "../../tests/unit/event_fold_aot.rs"] -mod event_fold_aot_tests; diff --git a/rust/gerbil-parser-rowan/src/engine/rowan_tree.rs b/rust/gerbil-parser-rowan/src/engine/rowan_tree.rs deleted file mode 100644 index e1e98253..00000000 --- a/rust/gerbil-parser-rowan/src/engine/rowan_tree.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! Lossless Rowan green-tree emission. - -use rowan::{GreenNode, GreenNodeBuilder}; - -use super::model::{LanguageSpec, Token, Value}; - -pub(crate) fn build_green( - spec: &LanguageSpec, - source: &str, - tokens: &[Token<'_>], - root: &Value, -) -> Result { - let Value::Node { kind, children, .. } = root else { - return Err("accepted semantic value is not a syntax node".into()); - }; - if *kind != spec.root_kind { - return Err("accepted syntax node does not have the declared root kind".into()); - } - let mut builder = GreenNodeBuilder::new(); - let mut next_token = 0; - builder.start_node(rowan::SyntaxKind(*kind)); - for child in children { - emit_node(&mut builder, tokens, &child.value, &mut next_token)?; - } - while next_token < tokens.len() { - emit_token(&mut builder, &tokens[next_token]); - next_token += 1; - } - builder.finish_node(); - let green = builder.finish(); - // The lexer covers source monotonically with borrowed slices and emit_node - // consumes every token exactly once. Length equality therefore proves the - // lossless roundtrip without allocating a second complete source string. - if usize::from(green.text_len()) != source.len() { - return Err("Rowan tree does not cover the complete source".into()); - } - Ok(green) -} - -fn emit_node( - builder: &mut GreenNodeBuilder<'_>, - tokens: &[Token<'_>], - value: &Value, - next_token: &mut usize, -) -> Result<(), String> { - match value { - Value::Token(index) => { - while *next_token < *index { - emit_token(builder, &tokens[*next_token]); - *next_token += 1; - } - if *next_token != *index { - return Err("recognition token order is not monotonic".into()); - } - emit_token(builder, &tokens[*index]); - *next_token += 1; - } - Value::Node { kind, children, .. } => { - builder.start_node(rowan::SyntaxKind(*kind)); - for child in children { - emit_node(builder, tokens, &child.value, next_token)?; - } - builder.finish_node(); - } - Value::Fragment(children) => { - for child in children { - emit_node(builder, tokens, &child.value, next_token)?; - } - } - } - Ok(()) -} - -fn emit_token(builder: &mut GreenNodeBuilder<'_>, token: &Token<'_>) { - builder.token(rowan::SyntaxKind(token.syntax_kind), token.text); -} diff --git a/rust/gerbil-parser-rowan/src/lib.rs b/rust/gerbil-parser-rowan/src/lib.rs deleted file mode 100644 index 06fb8b7f..00000000 --- a/rust/gerbil-parser-rowan/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Lossless Rowan runtime for parser tables generated by `gerbil-parser`. -//! -//! Gerbil owns grammar declarations and LR conflict resolution at AOT time; -//! this crate executes the resulting immutable Rust tables. - -#![forbid(unsafe_code)] - -extern crate self as gerbil_parser_rowan; - -mod engine; - -pub use engine::{ - ActionEntry, BlockContents, BlockHeaderRule, BlockLineRule, BlockOpeningMode, Diagnostic, - GerbilLanguage, GotoEntry, GraphFieldMode, GraphFieldRule, GraphFieldValue, GraphNodeRule, - GraphProjectionSpec, GraphRecord, HeadingFieldsRule, HeadingLineRule, InlineLinkRule, - KeyLineContext, KeyLineMode, KeyLineRule, KeyValueLineRule, KindCategory, KindSpec, - LanguageSpec, LexicalExpr, LexicalRule, LineStructureSpec, ListLineRule, Operand, - OperandAction, Parse, ParseError, ParseReceipt, ParserAction, Production, Reduction, - ScannedToken, SelectiveGlrReceipt, Symbol, SyntaxKind, SyntaxNode, SyntaxToken, TableLineRule, - Terminal, TerminalSpec, TreeEvent, UnclosedBlockPolicy, build_rowan_events, parse, - parse_generated_events, parse_scanned, parse_structural_lines, project_syntax_graph, -}; diff --git a/rust/gerbil-parser-rowan/tests/unit/lexical.rs b/rust/gerbil-parser-rowan/tests/unit/lexical.rs deleted file mode 100644 index 929e2cfa..00000000 --- a/rust/gerbil-parser-rowan/tests/unit/lexical.rs +++ /dev/null @@ -1,83 +0,0 @@ -use super::{lexer::lexical_end, model::LexicalExpr}; - -const NUMBER: LexicalExpr = LexicalExpr::NumberLiteral { - prefixes: &["0x", "0o", "0b"], - separator: "_", - suffixes: &["M", "F", "D"], - leading_period: true, - trailing_period: true, -}; - -#[test] -fn profiled_numbers_match_the_scheme_boundaries() { - assert_eq!(lexical_end(&NUMBER, "0xCA_FE", 0), Some(7)); - assert_eq!(lexical_end(&NUMBER, "0o7_55", 0), Some(6)); - assert_eq!(lexical_end(&NUMBER, "0b10_01", 0), Some(7)); - assert_eq!(lexical_end(&NUMBER, "1_000.5E+2F", 0), Some(11)); - assert_eq!(lexical_end(&NUMBER, ".5M", 0), Some(3)); - assert_eq!(lexical_end(&NUMBER, "0x", 0), Some(1)); -} - -#[test] -fn quoted_strings_preserve_doubled_and_backslash_escapes() { - let expression = LexicalExpr::QuotedString(&["\"", "'", "`"]); - assert_eq!(lexical_end(&expression, "`a``b`", 0), Some(6)); - assert_eq!(lexical_end(&expression, "'Ada''s graph'", 0), Some(14)); - assert_eq!(lexical_end(&expression, r#""a\"b""#, 0), Some(6)); - assert_eq!(lexical_end(&expression, "`unterminated", 0), None); -} - -#[test] -fn escaped_quoted_strings_leave_adjacent_strings_separate() { - let expression = LexicalExpr::EscapedQuotedString(&["\""]); - assert_eq!(lexical_end(&expression, "\"a\"\"b\"", 0), Some(3)); - assert_eq!(lexical_end(&expression, r#""a\"b""#, 0), Some(6)); - assert_eq!(lexical_end(&expression, "\"unterminated", 0), None); -} - -#[test] -fn comments_and_choices_take_the_longest_complete_match() { - let expression = LexicalExpr::Choice(&[ - LexicalExpr::LineComment(&["//", "#"]), - LexicalExpr::BlockComment { - opening: "/*", - closing: "*/", - }, - ]); - assert_eq!(lexical_end(&expression, "// note\nnext", 0), Some(7)); - assert_eq!(lexical_end(&expression, "/* note */next", 0), Some(10)); - assert_eq!(lexical_end(&expression, "/* open", 0), None); -} - -#[test] -fn nested_comments_and_heredocs_close_losslessly() { - let nested = LexicalExpr::NestedBlockComment { - opening: "/*", - closing: "*/", - }; - assert_eq!(lexical_end(&nested, "/* /* */ */", 0), Some(11)); - assert_eq!( - lexical_end(&LexicalExpr::Heredoc, "<>(), - ["SCHEDULED", "DEADLINE"] - ); - assert_eq!( - planning.values("value").collect::>(), - ["<2026-09-24 Thu>", "<2026-09-25 Fri>"] - ); -} diff --git a/rust/gerbil-parser-runtime-external/Cargo.toml b/rust/gerbil-parser-runtime-external/Cargo.toml new file mode 100644 index 00000000..f492bc2c --- /dev/null +++ b/rust/gerbil-parser-runtime-external/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "gerbil-parser-runtime-external" +version.workspace = true +description = "Opt-in lossless indexed adapter for externally maintained parsers" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +gerbil-parser-runtime.workspace = true +gerbil-parser-artifact.workspace = true +sha2.workspace = true + +[[test]] +name = "adapter" +path = "tests/integration/adapter.rs" + +[lints] +workspace = true diff --git a/rust/gerbil-parser-runtime-external/src/adapter.rs b/rust/gerbil-parser-runtime-external/src/adapter.rs new file mode 100644 index 00000000..c88f175a --- /dev/null +++ b/rust/gerbil-parser-runtime-external/src/adapter.rs @@ -0,0 +1,151 @@ +//! Opt-in event handoff for externally maintained parsers. +//! +//! This crate validates a lossless CST and reports the external provider as +//! syntax authority. It does not recognize source text or claim Gerbil Parser +//! IR, generated-LR, or Scheme-AOT parity. + +use gerbil_parser_artifact::syntax::SyntaxTree; +use gerbil_parser_runtime::{ + Diagnostic, EventCatalog, SyntaxKind, SyntaxNode, TreeEvent, build_syntax_events_catalog, +}; +use sha2::{Digest, Sha256}; + +/// Immutable identity and syntax-kind catalog supplied by an external pack. +#[derive(Clone, Copy, Debug)] +pub struct ExternalLanguageSpec { + pub language: &'static str, + pub version: &'static str, + pub contract: &'static str, + pub provider: &'static str, + pub provider_version: &'static str, + /// Build-bound digest of the external parser artifact. + pub provider_digest: &'static str, + pub catalog: EventCatalog, +} + +/// Source-bound receipt whose authority cannot be mistaken for Grammar IR. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExternalParseReceipt { + pub authority: &'static str, + pub language: &'static str, + pub version: &'static str, + pub contract: &'static str, + pub provider: &'static str, + pub provider_version: &'static str, + pub provider_digest: &'static str, + pub source_digest: String, +} + +/// A validated external CST with its own provenance receipt. +#[derive(Clone, Debug)] +pub struct ExternalParse { + tree: SyntaxTree, + spec: &'static ExternalLanguageSpec, + receipt: ExternalParseReceipt, +} + +impl ExternalParse { + #[must_use] + pub fn syntax(&self) -> SyntaxNode { + self.tree.root() + } + + #[must_use] + pub fn receipt(&self) -> &ExternalParseReceipt { + &self.receipt + } + + #[must_use] + pub fn kind_name(&self, kind: SyntaxKind) -> Option<&str> { + self.spec + .catalog + .kinds + .get(usize::from(kind.0)) + .map(|entry| entry.name) + } +} + +/// Typed failure carrying the same external provenance and source identity. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExternalParseError { + pub receipt: Box, + pub diagnostic: Diagnostic, +} + +impl ExternalParseError { + /// Wrap a rejection produced before an external parser emitted events. + #[must_use] + pub fn from_provider( + spec: &ExternalLanguageSpec, + source: &str, + diagnostic: Diagnostic, + ) -> Self { + Self { + receipt: Box::new(external_receipt(spec, source)), + diagnostic, + } + } +} + +/// Derive an external-authority receipt from the provider descriptor and source. +#[must_use] +pub fn external_receipt(spec: &ExternalLanguageSpec, source: &str) -> ExternalParseReceipt { + let digest = Sha256::digest(source.as_bytes()); + ExternalParseReceipt { + authority: "external-parser", + language: spec.language, + version: spec.version, + contract: spec.contract, + provider: spec.provider, + provider_version: spec.provider_version, + provider_digest: spec.provider_digest, + source_digest: format!("sha256:{digest:x}"), + } +} + +/// Validate external events and return one source-exact indexed syntax. +/// +/// # Errors +/// +/// Rejects incomplete provider identity, an invalid artifact digest, or an +/// event stream that fails the generic catalog and source-coverage checks. +pub fn parse_external_events( + spec: &'static ExternalLanguageSpec, + source: &str, + events: &[TreeEvent], +) -> Result { + let receipt = external_receipt(spec, source); + let failure = |diagnostic| ExternalParseError { + receipt: Box::new(receipt.clone()), + diagnostic, + }; + if [ + spec.language, + spec.version, + spec.contract, + spec.provider, + spec.provider_version, + ] + .iter() + .any(|value| value.is_empty()) + || !canonical_sha256(spec.provider_digest) + { + return Err(failure(Diagnostic { + reason_kind: "invalid-external-provider", + byte_offset: 0, + message: "external provider identity or artifact digest is invalid".into(), + })); + } + let tree = build_syntax_events_catalog(&spec.catalog, source, events).map_err(failure)?; + Ok(ExternalParse { + tree, + spec, + receipt, + }) +} + +fn canonical_sha256(digest: &str) -> bool { + digest.len() == 71 + && digest.starts_with("sha256:") + && digest[7..].bytes().all(|byte| byte.is_ascii_hexdigit()) +} diff --git a/rust/gerbil-parser-runtime-external/src/lib.rs b/rust/gerbil-parser-runtime-external/src/lib.rs new file mode 100644 index 00000000..ab291b56 --- /dev/null +++ b/rust/gerbil-parser-runtime-external/src/lib.rs @@ -0,0 +1,10 @@ +//! Opt-in event handoff for externally maintained parsers. + +#![forbid(unsafe_code)] + +mod adapter; + +pub use adapter::{ + ExternalLanguageSpec, ExternalParse, ExternalParseError, ExternalParseReceipt, + external_receipt, parse_external_events, +}; diff --git a/rust/gerbil-parser-runtime-external/tests/integration/adapter.rs b/rust/gerbil-parser-runtime-external/tests/integration/adapter.rs new file mode 100644 index 00000000..bcf4101c --- /dev/null +++ b/rust/gerbil-parser-runtime-external/tests/integration/adapter.rs @@ -0,0 +1,94 @@ +use gerbil_parser_runtime::{Diagnostic, EventCatalog, KindCategory, KindSpec, TreeEvent}; +use gerbil_parser_runtime_external::{ + ExternalLanguageSpec, ExternalParseError, parse_external_events, +}; + +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "Document", + category: KindCategory::Node, + }, + KindSpec { + name: "Text", + category: KindCategory::Token, + }, +]; +static EXTERNAL: ExternalLanguageSpec = ExternalLanguageSpec { + language: "independent-text", + version: "edition-a", + contract: "independent-text.syntax-a", + provider: "independent-parser", + provider_version: "0.2", + provider_digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111", + catalog: EventCatalog { + root_kind: 0, + kinds: KINDS, + }, +}; +static INVALID: ExternalLanguageSpec = ExternalLanguageSpec { + provider_digest: "stale", + ..EXTERNAL +}; + +#[test] +fn independent_provider_builds_lossless_tree_with_external_receipt() { + let source = "é\n"; + let events = [ + TreeEvent::StartNode(0), + TreeEvent::Token { + kind: 1, + start: 0, + end: source.len(), + }, + TreeEvent::FinishNode, + ]; + let parsed = parse_external_events(&EXTERNAL, source, &events).unwrap(); + assert_eq!(parsed.syntax().to_string(), source); + assert_eq!(parsed.receipt().authority, "external-parser"); + assert_eq!(parsed.receipt().provider, "independent-parser"); + assert_eq!(parsed.receipt().provider_digest, EXTERNAL.provider_digest); + assert_eq!( + parsed.receipt().source_digest, + "sha256:edd3a863872a04239eb29ad4bc12fc892b3d4ae57cc7e786a3697816f8e141c2" + ); + assert_eq!( + parsed.kind_name(gerbil_parser_runtime::SyntaxKind(1)), + Some("Text") + ); +} + +#[test] +fn invalid_provider_and_event_ranges_fail_with_source_bound_receipts() { + let source = "é"; + let split = [ + TreeEvent::StartNode(0), + TreeEvent::Token { + kind: 1, + start: 0, + end: 1, + }, + TreeEvent::FinishNode, + ]; + let error = parse_external_events(&EXTERNAL, source, &split).unwrap_err(); + assert_eq!(error.diagnostic.reason_kind, "event-range"); + assert_eq!(error.receipt.authority, "external-parser"); + assert_eq!( + error.receipt.source_digest, + "sha256:4a99557e4033c3539de2eb65472017cad5f9557f7a0625a09f1c3f6e2ba69c4c" + ); + + let error = parse_external_events(&INVALID, source, &split).unwrap_err(); + assert_eq!(error.diagnostic.reason_kind, "invalid-external-provider"); + + let rejection = ExternalParseError::from_provider( + &EXTERNAL, + source, + Diagnostic { + reason_kind: "provider-rejected", + byte_offset: 0, + message: "source is outside this parser's grammar".into(), + }, + ); + assert_eq!(rejection.receipt.authority, "external-parser"); + assert_eq!(rejection.diagnostic.reason_kind, "provider-rejected"); +} diff --git a/rust/gerbil-parser-rowan/Cargo.toml b/rust/gerbil-parser-runtime/Cargo.toml similarity index 58% rename from rust/gerbil-parser-rowan/Cargo.toml rename to rust/gerbil-parser-runtime/Cargo.toml index 208eefc8..9098cb58 100644 --- a/rust/gerbil-parser-rowan/Cargo.toml +++ b/rust/gerbil-parser-runtime/Cargo.toml @@ -1,14 +1,14 @@ [package] -name = "gerbil-parser-rowan" +name = "gerbil-parser-runtime" version = "0.1.0" -description = "Rust/Rowan runtime for gerbil-parser AOT grammar artifacts" +description = "Rust runtime for gerbil-parser AOT grammar artifacts" edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true [dependencies] -rowan.workspace = true +gerbil-parser-artifact.workspace = true sha2.workspace = true [lints] diff --git a/rust/gerbil-parser-runtime/src/engine/command_model.rs b/rust/gerbil-parser-runtime/src/engine/command_model.rs new file mode 100644 index 00000000..1f7c6872 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/command_model.rs @@ -0,0 +1,117 @@ +//! Closed command product types and the engine token/word host boundary. +use super::{Diagnostic, ProjectedValue, TextProfile}; + +/// Closed token-selection algebra; literals are source text, never procedures. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CommandTrigger { + None, + Manual(Option<&'static Self>), + Token { + kind: &'static str, + literals: &'static [&'static str], + }, + Role(&'static str), + Or(&'static [Self]), + Lookahead(&'static [Self]), + Adjacent(&'static [Self]), +} +/// One ordered branch of a command choice. +#[derive(Clone, Copy, Debug)] +pub struct CommandChoice { + pub trigger: CommandTrigger, + pub program: &'static [CommandInstruction], +} +/// Closed command construction and control-flow operations. +#[derive(Clone, Copy, Debug)] +pub enum CommandInstruction { + As(&'static str), + Raw(&'static str), + Word(&'static str), + Command(&'static str), + Take { + field: &'static str, + trigger: CommandTrigger, + }, + List { + field: &'static str, + until: CommandTrigger, + allow_empty: bool, + }, + Call { + field: &'static str, + form: &'static str, + }, + Node { + field: &'static str, + kind: &'static str, + program: &'static [Self], + }, + Optional { + trigger: CommandTrigger, + program: &'static [Self], + }, + Many { + trigger: CommandTrigger, + program: &'static [Self], + }, + Until { + trigger: CommandTrigger, + program: &'static [Self], + }, + Branch { + trigger: CommandTrigger, + yes: &'static [Self], + no: &'static [Self], + }, + Choose(&'static [CommandChoice]), + Balance { + open: CommandTrigger, + close: CommandTrigger, + depth: usize, + open_field: &'static str, + close_field: &'static str, + program: &'static [Self], + }, +} +/// A named declared form; selector precedence is resolved by the driver. +#[derive(Clone, Copy, Debug)] +pub struct CommandForm { + pub id: &'static str, + pub trigger: CommandTrigger, + pub priority: u16, + pub kind: &'static str, + pub program: &'static [CommandInstruction], +} +/// A token class within one declared command role. +#[derive(Clone, Copy, Debug)] +pub struct CommandTokenClass { + pub kind: &'static str, + pub literals: &'static [&'static str], +} +/// Engine role membership lowered from a language declaration. +#[derive(Clone, Copy, Debug)] +pub struct CommandRole { + pub name: &'static str, + pub classes: &'static [CommandTokenClass], +} +/// Immutable node roles and forms lowered from the Scheme `CommandProfile`. +#[derive(Clone, Copy, Debug)] +pub struct CommandProgramSpec { + pub assignment_tail: &'static str, + pub pipeline_head: &'static str, + pub forms: &'static [CommandForm], + pub nodes: &'static [(&'static str, &'static str)], + pub roles: &'static [CommandRole], + pub descriptor: &'static TextProfile, +} +/// Engine-owned token/Word boundary. Source uses continuation requests instead. +pub(crate) trait CommandHost<'source> { + fn source(&self) -> &'source str; + fn start(&mut self) -> Result; + fn end(&self) -> usize; + fn position(&self) -> usize; + fn available(&mut self) -> Result; + fn matches(&mut self, trigger: CommandTrigger) -> Result; + fn raw(&mut self) -> Result, Diagnostic>; + fn word(&mut self) -> Result, Diagnostic>; +} diff --git a/rust/gerbil-parser-runtime/src/engine/command_program.rs b/rust/gerbil-parser-runtime/src/engine/command_program.rs new file mode 100644 index 00000000..3dad0a9c --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/command_program.rs @@ -0,0 +1,942 @@ +//! Closed Scheme command control programs, executed with an explicit frame stack. +use super::{ + Diagnostic, KindCategory, PreparedResultNode, PreparedResultProfile, PreparedTextProfile, + ProjectedNode, ProjectedValue, ResultChildCapture, ResultProfileSpec, +}; +use std::collections::{HashMap, HashSet}; + +pub(crate) use super::command_model::CommandHost; +pub use super::command_model::{ + CommandChoice, CommandForm, CommandInstruction, CommandProgramSpec, CommandRole, + CommandTokenClass, CommandTrigger, +}; + +/// Prepared node constructors and named forms; no per-operation catalog binding. +pub struct PreparedCommandProgram { + pub(super) spec: &'static CommandProgramSpec, + pub(super) result_owner: &'static ResultProfileSpec, + widths: HashMap<&'static str, usize>, + pub(super) assignment_tail: usize, + pub(super) pipeline_head: usize, + pub(super) forms: HashMap<&'static str, usize>, + pub(super) descriptor: PreparedTextProfile, + pub(super) tokens: HashMap<&'static str, u16>, + pub(super) selectors: HashMap<&'static str, HashMap<&'static str, usize>>, + pub(super) fallback: Vec, + pub(super) nodes: HashMap<&'static str, PreparedResultNode>, +} +fn error(at: usize, message: &str) -> Diagnostic { + Diagnostic { + reason_kind: "invalid-command-program", + byte_offset: at, + message: message.into(), + } +} +fn trigger_valid(trigger: CommandTrigger, depth: usize) -> bool { + if depth >= 32 { + return false; + } + match trigger { + CommandTrigger::None | CommandTrigger::Manual(None) => true, + CommandTrigger::Manual(Some(t)) => { + matches!( + t, + CommandTrigger::Token { + kind: "word" | "operator", + .. + } + ) && trigger_valid(*t, depth + 1) + } + CommandTrigger::Token { kind, literals } => { + ["word", "operator", "newline"].contains(&kind) + && literals.iter().all(|s| !s.is_empty()) + } + CommandTrigger::Role(role) => [ + "trivia", + "separator", + "redirect", + "here-redirect", + "strip-redirect", + "reserved", + "conditional-operator", + "pipeline", + "and-or", + "case-end", + ] + .contains(&role), + CommandTrigger::Or(rows) => { + !rows.is_empty() && rows.iter().all(|t| trigger_valid(*t, depth + 1)) + } + CommandTrigger::Lookahead(rows) | CommandTrigger::Adjacent(rows) => { + !rows.is_empty() + && rows.len() <= 32 + && rows.iter().all(|t| { + matches!( + t, + CommandTrigger::Token { + kind: "word" | "operator", + .. + } + ) && trigger_valid(*t, depth + 1) + }) + } + } +} +struct Frame<'s> { + kind: &'static str, + start: usize, + field: Option<&'static str>, + children: Vec>, +} +enum Task { + Program(&'static [CommandInstruction], usize), + Finish, + Repeat { + trigger: CommandTrigger, + program: &'static [CommandInstruction], + until: bool, + before: Option, + }, + Balance { + open: CommandTrigger, + close: CommandTrigger, + depth: usize, + open_field: &'static str, + close_field: &'static str, + program: &'static [CommandInstruction], + before: Option, + }, +} +/// Request-local VM storage survives a Source suspension. +pub(super) struct CommandExecution<'s> { + frames: Vec>, + tasks: Vec, +} +pub(super) enum CommandRequest { + Compound(&'static str), + List { + field: &'static str, + until: CommandTrigger, + allow_empty: bool, + }, +} +pub(super) enum CommandProgress<'s> { + Complete(ProjectedNode<'s>), + Suspend(CommandRequest), +} +impl<'s> CommandExecution<'s> { + pub(super) fn publish( + &mut self, + field: &'static str, + value: ProjectedNode<'s>, + ) -> Result<(), Diagnostic> { + publish(&mut self.frames, field, ProjectedValue::Node(value)) + } + pub(super) fn live_frames(&self) -> usize { + self.frames.len() + self.tasks.len() + } +} +impl PreparedCommandProgram { + /// Admit all forms, field uses, consuming loops and manual-call graph once. + /// # Errors + /// Rejects foreign fields, malformed selectors, nonprogressing bodies and call cycles. + pub fn new( + spec: &'static CommandProgramSpec, + results: &'static ResultProfileSpec, + ) -> Result { + if spec.forms.is_empty() || spec.nodes.is_empty() { + return Err(error(0, "empty command specification")); + } + let results_plan = PreparedResultProfile::new(results)?; + let mut nodes = HashMap::new(); + for &(role, kind) in spec.nodes { + if role.is_empty() || nodes.insert(role, results_plan.bind_node(kind)?).is_some() { + return Err(error(0, "duplicate command node role")); + } + } + let mut forms = HashMap::new(); + for (index, form) in spec.forms.iter().enumerate() { + if form.id.is_empty() + || form.priority > 1000 + || !trigger_valid(form.trigger, 0) + || !nodes.contains_key(form.kind) + || forms.insert(form.id, index).is_some() + { + return Err(error(0, "invalid command form")); + } + } + let mut tokens = HashMap::new(); + for (index, kind) in results + .kinds + .iter() + .enumerate() + .filter(|(_, k)| k.category == KindCategory::Token) + { + tokens.insert( + kind.name, + u16::try_from(index).map_err(|_| error(0, "token kind overflow"))?, + ); + } + admit_roles(spec, &tokens)?; + admit_shapes(spec, results)?; + let descriptor = PreparedTextProfile::new(spec.descriptor) + .ok_or_else(|| error(0, "invalid descriptor text profile"))?; + let (selectors, fallback) = admit_selectors(spec)?; + let assignment_tail = *forms + .get(spec.assignment_tail) + .ok_or_else(|| error(0, "missing assignment entrypoint"))?; + let pipeline_head = *forms + .get(spec.pipeline_head) + .ok_or_else(|| error(0, "missing pipeline entrypoint"))?; + let mut prepared = Self { + spec, + result_owner: results, + widths: HashMap::new(), + forms, + assignment_tail, + pipeline_head, + descriptor, + tokens, + selectors, + fallback, + nodes, + }; + let mut done = HashSet::new(); + let mut order = Vec::new(); + for form in spec.forms { + prepared.visit(form.id, &mut done, &mut order)?; + } + for id in order { + let form = &spec.forms[prepared.forms[id]]; + let width = prepared.validate(form.program, form.kind, results, 0)?; + prepared.widths.insert(id, width); + if !matches!(form.trigger, CommandTrigger::Manual(_)) && width == 0 { + return Err(error(0, "command form does not consume input")); + } + } + prepared.admit_entries(results)?; + Ok(prepared) + } + fn fields( + &self, + role: &str, + results: &ResultProfileSpec, + ) -> Result<&'static [&'static str], Diagnostic> { + let target = self + .spec + .nodes + .iter() + .find(|(r, _)| *r == role) + .ok_or_else(|| error(0, "unknown command node role"))? + .1; + results + .nodes + .iter() + .find(|n| results.kinds[usize::from(n.kind)].name == target) + .map(|n| n.fields) + .ok_or_else(|| error(0, "missing command fields")) + } + fn validate( + &self, + program: &[CommandInstruction], + kind: &str, + results: &ResultProfileSpec, + depth: usize, + ) -> Result { + if depth >= 128 { + return Err(error(0, "invalid command program depth")); + } + let fields = self.fields(kind, results)?; + let field = |name| { + if fields.contains(&name) { + Ok(()) + } else { + Err(error(0, "foreign command field")) + } + }; + let trigger = |t| { + if trigger_valid(t, 0) { + Ok(()) + } else { + Err(error(0, "invalid command trigger")) + } + }; + let body = |p, k| self.validate(p, k, results, depth + 1); + let mut width = 0usize; + for instruction in program { + let consumed = match *instruction { + CommandInstruction::As(k) => { + if !fields + .iter() + .all(|f| self.fields(k, results).is_ok_and(|v| v.contains(f))) + { + return Err(error(0, "projection branch loses fields")); + } + 0 + } + leaf @ (CommandInstruction::Raw(_) + | CommandInstruction::Word(_) + | CommandInstruction::Command(_) + | CommandInstruction::Take { .. } + | CommandInstruction::List { .. } + | CommandInstruction::Call { .. }) => self.validate_leaf(leaf, &field, &trigger)?, + CommandInstruction::Node { + field: f, + kind: k, + program: p, + } => { + field(f)?; + body(p, k)? + } + CommandInstruction::Optional { + trigger: t, + program: p, + } => { + trigger(t)?; + body(p, kind)?; + 0 + } + CommandInstruction::Many { + trigger: t, + program: p, + } + | CommandInstruction::Until { + trigger: t, + program: p, + } => { + trigger(t)?; + if body(p, kind)? == 0 { + return Err(error(0, "command repetition does not consume input")); + } + 0 + } + CommandInstruction::Branch { + trigger: t, + yes, + no, + } => { + trigger(t)?; + body(yes, kind)?.min(body(no, kind)?) + } + CommandInstruction::Choose(rows) => { + if rows.is_empty() { + return Err(error(0, "empty command choice")); + } + let mut minimum = usize::MAX; + for row in rows { + trigger(row.trigger)?; + minimum = minimum.min(body(row.program, kind)?); + } + minimum + } + balance @ CommandInstruction::Balance { .. } => { + Self::validate_balance(balance, &field, &trigger, &|p| body(p, kind))? + } + }; + width = width + .checked_add(consumed) + .ok_or_else(|| error(0, "command width overflow"))?; + } + Ok(width) + } + fn validate_balance( + instruction: CommandInstruction, + field: &impl Fn(&'static str) -> Result<(), Diagnostic>, + trigger: &impl Fn(CommandTrigger) -> Result<(), Diagnostic>, + body: &impl Fn(&'static [CommandInstruction]) -> Result, + ) -> Result { + let CommandInstruction::Balance { + open, + close, + depth, + open_field, + close_field, + program, + } = instruction + else { + return Err(error(0, "invalid balanced instruction")); + }; + trigger(open)?; + trigger(close)?; + field(open_field)?; + field(close_field)?; + if depth == 0 || body(program)? == 0 { + return Err(error(0, "invalid balanced command body")); + } + Ok(1) + } + fn validate_leaf( + &self, + instruction: CommandInstruction, + field: &impl Fn(&'static str) -> Result<(), Diagnostic>, + trigger: &impl Fn(CommandTrigger) -> Result<(), Diagnostic>, + ) -> Result { + Ok(match instruction { + CommandInstruction::Raw(f) + | CommandInstruction::Word(f) + | CommandInstruction::Command(f) => { + field(f)?; + 1 + } + CommandInstruction::Take { + field: f, + trigger: t, + } => { + field(f)?; + trigger(t)?; + 1 + } + CommandInstruction::List { + field: f, + until, + allow_empty, + } => { + field(f)?; + trigger(until)?; + usize::from(!allow_empty) + } + CommandInstruction::Call { field: f, form } => { + field(f)?; + if !self.forms.contains_key(form) { + return Err(error(0, "unknown command form call")); + } + if self.widths.get(form).is_none_or(|width| *width == 0) { + return Err(error(0, "command call requires consuming target")); + } + 1 + } + _ => return Err(error(0, "invalid command leaf")), + }) + } + fn admit_entries(&self, results: &ResultProfileSpec) -> Result<(), Diagnostic> { + for (id, kind) in [ + (self.spec.assignment_tail, "ArrayAssignment"), + (self.spec.pipeline_head, "Pipeline"), + ] { + let form = self + .forms + .get(id) + .map(|&i| &self.spec.forms[i]) + .ok_or_else(|| error(0, "missing manual entrypoint"))?; + if !matches!(form.trigger, CommandTrigger::Manual(_)) || form.kind != kind { + return Err(error(0, "invalid manual entrypoint")); + } + let width = self.validate(form.program, kind, results, 0)?; + if id == self.spec.assignment_tail && width == 0 { + return Err(error(0, "array body must consume input")); + } + if id == self.spec.pipeline_head { + for step in form.program { + let (CommandInstruction::Optional { program: body, .. } + | CommandInstruction::Many { program: body, .. }) = step + else { + return Err(error(0, "pipeline header requires guarded operations")); + }; + if self.validate(body, kind, results, 0)? == 0 { + return Err(error(0, "pipeline guard must consume input")); + } + } + } + } + Ok(()) + } + fn visit( + &self, + id: &'static str, + done: &mut HashSet<&'static str>, + order: &mut Vec<&'static str>, + ) -> Result<(), Diagnostic> { + let mut active = HashSet::new(); + let mut tasks = vec![(id, false)]; + while let Some((id, finish)) = tasks.pop() { + if finish { + active.remove(id); + done.insert(id); + order.push(id); + continue; + } + if done.contains(id) { + continue; + } + if !active.insert(id) { + return Err(error(0, "cyclic command form call")); + } + tasks.push((id, true)); + let index = *self + .forms + .get(id) + .ok_or_else(|| error(0, "unknown command form call"))?; + let mut programs = vec![(self.spec.forms[index].program, 0)]; + while let Some((program, depth)) = programs.pop() { + if depth >= 128 { + return Err(error(0, "invalid command program depth")); + } + for instruction in program { + match instruction { + CommandInstruction::Call { form, .. } => tasks.push((form, false)), + CommandInstruction::Node { program, .. } + | CommandInstruction::Optional { program, .. } + | CommandInstruction::Many { program, .. } + | CommandInstruction::Until { program, .. } + | CommandInstruction::Balance { program, .. } => { + programs.push((program, depth + 1)); + } + CommandInstruction::Branch { yes, no, .. } => { + programs.push((yes, depth + 1)); + programs.push((no, depth + 1)); + } + CommandInstruction::Choose(rows) => { + programs.extend(rows.iter().map(|r| (r.program, depth + 1))); + } + _ => {} + } + } + } + } + Ok(()) + } + pub(super) fn begin<'s>( + &self, + id: &str, + start: usize, + ) -> Result, Diagnostic> { + let form = &self.spec.forms[*self + .forms + .get(id) + .ok_or_else(|| error(start, "unknown command entry"))?]; + Ok(Self::begin_initial(form, start, Vec::new())) + } + pub(super) fn begin_initial<'s>( + form: &CommandForm, + start: usize, + children: Vec>, + ) -> CommandExecution<'s> { + CommandExecution { + frames: vec![Frame { + kind: form.kind, + start, + field: None, + children, + }], + tasks: vec![Task::Finish, Task::Program(form.program, 0)], + } + } + pub(super) fn resume<'s>( + &self, + execution: &mut CommandExecution<'s>, + host: &mut impl CommandHost<'s>, + frame_budget: usize, + ) -> Result, Diagnostic> { + let CommandExecution { frames, tasks } = execution; + while let Some(task) = tasks.pop() { + if frames.len() + tasks.len() + 1 > frame_budget { + return Err(Diagnostic { + reason_kind: "command-source-resource-limit", + byte_offset: host.end(), + message: "Source continuation frame budget exceeded".into(), + }); + } + match task { + Task::Program(program, at) => { + let Some(instruction) = program.get(at) else { + continue; + }; + tasks.push(Task::Program(program, at + 1)); + if let Some(request) = self.step(*instruction, host, frames, tasks)? { + return Ok(CommandProgress::Suspend(request)); + } + } + Task::Finish => { + if let Some(node) = self.finish(host, frames, tasks)? { + return Ok(CommandProgress::Complete(node)); + } + } + Task::Repeat { + trigger, + program, + until, + before, + } => { + progress(host.position(), before, host.end())?; + if host.matches(trigger)? != until { + if !host.available()? { + return Err(error(host.end(), "unterminated command repetition")); + } + tasks.push(Task::Repeat { + trigger, + program, + until, + before: Some(host.position()), + }); + tasks.push(Task::Program(program, 0)); + } + } + Task::Balance { + open, + close, + mut depth, + open_field, + close_field, + program, + before, + } => { + progress(host.position(), before, host.end())?; + if !host.available()? { + return Err(error(host.end(), "unterminated balanced command body")); + } + if host.matches(open)? { + publish(frames, open_field, host.raw()?)?; + depth = depth + .checked_add(1) + .ok_or_else(|| error(host.end(), "command depth overflow"))?; + } else if host.matches(close)? { + publish(frames, close_field, host.raw()?)?; + depth -= 1; + if depth == 0 { + continue; + } + } else { + tasks.push(Task::Balance { + open, + close, + depth, + open_field, + close_field, + program, + before: Some(host.position()), + }); + tasks.push(Task::Program(program, 0)); + continue; + } + tasks.push(Task::Balance { + open, + close, + depth, + open_field, + close_field, + program, + before: None, + }); + } + } + } + Err(error(host.end(), "missing command result")) + } + fn finish<'s>( + &self, + host: &impl CommandHost<'s>, + frames: &mut Vec>, + tasks: &[Task], + ) -> Result>, Diagnostic> { + let frame = frames + .pop() + .ok_or_else(|| error(host.end(), "missing command frame"))?; + let node = + self.nodes[frame.kind].build(host.source(), frame.start..host.end(), frame.children)?; + if let Some(field) = frame.field { + publish(frames, field, ProjectedValue::Node(node))?; + Ok(None) + } else if frames.is_empty() && tasks.is_empty() { + Ok(Some(node)) + } else { + Err(error(host.end(), "invalid command publication stack")) + } + } + fn step<'s>( + &self, + instruction: CommandInstruction, + host: &mut impl CommandHost<'s>, + frames: &mut Vec>, + tasks: &mut Vec, + ) -> Result, Diagnostic> { + match instruction { + CommandInstruction::As(kind) => { + frames + .last_mut() + .ok_or_else(|| error(host.end(), "missing command frame"))? + .kind = kind; + } + CommandInstruction::Raw(field) => publish(frames, field, host.raw()?)?, + CommandInstruction::Word(field) => publish(frames, field, host.word()?)?, + CommandInstruction::Command(field) => return Ok(Some(CommandRequest::Compound(field))), + CommandInstruction::Take { field, trigger } => { + if !host.matches(trigger)? { + return Err(error(host.end(), "expected declared command token")); + } + publish(frames, field, host.raw()?)?; + } + CommandInstruction::List { + field, + until, + allow_empty, + } => { + return Ok(Some(CommandRequest::List { + field, + until, + allow_empty, + })); + } + CommandInstruction::Call { field, form } => { + let row = &self.spec.forms[self.forms[form]]; + enter(frames, tasks, host.start()?, field, row.kind, row.program); + } + CommandInstruction::Node { + field, + kind, + program, + } => enter(frames, tasks, host.start()?, field, kind, program), + CommandInstruction::Optional { trigger, program } => { + if host.matches(trigger)? { + tasks.push(Task::Program(program, 0)); + } + } + CommandInstruction::Many { trigger, program } => tasks.push(Task::Repeat { + trigger, + program, + until: false, + before: None, + }), + CommandInstruction::Until { trigger, program } => tasks.push(Task::Repeat { + trigger, + program, + until: true, + before: None, + }), + CommandInstruction::Branch { trigger, yes, no } => tasks.push(Task::Program( + if host.matches(trigger)? { yes } else { no }, + 0, + )), + CommandInstruction::Choose(rows) => { + let mut selected = None; + for row in rows { + if host.matches(row.trigger)? { + selected = Some(row.program); + break; + } + } + tasks.push(Task::Program( + selected.ok_or_else(|| error(host.end(), "no declared command branch"))?, + 0, + )); + } + CommandInstruction::Balance { + open, + close, + depth, + open_field, + close_field, + program, + } => tasks.push(Task::Balance { + open, + close, + depth, + open_field, + close_field, + program, + before: None, + }), + } + Ok(None) + } +} +fn publish<'s>( + frames: &mut [Frame<'s>], + field: &'static str, + value: ProjectedValue<'s>, +) -> Result<(), Diagnostic> { + frames + .last_mut() + .ok_or_else(|| error(0, "missing command frame"))? + .children + .push(ResultChildCapture { field, value }); + Ok(()) +} +fn enter( + frames: &mut Vec>, + tasks: &mut Vec, + start: usize, + field: &'static str, + kind: &'static str, + program: &'static [CommandInstruction], +) { + frames.push(Frame { + kind, + start, + field: Some(field), + children: Vec::new(), + }); + tasks.push(Task::Finish); + tasks.push(Task::Program(program, 0)); +} +fn progress(at: usize, before: Option, byte_offset: usize) -> Result<(), Diagnostic> { + if before.is_some_and(|before| at <= before) { + Err(error(byte_offset, "command program did not advance input")) + } else { + Ok(()) + } +} + +const ROLES: &[&str] = &[ + "trivia", + "separator", + "redirect", + "here-redirect", + "strip-redirect", + "reserved", + "conditional-operator", + "pipeline", + "and-or", + "case-end", +]; +fn admit_roles(spec: &CommandProgramSpec, tokens: &HashMap<&str, u16>) -> Result<(), Diagnostic> { + let mut seen = HashSet::new(); + for role in spec.roles { + if !ROLES.contains(&role.name) || !seen.insert(role.name) || role.classes.is_empty() { + return Err(error(0, "invalid command role")); + } + let mut kinds = HashSet::new(); + for class in role.classes { + let mut literals = HashSet::new(); + if !tokens.contains_key(class.kind) + || !kinds.insert(class.kind) + || !class + .literals + .iter() + .all(|s| !s.is_empty() && literals.insert(*s)) + { + return Err(error(0, "invalid role terminal or literal")); + } + } + } + if seen.len() != ROLES.len() { + return Err(error(0, "missing command role")); + } + Ok(()) +} +type Selectors = HashMap<&'static str, HashMap<&'static str, usize>>; +fn admit_selectors(spec: &CommandProgramSpec) -> Result<(Selectors, Vec), Diagnostic> { + let mut selectors: Selectors = HashMap::new(); + let mut fallback: Vec = Vec::new(); + for (index, form) in spec.forms.iter().enumerate() { + match form.trigger { + CommandTrigger::Manual(_) => {} + CommandTrigger::Token { + kind: "word" | "operator", + literals, + } if !literals.is_empty() => { + let CommandTrigger::Token { kind, .. } = form.trigger else { + unreachable!() + }; + for literal in literals { + if selectors + .entry(kind) + .or_default() + .insert(literal, index) + .is_some() + { + return Err(error(0, "ambiguous command form literal")); + } + } + } + CommandTrigger::Lookahead(_) | CommandTrigger::Adjacent(_) => { + if fallback + .iter() + .any(|&i| spec.forms[i].trigger == form.trigger) + { + return Err(error(0, "duplicate command lookahead selector")); + } + fallback.push(index); + } + _ => return Err(error(0, "unregistered command entry selector")), + } + } + fallback.reverse(); + Ok((selectors, fallback)) +} + +const SHAPES: &[(&str, &[&str])] = &[ + ("Redirection", &["descriptor", "operator", "target"]), + ( + "ArrayAssignment", + &["assignment", "open", "close", "separator", "element"], + ), + ( + "SimpleCommand", + &["assignment", "name", "argument", "redirect"], + ), + ("CommandList", &["command", "separator", "here-document"]), + ("IfCommand", &["keyword", "condition", "body", "else-body"]), + ("WhileCommand", &["keyword", "condition", "body"]), + ("UntilCommand", &["keyword", "condition", "body"]), + ( + "ForCommand", + &["keyword", "header", "variable", "item", "separator", "body"], + ), + ( + "SelectCommand", + &["keyword", "header", "variable", "item", "separator", "body"], + ), + ( + "ArithmeticForCommand", + &["keyword", "header", "variable", "item", "separator", "body"], + ), + ( + "CaseCommand", + &["keyword", "subject", "separator", "clause"], + ), + ( + "CaseClause", + &[ + "open", + "pattern", + "alternate", + "close", + "body", + "terminator", + ], + ), + ( + "FunctionDefinition", + &["keyword", "name", "open", "close", "body"], + ), + ( + "ConditionalCommand", + &["open", "close", "operator", "operand"], + ), + ( + "ArithmeticCommand", + &["open", "close", "expression", "operator"], + ), + ("RedirectedCommand", &["command", "redirect"]), + ("BraceGroup", &["open", "body", "close"]), + ("Subshell", &["open", "body", "close"]), + ( + "Pipeline", + &["keyword", "option", "negate", "command", "operator"], + ), + ("AndOrList", &["command", "operator"]), + ("HereDocument", &["delimiter", "content"]), + ("BashFile", &["command", "separator", "here-document"]), +]; +fn admit_shapes(spec: &CommandProgramSpec, results: &ResultProfileSpec) -> Result<(), Diagnostic> { + if spec.nodes.len() != SHAPES.len() { + return Err(error(0, "missing or foreign Source projection")); + } + for &(role, fields) in SHAPES { + let name = spec + .nodes + .iter() + .find(|&&(r, _)| r == role) + .map(|&(_, n)| n) + .ok_or_else(|| error(0, "missing Source projection"))?; + let node = results + .nodes + .iter() + .find(|n| results.kinds[usize::from(n.kind)].name == name) + .ok_or_else(|| error(0, "unknown Source projection"))?; + if !fields.iter().all(|f| node.fields.contains(f)) { + return Err(error(0, "Source projection lacks required fields")); + } + } + Ok(()) +} diff --git a/rust/gerbil-parser-runtime/src/engine/command_source/execution.rs b/rust/gerbil-parser-runtime/src/engine/command_source/execution.rs new file mode 100644 index 00000000..71aeb2cd --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/command_source/execution.rs @@ -0,0 +1,863 @@ +//! Engine-owned Source cursor for closed command programs and Word recognition. +use crate::engine::command_program::{ + CommandExecution, CommandHost, CommandInstruction, CommandProgress, CommandRequest, + CommandTrigger, PreparedCommandProgram, +}; +use crate::engine::{ + CommandProgramSpec, Diagnostic, PartProfileSpec, PreparedPartProfile, ProjectedNode, + ProjectedValue, ResultChildCapture, ResultProfileSpec, ScannedToken, +}; +use std::collections::{HashMap, VecDeque}; + +use super::publication::{CommandParse, error}; + +const MAX_SOURCE_FRAMES: usize = 16_384; + +enum SourceTask<'s> { + Compound, + CompoundAfter, + Pipeline, + PipelinePrefixAfter { + start: usize, + }, + Simple { + start: usize, + named: bool, + children: Vec>, + }, + SimpleAfter { + start: usize, + named: bool, + children: Vec>, + }, + PipelineAfter { + start: usize, + children: Vec>, + }, + AndOr, + AndOrAfter { + start: Option, + children: Vec>, + }, + List(ListFrame<'s>), + Vm(CommandExecution<'s>), + VmAfter { + execution: CommandExecution<'s>, + field: &'static str, + }, +} +struct ListFrame<'s> { + start: usize, + until: CommandTrigger, + allow_empty: bool, + root: bool, + children: Vec>, + before: Option<(usize, usize)>, +} +struct SourceExecution<'s> { + tasks: Vec>, + live_frames: usize, + returned: Option>, +} +impl<'s> SourceExecution<'s> { + fn schedule(&mut self, task: SourceTask<'s>, at: usize) -> Result<(), Diagnostic> { + schedule(&mut self.tasks, &mut self.live_frames, task, at) + } +} +impl SourceTask<'_> { + fn live_frames(&self) -> usize { + 1 + match self { + Self::Vm(execution) | Self::VmAfter { execution, .. } => execution.live_frames(), + _ => 0, + } + } +} +fn schedule<'s>( + tasks: &mut Vec>, + live: &mut usize, + task: SourceTask<'s>, + at: usize, +) -> Result<(), Diagnostic> { + let next = live + .checked_add(task.live_frames()) + .filter(|n| *n <= MAX_SOURCE_FRAMES) + .ok_or_else(|| Diagnostic { + reason_kind: "command-source-resource-limit", + byte_offset: at, + message: "Source continuation frame budget exceeded".into(), + })?; + *live = next; + tasks.push(task); + Ok(()) +} +fn take_result<'s>( + value: &mut Option>, + at: usize, +) -> Result, Diagnostic> { + value + .take() + .ok_or_else(|| error(at, "missing Source continuation result")) +} +struct Cursor<'p, 's> { + plan: &'p PreparedCommandProgram, + source: &'s str, + tokens: &'p [ScannedToken], + parts: &'p PreparedPartProfile, + kinds: &'p HashMap<&'static str, u16>, + + at: usize, + end: usize, + markers: VecDeque<(usize, bool)>, + links: Vec<(usize, usize)>, +} +impl PreparedCommandProgram { + /// Recognize a full Source from a validated scanner tape using declared roles. + /// The cursor and command/Word/list bridge are owned entirely by the engine. + /// # Errors + /// Rejects invalid token coverage, unexpected selectors and unmatched deferred bodies. + pub fn parse_scanned<'s>( + &self, + source: &'s str, + tokens: &[ScannedToken], + parts: &PreparedPartProfile, + ) -> Result, Diagnostic> { + if !std::ptr::eq(self.result_owner, parts.result_owner) { + return Err(error(0, "foreign Word result catalog")); + } + let kinds = &self.tokens; + let mut end = 0; + for token in tokens { + if token.start != end + || token.end <= end + || source.get(token.start..token.end).is_none() + || !kinds.contains_key(token.terminal) + { + return Err(error(end, "invalid command scanner tape")); + } + end = token.end; + } + if end != source.len() { + return Err(error(end, "incomplete command scanner tape")); + } + let mut cursor = Cursor { + plan: self, + source, + tokens, + parts, + kinds, + at: 0, + end: 0, + markers: VecDeque::new(), + links: Vec::new(), + }; + let root = cursor.run_source()?; + if !cursor.markers.is_empty() { + return Err(error(cursor.end, "missing deferred body")); + } + let trivia = tokens + .iter() + .filter(|t| cursor.role("trivia", t)) + .map(|t| (cursor.kinds[t.terminal], t.start..t.end)) + .collect(); + Ok(CommandParse { + root, + here_documents: cursor.links, + trivia, + source, + result_owner: self.result_owner, + }) + } +} +fn child<'s>(field: &'static str, value: ProjectedValue<'s>) -> ResultChildCapture<'s> { + ResultChildCapture { field, value } +} +fn node<'s>(field: &'static str, value: ProjectedNode<'s>) -> ResultChildCapture<'s> { + child(field, ProjectedValue::Node(value)) +} +impl<'s> Cursor<'_, 's> { + fn text(&self, token: &ScannedToken) -> &'s str { + &self.source[token.start..token.end] + } + fn role(&self, role: &str, token: &ScannedToken) -> bool { + self.plan + .spec + .roles + .iter() + .find(|r| r.name == role) + .is_some_and(|r| { + r.classes.iter().any(|c| { + c.kind == token.terminal + && (c.literals.is_empty() || c.literals.contains(&self.text(token))) + }) + }) + } + fn peek(&mut self) -> Option { + while self + .tokens + .get(self.at) + .is_some_and(|t| self.role("trivia", t)) + { + self.end = self.tokens[self.at].end; + self.at += 1; + } + self.tokens.get(self.at).copied() + } + fn token(&mut self) -> Result { + let token = self + .peek() + .ok_or_else(|| error(self.end, "expected command token"))?; + self.at += 1; + self.end = token.end; + Ok(token) + } + fn raw_value(&self, token: ScannedToken) -> ProjectedValue<'s> { + ProjectedValue::Token { + kind: self.kinds[token.terminal], + span: token.start..token.end, + } + } + fn test(&self, trigger: CommandTrigger, at: usize) -> bool { + match trigger { + CommandTrigger::None | CommandTrigger::Manual(None) => false, + CommandTrigger::Manual(Some(t)) => self.test(*t, at), + CommandTrigger::Token { kind, literals } => self.tokens.get(at).is_some_and(|t| { + t.terminal == kind && (literals.is_empty() || literals.contains(&self.text(t))) + }), + CommandTrigger::Role(role) => self.tokens.get(at).is_some_and(|t| self.role(role, t)), + CommandTrigger::Or(rows) => rows.iter().any(|t| self.test(*t, at)), + CommandTrigger::Lookahead(rows) | CommandTrigger::Adjacent(rows) => { + let adjacent = matches!(trigger, CommandTrigger::Adjacent(_)); + let mut pos = at; + let mut previous = None; + for row in rows { + if !adjacent { + while self.tokens.get(pos).is_some_and(|t| self.role("trivia", t)) { + pos += 1; + } + } + let Some(token) = self.tokens.get(pos) else { + return false; + }; + if adjacent && previous.is_some_and(|end| end != token.start) + || !self.test(*row, pos) + { + return false; + } + previous = Some(token.end); + pos += 1; + } + true + } + } + } + fn build( + &self, + role: &str, + start: usize, + end: usize, + children: Vec>, + ) -> Result, Diagnostic> { + self.plan + .nodes + .get(role) + .ok_or_else(|| error(start, "missing Source node role"))? + .build(self.source, start..end, children) + } + fn selected(&self) -> Option<&'static str> { + let token = self.tokens.get(self.at)?; + let mut selected = self + .plan + .selectors + .get(token.terminal) + .and_then(|rows| rows.get(self.text(token))) + .map(|&index| &self.plan.spec.forms[index]); + for &index in &self.plan.fallback { + let form = &self.plan.spec.forms[index]; + if self.test(form.trigger, self.at) + && selected.is_none_or(|s| form.priority > s.priority) + { + selected = Some(form); + } + } + selected.map(|f| f.id) + } + fn descriptor_before(&self) -> bool { + self.tokens.get(self.at).is_some_and(|t| { + t.terminal == "word" + && self + .plan + .descriptor + .match_prefix(self.source, t.start, t.end) + == Some(t.end) + && self + .tokens + .get(self.at + 1) + .is_some_and(|n| self.role("redirect", n) && t.end == n.start) + }) + } + fn redirection( + &mut self, + descriptor: Option, + ) -> Result, Diagnostic> { + let operator = self.token()?; + if !self.role("redirect", &operator) { + return Err(error(operator.start, "expected redirection")); + } + let start = descriptor.map_or(operator.start, |t| t.start); + let mut children = Vec::new(); + if let Some(d) = descriptor { + children.push(child("descriptor", self.raw_value(d))); + } + let here = self.role("here-redirect", &operator); + children.push(child("operator", self.raw_value(operator))); + let target = self + .peek() + .ok_or_else(|| error(self.end, "missing redirection target"))?; + let value = if here { + if target.terminal != "heredoc-marker" { + return Err(error(target.start, "expected deferred marker")); + } + let (_, quoted) = crate::scanner::shell_delimiter(self.text(&target), target.start)?; + self.markers.push_back((target.start, quoted)); + self.raw()? + } else { + self.word()? + }; + children.push(child("target", value)); + self.build("Redirection", start, self.end, children) + } + // One outer loop drives Source and the existing Command VM. Child results + // move through a single slot and are published once by their continuation. + fn run_source(&mut self) -> Result, Diagnostic> { + let mut execution = SourceExecution { + tasks: Vec::new(), + live_frames: 0, + returned: None, + }; + execution.schedule( + SourceTask::List(ListFrame { + start: 0, + until: CommandTrigger::None, + allow_empty: true, + root: true, + children: Vec::new(), + before: None, + }), + self.end, + )?; + while let Some(task) = execution.tasks.pop() { + execution.live_frames -= task.live_frames(); + self.step_source(task, &mut execution)?; + } + take_result(&mut execution.returned, self.end) + } + fn step_source( + &mut self, + task: SourceTask<'s>, + execution: &mut SourceExecution<'s>, + ) -> Result<(), Diagnostic> { + match task { + SourceTask::Compound => self.step_compound(execution)?, + SourceTask::CompoundAfter => self.finish_compound(execution)?, + SourceTask::Simple { + start, + named, + children, + } => self.step_simple(start, named, children, execution)?, + SourceTask::Pipeline => self.step_pipeline(execution)?, + SourceTask::PipelineAfter { start, children } => { + self.step_pipeline_after(start, children, execution)?; + } + SourceTask::AndOrAfter { start, children } => { + self.step_and_or_after(start, children, execution)?; + } + SourceTask::List(frame) => self.step_list(frame, execution)?, + SourceTask::Vm(vm) => self.step_vm(vm, execution)?, + SourceTask::SimpleAfter { + start, + named, + mut children, + } => { + children.push(node( + "assignment", + take_result(&mut execution.returned, self.end)?, + )); + execution.schedule( + SourceTask::Simple { + start, + named, + children, + }, + self.end, + )?; + } + SourceTask::PipelinePrefixAfter { start } => { + let children = take_result(&mut execution.returned, self.end)?.into_captures(); + execution.schedule(SourceTask::PipelineAfter { start, children }, self.end)?; + execution.schedule(SourceTask::Compound, self.end)?; + } + SourceTask::AndOr => { + execution.schedule( + SourceTask::AndOrAfter { + start: None, + children: Vec::new(), + }, + self.end, + )?; + execution.schedule(SourceTask::Pipeline, self.end)?; + } + SourceTask::VmAfter { + execution: mut vm, + field, + } => { + vm.publish(field, take_result(&mut execution.returned, self.end)?)?; + execution.schedule(SourceTask::Vm(vm), self.end)?; + } + } + Ok(()) + } + fn step_compound(&mut self, execution: &mut SourceExecution<'s>) -> Result<(), Diagnostic> { + self.peek(); + if let Some(id) = self.selected() { + let vm = self.plan.begin(id, self.start()?)?; + execution.schedule(SourceTask::CompoundAfter, self.end)?; + execution.schedule(SourceTask::Vm(vm), self.end)?; + } else { + let first = self + .peek() + .ok_or_else(|| error(self.end, "expected command"))?; + if self.role("reserved", &first) { + return Err(error(first.start, "unexpected reserved token")); + } + execution.schedule( + SourceTask::Simple { + start: first.start, + named: false, + children: Vec::new(), + }, + self.end, + )?; + } + Ok(()) + } + fn finish_compound(&mut self, execution: &mut SourceExecution<'s>) -> Result<(), Diagnostic> { + let command = take_result(&mut execution.returned, self.end)?; + let start = command.span().start; + let mut children = vec![node("command", command)]; + while let Some(token) = self.peek() { + if self.descriptor_before() { + let d = self.token()?; + children.push(node("redirect", self.redirection(Some(d))?)); + } else if self.role("redirect", &token) { + children.push(node("redirect", self.redirection(None)?)); + } else { + break; + } + } + execution.returned = Some(self.finish_chain("RedirectedCommand", start, children)?); + Ok(()) + } + fn step_simple( + &mut self, + start: usize, + mut named: bool, + mut children: Vec>, + execution: &mut SourceExecution<'s>, + ) -> Result<(), Diagnostic> { + while let Some(token) = self.peek() { + if self.descriptor_before() { + let descriptor = self.token()?; + children.push(node("redirect", self.redirection(Some(descriptor))?)); + } else if self.role("redirect", &token) { + children.push(node("redirect", self.redirection(None)?)); + } else if token.terminal == "word" { + if !named + && let Some(assignment) = + self.parts.assignment(self.source, token.start..token.end)? + { + self.token()?; + let form = &self.plan.spec.forms[self.plan.assignment_tail]; + if self + .tokens + .get(self.at) + .is_some_and(|n| n.start == token.end) + && self.test(form.trigger, self.at) + { + let vm = PreparedCommandProgram::begin_initial( + form, + assignment.span().start, + vec![node("assignment", assignment)], + ); + execution.schedule( + SourceTask::SimpleAfter { + start, + named, + children, + }, + self.end, + )?; + execution.schedule(SourceTask::Vm(vm), self.end)?; + return Ok(()); + } + children.push(node("assignment", assignment)); + } else { + children.push(child(if named { "argument" } else { "name" }, self.word()?)); + named = true; + } + } else { + break; + } + } + if children.is_empty() { + return Err(error(start, "expected command or redirection")); + } + execution.returned = Some(self.build("SimpleCommand", start, self.end, children)?); + Ok(()) + } + fn step_pipeline(&mut self, execution: &mut SourceExecution<'s>) -> Result<(), Diagnostic> { + let start = self.start()?; + let form = &self.plan.spec.forms[self.plan.pipeline_head]; + if form.program.iter().any(|i| match i { + CommandInstruction::Optional { trigger, .. } + | CommandInstruction::Many { trigger, .. } => self.test(*trigger, self.at), + _ => false, + }) { + let vm = PreparedCommandProgram::begin_initial(form, start, Vec::new()); + execution.schedule(SourceTask::PipelinePrefixAfter { start }, self.end)?; + execution.schedule(SourceTask::Vm(vm), self.end)?; + } else { + execution.schedule( + SourceTask::PipelineAfter { + start, + children: Vec::new(), + }, + self.end, + )?; + execution.schedule(SourceTask::Compound, self.end)?; + } + Ok(()) + } + fn step_pipeline_after( + &mut self, + start: usize, + mut children: Vec>, + execution: &mut SourceExecution<'s>, + ) -> Result<(), Diagnostic> { + children.push(node( + "command", + take_result(&mut execution.returned, self.end)?, + )); + if self.peek().is_some_and(|t| self.role("pipeline", &t)) { + children.push(child("operator", self.raw()?)); + execution.schedule(SourceTask::PipelineAfter { start, children }, self.end)?; + execution.schedule(SourceTask::Compound, self.end)?; + } else { + execution.returned = Some(self.finish_chain("Pipeline", start, children)?); + } + Ok(()) + } + fn step_and_or_after( + &mut self, + start: Option, + mut children: Vec>, + execution: &mut SourceExecution<'s>, + ) -> Result<(), Diagnostic> { + let command = take_result(&mut execution.returned, self.end)?; + let start = start.unwrap_or(command.span().start); + children.push(node("command", command)); + if self.peek().is_some_and(|t| self.role("and-or", &t)) { + children.push(child("operator", self.raw()?)); + execution.schedule( + SourceTask::AndOrAfter { + start: Some(start), + children, + }, + self.end, + )?; + execution.schedule(SourceTask::Pipeline, self.end)?; + } else { + execution.returned = Some(self.finish_chain("AndOrList", start, children)?); + } + Ok(()) + } + fn step_list( + &mut self, + frame: ListFrame<'s>, + execution: &mut SourceExecution<'s>, + ) -> Result<(), Diagnostic> { + let ListFrame { + start, + until, + allow_empty, + root, + mut children, + before, + } = frame; + + if let Some((at, offset)) = before { + children.push(node( + "command", + take_result(&mut execution.returned, self.end)?, + )); + if self.at <= at { + return Err(error(offset, "Source did not advance")); + } + } + while let Some(token) = self.peek() { + if self.test(until, self.at) { + break; + } + let at = self.at; + if self.role("separator", &token) { + if token.terminal == "operator" && !children.iter().any(|c| c.field == "command") { + return Err(error(token.start, "separator has no command")); + } + children.push(child("separator", self.raw()?)); + if token.terminal == "newline" { + children.extend(self.here()?); + } + if self.at <= at { + return Err(error(token.start, "Source did not advance")); + } + } else { + execution.schedule( + SourceTask::List(ListFrame { + start, + until, + allow_empty, + root, + children, + before: Some((at, token.start)), + }), + self.end, + )?; + execution.schedule(SourceTask::AndOr, self.end)?; + // The list moved to its continuation; no partial result exists. + execution.returned = None; + return Ok(()); + } + } + if !allow_empty && !children.iter().any(|c| c.field == "command") { + return Err(error(start, "empty compound list")); + } + let end = if root { self.source.len() } else { self.end }; + execution.returned = Some(self.build( + if root { "BashFile" } else { "CommandList" }, + start, + end, + children, + )?); + Ok(()) + } + fn step_vm( + &mut self, + mut vm: CommandExecution<'s>, + execution: &mut SourceExecution<'s>, + ) -> Result<(), Diagnostic> { + let plan = self.plan; + match plan.resume(&mut vm, self, MAX_SOURCE_FRAMES - execution.live_frames - 1)? { + CommandProgress::Complete(result) => execution.returned = Some(result), + CommandProgress::Suspend(request) => { + let (field, next) = match request { + CommandRequest::Compound(field) => (field, SourceTask::Compound), + CommandRequest::List { + field, + until, + allow_empty, + } => ( + field, + SourceTask::List(ListFrame { + start: self.end, + until, + allow_empty, + root: false, + children: Vec::new(), + before: None, + }), + ), + }; + execution.schedule( + SourceTask::VmAfter { + execution: vm, + field, + }, + self.end, + )?; + execution.schedule(next, self.end)?; + } + } + Ok(()) + } + fn finish_chain( + &self, + kind: &str, + start: usize, + mut children: Vec>, + ) -> Result, Diagnostic> { + if children.len() == 1 { + let ProjectedValue::Node(command) = children.remove(0).value else { + unreachable!() + }; + Ok(command) + } else { + self.build(kind, start, self.end, children) + } + } + fn here(&mut self) -> Result>, Diagnostic> { + let mut nodes = Vec::new(); + while self + .peek() + .is_some_and(|t| matches!(t.terminal, "heredoc-content" | "heredoc-end")) + { + let (marker, quoted) = self + .markers + .pop_front() + .ok_or_else(|| error(self.end, "deferred body has no marker"))?; + let start = self.start()?; + let mut children = Vec::new(); + loop { + let token = self.token()?; + if !matches!(token.terminal, "heredoc-content" | "heredoc-end") { + return Err(error(token.start, "unterminated deferred body")); + } + let done = token.terminal == "heredoc-end"; + let value = if done || quoted { + self.raw_value(token) + } else { + ProjectedValue::Node( + self.parts + .here_content(self.source, token.start..token.end)?, + ) + }; + children.push(child(if done { "delimiter" } else { "content" }, value)); + if done { + break; + } + } + self.links.push((marker, start)); + nodes.push(node( + "here-document", + self.build("HereDocument", start, self.end, children)?, + )); + } + Ok(nodes) + } +} +impl<'s> CommandHost<'s> for Cursor<'_, 's> { + fn source(&self) -> &'s str { + self.source + } + fn start(&mut self) -> Result { + self.peek() + .map(|t| t.start) + .ok_or_else(|| error(self.end, "expected command start")) + } + fn end(&self) -> usize { + self.end + } + fn position(&self) -> usize { + self.at + } + fn available(&mut self) -> Result { + Ok(self.peek().is_some()) + } + fn matches(&mut self, trigger: CommandTrigger) -> Result { + self.peek(); + Ok(self.test(trigger, self.at)) + } + fn raw(&mut self) -> Result, Diagnostic> { + let token = self.token()?; + Ok(self.raw_value(token)) + } + fn word(&mut self) -> Result, Diagnostic> { + let token = self.token()?; + if token.terminal != "word" { + return Err(error(token.start, "expected Word")); + } + Ok(ProjectedValue::Node( + self.parts.word(self.source, token.start..token.end)?, + )) + } +} + +/// One generated product of the declared scanner, Word, Command and result slots. +#[derive(Clone, Copy, Debug)] +pub struct CommandSourceSpec { + pub scanner: &'static crate::scanner::ScannerSpec, + pub parts: &'static PartProfileSpec, + pub regions: &'static crate::scanner::RegionSpec, + pub results: &'static ResultProfileSpec, + pub commands: &'static CommandProgramSpec, +} +/// Reusable admitted Source product. Languages contribute immutable data only. +pub struct PreparedCommandSource { + spec: &'static CommandSourceSpec, + commands: PreparedCommandProgram, + parts: PreparedPartProfile, +} +impl PreparedCommandSource { + /// Prepare all language slots once and check role terminals against the scanner. + /// # Errors + /// Rejects malformed profiles or role terminals absent from the scanner. + pub fn new(spec: &'static CommandSourceSpec) -> Result { + crate::scanner::ContextualScanner::new(spec.scanner, "")?; + if !spec.scanner.positions.contains(&"source") { + return Err(error(0, "scanner lacks Source position")); + } + for role in spec.commands.roles { + if !role + .classes + .iter() + .all(|c| spec.scanner.cells.iter().any(|r| r.terminal == c.kind)) + { + return Err(error(0, "command role is not declared by scanner")); + } + } + let commands = PreparedCommandProgram::new(spec.commands, spec.results)?; + if !spec + .scanner + .cells + .iter() + .all(|cell| commands.tokens.contains_key(cell.terminal)) + { + return Err(error(0, "scanner terminal is absent from result catalog")); + } + let parts = PreparedPartProfile::new(spec.parts, spec.results, spec.regions)?; + Ok(Self { + spec, + commands, + parts, + }) + } + /// Scan and recognize immutable source through the engine-owned cursor. + /// # Errors + /// Returns the first scanner or command diagnostic for rejected syntax. + pub fn parse<'s>(&self, source: &'s str) -> Result, Diagnostic> { + let tokens = + crate::scanner::ContextualScanner::new(self.spec.scanner, source)?.scan("source")?; + self.commands.parse_scanned(source, &tokens, &self.parts) + } + /// Own a complete Source scan, reusing certified lexical history after edits. + /// # Errors + /// Returns the first scanner diagnostic; no failed scan becomes a cache. + pub fn scan_session( + &self, + source: String, + previous: Option<&crate::scanner::ContextualScanSession>, + ) -> Result { + crate::scanner::ContextualScanSession::new(self.spec.scanner, source, "source", previous) + } + /// Recognize and publish against the session's owned source and tokens. + /// # Errors + /// Rejects foreign scanner products or command syntax errors. + pub fn parse_session<'s>( + &self, + session: &'s crate::scanner::ContextualScanSession, + ) -> Result, Diagnostic> { + if !session.admits(self.spec.scanner, "source") { + return Err(error(0, "foreign Source scanning session")); + } + self.commands + .parse_scanned(session.source(), session.tokens(), &self.parts) + } +} diff --git a/rust/gerbil-parser-runtime/src/engine/command_source/mod.rs b/rust/gerbil-parser-runtime/src/engine/command_source/mod.rs new file mode 100644 index 00000000..973664d9 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/command_source/mod.rs @@ -0,0 +1,6 @@ +//! Command Source execution and canonical publication owner. +mod execution; +mod publication; + +pub use execution::{CommandSourceSpec, PreparedCommandSource}; +pub use publication::{CommandEvent, CommandParse}; diff --git a/rust/gerbil-parser-runtime/src/engine/command_source/publication.rs b/rust/gerbil-parser-runtime/src/engine/command_source/publication.rs new file mode 100644 index 00000000..ad135f9c --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/command_source/publication.rs @@ -0,0 +1,186 @@ +//! Canonical command Source event publication and borrowed result ownership. +use crate::engine::{Diagnostic, ProjectedNode, ProjectedValue, ResultProfileSpec, TreeEvent}; + +pub(super) fn error(at: usize, message: &str) -> Diagnostic { + Diagnostic { + reason_kind: "invalid-command-source", + byte_offset: at, + message: message.into(), + } +} +/// Recognized Source and FIFO deferred-body links. Fields retain declaration order. +pub struct CommandParse<'s> { + pub root: ProjectedNode<'s>, + pub here_documents: Vec<(usize, usize)>, + pub(super) trivia: Vec<(u16, std::ops::Range)>, + pub(super) source: &'s str, + pub(super) result_owner: &'static ResultProfileSpec, +} +/// Canonical Source publication, retaining fields and producer-order identifiers. +/// Kind indexes refer to the admitted result profile; fields are declaration names. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CommandEvent { + StartNode { + id: u64, + kind: u16, + offset: usize, + }, + FinishNode { + id: u64, + kind: u16, + offset: usize, + }, + StartField { + name: &'static str, + offset: usize, + }, + FinishField { + name: &'static str, + offset: usize, + }, + Token { + id: u64, + kind: u16, + start: usize, + end: usize, + }, +} +enum CommandWalk<'a> { + Node(&'a ProjectedNode<'a>), + Value(&'a ProjectedValue<'a>, &'static str), + FinishNode(u64, u16, usize), + FinishField(&'static str, usize), +} +impl CommandWalk<'_> { + fn boundary(&self) -> usize { + match self { + Self::Node(n) => n.span().start, + Self::Value(ProjectedValue::Node(n), _) => n.span().start, + Self::Value(ProjectedValue::Token { span, .. }, _) => span.start, + Self::FinishNode(_, _, end) | Self::FinishField(_, end) => *end, + } + } +} +impl CommandParse<'_> { + /// Exact borrowed source admitted with this request's token tape. + #[must_use] + pub fn source(&self) -> &str { + self.source + } + + /// Walk canonical records once, without building an intermediate event tape. + /// Trivia precedes a child's field scope; trailing trivia belongs to its node. + /// # Errors + /// Rejects overlapping captures or trivia outside the root. A sink may have + /// received a prefix on failure, so callers must commit publication on success. + pub fn walk_events(&self, mut emit: impl FnMut(CommandEvent)) -> Result<(), Diagnostic> { + if !self.root.bound_to(self.result_owner, self.source) + || self.root.span() != (0..self.source.len()) + { + return Err(error(0, "foreign command Source root")); + } + let mut tasks = vec![CommandWalk::Node(&self.root)]; + let mut trivia = self.trivia.iter().peekable(); + let (mut offset, mut node_id, mut token_id) = (0, 0, 0); + while let Some(task) = tasks.pop() { + // Closing fields never claim trivia beyond the value's end. + let boundary = task.boundary(); + while let Some((kind, span)) = trivia.peek() { + if span.end > boundary { + break; + } + if span.start != offset { + return Err(error(offset, "source trivia does not cover gap")); + } + emit(CommandEvent::Token { + id: token_id, + kind: *kind, + start: span.start, + end: span.end, + }); + token_id += 1; + offset = span.end; + trivia.next(); + } + match task { + CommandWalk::Node(node) => { + if node.span().start != offset { + return Err(error(offset, "command node is outside source order")); + } + let id = node_id; + node_id += 1; + emit(CommandEvent::StartNode { + id, + kind: node.kind(), + offset, + }); + tasks.push(CommandWalk::FinishNode(id, node.kind(), node.span().end)); + tasks.extend( + node.children() + .iter() + .rev() + .map(|c| CommandWalk::Value(c.value(), c.field())), + ); + } + CommandWalk::Value(value, field) => { + let span = match value { + ProjectedValue::Node(node) => node.span(), + ProjectedValue::Token { span, .. } => span.clone(), + }; + if span.start != offset { + return Err(error(offset, "command value is outside source order")); + } + emit(CommandEvent::StartField { + name: field, + offset, + }); + tasks.push(CommandWalk::FinishField(field, span.end)); + match value { + ProjectedValue::Node(node) => tasks.push(CommandWalk::Node(node)), + ProjectedValue::Token { kind, span } => { + emit(CommandEvent::Token { + id: token_id, + kind: *kind, + start: span.start, + end: span.end, + }); + token_id += 1; + offset = span.end; + } + } + } + CommandWalk::FinishNode(id, kind, end) => { + if end != offset { + return Err(error(offset, "unclaimed significant source range")); + } + emit(CommandEvent::FinishNode { id, kind, offset }); + } + CommandWalk::FinishField(name, end) => { + if end != offset { + return Err(error(offset, "field is outside source order")); + } + emit(CommandEvent::FinishField { name, offset }); + } + } + } + if trivia.next().is_some() { + return Err(error(offset, "trivia remains outside source")); + } + Ok(()) + } + /// Publish the lossless CST tape from the same canonical event walk. + /// # Errors + /// Rejects overlapping captures or trivia outside the root. + pub fn events(&self) -> Result, Diagnostic> { + let mut events = Vec::new(); + self.walk_events(|event| match event { + CommandEvent::StartNode { kind, .. } => events.push(TreeEvent::StartNode(kind)), + CommandEvent::FinishNode { .. } => events.push(TreeEvent::FinishNode), + CommandEvent::Token { + kind, start, end, .. + } => events.push(TreeEvent::Token { kind, start, end }), + CommandEvent::StartField { .. } | CommandEvent::FinishField { .. } => {} + })?; + Ok(events) + } +} diff --git a/rust/gerbil-parser-runtime/src/engine/contextual_parser.rs b/rust/gerbil-parser-runtime/src/engine/contextual_parser.rs new file mode 100644 index 00000000..623b241e --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/contextual_parser.rs @@ -0,0 +1,182 @@ +//! Deterministic LR pulls from the shared scanner at the generated state position. +use super::contextual_plan::{ContextualPlan, prepare_once}; +use super::model::{ + Diagnostic, LanguageSpec, Parse, ParseError, ParseReceipt, ParserAction, SelectiveGlrReceipt, + Token, Value, +}; +use super::parser::{ParserConfiguration, apply_reduce, find_action}; +use super::recognition_index::build_syntax; +use super::validation::receipt; +use crate::scanner::{ContextualScanner, ScannerCheckpoint, ScannerSpec}; + +/// A Scheme-compiled contextual parser binds scanner and LR identities. +#[derive(Debug)] +pub struct ContextualParserSpec { + pub language: &'static LanguageSpec, + pub scanner: &'static ScannerSpec, + pub parser_digest: &'static str, + pub state_positions: &'static [&'static str], +} +/// Parse with generated LR positions and shared scanner IR. +/// +/// # Errors +/// Returns a diagnostic for invalid products, lexical/parse rejection or a GLR +/// fork. Contextual branch-local scanner state is not yet admitted for GLR. +pub fn parse_contextual( + product: &'static ContextualParserSpec, + source: &str, +) -> Result { + let spec = product.language; + let bound_receipt = receipt( + spec, + source, + Some(product.parser_digest), + Some(product.scanner.digest), + ); + let failure = |diagnostic| ParseError { + receipt: Box::new(bound_receipt.clone()), + diagnostic: Box::new(diagnostic), + selective_glr: None, + }; + let reject = |offset, message: &str| Diagnostic { + reason_kind: "contextual-parser", + byte_offset: offset, + message: message.into(), + }; + let plan = prepare_once(product).map_err(failure)?; + let scanner = ContextualScanner::new(product.scanner, source).map_err(failure)?; + let mut state = scanner.initial_state(); + let mut tokens = Vec::new(); + let mut configuration = ParserConfiguration { + states: vec![0], + values: Vec::new(), + cursor: 0, + score: 0, + }; + let mut lookahead = None; + let mut eof = false; + loop { + let lr_state = *configuration + .states + .last() + .ok_or_else(|| failure(reject(state.byte_offset(), "empty LR stack")))?; + if lookahead.is_none() && !eof { + let position = product + .state_positions + .get(lr_state as usize) + .ok_or_else(|| failure(reject(state.byte_offset(), "undeclared LR state")))?; + lookahead = + pull_token(&scanner, &mut state, position, &plan, &mut tokens).map_err(failure)?; + eof = lookahead.is_none(); + } + + let token = lookahead.map(|index| &tokens[index]); + let action = find_action(spec, lr_state, token).ok_or_else(|| { + failure(reject( + token.map_or(source.len(), |t| t.start), + "no contextual LR action", + )) + })?; + match action { + ParserAction::Shift(next) => { + let index = lookahead + .take() + .ok_or_else(|| failure(reject(source.len(), "LR table shifts EOF")))?; + configuration.states.push(next); + configuration.values.push(Value::Token(index)); + configuration.cursor += 1; + } + ParserAction::Reduce(production) => { + apply_reduce(spec, &tokens, token, &mut configuration, production) + .map_err(failure)?; + } + ParserAction::Accept => { + if !eof || lookahead.is_some() || configuration.values.len() != 1 { + return Err(failure(reject( + state.byte_offset(), + "accept did not cover one complete root", + ))); + } + return finish_parse(spec, source, &tokens, configuration, bound_receipt.clone()) + .map_err(failure); + } + + ParserAction::Fork(_) => { + return Err(failure(reject( + state.byte_offset(), + "contextual GLR branch scanner state is not admitted", + ))); + } + ParserAction::RejectNonAssoc => { + return Err(failure(reject( + state.byte_offset(), + "non-associative operator rejected", + ))); + } + } + } +} + +fn pull_token<'scanner, 'source>( + scanner: &'scanner ContextualScanner<'source>, + state: &mut ScannerCheckpoint<'scanner, 'source>, + position: &str, + plan: &ContextualPlan, + tokens: &mut Vec>, +) -> Result, Diagnostic> { + loop { + let (lexeme, next) = scanner.step(state, position)?; + *state = next; + let Some(lexeme) = lexeme else { + return Ok(None); + }; + let terminal = plan.terminals.get(lexeme.terminal).ok_or_else(|| { + crate::scanner::error(lexeme.start, "scanner terminal is outside language") + })?; + let rule = plan.lexical.get(lexeme.terminal).ok_or_else(|| { + crate::scanner::error(lexeme.start, "scanner terminal has no lexical rule") + })?; + let index = tokens.len(); + tokens.push(Token { + terminal: terminal.name, + syntax_kind: terminal.syntax_kind, + text: scanner.source_slice(lexeme.start, lexeme.end), + start: lexeme.start, + end: lexeme.end, + }); + if !rule.extra { + return Ok(Some(index)); + } + } +} + +fn finish_parse( + spec: &LanguageSpec, + source: &str, + tokens: &[Token<'_>], + mut configuration: ParserConfiguration, + bound_receipt: ParseReceipt, +) -> Result { + let root = configuration + .values + .pop() + .ok_or_else(|| crate::scanner::error(0, "missing root"))?; + let tree = build_syntax(spec, source, tokens, &root) + .map_err(|message| crate::scanner::error(0, &message))?; + Ok(Parse { + tree, + kinds: spec.kinds, + receipt: bound_receipt, + selective_glr: SelectiveGlrReceipt { + branch_budget: 0, + branches_explored: 1, + speculative_branches_explored: 0, + max_speculative_depth: 0, + merged_branches: 0, + successful_completions: 1, + distinct_completions: 1, + winner_reason: "contextual-deterministic", + dynamic_score: configuration.score, + }, + }) +} diff --git a/rust/gerbil-parser-runtime/src/engine/contextual_plan.rs b/rust/gerbil-parser-runtime/src/engine/contextual_plan.rs new file mode 100644 index 00000000..51442d95 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/contextual_plan.rs @@ -0,0 +1,60 @@ +//! Immutable contextual admission and token catalog shared across inputs. +use super::contextual_parser::ContextualParserSpec; +use super::model::{Diagnostic, LexicalRule, TerminalSpec}; +use super::validation::{canonical_sha256_digest, validate_spec_once}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +#[derive(Debug)] +pub(super) struct ContextualPlan { + pub(super) terminals: HashMap<&'static str, &'static TerminalSpec>, + pub(super) lexical: HashMap<&'static str, &'static LexicalRule>, +} + +pub(super) fn prepare_once( + product: &'static ContextualParserSpec, +) -> Result, Diagnostic> { + static PLANS: OnceLock>>> = OnceLock::new(); + let key = std::ptr::from_ref(product) as usize; + let mut plans = PLANS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .map_err(|_| reject("contextual plan registry poisoned"))?; + if let Some(plan) = plans.get(&key) { + return Ok(Arc::clone(plan)); + } + let spec = product.language; + validate_spec_once(spec).map_err(|message| reject(&message))?; + if product.scanner.base_grammar_digest != Some(spec.grammar_digest) + || !canonical_sha256_digest(product.parser_digest) + || product.state_positions.len() != spec.actions.len() + || product + .state_positions + .iter() + .any(|p| !product.scanner.positions.contains(p)) + { + return Err(reject( + "invalid contextual parser identity or state positions", + )); + } + // Preserve the existing first-declared lookup semantics for duplicate keys. + let mut terminals = HashMap::new(); + for terminal in spec.terminals { + terminals.entry(terminal.name).or_insert(terminal); + } + let mut lexical = HashMap::new(); + for rule in spec.lexical_rules { + lexical.entry(rule.terminal).or_insert(rule); + } + let plan = Arc::new(ContextualPlan { terminals, lexical }); + plans.insert(key, Arc::clone(&plan)); + Ok(plan) +} + +fn reject(message: &str) -> Diagnostic { + Diagnostic { + reason_kind: "contextual-parser", + byte_offset: 0, + message: message.into(), + } +} diff --git a/rust/gerbil-parser-rowan/src/engine/event_tree.rs b/rust/gerbil-parser-runtime/src/engine/event_tree.rs similarity index 52% rename from rust/gerbil-parser-rowan/src/engine/event_tree.rs rename to rust/gerbil-parser-runtime/src/engine/event_tree.rs index 69ef6447..37698d56 100644 --- a/rust/gerbil-parser-rowan/src/engine/event_tree.rs +++ b/rust/gerbil-parser-runtime/src/engine/event_tree.rs @@ -1,11 +1,11 @@ -//! Validated Rowan sink for structure emitted by Scheme-AOT parsers. +//! Validated indexed syntax sink for source-backed structure events. -use rowan::{GreenNode, GreenToken, NodeOrToken}; +use gerbil_parser_artifact::syntax::{IndexedEntry, SyntaxTree}; -use super::model::{Diagnostic, KindCategory, LanguageSpec, TreeEvent}; +use super::model::{Diagnostic, EventCatalog, KindCategory, LanguageSpec, TreeEvent}; use super::validation::validate_spec_once; -/// Build one lossless Rowan tree from AOT parser events. +/// Build one lossless indexed syntax tree from AOT parser events. /// /// The generic engine checks generated kind identities, balanced node events, /// and ordered UTF-8 token coverage. Language-specific structure and recovery @@ -15,38 +15,79 @@ use super::validation::validate_spec_once; /// /// Returns a typed diagnostic if the generated specification or event stream /// cannot form one root covering the exact source. -pub fn build_rowan_events( +pub fn build_syntax_events( spec: &'static LanguageSpec, source: &str, events: &[TreeEvent], -) -> Result { +) -> Result { validate_spec_once(spec).map_err(|message| diagnostic("invalid-aot-artifact", 0, message))?; - let mut frames: Vec<(u16, Vec>)> = Vec::new(); + build_syntax_events_catalog( + &EventCatalog { + root_kind: spec.root_kind, + kinds: spec.kinds, + }, + source, + events, + ) +} + +/// Build one lossless tree from an independently authored event producer. +/// This validates structure and source coverage, not the producer's grammar +/// or its authority receipt. +/// +/// # Errors +/// +/// Rejects an invalid kind catalog or event stream. +pub fn build_syntax_events_catalog( + catalog: &EventCatalog, + source: &str, + events: &[TreeEvent], +) -> Result { + build_syntax_event_iter(catalog, source, events.iter().copied()) +} + +pub(crate) fn build_syntax_event_iter( + catalog: &EventCatalog, + source: &str, + events: impl IntoIterator, +) -> Result { + validate_catalog(catalog)?; + let mut frames: Vec<(u16, usize)> = Vec::new(); + let mut entries: Vec = Vec::new(); let mut root = None; let mut offset = 0usize; for event in events { - match *event { + match event { TreeEvent::StartNode(kind) => { - if root.is_some() || (frames.is_empty() && kind != spec.root_kind) { + if root.is_some() || (frames.is_empty() && kind != catalog.root_kind) { return Err(diagnostic( "event-root", offset, "events must contain exactly one declared root node", )); } - validate_kind(spec, kind, KindCategory::Node, offset)?; - frames.push((kind, Vec::new())); + validate_kind(catalog, kind, KindCategory::Node, offset)?; + let at = entries.len(); + entries.push(IndexedEntry { + kind, + token: false, + start: offset, + end: 0, + parent: frames.last().map(|f| f.1), + subtree_end: 0, + }); + frames.push((kind, at)); } TreeEvent::Token { kind, start, end } => { - let Some((_, children)) = frames.last_mut() else { + let Some((_, parent)) = frames.last() else { return Err(diagnostic( "event-nesting", start, "a token must be inside the root node", )); }; - validate_kind(spec, kind, KindCategory::Token, start)?; + validate_kind(catalog, kind, KindCategory::Token, start)?; if start != offset || end <= start || end > source.len() @@ -59,28 +100,34 @@ pub fn build_rowan_events( "tokens must cover ordered, nonempty UTF-8 source ranges", )); } - children.push(GreenToken::new(rowan::SyntaxKind(kind), &source[start..end]).into()); + entries.push(IndexedEntry { + kind, + token: true, + start, + end, + parent: Some(*parent), + subtree_end: entries.len() + 1, + }); offset = end; } TreeEvent::FinishNode => { - let Some((kind, children)) = frames.pop() else { + let Some((_kind, at)) = frames.pop() else { return Err(diagnostic( "event-nesting", offset, "node finish has no matching start", )); }; - let node = GreenNode::new(rowan::SyntaxKind(kind), children); - if let Some((_, parent)) = frames.last_mut() { - parent.push(node.into()); - } else { - root = Some(node); + entries[at].end = offset; + entries[at].subtree_end = entries.len(); + if frames.is_empty() { + root = Some(at); } } } } - let Some(green) = root else { + let Some(_root) = root else { return Err(diagnostic( "event-nesting", offset, @@ -101,23 +148,33 @@ pub fn build_rowan_events( "event stream does not cover the source suffix", )); } - if usize::from(green.text_len()) != source.len() { + SyntaxTree::from_index(source, entries) + .map_err(|message| diagnostic("event-range", offset, message)) +} + +fn validate_catalog(catalog: &EventCatalog) -> Result<(), Diagnostic> { + if catalog.kinds.len() > usize::from(u16::MAX) + 1 + || catalog + .kinds + .get(usize::from(catalog.root_kind)) + .is_none_or(|root| root.category != KindCategory::Node) + { return Err(diagnostic( - "event-range", - offset, - "Rowan tree does not cover the complete source", + "invalid-event-catalog", + 0, + "event catalog requires a declared node root and bounded kinds", )); } - Ok(green) + Ok(()) } fn validate_kind( - spec: &LanguageSpec, + catalog: &EventCatalog, kind: u16, category: KindCategory, offset: usize, ) -> Result<(), Diagnostic> { - if spec + if catalog .kinds .get(usize::from(kind)) .is_none_or(|entry| entry.category != category) diff --git a/rust/gerbil-parser-rowan/src/engine/generated_events.rs b/rust/gerbil-parser-runtime/src/engine/generated_events.rs similarity index 89% rename from rust/gerbil-parser-rowan/src/engine/generated_events.rs rename to rust/gerbil-parser-runtime/src/engine/generated_events.rs index 8eff353b..0c410698 100644 --- a/rust/gerbil-parser-rowan/src/engine/generated_events.rs +++ b/rust/gerbil-parser-runtime/src/engine/generated_events.rs @@ -1,6 +1,6 @@ -//! The Rowan handoff for event functions generated from Scheme parser strategies. +//! The source-index handoff for event functions generated from Scheme parser strategies. -use super::event_tree::build_rowan_events; +use super::event_tree::build_syntax_events; use super::model::{Diagnostic, LanguageSpec, Parse, ParseError, SelectiveGlrReceipt, TreeEvent}; use super::validation::{canonical_sha256_digest, receipt}; @@ -32,9 +32,9 @@ pub fn parse_generated_events( message: "generated parser digest is not canonical SHA-256".into(), })); } - let green = build_rowan_events(language, source, events).map_err(with_receipt)?; + let tree = build_syntax_events(language, source, events).map_err(with_receipt)?; Ok(Parse { - green, + tree, kinds: language.kinds, receipt: parse_receipt, selective_glr: SelectiveGlrReceipt { diff --git a/rust/gerbil-parser-runtime/src/engine/graph_index.rs b/rust/gerbil-parser-runtime/src/engine/graph_index.rs new file mode 100644 index 00000000..e5e74cd0 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/graph_index.rs @@ -0,0 +1,193 @@ +//! Language-neutral, validated preorder index for AOT indexed graph queries. + +use std::collections::HashSet; + +use super::graph_projection::GraphRecord; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GraphIndexError { + InvalidRecord, + InvalidScope, + InvalidTarget, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GraphRelation { + Any, + At, + ChildOf, + DescendantOf, +} + +#[derive(Clone, Debug)] +pub struct GraphIndex { + subtree_end: Vec, +} + +impl GraphIndex { + /// Validate preorder ancestry once, then reuse its subtree intervals. + /// + /// # Errors + /// Rejects out-of-order IDs, non-preorder parents, and inconsistent child IDs. + pub fn new(records: &[GraphRecord]) -> Result { + let mut subtree_end = vec![records.len(); records.len()]; + let mut stack = Vec::::new(); + let mut child_counts = vec![0usize; records.len()]; + for (id, record) in records.iter().enumerate() { + if record.id != id { + return Err(GraphIndexError::InvalidRecord); + } + while stack.last().copied() != record.parent_id { + let Some(closed) = stack.pop() else { + return Err(GraphIndexError::InvalidRecord); + }; + subtree_end[closed] = id; + } + if let Some(parent) = record.parent_id { + let children = &records[parent].child_ids; + if children.get(child_counts[parent]) != Some(&id) { + return Err(GraphIndexError::InvalidRecord); + } + child_counts[parent] += 1; + } + stack.push(id); + } + for id in stack { + subtree_end[id] = records.len(); + } + if records + .iter() + .enumerate() + .any(|(id, record)| record.child_ids.len() != child_counts[id]) + { + return Err(GraphIndexError::InvalidRecord); + } + Ok(Self { subtree_end }) + } + + #[must_use] + pub fn subtree_end(&self, id: usize) -> Option { + self.subtree_end.get(id).copied() + } + + #[must_use] + pub fn contains(&self, ancestor: usize, node: usize) -> bool { + self.subtree_end(ancestor) + .is_some_and(|end| ancestor <= node && node < end) + } + + /// Find the nearest matching *strict* ancestor of every preorder record. + /// Each predicate result is computed once; the projection is linear in the + /// graph size, regardless of nesting depth. + /// + /// # Errors + /// Rejects records that do not belong to an index of this size. + pub fn nearest_ancestors_matching( + &self, + records: &[GraphRecord], + mut predicate: F, + ) -> Result>, GraphIndexError> + where + F: FnMut(&GraphRecord) -> bool, + { + if records.len() != self.subtree_end.len() { + return Err(GraphIndexError::InvalidRecord); + } + let mut nearest = Vec::with_capacity(records.len()); + let mut matches = Vec::with_capacity(records.len()); + for record in records { + let ancestor = match record.parent_id { + Some(parent) if parent < nearest.len() => { + if matches[parent] { + Some(parent) + } else { + nearest[parent] + } + } + Some(_) => return Err(GraphIndexError::InvalidRecord), + None => None, + }; + nearest.push(ancestor); + matches.push(predicate(record)); + } + Ok(nearest) + } + + /// Select graph records inside one scope using a language-owned predicate. + /// + /// # Errors + /// Rejects a missing scope or target; records must match this index. + pub fn select( + &self, + records: &[GraphRecord], + scope: usize, + kind: &str, + relation: GraphRelation, + targets: &[usize], + mut predicate: F, + ) -> Result, GraphIndexError> + where + F: FnMut(&GraphRecord) -> bool, + { + let Some(end) = self.subtree_end(scope) else { + return Err(GraphIndexError::InvalidScope); + }; + if records.len() != self.subtree_end.len() { + return Err(GraphIndexError::InvalidRecord); + } + if targets.iter().any(|&id| id >= records.len()) { + return Err(GraphIndexError::InvalidTarget); + } + let intervals = + (relation == GraphRelation::DescendantOf).then(|| self.descendant_intervals(targets)); + let target_set = (targets.len() > 8 + && matches!(relation, GraphRelation::At | GraphRelation::ChildOf)) + .then(|| targets.iter().copied().collect::>()); + let contains_target = |id| { + target_set + .as_ref() + .map_or_else(|| targets.contains(&id), |set| set.contains(&id)) + }; + Ok(records[scope..end] + .iter() + .filter(|record| { + record.kind == kind + && match relation { + GraphRelation::Any => true, + GraphRelation::At => contains_target(record.id), + GraphRelation::ChildOf => record.parent_id.is_some_and(contains_target), + GraphRelation::DescendantOf => { + let ranges = intervals.as_deref().unwrap_or_default(); + let position = ranges.partition_point(|&(start, _)| start <= record.id); + position > 0 && record.id < ranges[position - 1].1 + } + } + && predicate(record) + }) + .map(|record| record.id) + .collect()) + } + + fn descendant_intervals(&self, targets: &[usize]) -> Vec<(usize, usize)> { + let mut ranges: Vec<_> = targets + .iter() + .filter_map(|&target| { + let start = target + 1; + let end = self.subtree_end[target]; + (start < end).then_some((start, end)) + }) + .collect(); + ranges.sort_unstable_by_key(|&(start, _)| start); + let mut merged = Vec::<(usize, usize)>::with_capacity(ranges.len()); + for (start, end) in ranges { + if let Some(last) = merged.last_mut() + && start <= last.1 + { + last.1 = last.1.max(end); + } else { + merged.push((start, end)); + } + } + merged + } +} diff --git a/rust/gerbil-parser-rowan/src/engine/graph_projection.rs b/rust/gerbil-parser-runtime/src/engine/graph_projection.rs similarity index 68% rename from rust/gerbil-parser-rowan/src/engine/graph_projection.rs rename to rust/gerbil-parser-runtime/src/engine/graph_projection.rs index cb85e216..5bd042a2 100644 --- a/rust/gerbil-parser-rowan/src/engine/graph_projection.rs +++ b/rust/gerbil-parser-runtime/src/engine/graph_projection.rs @@ -1,12 +1,12 @@ -//! Linear, language-neutral projection of a validated Rowan CST into records. +//! Linear, language-neutral projection of a validated indexed CST into records. -use rowan::{NodeOrToken, TextRange, WalkEvent}; +use gerbil_parser_artifact::syntax::{SyntaxElement, TextRange, WalkEvent}; use super::model::{Diagnostic, KindCategory, LanguageSpec, SyntaxNode}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct GraphFieldRule { - /// Syntax token whose text is projected into this field. + /// Syntax token, or a node for a node-text mode, whose text is projected. pub token_kind: u16, /// Public field name on the graph record. pub name: &'static str, @@ -14,7 +14,7 @@ pub struct GraphFieldRule { pub mode: GraphFieldMode, } -/// AOT-declared cardinality for repeated graph field tokens. +/// AOT-declared source and cardinality for graph fields. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum GraphFieldMode { /// Join adjacent token text into one field value. @@ -23,6 +23,10 @@ pub enum GraphFieldMode { Each, /// Join token text, or project an empty value when no token is present. AppendOrEmpty, + /// Project the complete source text of a descendant syntax node. + NodeText, + /// Keep each descendant syntax node's complete text as a separate field. + EachNodeText, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -56,6 +60,8 @@ pub struct GraphRecord { pub struct GraphFieldValue { pub name: &'static str, pub value: String, + /// Smallest source span covering all tokens or nodes in this field. + pub range: TextRange, } impl GraphRecord { @@ -75,9 +81,18 @@ impl GraphRecord { .find(|field| field.name == name) .map(|field| field.value.as_str()) } + + /// Return the source span of the first projected field value. + #[must_use] + pub fn field_range(&self, name: &str) -> Option { + self.fields + .iter() + .find(|field| field.name == name) + .map(|field| field.range) + } } -/// Project one Rowan tree using only AOT-resolved syntax kinds and fields. +/// Project one indexed syntax using only AOT-resolved syntax kinds and fields. /// /// The executor never recognizes language syntax or interprets field values. /// It walks each node/token once and attaches a token to its nearest projected @@ -104,7 +119,7 @@ pub fn project_syntax_graph( let mut projected_stack = Vec::::new(); for event in root.preorder_with_tokens() { match event { - WalkEvent::Enter(NodeOrToken::Node(node)) => { + WalkEvent::Enter(SyntaxElement::Node(node)) => { let kind = usize::from(node.kind().0); let Some(rule) = rules.get(kind).and_then(|rule| *rule) else { continue; @@ -126,7 +141,7 @@ pub fn project_syntax_graph( } projected_stack.push(id); } - WalkEvent::Enter(NodeOrToken::Token(token)) => { + WalkEvent::Enter(SyntaxElement::Token(token)) => { let Some(id) = projected_stack.last().copied() else { continue; }; @@ -141,17 +156,19 @@ pub fn project_syntax_graph( .iter_mut() .find(|value| value.name == field.name) { - value.value.push_str(token.text()); + append_field_value(value, token.text(), token.text_range()); continue; } records[id].fields.push(GraphFieldValue { name: field.name, value: token.text().to_owned(), + range: token.text_range(), }); } } } - WalkEvent::Leave(NodeOrToken::Node(node)) => { + WalkEvent::Leave(SyntaxElement::Node(node)) => { + project_node_text_fields(&mut records, &rules, &projected_stack, &node); if let Some(rule) = rules.get(usize::from(node.kind().0)).and_then(|rule| *rule) { if let Some(&id) = projected_stack.last() { for field in rule @@ -164,9 +181,11 @@ pub fn project_syntax_graph( .iter() .any(|value| value.name == field.name) { + let empty_range = TextRange::empty(records[id].range.end()); records[id].fields.push(GraphFieldValue { name: field.name, value: String::new(), + range: empty_range, }); } } @@ -174,12 +193,64 @@ pub fn project_syntax_graph( projected_stack.pop(); } } - WalkEvent::Leave(NodeOrToken::Token(_)) => {} + WalkEvent::Leave(SyntaxElement::Token(_)) => {} } } Ok(records) } +fn project_node_text_fields( + records: &mut [GraphRecord], + rules: &[Option<&GraphNodeRule>], + projected_stack: &[usize], + node: &SyntaxNode, +) { + let Some(&id) = projected_stack.last() else { + return; + }; + let Some(rule) = rules + .get(usize::from(records[id].syntax_kind)) + .and_then(|rule| *rule) + else { + return; + }; + for field in rule.fields.iter().filter(|field| { + matches!( + field.mode, + GraphFieldMode::NodeText | GraphFieldMode::EachNodeText + ) && field.token_kind == node.kind().0 + }) { + let value = node.text().to_string(); + if field.mode == GraphFieldMode::EachNodeText { + records[id].fields.push(GraphFieldValue { + name: field.name, + value, + range: node.text_range(), + }); + } else if let Some(existing) = records[id] + .fields + .iter_mut() + .find(|existing| existing.name == field.name) + { + append_field_value(existing, &value, node.text_range()); + } else { + records[id].fields.push(GraphFieldValue { + name: field.name, + value, + range: node.text_range(), + }); + } + } +} + +fn append_field_value(field: &mut GraphFieldValue, text: &str, range: TextRange) { + field.value.push_str(text); + field.range = TextRange::new( + field.range.start().min(range.start()), + field.range.end().max(range.end()), + ); +} + fn validate_rules<'a>( language: &LanguageSpec, spec: &'a GraphProjectionSpec, @@ -213,10 +284,18 @@ fn validate_rules<'a>( return Err(invalid("graph node kind is invalid or declared twice")); } for field in rule.fields { + let expected_category = if matches!( + field.mode, + GraphFieldMode::NodeText | GraphFieldMode::EachNodeText + ) { + KindCategory::Node + } else { + KindCategory::Token + }; if language .kinds .get(usize::from(field.token_kind)) - .is_none_or(|kind| kind.category != KindCategory::Token) + .is_none_or(|kind| kind.category != expected_category) || field.name.is_empty() { return Err(invalid("graph field references an invalid token or name")); diff --git a/rust/gerbil-parser-rowan/src/engine/lexer.rs b/rust/gerbil-parser-runtime/src/engine/lexer.rs similarity index 59% rename from rust/gerbil-parser-rowan/src/engine/lexer.rs rename to rust/gerbil-parser-runtime/src/engine/lexer.rs index c3669264..1bf1d068 100644 --- a/rust/gerbil-parser-rowan/src/engine/lexer.rs +++ b/rust/gerbil-parser-runtime/src/engine/lexer.rs @@ -4,6 +4,29 @@ use std::collections::HashMap; use super::model::{Diagnostic, LanguageSpec, LexicalExpr, LexicalRule, ScannedToken, Token}; +/// An admitted nonnullable profile shared by lexical and embedded recognition. +/// Preparation validates the static IR once; calls return absolute byte ends. +#[derive(Debug)] +pub struct PreparedTextProfile { + profile: &'static super::model::TextProfile, +} +impl PreparedTextProfile { + #[must_use] + pub fn new(profile: &'static super::model::TextProfile) -> Option { + (super::validation::text_profile_width(profile, 0)? > 0).then_some(Self { profile }) + } + /// Match within `[start, limit)`. Invalid UTF-8 boundaries and no match + /// both return `None`, matching the bounded Scheme scanner contract. + #[must_use] + pub fn match_prefix(&self, source: &str, start: usize, limit: usize) -> Option { + if start > limit || !source.is_char_boundary(start) { + return None; + } + let bounded = source.get(..limit)?; + text_profile_end(self.profile, bounded, start).filter(|end| *end > start) + } +} + pub(crate) fn lex_scanned<'source>( spec: &LanguageSpec, source: &'source str, @@ -69,6 +92,10 @@ pub(crate) fn lex<'source>( spec: &LanguageSpec, source: &'source str, ) -> Result<(Vec>, Vec), Diagnostic> { + let prepared = PreparedLexicalSource::new( + source, + spec.lexical_rules.iter().map(|rule| &rule.expression), + ); let mut offset = 0; let token_capacity = source.len().min(256); let mut tokens = Vec::with_capacity(token_capacity); @@ -83,7 +110,7 @@ pub(crate) fn lex<'source>( } let mut selected: Option<(&LexicalRule, usize, usize)> = None; for (order, rule) in spec.lexical_rules.iter().enumerate() { - let Some(end) = lexical_end(&rule.expression, source, offset) else { + let Some(end) = lexical_end_prepared(&rule.expression, offset, &prepared) else { continue; }; if end <= offset { @@ -134,9 +161,217 @@ pub(crate) fn lex<'source>( Ok((tokens, significant)) } +fn text_class_matches(class: &super::model::TextClass, character: char) -> bool { + use super::model::TextClass; + match class { + TextClass::Alphabetic => super::unicode_alphabetic::is_scheme_alphabetic(character), + TextClass::Numeric => super::unicode_numeric::is_scheme_numeric(character), + TextClass::AsciiLetter => character.is_ascii_alphabetic(), + TextClass::Characters(values) => values.contains(character), + TextClass::Union(classes) => classes + .iter() + .any(|class| text_class_matches(class, character)), + } +} + +fn text_profile_end( + profile: &super::model::TextProfile, + source: &str, + start: usize, +) -> Option { + use super::model::TextProfile; + let suffix = source.get(start..)?; + match profile { + TextProfile::Literal(value) => suffix.starts_with(value).then_some(start + value.len()), + TextProfile::Run { + class, + minimum, + maximum, + } => { + let mut count = 0; + let mut end = start; + for character in suffix.chars() { + if maximum.is_some_and(|maximum| count >= maximum) + || !text_class_matches(class, character) + { + break; + } + count += 1; + end += character.len_utf8(); + } + (count >= *minimum).then_some(end) + } + TextProfile::RunContaining { + class, + required, + minimum, + maximum, + } => { + let mut count = 0; + let mut end = start; + let mut seen = false; + for character in suffix.chars() { + if maximum.is_some_and(|maximum| count >= maximum) + || !text_class_matches(class, character) + { + break; + } + seen = seen || text_class_matches(required, character); + count += 1; + end += character.len_utf8(); + } + (seen && count >= *minimum).then_some(end) + } + TextProfile::EndsIn { + class, + body, + positive, + } => { + let end = text_profile_end(body, source, start)?; + let last = source.get(start..end)?.chars().next_back()?; + (text_class_matches(class, last) == *positive).then_some(end) + } + TextProfile::Sequence(steps) => { + let mut end = start; + for step in *steps { + end = text_profile_end(step, source, end)?; + } + Some(end) + } + TextProfile::Optional(step) => Some(text_profile_end(step, source, start).unwrap_or(start)), + TextProfile::IfNext { + class, + body, + otherwise, + } => { + if suffix + .chars() + .next() + .is_some_and(|character| text_class_matches(class, character)) + { + text_profile_end(body, source, start) + } else { + otherwise.map_or(Some(start), |step| text_profile_end(step, source, start)) + } + } + TextProfile::NotNext(class) => (!suffix + .chars() + .next() + .is_some_and(|character| text_class_matches(class, character))) + .then_some(start), + } +} + +// Borrow the bounded delimiter span directly from the source, without allocation. +fn header_delimiters<'a>(source: &'a str, prefix: &str, count: usize) -> Option<&'a str> { + if prefix.is_empty() || !(1..=32).contains(&count) { + return None; + } + let suffix = source.strip_prefix(prefix)?; + let mut end = 0; + for (at, character) in suffix.char_indices().take(count) { + if super::unicode_alphabetic::is_scheme_alphabetic(character) + || super::unicode_numeric::is_scheme_numeric(character) + || suffix[..at].contains(character) + { + return None; + } + end = at + character.len_utf8(); + } + let delimiters = &suffix[..end]; + (delimiters.chars().count() == count).then_some(delimiters) +} + +// Request-owned captures borrow their source. Invalid headers are cached too. +struct PreparedLexicalSource<'source> { + source: &'source str, + headers: Vec<(&'static str, usize, Option<&'source str>)>, + modules: Vec<( + &'static super::model::ModuleTextProfile, + super::module_source::PreparedModuleSource<'source>, + )>, +} +impl<'source> PreparedLexicalSource<'source> { + fn new( + source: &'source str, + expressions: impl IntoIterator, + ) -> Self { + let mut prepared = Self { + source, + headers: Vec::new(), + modules: Vec::new(), + }; + for expression in expressions { + prepared.admit(source, expression); + } + prepared + } + fn admit(&mut self, source: &'source str, expression: &LexicalExpr) { + match expression { + LexicalExpr::ModuleText(profile) + if !self.modules.iter().any(|(recipe, _)| recipe == profile) => + { + self.modules.push(( + profile, + super::module_source::PreparedModuleSource::new(source, profile), + )); + } + LexicalExpr::HeaderDelimiter { prefix, count, .. } + | LexicalExpr::HeaderData { prefix, count, .. } + if !self + .headers + .iter() + .any(|(name, width, _)| name == prefix && width == count) => + { + self.headers + .push((prefix, *count, header_delimiters(source, prefix, *count))); + } + LexicalExpr::Choice(expressions) => { + for child in *expressions { + self.admit(source, child); + } + } + _ => {} + } + } + fn module_end( + &self, + profile: &super::model::ModuleTextProfile, + offset: usize, + ) -> Option { + self.modules + .iter() + .find(|(recipe, _)| *recipe == profile)? + .1 + .end(offset) + } + fn header(&self, prefix: &str, count: usize) -> Option<&'source str> { + self.headers + .iter() + .find(|(name, width, _)| *name == prefix && *width == count)? + .2 + } +} + +#[cfg(test)] pub(crate) fn lexical_end(expression: &LexicalExpr, source: &str, offset: usize) -> Option { - let suffix = &source[offset..]; + let prepared = PreparedLexicalSource::new(source, [expression]); + lexical_end_prepared(expression, offset, &prepared) +} + +fn lexical_end_prepared( + expression: &LexicalExpr, + offset: usize, + prepared: &PreparedLexicalSource<'_>, +) -> Option { + let source = prepared.source; + + let suffix = source.get(offset..)?; match expression { + LexicalExpr::ModuleText(profile) => prepared.module_end(profile, offset), + LexicalExpr::TextProfile(profile) => { + text_profile_end(profile, source, offset).filter(|end| *end > offset) + } LexicalExpr::Whitespace => consume_while(source, offset, char::is_whitespace), LexicalExpr::HorizontalWhitespace => { consume_while(source, offset, |character| matches!(character, ' ' | '\t')) @@ -163,15 +398,48 @@ pub(crate) fn lexical_end(expression: &LexicalExpr, source: &str, offset: usize) *trailing_period, ), LexicalExpr::Identifier => identifier_end(source, offset), + LexicalExpr::HeaderDelimiter { + prefix, + count, + index, + } => { + if index >= count { + return None; + } + let delimiter = prepared.header(prefix, *count)?.chars().nth(*index)?; + suffix + .starts_with(delimiter) + .then_some(offset + delimiter.len_utf8()) + } + LexicalExpr::HeaderData { + prefix, + count, + stops, + } => { + let delimiters = prepared.header(prefix, *count)?; + consume_while(source, offset, |character| { + !stops.contains(character) && !delimiters.contains(character) + }) + } LexicalExpr::UntilDelimiters(delimiters) => consume_while(source, offset, |character| { !character.is_whitespace() && !delimiters.contains(character) }), + LexicalExpr::CharacterRun { character, minimum } => { + let symbol = character.chars().next()?; + let end = consume_while(source, offset, |ch| ch == symbol)?; + (source[offset..end].chars().count() >= *minimum).then_some(end) + } LexicalExpr::QuotedString(delimiters) => delimiters .iter() .find_map(|delimiter| quoted_string_end(source, offset, delimiter, true)), LexicalExpr::EscapedQuotedString(delimiters) => delimiters .iter() .find_map(|delimiter| quoted_string_end(source, offset, delimiter, false)), + LexicalExpr::QuotedStringProfile { + delimiter, + escapes, + unicode_width, + } => quoted_string_profile_end(source, offset, delimiter, escapes, *unicode_width), LexicalExpr::Heredoc => heredoc_end(source, offset), LexicalExpr::LineComment(prefixes) => line_comment_end(source, offset, prefixes), LexicalExpr::BlockComment { opening, closing } => { @@ -182,7 +450,7 @@ pub(crate) fn lexical_end(expression: &LexicalExpr, source: &str, offset: usize) } LexicalExpr::Choice(expressions) => expressions .iter() - .filter_map(|expression| lexical_end(expression, source, offset)) + .filter_map(|expression| lexical_end_prepared(expression, offset, prepared)) .max(), LexicalExpr::Literals(values) => values .iter() @@ -471,3 +739,47 @@ fn consume_while(source: &str, offset: usize, predicate: impl Fn(char) -> bool) } (end > offset).then_some(end) } + +fn quoted_string_profile_end( + source: &str, + offset: usize, + delimiter: &str, + escapes: &str, + unicode_width: usize, +) -> Option { + let quote = delimiter.chars().next()?; + if delimiter.chars().count() != 1 || unicode_width > 8 { + return None; + } + let mut characters = source[offset..].char_indices(); + if characters.next()?.1 != quote { + return None; + } + while let Some((position, character)) = characters.next() { + if character == quote { + return Some(offset + position + character.len_utf8()); + } + if character == '\\' { + let escape = characters.next()?.1; + if escapes.contains(escape) { + continue; + } + if escape != 'u' || unicode_width == 0 { + return None; + } + for _ in 0..unicode_width { + let digit = characters.next()?.1; + if !super::unicode_numeric::is_scheme_numeric(digit) + && !matches!(digit, 'a'..='f' | 'A'..='F') + { + return None; + } + } + } + } + None +} + +#[cfg(test)] +#[path = "../../tests/unit/prepared_source.rs"] +mod prepared_source_tests; diff --git a/rust/gerbil-parser-runtime/src/engine/mod.rs b/rust/gerbil-parser-runtime/src/engine/mod.rs new file mode 100644 index 00000000..fd5273fe --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/mod.rs @@ -0,0 +1,120 @@ +//! Generated-table execution owners for the parser runtime. + +mod command_model; +mod command_program; +mod command_source; +pub use command_source::{CommandEvent, CommandParse, CommandSourceSpec, PreparedCommandSource}; +mod contextual_parser; +pub use command_program::{ + CommandChoice, CommandForm, CommandInstruction, CommandProgramSpec, CommandRole, + CommandTokenClass, CommandTrigger, PreparedCommandProgram, +}; +mod contextual_plan; +pub(crate) mod event_tree; +mod generated_events; +mod graph_index; +mod graph_projection; +mod lexer; +mod model; +mod module_source; +mod parser; +mod part_profile; +mod word_parts; +pub use part_profile::{ + BindingSpec, PartGuard, PartOpcode, PartProfileSpec, PartRule, PreparedPartProfile, + SubscriptSpec, +}; +mod recognition_index; +mod result_projection; +mod structural_block_opening; +mod structural_key_line; +mod structural_lines; +mod structural_list; +mod structural_table; +mod unicode_alphabetic; +mod unicode_numeric; +mod unicode_whitespace; +pub(crate) use unicode_whitespace::is_scheme_whitespace; +mod validation; + +pub use contextual_parser::{ContextualParserSpec, parse_contextual}; +pub use event_tree::{build_syntax_events, build_syntax_events_catalog}; +pub use generated_events::parse_generated_events; +pub use graph_index::{GraphIndex, GraphIndexError, GraphRelation}; +pub use graph_projection::{ + GraphFieldMode, GraphFieldRule, GraphFieldValue, GraphNodeRule, GraphProjectionSpec, + GraphRecord, project_syntax_graph, +}; +pub use lexer::PreparedTextProfile; +pub use model::{ + ActionEntry, BlockContents, BlockHeaderRule, BlockLineRule, BlockOpeningMode, Diagnostic, + EventCatalog, GotoEntry, HeadingFieldsRule, HeadingLineRule, InlineLinkRule, KeyLineContext, + KeyLineMode, KeyLineRule, KeyValueLineRule, KindCategory, KindSpec, LanguageSpec, LexicalExpr, + LexicalRule, LineStructureSpec, ListLineRule, ModuleTextProfile, Operand, OperandAction, Parse, + ParseError, ParseReceipt, ParserAction, Production, Reduction, ScannedToken, + SelectiveGlrReceipt, Symbol, SyntaxKind, SyntaxNode, SyntaxToken, TableLineRule, Terminal, + TerminalSpec, TextClass, TextProfile, TreeEvent, UnclosedBlockPolicy, +}; +pub use parser::{parse, parse_scanned}; +pub use result_projection::{ + CaptureKind, CaptureSpec, PreparedResultNode, PreparedResultProfile, PreparedResultProjection, + ProjectedChild, ProjectedNode, ProjectedValue, ProjectionInstruction, ProjectionOpcode, + ResultCapture, ResultChildCapture, ResultNodeSpec, ResultProfileSpec, ResultProjection, +}; +pub use structural_lines::parse_structural_lines; + +#[cfg(test)] +#[path = "../../tests/unit/lexical.rs"] +mod lexical_tests; + +#[cfg(test)] +#[path = "../../tests/unit/selective_glr.rs"] +mod selective_glr_tests; + +#[cfg(test)] +#[path = "../../tests/unit/event_tree.rs"] +mod event_tree_tests; + +#[cfg(test)] +#[path = "../../tests/unit/structural_lines.rs"] +mod structural_lines_tests; + +#[cfg(test)] +#[path = "../../tests/unit/pure_function_aot.rs"] +mod pure_function_aot_tests; + +#[cfg(test)] +#[path = "../../tests/unit/event_strategy_aot.rs"] +mod event_strategy_aot_tests; + +#[cfg(test)] +#[path = "../../tests/unit/event_fold_aot.rs"] +mod event_fold_aot_tests; + +#[cfg(test)] +#[path = "../../tests/unit/graph_index.rs"] +mod graph_index_tests; + +#[cfg(test)] +#[path = "../../tests/unit/header_delimiters.rs"] +mod header_delimiter_tests; + +#[cfg(test)] +#[path = "../../tests/unit/binding_names.rs"] +mod binding_names_tests; + +#[cfg(test)] +#[path = "../../tests/unit/result_projection.rs"] +mod result_projection_tests; + +#[cfg(test)] +#[path = "../../tests/unit/word_parts.rs"] +mod word_parts_tests; + +#[cfg(test)] +#[path = "../../tests/unit/ordered_results.rs"] +mod ordered_results_tests; + +#[cfg(test)] +#[path = "../../tests/unit/command_source.rs"] +mod command_source_tests; diff --git a/rust/gerbil-parser-rowan/src/engine/model.rs b/rust/gerbil-parser-runtime/src/engine/model.rs similarity index 82% rename from rust/gerbil-parser-rowan/src/engine/model.rs rename to rust/gerbil-parser-runtime/src/engine/model.rs index 40cbfa4b..98b2568c 100644 --- a/rust/gerbil-parser-rowan/src/engine/model.rs +++ b/rust/gerbil-parser-runtime/src/engine/model.rs @@ -1,6 +1,6 @@ //! Public generated-table schema and internal parse values. -use rowan::GreenNode; +use gerbil_parser_artifact::syntax::SyntaxTree; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum KindCategory { @@ -14,14 +14,77 @@ pub struct KindSpec { pub category: KindCategory, } +/// The structural vocabulary needed by a lossless event producer. It does +/// not imply that the producer uses Gerbil Grammar IR or generated LR tables. +#[derive(Clone, Copy, Debug)] +pub struct EventCatalog { + pub root_kind: u16, + pub kinds: &'static [KindSpec], +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct TerminalSpec { pub name: &'static str, pub syntax_kind: u16, } +/// Closed character predicates shared with Scheme text-profile IR. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TextClass { + Numeric, + Alphabetic, + AsciiLetter, + Characters(&'static str), + Union(&'static [TextClass]), +} + +/// Offset-only lexical control: optional rollback and guarded commitment are +/// distinct operations. Root profiles must have positive minimum width. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TextProfile { + Literal(&'static str), + Run { + class: TextClass, + minimum: usize, + maximum: Option, + }, + RunContaining { + class: TextClass, + required: TextClass, + minimum: usize, + maximum: Option, + }, + EndsIn { + class: TextClass, + body: &'static TextProfile, + positive: bool, + }, + Sequence(&'static [TextProfile]), + Optional(&'static TextProfile), + IfNext { + class: TextClass, + body: &'static TextProfile, + otherwise: Option<&'static TextProfile>, + }, + NotNext(TextClass), +} + +/// Closed, admitted framing values for a source-owned module index. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ModuleTextProfile { + pub header_border: &'static str, + pub header_word: &'static str, + pub end_border: &'static str, + pub block_open: &'static str, + pub block_close: &'static str, + pub line_comment: &'static str, + pub name_extra: &'static str, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum LexicalExpr { + ModuleText(&'static ModuleTextProfile), + TextProfile(&'static TextProfile), Whitespace, HorizontalWhitespace, Newline, @@ -37,8 +100,27 @@ pub enum LexicalExpr { }, Identifier, UntilDelimiters(&'static str), + HeaderDelimiter { + prefix: &'static str, + count: usize, + index: usize, + }, + HeaderData { + prefix: &'static str, + count: usize, + stops: &'static str, + }, + CharacterRun { + character: &'static str, + minimum: usize, + }, QuotedString(&'static [&'static str]), EscapedQuotedString(&'static [&'static str]), + QuotedStringProfile { + delimiter: &'static str, + escapes: &'static str, + unicode_width: usize, + }, Heredoc, LineComment(&'static [&'static str]), BlockComment { @@ -317,26 +399,7 @@ pub struct LanguageSpec { pub productions: &'static [Production], } -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct SyntaxKind(pub u16); - -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub enum GerbilLanguage {} - -impl rowan::Language for GerbilLanguage { - type Kind = SyntaxKind; - - fn kind_from_raw(raw: rowan::SyntaxKind) -> Self::Kind { - SyntaxKind(raw.0) - } - - fn kind_to_raw(kind: Self::Kind) -> rowan::SyntaxKind { - rowan::SyntaxKind(kind.0) - } -} - -pub type SyntaxNode = rowan::SyntaxNode; -pub type SyntaxToken = rowan::SyntaxToken; +pub use gerbil_parser_artifact::syntax::{SyntaxKind, SyntaxNode, SyntaxToken}; #[derive(Clone, Debug, Eq, PartialEq)] pub struct ParseReceipt { @@ -361,7 +424,8 @@ pub struct SelectiveGlrReceipt { pub successful_completions: usize, pub distinct_completions: usize, pub winner_reason: &'static str, - pub dynamic_score: i32, + /// Exact accumulated score within the runtime's admitted i64 domain. + pub dynamic_score: i64, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -373,7 +437,7 @@ pub struct Diagnostic { #[derive(Clone, Debug)] pub struct Parse { - pub(crate) green: GreenNode, + pub(crate) tree: SyntaxTree, pub(crate) kinds: &'static [KindSpec], pub(crate) receipt: ParseReceipt, pub(crate) selective_glr: SelectiveGlrReceipt, @@ -382,7 +446,7 @@ pub struct Parse { impl Parse { #[must_use] pub fn syntax(&self) -> SyntaxNode { - SyntaxNode::new_root(self.green.clone()) + self.tree.root() } #[must_use] diff --git a/rust/gerbil-parser-runtime/src/engine/module_source.rs b/rust/gerbil-parser-runtime/src/engine/module_source.rs new file mode 100644 index 00000000..8d303147 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/module_source.rs @@ -0,0 +1,198 @@ +//! Source-owned module framing indexes. Border runs remain compact intervals. +use super::model::ModuleTextProfile; + +struct HeaderInterval { + first: usize, + last: usize, + end: usize, +} +struct DepthEvent { + at: usize, + depth: i64, +} +pub(super) struct PreparedModuleSource<'source> { + source: &'source str, + profile: &'static ModuleTextProfile, + headers: Vec, + events: Vec, +} +fn prefix_at(source: &str, at: usize, text: &str) -> bool { + source + .get(at..) + .is_some_and(|suffix| suffix.starts_with(text)) +} +fn run_end(source: &str, start: usize, accept: impl Fn(char) -> bool) -> usize { + let mut end = start; + for ch in source[start..].chars() { + if !accept(ch) { + break; + } + end += ch.len_utf8(); + } + end +} +fn whitespace(ch: char) -> bool { + super::unicode_whitespace::is_scheme_whitespace(ch) +} +fn name_end(source: &str, at: usize, extra: char) -> Option { + let mut end = at; + let mut seen = false; + for ch in source[at..].chars() { + let numeric = super::unicode_numeric::is_scheme_numeric(ch); + let alphabetic = super::unicode_alphabetic::is_scheme_alphabetic(ch); + if !numeric && !alphabetic && ch != extra { + break; + } + seen |= alphabetic || ch == extra; + end += ch.len_utf8(); + } + seen.then_some(end) +} +fn header_tail_end(source: &str, at: usize, profile: &ModuleTextProfile) -> Option { + let word = run_end(source, at, whitespace); + if !prefix_at(source, word, profile.header_word) { + return None; + } + let name = run_end(source, word + profile.header_word.len(), whitespace); + let name_stop = name_end(source, name, profile.name_extra.chars().next()?)?; + let close = run_end(source, name_stop, whitespace); + if !prefix_at(source, close, profile.header_border) { + return None; + } + let border = profile.header_border.chars().next()?; + Some(run_end(source, close, |ch| ch == border)) +} +fn header_intervals(source: &str, profile: &ModuleTextProfile) -> Vec { + let border = profile.header_border.chars().next().unwrap(); + let mut rows = Vec::new(); + let mut at = 0; + while at < source.len() { + let ch = source[at..].chars().next().unwrap(); + if ch == border { + let end = run_end(source, at, |ch| ch == border); + if end - at >= profile.header_border.len() + && let Some(stop) = header_tail_end(source, end, profile) + { + rows.push(HeaderInterval { + first: at, + last: end - profile.header_border.len(), + end: stop, + }); + } + at = end; + } else { + at += ch.len_utf8(); + } + } + rows +} +fn skip_block(source: &str, start: usize, open: &str, close: &str) -> usize { + let mut at = start + open.len(); + let mut depth = 1; + while at < source.len() { + if prefix_at(source, at, open) { + depth += 1; + at += open.len(); + } else if prefix_at(source, at, close) { + at += close.len(); + depth -= 1; + if depth == 0 { + break; + } + } else { + at += source[at..].chars().next().unwrap().len_utf8(); + } + } + at +} +fn skip_string(source: &str, start: usize) -> usize { + let mut at = start + 1; + while at < source.len() { + let ch = source[at..].chars().next().unwrap(); + at += ch.len_utf8(); + if ch == '"' { + break; + } + if ch == '\\' + && let Some(next) = source[at..].chars().next() + { + at += next.len_utf8(); + } + } + at +} +fn depth_events( + source: &str, + profile: &ModuleTextProfile, + headers: &[HeaderInterval], +) -> Vec { + let mut events = Vec::new(); + let mut at = 0; + let mut cursor = 0; + let mut depth = 0; + while at < source.len() { + while headers.get(cursor).is_some_and(|row| row.last < at) { + cursor += 1; + } + if prefix_at(source, at, profile.block_open) { + at = skip_block(source, at, profile.block_open, profile.block_close); + } else if source[at..].starts_with('"') { + at = skip_string(source, at); + } else if prefix_at(source, at, profile.line_comment) { + at = run_end(source, at, |ch| ch != '\n'); + } else if let Some(row) = headers.get(cursor).filter(|row| row.first <= at) { + depth += 1; + events.push(DepthEvent { at, depth }); + at = row.end; + } else if prefix_at(source, at, profile.end_border) { + depth -= 1; + events.push(DepthEvent { at, depth }); + let ch = profile.end_border.chars().next().unwrap(); + at = run_end(source, at, |item| item == ch); + } else { + at += source[at..].chars().next().unwrap().len_utf8(); + } + } + events +} +impl<'source> PreparedModuleSource<'source> { + pub(super) fn new(source: &'source str, profile: &'static ModuleTextProfile) -> Self { + let headers = header_intervals(source, profile); + let events = depth_events(source, profile, &headers); + Self { + source, + profile, + headers, + events, + } + } + pub(super) fn end(&self, start: usize) -> Option { + if start >= self.source.len() || !self.source.is_char_boundary(start) { + return None; + } + if start != 0 { + let prior = self.source[..start] + .char_indices() + .rev() + .find(|&(_, ch)| !whitespace(ch))?; + let prior_end = prior.0 + prior.1.len_utf8(); + if !self.source[..prior_end].ends_with(self.profile.end_border) { + return None; + } + let index = self.events.partition_point(|row| row.at < start); + if index > 0 && self.events[index - 1].depth != 0 { + return None; + } + } + let index = self.headers.partition_point(|row| row.last < start); + let end = self + .headers + .get(index) + .map_or(self.source.len(), |row| row.first.max(start)); + (end > start).then_some(end) + } +} + +#[cfg(test)] +#[path = "../../tests/unit/prepared_module.rs"] +mod tests; diff --git a/rust/gerbil-parser-rowan/src/engine/parser.rs b/rust/gerbil-parser-runtime/src/engine/parser.rs similarity index 92% rename from rust/gerbil-parser-rowan/src/engine/parser.rs rename to rust/gerbil-parser-runtime/src/engine/parser.rs index d4dcbf36..7c321e7f 100644 --- a/rust/gerbil-parser-rowan/src/engine/parser.rs +++ b/rust/gerbil-parser-runtime/src/engine/parser.rs @@ -6,7 +6,7 @@ use super::model::{ ParserFailure, Production, Reduction, ScannedToken, SelectiveGlrReceipt, Terminal, Token, Value, }; -use super::rowan_tree::build_green; +use super::recognition_index::build_syntax; use super::validation::{canonical_sha256_digest, receipt, validate_spec_once}; /// Parse `source` with one generated language specification. @@ -23,7 +23,7 @@ pub fn parse(spec: &'static LanguageSpec, source: &str) -> Result( diagnostic: Box::new(failure.diagnostic), selective_glr: failure.selective_glr, })?; - let green = build_green(spec, source, &tokens, &root).map_err(|message| ParseError { + let tree = build_syntax(spec, source, &tokens, &root).map_err(|message| ParseError { receipt: Box::new(receipt.clone()), diagnostic: Box::new(Diagnostic { reason_kind: "invalid-cst", @@ -89,7 +89,7 @@ fn parse_inner<'source>( selective_glr: None, })?; Ok(Parse { - green, + tree, kinds: spec.kinds, receipt, selective_glr, @@ -97,19 +97,27 @@ fn parse_inner<'source>( } const SELECTIVE_GLR_BRANCH_BUDGET: usize = 256; +#[cfg(test)] +#[path = "../../tests/unit/precedence_capacity.rs"] +mod score_tests; + +#[cfg(test)] +#[path = "../../tests/unit/nullable_boundary.rs"] +mod nullable_boundary_tests; + #[derive(Clone, Debug)] -struct ParserConfiguration { - states: Vec, - values: Vec, - cursor: usize, - score: i32, +pub(super) struct ParserConfiguration { + pub(super) states: Vec, + pub(super) values: Vec, + pub(super) cursor: usize, + pub(super) score: i64, } #[derive(Clone, Debug)] struct ParseCandidate { value: Value, cursor: usize, - score: i32, + score: i64, ambiguities: usize, winner_reason: &'static str, } @@ -181,7 +189,7 @@ fn parse_tokens( fn selective_glr_receipt( context: &GlrContext, winner_reason: &'static str, - dynamic_score: i32, + dynamic_score: i64, ) -> SelectiveGlrReceipt { SelectiveGlrReceipt { branch_budget: SELECTIVE_GLR_BRANCH_BUDGET, @@ -281,7 +289,7 @@ fn run_configuration( } } -fn apply_reduce( +pub(super) fn apply_reduce( spec: &LanguageSpec, tokens: &[Token<'_>], token: Option<&Token<'_>>, @@ -304,6 +312,17 @@ fn apply_reduce( message: format!("production {production_id} underflows the LR stack"), }); } + let score = configuration + .score + .checked_add(i64::from(production.dynamic_precedence)) + .ok_or_else(|| Diagnostic { + reason_kind: "dynamic-precedence-overflow", + byte_offset: token.map_or_else( + || tokens.last().map_or(0, |last| last.end), + |lookahead| lookahead.start, + ), + message: "accumulated dynamic precedence exceeds signed 64-bit capacity".into(), + })?; configuration .states .truncate(configuration.states.len() - count); @@ -330,7 +349,7 @@ fn apply_reduce( })?; configuration.states.push(next); configuration.values.push(reduced); - configuration.score += production.dynamic_precedence; + configuration.score = score; Ok(()) } @@ -374,7 +393,9 @@ fn explore_fork( Err(diagnostic) if matches!( diagnostic.reason_kind, - "invalid-aot-artifact" | "selective-glr-budget-exhausted" + "invalid-aot-artifact" + | "selective-glr-budget-exhausted" + | "dynamic-precedence-overflow" ) => { return Err(diagnostic); @@ -546,7 +567,11 @@ fn current_offset(tokens: &[Token<'_>], significant: &[usize], cursor: usize) -> ) } -fn find_action(spec: &LanguageSpec, state: u32, token: Option<&Token<'_>>) -> Option { +pub(super) fn find_action( + spec: &LanguageSpec, + state: u32, + token: Option<&Token<'_>>, +) -> Option { let row = spec.actions.get(state as usize)?; if let Some(token) = token { let mut token_action = None; @@ -579,7 +604,7 @@ fn find_action(spec: &LanguageSpec, state: u32, token: Option<&Token<'_>>) -> Op fn reduce( production: &Production, - mut values: impl ExactSizeIterator, + mut values: impl ExactSizeIterator + DoubleEndedIterator, tokens: &[Token<'_>], fallback: usize, ) -> Value { @@ -592,9 +617,12 @@ fn reduce( ); } let mut groups = Vec::with_capacity(values.len()); - for (operand, value) in production.rhs.iter().zip(values) { - groups.push(apply_operand_actions(operand, value, tokens, fallback)); + let mut boundary = fallback; + for (operand, value) in production.rhs.iter().zip(values).rev() { + boundary = value.start(tokens, boundary); + groups.push(apply_operand_actions(operand, value, tokens, boundary)); } + groups.reverse(); match production.reduction { Reduction::Pass | Reduction::Concat => { Value::Fragment(groups.into_iter().flat_map(Value::into_children).collect()) diff --git a/rust/gerbil-parser-runtime/src/engine/part_profile.rs b/rust/gerbil-parser-runtime/src/engine/part_profile.rs new file mode 100644 index 00000000..5436f3aa --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/part_profile.rs @@ -0,0 +1,418 @@ +//! Admission and first-character dispatch for Scheme-owned part/binding tables. +use super::{ + CaptureKind, Diagnostic, PreparedResultProfile, PreparedResultProjection, PreparedTextProfile, + ResultProfileSpec, TextProfile, +}; +use crate::scanner::{PreparedRegionPlan, RegionScope, RegionSpec}; +use std::collections::{HashMap, HashSet}; + +/// Declared subscript opener with the Scheme-derived restricted region plan. +#[derive(Clone, Copy, Debug)] +pub struct SubscriptSpec { + pub prefix: &'static str, + pub closing: char, + pub regions: &'static RegionSpec, + pub scopes: &'static [RegionScope], +} +/// A binding recognizer and its bounded prefix/operator vocabulary. +#[derive(Clone, Copy, Debug)] +pub struct BindingSpec { + pub name: &'static TextProfile, + pub prefixes: &'static [&'static str], + pub operators: &'static [&'static str], + pub subscript: Option, +} +/// Guard on the character following a declared prefix. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PartGuard { + Any, + Next, + Characters(&'static str), +} +/// Closed part recognition operations; node vocabulary belongs to projections. +#[derive(Clone, Copy, Debug)] +pub enum PartOpcode { + Quote { delimiter: char }, + QuotedBody { delimiter: char }, + Pair, + Parameter { binding: usize }, + Name { binding: usize }, + Escape, +} +/// Ordered rule shared by one or more declared contexts. +#[derive(Clone, Copy, Debug)] +pub struct PartRule { + pub contexts: &'static [&'static str], + pub prefix: &'static str, + pub guard: PartGuard, + pub opcode: PartOpcode, + pub projection: &'static str, + pub opening: usize, + pub closing: usize, +} +/// Immutable part/binding tables lowered from an admitted POO profile. +#[derive(Clone, Copy, Debug)] +pub struct PartProfileSpec { + pub contexts: &'static [&'static str], + pub rules: &'static [PartRule], + pub bindings: &'static [BindingSpec], + pub assignment: Option, + pub literal: &'static str, +} +struct PartContext { + ascii: Box<[Vec; 128]>, + wide: HashMap>, +} +impl PartContext { + fn entries(&mut self, first: char) -> &mut Vec { + let code = first as usize; + if code < 128 { + &mut self.ascii[code] + } else { + self.wide.entry(first).or_default() + } + } + fn rules(&self, first: char) -> &[usize] { + let code = first as usize; + if code < 128 { + &self.ascii[code] + } else { + self.wide.get(&first).map_or(&[], Vec::as_slice) + } + } +} +fn prepare_dispatch(spec: &PartProfileSpec) -> Result, Diagnostic> { + let mut dispatch: Vec<_> = spec + .contexts + .iter() + .map(|_| PartContext { + ascii: Box::new(std::array::from_fn(|_| Vec::new())), + wide: HashMap::new(), + }) + .collect(); + for (index, rule) in spec.rules.iter().enumerate() { + let first = rule + .prefix + .chars() + .next() + .ok_or_else(|| error(0, "empty part prefix"))?; + for context in rule.contexts { + let slot = context_index(spec, context)?; + let entries = dispatch[slot].entries(first); + if entries.iter().any(|i| { + let old = &spec.rules[*i]; + old.prefix == rule.prefix && old.guard == rule.guard + }) { + return Err(error(0, "duplicate part recognition rule")); + } + entries.push(index); + } + } + Ok(dispatch) +} +fn context_index(spec: &PartProfileSpec, name: &str) -> Result { + spec.contexts + .iter() + .position(|c| *c == name) + .ok_or_else(|| error(0, "unknown part context")) +} +/// Once-admitted recognition tables and bound result instructions. +pub struct PreparedPartProfile { + pub(super) spec: &'static PartProfileSpec, + pub(super) result_owner: &'static ResultProfileSpec, + pub(super) projections: Vec, + pub(super) rule_projections: Vec, + pub(super) literal: usize, + pub(super) word: usize, + pub(super) here: usize, + pub(super) assignment: Option, + pub(super) subscript: Option, + pub(super) names: Vec, + pub(super) regions: PreparedRegionPlan, + pub(super) subscripts: Vec>, + pub(super) word_context: usize, + pub(super) here_context: usize, + pub(super) quote_contexts: Vec>, + dispatch: Vec, +} +pub(super) fn error(at: usize, message: &str) -> Diagnostic { + Diagnostic { + reason_kind: "invalid-part-recognition", + byte_offset: at, + message: message.into(), + } +} +fn unique(values: &[&str]) -> bool { + let mut seen = HashSet::new(); + values.iter().all(|v| !v.is_empty() && seen.insert(*v)) +} +const LEAF: &[(&str, CaptureKind, bool)] = &[("text", CaptureKind::Span, true)]; +const PARTS: &[(&str, CaptureKind, bool)] = &[("parts", CaptureKind::Parts, false)]; +const QUOTE: &[(&str, CaptureKind, bool)] = &[ + ("open", CaptureKind::Span, true), + ("parts", CaptureKind::Parts, false), + ("close", CaptureKind::Span, true), +]; +const OPAQUE: &[(&str, CaptureKind, bool)] = &[ + ("open", CaptureKind::Span, true), + ("body", CaptureKind::Span, false), + ("close", CaptureKind::Span, true), +]; +const PARAMETER: &[(&str, CaptureKind, bool)] = &[ + ("open", CaptureKind::Span, true), + ("prefix", CaptureKind::Span, false), + ("name", CaptureKind::Span, false), + ("subscript", CaptureKind::Node, false), + ("operator", CaptureKind::Span, false), + ("parts", CaptureKind::Parts, false), + ("close", CaptureKind::Span, true), +]; +const ASSIGNMENT: &[(&str, CaptureKind, bool)] = &[ + ("name", CaptureKind::Span, true), + ("operator", CaptureKind::Span, true), + ("parts", CaptureKind::Parts, false), +]; +fn bind( + results: &PreparedResultProfile, + projections: &mut Vec, + id: &str, + signature: &[(&str, CaptureKind, bool)], +) -> Result { + let projection = results.bind(id)?; + if !projection.has_signature(signature) { + return Err(error(0, "part projection signature mismatch")); + } + let index = projections.len(); + projections.push(projection); + Ok(index) +} +fn valid_subscript(sub: SubscriptSpec, base: &RegionSpec) -> bool { + if sub.prefix.is_empty() + || sub.regions.stops != base.stops + || sub.regions.quotes != base.quotes + || sub.regions.consume_initial_stop != base.consume_initial_stop + { + return false; + } + let Some(opener) = sub.prefix.chars().last() else { + return false; + }; + let expected = crate::scanner::RegionPair { + prefix: sub.prefix, + opening: opener, + closing: sub.closing, + depth: 1, + }; + let existing = base.pairs.iter().find(|p| p.prefix == sub.prefix); + if existing.is_some_and(|p| *p != expected) { + return false; + } + if sub.regions.pairs.len() != base.pairs.len() + usize::from(existing.is_none()) + || !base.pairs.iter().all(|p| sub.regions.pairs.contains(p)) + || !sub.regions.pairs.contains(&expected) + { + return false; + } + let base_names: Vec<_> = base + .pairs + .iter() + .filter(|p| p.prefix != sub.prefix) + .map(|p| p.prefix) + .collect(); + sub.scopes.len() == base_names.len() + 1 + && sub.scopes.iter().all(|s| { + if s.prefix == sub.prefix { + s.pairs.len() == base_names.len() + 1 + && s.pairs.contains(&sub.prefix) + && base_names.iter().all(|n| s.pairs.contains(n)) + } else { + base_names.contains(&s.prefix) && s.pairs == base_names + } + }) +} +fn rule_signature( + rule: &PartRule, + spec: &PartProfileSpec, + regions: &RegionSpec, +) -> Result<&'static [(&'static str, CaptureKind, bool)], Diagnostic> { + Ok(match rule.opcode { + PartOpcode::Name { binding } => { + if rule.opening != 1 || rule.closing != 0 || binding >= spec.bindings.len() { + return Err(error(0, "invalid name part")); + } + LEAF + } + PartOpcode::Escape => { + if rule.opening != 1 || rule.closing != 0 { + return Err(error(0, "invalid escape part")); + } + LEAF + } + PartOpcode::Quote { delimiter } | PartOpcode::QuotedBody { delimiter } => { + if rule.closing != 1 + || !rule.prefix.ends_with(delimiter) + || !regions.quotes.iter().any(|q| q.delimiter == delimiter) + { + return Err(error(0, "invalid quote part")); + } + if matches!(rule.opcode, PartOpcode::Quote { .. }) { + if !spec.contexts.contains(&rule.projection) { + return Err(error(0, "missing quote context")); + } + QUOTE + } else { + OPAQUE + } + } + PartOpcode::Pair => { + if !regions + .pairs + .iter() + .any(|p| p.prefix == rule.prefix && p.depth == rule.closing) + { + return Err(error(0, "invalid pair part")); + } + OPAQUE + } + PartOpcode::Parameter { binding } => { + if binding >= spec.bindings.len() + || rule.opening != 2 + || rule.closing != 1 + || !regions + .pairs + .iter() + .any(|p| p.prefix == rule.prefix && p.depth == rule.closing) + { + return Err(error(0, "invalid parameter part")); + } + PARAMETER + } + }) +} +impl PreparedPartProfile { + /// Admit recognition, region, binding and projection contracts once. + /// # Errors + /// Rejects malformed rules, nullable names, region conflicts and projection signatures. + pub fn new( + spec: &'static PartProfileSpec, + results: &'static ResultProfileSpec, + regions: &'static RegionSpec, + ) -> Result { + if !unique(spec.contexts) + || !spec.contexts.contains(&"word") + || !spec.contexts.contains(&"HereDocument") + { + return Err(error(0, "part profile lacks required unique contexts")); + } + let result_owner = results; + let results = PreparedResultProfile::new(results)?; + let mut projections = Vec::new(); + let word = bind(&results, &mut projections, "Word", PARTS)?; + let here = bind(&results, &mut projections, "HereDocumentLine", PARTS)?; + let literal = bind(&results, &mut projections, spec.literal, LEAF)?; + let regions_plan = PreparedRegionPlan::new(regions, &[])?; + let mut names = Vec::new(); + let mut subscripts = Vec::new(); + for binding in spec.bindings { + if !unique(binding.prefixes) || !unique(binding.operators) { + return Err(error(0, "invalid binding vocabulary")); + } + names.push( + PreparedTextProfile::new(binding.name) + .ok_or_else(|| error(0, "invalid binding name profile"))?, + ); + subscripts.push(if let Some(sub) = binding.subscript { + if !valid_subscript(sub, regions) { + return Err(error(0, "invalid derived binding region")); + } + Some(PreparedRegionPlan::new(sub.regions, sub.scopes)?) + } else { + None + }); + } + let assignment = if let Some(index) = spec.assignment { + let b = spec + .bindings + .get(index) + .ok_or_else(|| error(0, "unknown assignment binding"))?; + if b.operators.is_empty() || !b.prefixes.is_empty() || b.subscript.is_some() { + return Err(error(0, "invalid assignment binding")); + } + Some(bind(&results, &mut projections, "Assignment", ASSIGNMENT)?) + } else { + None + }; + + let mut rule_projections = Vec::new(); + let mut subscript = None; + for rule in spec.rules { + if rule.opening != rule.prefix.chars().count() + || !unique(rule.contexts) + || rule.contexts.is_empty() + || !rule.contexts.iter().all(|c| spec.contexts.contains(c)) + { + return Err(error(0, "invalid part opening or contexts")); + } + let signature = rule_signature(rule, spec, regions)?; + if let PartOpcode::Parameter { binding } = rule.opcode + && subscripts[binding].is_some() + && subscript.is_none() + { + subscript = Some(bind(&results, &mut projections, "ArraySubscript", QUOTE)?); + } + rule_projections.push(bind( + &results, + &mut projections, + rule.projection, + signature, + )?); + } + Ok(Self { + spec, + result_owner, + projections, + rule_projections, + literal, + word, + here, + assignment, + subscript, + names, + regions: regions_plan, + subscripts, + dispatch: prepare_dispatch(spec)?, + word_context: context_index(spec, "word")?, + here_context: context_index(spec, "HereDocument")?, + quote_contexts: spec + .rules + .iter() + .map(|r| spec.contexts.iter().position(|c| *c == r.projection)) + .collect(), + }) + } + pub(super) fn matching( + &self, + source: &str, + at: usize, + end: usize, + context: usize, + ) -> Option { + let first = source.get(at..end)?.chars().next()?; + self.dispatch[context] + .rules(first) + .iter() + .copied() + .find(|i| { + let rule = &self.spec.rules[*i]; + if !source[at..end].starts_with(rule.prefix) { + return false; + } + let following = source[at + rule.prefix.len()..end].chars().next(); + match rule.guard { + PartGuard::Any => true, + PartGuard::Next => following.is_some(), + PartGuard::Characters(chars) => following.is_some_and(|ch| chars.contains(ch)), + } + }) + } +} diff --git a/rust/gerbil-parser-runtime/src/engine/recognition_index.rs b/rust/gerbil-parser-runtime/src/engine/recognition_index.rs new file mode 100644 index 00000000..97b97768 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/recognition_index.rs @@ -0,0 +1,72 @@ +//! Project an existing recognition value into source-range syntax indexes. +use super::event_tree::build_syntax_events_catalog; +use super::model::{EventCatalog, LanguageSpec, Token, TreeEvent, Value}; +use gerbil_parser_artifact::syntax::SyntaxTree; +enum Pending<'a> { + Value(&'a Value), + Finish, +} +pub(crate) fn build_syntax( + spec: &LanguageSpec, + source: &str, + tokens: &[Token<'_>], + root: &Value, +) -> Result { + let Value::Node { kind, children, .. } = root else { + return Err("accepted semantic value is not a syntax node".into()); + }; + if *kind != spec.root_kind { + return Err("accepted syntax root has an undeclared kind".into()); + } + let mut events = vec![TreeEvent::StartNode(*kind)]; + let mut pending: Vec<_> = children + .iter() + .rev() + .map(|c| Pending::Value(&c.value)) + .collect(); + let mut next_token = 0; + while let Some(item) = pending.pop() { + match item { + Pending::Finish => events.push(TreeEvent::FinishNode), + Pending::Value(Value::Node { kind, children, .. }) => { + events.push(TreeEvent::StartNode(*kind)); + pending.push(Pending::Finish); + pending.extend(children.iter().rev().map(|c| Pending::Value(&c.value))); + } + Pending::Value(Value::Fragment(children)) => { + pending.extend(children.iter().rev().map(|c| Pending::Value(&c.value))); + } + Pending::Value(Value::Token(at)) => { + if next_token > *at { + return Err("recognition token order is not monotonic".into()); + } + while next_token <= *at { + let token = tokens.get(next_token).ok_or("unknown recognition token")?; + events.push(TreeEvent::Token { + kind: token.syntax_kind, + start: token.start, + end: token.end, + }); + next_token += 1; + } + } + } + } + for token in &tokens[next_token..] { + events.push(TreeEvent::Token { + kind: token.syntax_kind, + start: token.start, + end: token.end, + }); + } + events.push(TreeEvent::FinishNode); + build_syntax_events_catalog( + &EventCatalog { + root_kind: spec.root_kind, + kinds: spec.kinds, + }, + source, + &events, + ) + .map_err(|d| d.message) +} diff --git a/rust/gerbil-parser-runtime/src/engine/result_projection.rs b/rust/gerbil-parser-runtime/src/engine/result_projection.rs new file mode 100644 index 00000000..47f786a4 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/result_projection.rs @@ -0,0 +1,467 @@ +//! Execution of Scheme-owned result projection tables. Captures move into the +//! tree; nested publication never copies a growing token/event suffix. +use super::model::{Diagnostic, KindCategory, KindSpec, TreeEvent}; +use std::collections::{HashMap, HashSet}; +use std::ops::Range; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CaptureKind { + Span, + Node, + Parts, +} +#[derive(Clone, Copy, Debug)] +pub struct CaptureSpec { + pub name: &'static str, + pub kind: CaptureKind, +} +#[derive(Clone, Copy, Debug)] +pub enum ProjectionOpcode { + Token { kind: u16, required: bool }, + One { required: bool }, + Many, +} +#[derive(Clone, Copy, Debug)] +pub struct ProjectionInstruction { + pub field: &'static str, + pub opcode: ProjectionOpcode, +} +#[derive(Clone, Copy, Debug)] +pub struct ResultProjection { + pub id: &'static str, + pub kind: u16, + pub captures: &'static [CaptureSpec], + pub instructions: &'static [ProjectionInstruction], +} +#[derive(Clone, Copy, Debug)] +pub struct ResultNodeSpec { + pub kind: u16, + pub fields: &'static [&'static str], +} +#[derive(Clone, Copy, Debug)] +pub struct ResultProfileSpec { + pub kinds: &'static [KindSpec], + pub nodes: &'static [ResultNodeSpec], + pub projections: &'static [ResultProjection], +} + +pub struct PreparedResultProfile { + spec: &'static ResultProfileSpec, + projections: HashMap<&'static str, usize>, + nodes: HashMap<&'static str, u16>, +} +/// A projection bound once when a source engine is prepared. +pub struct PreparedResultProjection { + spec: &'static ResultProfileSpec, + index: usize, +} +/// A catalog-bound constructor for nodes whose fields follow source order. +pub struct PreparedResultNode { + spec: &'static ResultProfileSpec, + kind: u16, + fields: &'static [&'static str], +} +/// Ordered typed child; repetitions preserve caller-supplied lexical order. +pub struct ResultChildCapture<'source> { + pub field: &'static str, + pub value: ProjectedValue<'source>, +} +/// A typed recognition capture. `Absent` is admitted only for optional slots. +pub enum ResultCapture<'source> { + Absent, + Span(Range), + Node(ProjectedNode<'source>), + Parts(Vec>), +} +/// Source and catalog ownership are private and survive nested composition. +pub struct ProjectedNode<'source> { + owner: &'static ResultProfileSpec, + source: &'source str, + kind: u16, + span: Range, + children: Vec>, +} +// Drain descendants before each node is dropped, bounding native call depth even +// when a successful parse or a partially assembled failure owns a deep tree. +impl Drop for ProjectedNode<'_> { + fn drop(&mut self) { + let mut pending = std::mem::take(&mut self.children); + while let Some(child) = pending.pop() { + if let ProjectedValue::Node(mut node) = child.value { + pending.append(&mut node.children); + } + } + } +} +pub struct ProjectedChild<'source> { + field: &'static str, + value: ProjectedValue<'source>, +} +pub enum ProjectedValue<'source> { + Token { kind: u16, span: Range }, + Node(ProjectedNode<'source>), +} +fn error(at: usize, message: &str) -> Diagnostic { + Diagnostic { + reason_kind: "invalid-result-projection", + byte_offset: at, + message: message.into(), + } +} +fn unique(values: impl IntoIterator) -> bool { + let mut seen = HashSet::new(); + values + .into_iter() + .all(|name| !name.is_empty() && seen.insert(name)) +} +impl PreparedResultProfile { + /// Admit the entire immutable catalog once, including unused projections. + /// # Errors + /// Rejects malformed kinds, fields, capture types or instruction cardinality. + pub fn new(spec: &'static ResultProfileSpec) -> Result { + if spec.nodes.is_empty() + || spec.kinds.len() > usize::from(u16::MAX) + 1 + || !unique( + spec.kinds + .iter() + .filter(|k| k.category == KindCategory::Node) + .map(|k| k.name), + ) + || !unique( + spec.kinds + .iter() + .filter(|k| k.category == KindCategory::Token) + .map(|k| k.name), + ) + { + return Err(error(0, "invalid result kind catalog")); + } + let category = |kind: u16, expected| { + spec.kinds + .get(usize::from(kind)) + .is_some_and(|k| k.category == expected) + }; + let mut nodes = HashMap::new(); + for node in spec.nodes { + if !category(node.kind, KindCategory::Node) + || !unique(node.fields.iter().copied()) + || nodes.insert(node.kind, node.fields).is_some() + { + return Err(error(0, "invalid result node fields")); + } + } + for (index, kind) in spec.kinds.iter().enumerate() { + let index = u16::try_from(index).map_err(|_| error(0, "result kind index overflow"))?; + if kind.category == KindCategory::Node && !nodes.contains_key(&index) { + return Err(error(0, "missing result node declaration")); + } + } + let mut projections = HashMap::new(); + for (index, projection) in spec.projections.iter().enumerate() { + let Some(fields) = nodes.get(&projection.kind) else { + return Err(error(0, "undeclared projection node")); + }; + if projection.id.is_empty() + || projections.insert(projection.id, index).is_some() + || !unique(projection.captures.iter().map(|c| c.name)) + || projection.captures.len() != projection.instructions.len() + { + return Err(error(0, "invalid projection identity or arity")); + } + for (capture, instruction) in projection.captures.iter().zip(projection.instructions) { + let valid = match instruction.opcode { + ProjectionOpcode::Token { kind, .. } => { + capture.kind == CaptureKind::Span && category(kind, KindCategory::Token) + } + ProjectionOpcode::One { .. } => capture.kind == CaptureKind::Node, + ProjectionOpcode::Many => capture.kind == CaptureKind::Parts, + }; + if !valid || !fields.contains(&instruction.field) { + return Err(error( + 0, + "projection instruction disagrees with capture or field", + )); + } + } + } + Ok(Self { + spec, + projections, + nodes: spec + .nodes + .iter() + .map(|n| (spec.kinds[usize::from(n.kind)].name, n.kind)) + .collect(), + }) + } + /// Bind a catalog node for ordered command/Source construction. + /// # Errors + /// Rejects unknown node identifiers, including names declared only as tokens. + pub fn bind_node(&self, name: &str) -> Result { + let kind = *self + .nodes + .get(name) + .ok_or_else(|| error(0, "unknown result node"))?; + let fields = self + .spec + .nodes + .iter() + .find(|n| n.kind == kind) + .ok_or_else(|| error(0, "missing result node fields"))? + .fields; + Ok(PreparedResultNode { + spec: self.spec, + kind, + fields, + }) + } + /// Resolve an instruction entry once during engine preparation. + /// # Errors + /// Rejects an undeclared projection identifier. + pub fn bind(&self, id: &str) -> Result { + let index = *self + .projections + .get(id) + .ok_or_else(|| error(0, "unknown result projection"))?; + Ok(PreparedResultProjection { + spec: self.spec, + index, + }) + } + /// Bind already recognized captures to the declared result fields. + /// # Errors + /// Rejects missing required captures, invalid UTF-8 spans and foreign nodes. + pub fn project<'source>( + &self, + id: &str, + source: &'source str, + span: Range, + captures: Vec>, + ) -> Result, Diagnostic> { + self.bind(id)?.project(source, span, captures) + } + fn project_at<'source>( + spec: &'static ResultProfileSpec, + index: usize, + source: &'source str, + span: Range, + captures: Vec>, + ) -> Result, Diagnostic> { + let projection = &spec.projections[index]; + if captures.len() != projection.captures.len() || !valid_span(source, &span) { + return Err(error(span.start, "invalid result projection frame")); + } + let mut children = Vec::with_capacity(captures.len()); + for (capture, instruction) in captures.into_iter().zip(projection.instructions) { + let value = match (instruction.opcode, capture) { + ( + ProjectionOpcode::Token { + required: false, .. + } + | ProjectionOpcode::One { required: false }, + ResultCapture::Absent, + ) => continue, + (ProjectionOpcode::Token { kind, .. }, ResultCapture::Span(child)) => { + if !inside(&child, &span) || !valid_span(source, &child) { + return Err(error(span.start, "invalid captured span")); + } + ProjectedValue::Token { kind, span: child } + } + (ProjectionOpcode::One { .. }, ResultCapture::Node(node)) => { + Self::check_node(spec, source, &span, &node)?; + ProjectedValue::Node(node) + } + (ProjectionOpcode::Many, ResultCapture::Parts(parts)) => { + for node in parts { + Self::check_node(spec, source, &span, &node)?; + children.push(ProjectedChild { + field: instruction.field, + value: ProjectedValue::Node(node), + }); + } + continue; + } + _ => return Err(error(span.start, "missing or mistyped result capture")), + }; + children.push(ProjectedChild { + field: instruction.field, + value, + }); + } + Ok(ProjectedNode { + owner: spec, + source, + kind: projection.kind, + span, + children, + }) + } + fn check_node( + spec: &'static ResultProfileSpec, + source: &str, + span: &Range, + node: &ProjectedNode<'_>, + ) -> Result<(), Diagnostic> { + if !std::ptr::eq(spec, node.owner) + || !std::ptr::eq(source, node.source) + || !inside(&node.span, span) + { + return Err(error(span.start, "foreign or out-of-frame result node")); + } + Ok(()) + } +} +impl PreparedResultProjection { + pub(super) fn has_signature(&self, expected: &[(&str, CaptureKind, bool)]) -> bool { + let p = &self.spec.projections[self.index]; + p.captures.len() == expected.len() + && p.captures.iter().zip(p.instructions).zip(expected).all( + |((capture, instruction), (name, kind, required))| { + let actual_required = match instruction.opcode { + ProjectionOpcode::Token { required, .. } + | ProjectionOpcode::One { required } => required, + ProjectionOpcode::Many => false, + }; + capture.name == *name && capture.kind == *kind && actual_required == *required + }, + ) + } + + /// Execute bound instructions without another identifier lookup. + /// # Errors + /// Rejects invalid captures, source boundaries or node ownership. + pub fn project<'source>( + &self, + source: &'source str, + span: Range, + captures: Vec>, + ) -> Result, Diagnostic> { + PreparedResultProfile::project_at(self.spec, self.index, source, span, captures) + } +} +fn valid_span(source: &str, span: &Range) -> bool { + span.start <= span.end + && span.end <= source.len() + && source.is_char_boundary(span.start) + && source.is_char_boundary(span.end) +} +fn inside(child: &Range, parent: &Range) -> bool { + parent.start <= child.start && child.start <= child.end && child.end <= parent.end +} +impl PreparedResultNode { + /// Move ordered captures into the admitted node without sorting or cloning. + /// # Errors + /// Rejects foreign fields, catalogs, sources, token kinds and invalid UTF-8 ranges. + pub fn build<'source>( + &self, + source: &'source str, + span: Range, + captures: Vec>, + ) -> Result, Diagnostic> { + if !valid_span(source, &span) { + return Err(error(span.start, "invalid ordered result frame")); + } + let mut children = Vec::with_capacity(captures.len()); + for capture in captures { + if !self.fields.contains(&capture.field) { + return Err(error(span.start, "undeclared ordered result field")); + } + match &capture.value { + ProjectedValue::Node(node) => { + PreparedResultProfile::check_node(self.spec, source, &span, node)?; + } + ProjectedValue::Token { kind, span: child } => { + if !self + .spec + .kinds + .get(usize::from(*kind)) + .is_some_and(|k| k.category == KindCategory::Token) + || !valid_span(source, child) + || !inside(child, &span) + { + return Err(error(span.start, "invalid ordered result token")); + } + } + } + children.push(ProjectedChild { + field: capture.field, + value: capture.value, + }); + } + Ok(ProjectedNode { + owner: self.spec, + source, + kind: self.kind, + span, + children, + }) + } +} +impl<'source> ProjectedNode<'source> { + pub(super) fn bound_to(&self, owner: &ResultProfileSpec, source: &str) -> bool { + std::ptr::eq(self.owner, owner) && std::ptr::eq(self.source, source) + } + + #[must_use] + pub(super) fn into_captures(mut self) -> Vec> { + std::mem::take(&mut self.children) + .into_iter() + .map(|child| ResultChildCapture { + field: child.field, + value: child.value, + }) + .collect() + } + #[must_use] + pub fn kind(&self) -> u16 { + self.kind + } + #[must_use] + pub fn span(&self) -> Range { + self.span.clone() + } + #[must_use] + pub fn children(&self) -> &[ProjectedChild<'_>] { + &self.children + } + /// Flatten exactly once at the public source-index handoff. Traversal is iterative. + #[must_use] + pub fn events(&self) -> Vec { + enum Visit<'a, 's> { + Node(&'a ProjectedNode<'s>), + Value(&'a ProjectedValue<'s>), + Finish, + } + let mut stack = vec![Visit::Node(self)]; + let mut events = Vec::new(); + while let Some(visit) = stack.pop() { + match visit { + Visit::Node(node) => { + events.push(TreeEvent::StartNode(node.kind)); + stack.push(Visit::Finish); + stack.extend(node.children.iter().rev().map(|c| Visit::Value(&c.value))); + } + Visit::Value(ProjectedValue::Node(node)) => stack.push(Visit::Node(node)), + Visit::Value(ProjectedValue::Token { kind, span }) => { + events.push(TreeEvent::Token { + kind: *kind, + start: span.start, + end: span.end, + }); + } + Visit::Finish => events.push(TreeEvent::FinishNode), + } + } + events + } +} +impl ProjectedChild<'_> { + #[must_use] + pub fn field(&self) -> &'static str { + self.field + } + #[must_use] + pub fn value(&self) -> &ProjectedValue<'_> { + &self.value + } +} diff --git a/rust/gerbil-parser-rowan/src/engine/structural_block_opening.rs b/rust/gerbil-parser-runtime/src/engine/structural_block_opening.rs similarity index 100% rename from rust/gerbil-parser-rowan/src/engine/structural_block_opening.rs rename to rust/gerbil-parser-runtime/src/engine/structural_block_opening.rs diff --git a/rust/gerbil-parser-rowan/src/engine/structural_key_line.rs b/rust/gerbil-parser-runtime/src/engine/structural_key_line.rs similarity index 98% rename from rust/gerbil-parser-rowan/src/engine/structural_key_line.rs rename to rust/gerbil-parser-runtime/src/engine/structural_key_line.rs index fb46daac..bb3147a3 100644 --- a/rust/gerbil-parser-rowan/src/engine/structural_key_line.rs +++ b/rust/gerbil-parser-runtime/src/engine/structural_key_line.rs @@ -1,4 +1,4 @@ -//! Bounded key/value line recognition and source-backed Rowan events. +//! Bounded key/value line recognition and source-backed syntax events. use super::model::{Diagnostic, KeyLineContext, KeyLineMode, KeyLineRule, KindCategory, TreeEvent}; diff --git a/rust/gerbil-parser-rowan/src/engine/structural_lines.rs b/rust/gerbil-parser-runtime/src/engine/structural_lines.rs similarity index 99% rename from rust/gerbil-parser-rowan/src/engine/structural_lines.rs rename to rust/gerbil-parser-runtime/src/engine/structural_lines.rs index 88cb0d11..a06d1356 100644 --- a/rust/gerbil-parser-rowan/src/engine/structural_lines.rs +++ b/rust/gerbil-parser-runtime/src/engine/structural_lines.rs @@ -1,6 +1,6 @@ //! Generic contextual line structure over AOT-resolved syntax kinds. -use super::event_tree::build_rowan_events; +use super::event_tree::build_syntax_events; use super::lexer::line_end; use super::model::{ BlockContents, BlockLineRule, BlockOpeningMode, Diagnostic, HeadingLineRule, InlineLinkRule, @@ -16,13 +16,13 @@ use super::validation::receipt; /// Parse one source into nested sections, Scheme-declared blocks, and text lines. /// /// The language pack owns every delimiter and kind identity. This engine only -/// executes the generic line/context transitions and emits validated Rowan +/// executes the generic line/context transitions and emits validated syntax /// events. An unclosed block follows its Scheme-declared recovery policy. /// /// # Errors /// /// Returns a parse error with the same language and source receipt as the -/// generated LR path when the structural table or Rowan event stream is invalid. +/// generated LR path when the structural table or syntax event stream is invalid. pub fn parse_structural_lines( language: &'static LanguageSpec, structure: &LineStructureSpec, @@ -421,9 +421,9 @@ fn finish_structural_parse( diagnostic: Box::new(diagnostic), selective_glr: None, }; - let green = build_rowan_events(language, source, events).map_err(with_receipt)?; + let tree = build_syntax_events(language, source, events).map_err(with_receipt)?; Ok(Parse { - green, + tree, kinds: language.kinds, receipt: parse_receipt, selective_glr: SelectiveGlrReceipt { diff --git a/rust/gerbil-parser-rowan/src/engine/structural_list.rs b/rust/gerbil-parser-runtime/src/engine/structural_list.rs similarity index 100% rename from rust/gerbil-parser-rowan/src/engine/structural_list.rs rename to rust/gerbil-parser-runtime/src/engine/structural_list.rs diff --git a/rust/gerbil-parser-rowan/src/engine/structural_table.rs b/rust/gerbil-parser-runtime/src/engine/structural_table.rs similarity index 100% rename from rust/gerbil-parser-rowan/src/engine/structural_table.rs rename to rust/gerbil-parser-runtime/src/engine/structural_table.rs diff --git a/rust/gerbil-parser-runtime/src/engine/unicode_alphabetic.rs b/rust/gerbil-parser-runtime/src/engine/unicode_alphabetic.rs new file mode 100644 index 00000000..94ff4182 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/unicode_alphabetic.rs @@ -0,0 +1,771 @@ +// @generated by t/generate-scheme-alphabetic-class.ss; do not edit. +// Source authority: pinned native Scheme char-alphabetic? class. +const RANGES: &[(u32, u32)] = &[ + (0x41, 0x5a), + (0x61, 0x7a), + (0xaa, 0xaa), + (0xb5, 0xb5), + (0xba, 0xba), + (0xc0, 0xd6), + (0xd8, 0xf6), + (0xf8, 0x2c1), + (0x2c6, 0x2d1), + (0x2e0, 0x2e4), + (0x2ec, 0x2ec), + (0x2ee, 0x2ee), + (0x345, 0x345), + (0x363, 0x374), + (0x376, 0x377), + (0x37a, 0x37d), + (0x37f, 0x37f), + (0x386, 0x386), + (0x388, 0x38a), + (0x38c, 0x38c), + (0x38e, 0x3a1), + (0x3a3, 0x3f5), + (0x3f7, 0x481), + (0x48a, 0x52f), + (0x531, 0x556), + (0x559, 0x559), + (0x560, 0x588), + (0x5b0, 0x5bd), + (0x5bf, 0x5bf), + (0x5c1, 0x5c2), + (0x5c4, 0x5c5), + (0x5c7, 0x5c7), + (0x5d0, 0x5ea), + (0x5ef, 0x5f2), + (0x610, 0x61a), + (0x620, 0x657), + (0x659, 0x65f), + (0x66e, 0x6d3), + (0x6d5, 0x6dc), + (0x6e1, 0x6e8), + (0x6ed, 0x6ef), + (0x6fa, 0x6fc), + (0x6ff, 0x6ff), + (0x710, 0x73f), + (0x74d, 0x7b1), + (0x7ca, 0x7ea), + (0x7f4, 0x7f5), + (0x7fa, 0x7fa), + (0x800, 0x817), + (0x81a, 0x82c), + (0x840, 0x858), + (0x860, 0x86a), + (0x870, 0x887), + (0x889, 0x88f), + (0x897, 0x897), + (0x8a0, 0x8c9), + (0x8d4, 0x8df), + (0x8e3, 0x8e9), + (0x8f0, 0x93b), + (0x93d, 0x94c), + (0x94e, 0x950), + (0x955, 0x963), + (0x971, 0x983), + (0x985, 0x98c), + (0x98f, 0x990), + (0x993, 0x9a8), + (0x9aa, 0x9b0), + (0x9b2, 0x9b2), + (0x9b6, 0x9b9), + (0x9bd, 0x9c4), + (0x9c7, 0x9c8), + (0x9cb, 0x9cc), + (0x9ce, 0x9ce), + (0x9d7, 0x9d7), + (0x9dc, 0x9dd), + (0x9df, 0x9e3), + (0x9f0, 0x9f1), + (0x9fc, 0x9fc), + (0xa01, 0xa03), + (0xa05, 0xa0a), + (0xa0f, 0xa10), + (0xa13, 0xa28), + (0xa2a, 0xa30), + (0xa32, 0xa33), + (0xa35, 0xa36), + (0xa38, 0xa39), + (0xa3e, 0xa42), + (0xa47, 0xa48), + (0xa4b, 0xa4c), + (0xa51, 0xa51), + (0xa59, 0xa5c), + (0xa5e, 0xa5e), + (0xa70, 0xa75), + (0xa81, 0xa83), + (0xa85, 0xa8d), + (0xa8f, 0xa91), + (0xa93, 0xaa8), + (0xaaa, 0xab0), + (0xab2, 0xab3), + (0xab5, 0xab9), + (0xabd, 0xac5), + (0xac7, 0xac9), + (0xacb, 0xacc), + (0xad0, 0xad0), + (0xae0, 0xae3), + (0xaf9, 0xafc), + (0xb01, 0xb03), + (0xb05, 0xb0c), + (0xb0f, 0xb10), + (0xb13, 0xb28), + (0xb2a, 0xb30), + (0xb32, 0xb33), + (0xb35, 0xb39), + (0xb3d, 0xb44), + (0xb47, 0xb48), + (0xb4b, 0xb4c), + (0xb56, 0xb57), + (0xb5c, 0xb5d), + (0xb5f, 0xb63), + (0xb71, 0xb71), + (0xb82, 0xb83), + (0xb85, 0xb8a), + (0xb8e, 0xb90), + (0xb92, 0xb95), + (0xb99, 0xb9a), + (0xb9c, 0xb9c), + (0xb9e, 0xb9f), + (0xba3, 0xba4), + (0xba8, 0xbaa), + (0xbae, 0xbb9), + (0xbbe, 0xbc2), + (0xbc6, 0xbc8), + (0xbca, 0xbcc), + (0xbd0, 0xbd0), + (0xbd7, 0xbd7), + (0xc00, 0xc0c), + (0xc0e, 0xc10), + (0xc12, 0xc28), + (0xc2a, 0xc39), + (0xc3d, 0xc44), + (0xc46, 0xc48), + (0xc4a, 0xc4c), + (0xc55, 0xc56), + (0xc58, 0xc5a), + (0xc5c, 0xc5d), + (0xc60, 0xc63), + (0xc80, 0xc83), + (0xc85, 0xc8c), + (0xc8e, 0xc90), + (0xc92, 0xca8), + (0xcaa, 0xcb3), + (0xcb5, 0xcb9), + (0xcbd, 0xcc4), + (0xcc6, 0xcc8), + (0xcca, 0xccc), + (0xcd5, 0xcd6), + (0xcdc, 0xcde), + (0xce0, 0xce3), + (0xcf1, 0xcf3), + (0xd00, 0xd0c), + (0xd0e, 0xd10), + (0xd12, 0xd3a), + (0xd3d, 0xd44), + (0xd46, 0xd48), + (0xd4a, 0xd4c), + (0xd4e, 0xd4e), + (0xd54, 0xd57), + (0xd5f, 0xd63), + (0xd7a, 0xd7f), + (0xd81, 0xd83), + (0xd85, 0xd96), + (0xd9a, 0xdb1), + (0xdb3, 0xdbb), + (0xdbd, 0xdbd), + (0xdc0, 0xdc6), + (0xdcf, 0xdd4), + (0xdd6, 0xdd6), + (0xdd8, 0xddf), + (0xdf2, 0xdf3), + (0xe01, 0xe3a), + (0xe40, 0xe46), + (0xe4d, 0xe4d), + (0xe81, 0xe82), + (0xe84, 0xe84), + (0xe86, 0xe8a), + (0xe8c, 0xea3), + (0xea5, 0xea5), + (0xea7, 0xeb9), + (0xebb, 0xebd), + (0xec0, 0xec4), + (0xec6, 0xec6), + (0xecd, 0xecd), + (0xedc, 0xedf), + (0xf00, 0xf00), + (0xf40, 0xf47), + (0xf49, 0xf6c), + (0xf71, 0xf83), + (0xf88, 0xf97), + (0xf99, 0xfbc), + (0x1000, 0x1036), + (0x1038, 0x1038), + (0x103b, 0x103f), + (0x1050, 0x108f), + (0x109a, 0x109d), + (0x10a0, 0x10c5), + (0x10c7, 0x10c7), + (0x10cd, 0x10cd), + (0x10d0, 0x10fa), + (0x10fc, 0x1248), + (0x124a, 0x124d), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125a, 0x125d), + (0x1260, 0x1288), + (0x128a, 0x128d), + (0x1290, 0x12b0), + (0x12b2, 0x12b5), + (0x12b8, 0x12be), + (0x12c0, 0x12c0), + (0x12c2, 0x12c5), + (0x12c8, 0x12d6), + (0x12d8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135a), + (0x1380, 0x138f), + (0x13a0, 0x13f5), + (0x13f8, 0x13fd), + (0x1401, 0x166c), + (0x166f, 0x167f), + (0x1681, 0x169a), + (0x16a0, 0x16ea), + (0x16ee, 0x16f8), + (0x1700, 0x1713), + (0x171f, 0x1733), + (0x1740, 0x1753), + (0x1760, 0x176c), + (0x176e, 0x1770), + (0x1772, 0x1773), + (0x1780, 0x17b3), + (0x17b6, 0x17c8), + (0x17d7, 0x17d7), + (0x17dc, 0x17dc), + (0x1820, 0x1878), + (0x1880, 0x18aa), + (0x18b0, 0x18f5), + (0x1900, 0x191e), + (0x1920, 0x192b), + (0x1930, 0x1938), + (0x1950, 0x196d), + (0x1970, 0x1974), + (0x1980, 0x19ab), + (0x19b0, 0x19c9), + (0x1a00, 0x1a1b), + (0x1a20, 0x1a5e), + (0x1a61, 0x1a74), + (0x1aa7, 0x1aa7), + (0x1abf, 0x1ac0), + (0x1acc, 0x1ace), + (0x1b00, 0x1b33), + (0x1b35, 0x1b43), + (0x1b45, 0x1b4c), + (0x1b80, 0x1ba9), + (0x1bac, 0x1baf), + (0x1bba, 0x1be5), + (0x1be7, 0x1bf1), + (0x1c00, 0x1c36), + (0x1c4d, 0x1c4f), + (0x1c5a, 0x1c7d), + (0x1c80, 0x1c8a), + (0x1c90, 0x1cba), + (0x1cbd, 0x1cbf), + (0x1ce9, 0x1cec), + (0x1cee, 0x1cf3), + (0x1cf5, 0x1cf6), + (0x1cfa, 0x1cfa), + (0x1d00, 0x1dbf), + (0x1dd3, 0x1df4), + (0x1e00, 0x1f15), + (0x1f18, 0x1f1d), + (0x1f20, 0x1f45), + (0x1f48, 0x1f4d), + (0x1f50, 0x1f57), + (0x1f59, 0x1f59), + (0x1f5b, 0x1f5b), + (0x1f5d, 0x1f5d), + (0x1f5f, 0x1f7d), + (0x1f80, 0x1fb4), + (0x1fb6, 0x1fbc), + (0x1fbe, 0x1fbe), + (0x1fc2, 0x1fc4), + (0x1fc6, 0x1fcc), + (0x1fd0, 0x1fd3), + (0x1fd6, 0x1fdb), + (0x1fe0, 0x1fec), + (0x1ff2, 0x1ff4), + (0x1ff6, 0x1ffc), + (0x2071, 0x2071), + (0x207f, 0x207f), + (0x2090, 0x209c), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210a, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211d), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212a, 0x212d), + (0x212f, 0x2139), + (0x213c, 0x213f), + (0x2145, 0x2149), + (0x214e, 0x214e), + (0x2160, 0x2188), + (0x24b6, 0x24e9), + (0x2c00, 0x2ce4), + (0x2ceb, 0x2cee), + (0x2cf2, 0x2cf3), + (0x2d00, 0x2d25), + (0x2d27, 0x2d27), + (0x2d2d, 0x2d2d), + (0x2d30, 0x2d67), + (0x2d6f, 0x2d6f), + (0x2d80, 0x2d96), + (0x2da0, 0x2da6), + (0x2da8, 0x2dae), + (0x2db0, 0x2db6), + (0x2db8, 0x2dbe), + (0x2dc0, 0x2dc6), + (0x2dc8, 0x2dce), + (0x2dd0, 0x2dd6), + (0x2dd8, 0x2dde), + (0x2de0, 0x2dff), + (0x2e2f, 0x2e2f), + (0x3005, 0x3007), + (0x3021, 0x3029), + (0x3031, 0x3035), + (0x3038, 0x303c), + (0x3041, 0x3096), + (0x309d, 0x309f), + (0x30a1, 0x30fa), + (0x30fc, 0x30ff), + (0x3105, 0x312f), + (0x3131, 0x318e), + (0x31a0, 0x31bf), + (0x31f0, 0x31ff), + (0x3400, 0x4dbf), + (0x4e00, 0xa48c), + (0xa4d0, 0xa4fd), + (0xa500, 0xa60c), + (0xa610, 0xa61f), + (0xa62a, 0xa62b), + (0xa640, 0xa66e), + (0xa674, 0xa67b), + (0xa67f, 0xa6ef), + (0xa717, 0xa71f), + (0xa722, 0xa788), + (0xa78b, 0xa7dc), + (0xa7f1, 0xa805), + (0xa807, 0xa827), + (0xa840, 0xa873), + (0xa880, 0xa8c3), + (0xa8c5, 0xa8c5), + (0xa8f2, 0xa8f7), + (0xa8fb, 0xa8fb), + (0xa8fd, 0xa8ff), + (0xa90a, 0xa92a), + (0xa930, 0xa952), + (0xa960, 0xa97c), + (0xa980, 0xa9b2), + (0xa9b4, 0xa9bf), + (0xa9cf, 0xa9cf), + (0xa9e0, 0xa9ef), + (0xa9fa, 0xa9fe), + (0xaa00, 0xaa36), + (0xaa40, 0xaa4d), + (0xaa60, 0xaa76), + (0xaa7a, 0xaabe), + (0xaac0, 0xaac0), + (0xaac2, 0xaac2), + (0xaadb, 0xaadd), + (0xaae0, 0xaaef), + (0xaaf2, 0xaaf5), + (0xab01, 0xab06), + (0xab09, 0xab0e), + (0xab11, 0xab16), + (0xab20, 0xab26), + (0xab28, 0xab2e), + (0xab30, 0xab5a), + (0xab5c, 0xab69), + (0xab70, 0xabea), + (0xac00, 0xd7a3), + (0xd7b0, 0xd7c6), + (0xd7cb, 0xd7fb), + (0xf900, 0xfa6d), + (0xfa70, 0xfad9), + (0xfb00, 0xfb06), + (0xfb13, 0xfb17), + (0xfb1d, 0xfb28), + (0xfb2a, 0xfb36), + (0xfb38, 0xfb3c), + (0xfb3e, 0xfb3e), + (0xfb40, 0xfb41), + (0xfb43, 0xfb44), + (0xfb46, 0xfbb1), + (0xfbd3, 0xfd3d), + (0xfd50, 0xfd8f), + (0xfd92, 0xfdc7), + (0xfdf0, 0xfdfb), + (0xfe70, 0xfe74), + (0xfe76, 0xfefc), + (0xff21, 0xff3a), + (0xff41, 0xff5a), + (0xff66, 0xffbe), + (0xffc2, 0xffc7), + (0xffca, 0xffcf), + (0xffd2, 0xffd7), + (0xffda, 0xffdc), + (0x10000, 0x1000b), + (0x1000d, 0x10026), + (0x10028, 0x1003a), + (0x1003c, 0x1003d), + (0x1003f, 0x1004d), + (0x10050, 0x1005d), + (0x10080, 0x100fa), + (0x10140, 0x10174), + (0x10280, 0x1029c), + (0x102a0, 0x102d0), + (0x10300, 0x1031f), + (0x1032d, 0x1034a), + (0x10350, 0x1037a), + (0x10380, 0x1039d), + (0x103a0, 0x103c3), + (0x103c8, 0x103cf), + (0x103d1, 0x103d5), + (0x10400, 0x1049d), + (0x104b0, 0x104d3), + (0x104d8, 0x104fb), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057a), + (0x1057c, 0x1058a), + (0x1058c, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105a1), + (0x105a3, 0x105b1), + (0x105b3, 0x105b9), + (0x105bb, 0x105bc), + (0x105c0, 0x105f3), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107b0), + (0x107b2, 0x107ba), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080a, 0x10835), + (0x10837, 0x10838), + (0x1083c, 0x1083c), + (0x1083f, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089e), + (0x108e0, 0x108f2), + (0x108f4, 0x108f5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10940, 0x10959), + (0x10980, 0x109b7), + (0x109be, 0x109bf), + (0x10a00, 0x10a03), + (0x10a05, 0x10a06), + (0x10a0c, 0x10a13), + (0x10a15, 0x10a17), + (0x10a19, 0x10a35), + (0x10a60, 0x10a7c), + (0x10a80, 0x10a9c), + (0x10ac0, 0x10ac7), + (0x10ac9, 0x10ae4), + (0x10b00, 0x10b35), + (0x10b40, 0x10b55), + (0x10b60, 0x10b72), + (0x10b80, 0x10b91), + (0x10c00, 0x10c48), + (0x10c80, 0x10cb2), + (0x10cc0, 0x10cf2), + (0x10d00, 0x10d27), + (0x10d4a, 0x10d65), + (0x10d69, 0x10d69), + (0x10d6f, 0x10d85), + (0x10e80, 0x10ea9), + (0x10eab, 0x10eac), + (0x10eb0, 0x10eb1), + (0x10ec2, 0x10ec7), + (0x10efa, 0x10efc), + (0x10f00, 0x10f1c), + (0x10f27, 0x10f27), + (0x10f30, 0x10f45), + (0x10f70, 0x10f81), + (0x10fb0, 0x10fc4), + (0x10fe0, 0x10ff6), + (0x11000, 0x11045), + (0x11071, 0x11075), + (0x11080, 0x110b8), + (0x110c2, 0x110c2), + (0x110d0, 0x110e8), + (0x11100, 0x11132), + (0x11144, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11180, 0x111bf), + (0x111c1, 0x111c4), + (0x111ce, 0x111cf), + (0x111da, 0x111da), + (0x111dc, 0x111dc), + (0x11200, 0x11211), + (0x11213, 0x11234), + (0x11237, 0x11237), + (0x1123e, 0x11241), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128a, 0x1128d), + (0x1128f, 0x1129d), + (0x1129f, 0x112a8), + (0x112b0, 0x112e8), + (0x11300, 0x11303), + (0x11305, 0x1130c), + (0x1130f, 0x11310), + (0x11313, 0x11328), + (0x1132a, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133d, 0x11344), + (0x11347, 0x11348), + (0x1134b, 0x1134c), + (0x11350, 0x11350), + (0x11357, 0x11357), + (0x1135d, 0x11363), + (0x11380, 0x11389), + (0x1138b, 0x1138b), + (0x1138e, 0x1138e), + (0x11390, 0x113b5), + (0x113b7, 0x113c0), + (0x113c2, 0x113c2), + (0x113c5, 0x113c5), + (0x113c7, 0x113ca), + (0x113cc, 0x113cd), + (0x113d1, 0x113d1), + (0x113d3, 0x113d3), + (0x11400, 0x11441), + (0x11443, 0x11445), + (0x11447, 0x1144a), + (0x1145f, 0x11461), + (0x11480, 0x114c1), + (0x114c4, 0x114c5), + (0x114c7, 0x114c7), + (0x11580, 0x115b5), + (0x115b8, 0x115be), + (0x115d8, 0x115dd), + (0x11600, 0x1163e), + (0x11640, 0x11640), + (0x11644, 0x11644), + (0x11680, 0x116b5), + (0x116b8, 0x116b8), + (0x11700, 0x1171a), + (0x1171d, 0x1172a), + (0x11740, 0x11746), + (0x11800, 0x11838), + (0x118a0, 0x118df), + (0x118ff, 0x11906), + (0x11909, 0x11909), + (0x1190c, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x11935), + (0x11937, 0x11938), + (0x1193b, 0x1193c), + (0x1193f, 0x11942), + (0x119a0, 0x119a7), + (0x119aa, 0x119d7), + (0x119da, 0x119df), + (0x119e1, 0x119e1), + (0x119e3, 0x119e4), + (0x11a00, 0x11a32), + (0x11a35, 0x11a3e), + (0x11a50, 0x11a97), + (0x11a9d, 0x11a9d), + (0x11ab0, 0x11af8), + (0x11b60, 0x11b67), + (0x11bc0, 0x11be0), + (0x11c00, 0x11c08), + (0x11c0a, 0x11c36), + (0x11c38, 0x11c3e), + (0x11c40, 0x11c40), + (0x11c72, 0x11c8f), + (0x11c92, 0x11ca7), + (0x11ca9, 0x11cb6), + (0x11d00, 0x11d06), + (0x11d08, 0x11d09), + (0x11d0b, 0x11d36), + (0x11d3a, 0x11d3a), + (0x11d3c, 0x11d3d), + (0x11d3f, 0x11d41), + (0x11d43, 0x11d43), + (0x11d46, 0x11d47), + (0x11d60, 0x11d65), + (0x11d67, 0x11d68), + (0x11d6a, 0x11d8e), + (0x11d90, 0x11d91), + (0x11d93, 0x11d96), + (0x11d98, 0x11d98), + (0x11db0, 0x11ddb), + (0x11ee0, 0x11ef6), + (0x11f00, 0x11f10), + (0x11f12, 0x11f3a), + (0x11f3e, 0x11f40), + (0x11fb0, 0x11fb0), + (0x12000, 0x12399), + (0x12400, 0x1246e), + (0x12480, 0x12543), + (0x12f90, 0x12ff0), + (0x13000, 0x1342f), + (0x13441, 0x13446), + (0x13460, 0x143fa), + (0x14400, 0x14646), + (0x16100, 0x1612e), + (0x16800, 0x16a38), + (0x16a40, 0x16a5e), + (0x16a70, 0x16abe), + (0x16ad0, 0x16aed), + (0x16b00, 0x16b2f), + (0x16b40, 0x16b43), + (0x16b63, 0x16b77), + (0x16b7d, 0x16b8f), + (0x16d40, 0x16d6c), + (0x16e40, 0x16e7f), + (0x16ea0, 0x16eb8), + (0x16ebb, 0x16ed3), + (0x16f00, 0x16f4a), + (0x16f4f, 0x16f87), + (0x16f8f, 0x16f9f), + (0x16fe0, 0x16fe1), + (0x16fe3, 0x16fe3), + (0x16ff0, 0x16ff6), + (0x17000, 0x18cd5), + (0x18cff, 0x18d1e), + (0x18d80, 0x18df2), + (0x1aff0, 0x1aff3), + (0x1aff5, 0x1affb), + (0x1affd, 0x1affe), + (0x1b000, 0x1b122), + (0x1b132, 0x1b132), + (0x1b150, 0x1b152), + (0x1b155, 0x1b155), + (0x1b164, 0x1b167), + (0x1b170, 0x1b2fb), + (0x1bc00, 0x1bc6a), + (0x1bc70, 0x1bc7c), + (0x1bc80, 0x1bc88), + (0x1bc90, 0x1bc99), + (0x1bc9e, 0x1bc9e), + (0x1d400, 0x1d454), + (0x1d456, 0x1d49c), + (0x1d49e, 0x1d49f), + (0x1d4a2, 0x1d4a2), + (0x1d4a5, 0x1d4a6), + (0x1d4a9, 0x1d4ac), + (0x1d4ae, 0x1d4b9), + (0x1d4bb, 0x1d4bb), + (0x1d4bd, 0x1d4c3), + (0x1d4c5, 0x1d505), + (0x1d507, 0x1d50a), + (0x1d50d, 0x1d514), + (0x1d516, 0x1d51c), + (0x1d51e, 0x1d539), + (0x1d53b, 0x1d53e), + (0x1d540, 0x1d544), + (0x1d546, 0x1d546), + (0x1d54a, 0x1d550), + (0x1d552, 0x1d6a5), + (0x1d6a8, 0x1d6c0), + (0x1d6c2, 0x1d6da), + (0x1d6dc, 0x1d6fa), + (0x1d6fc, 0x1d714), + (0x1d716, 0x1d734), + (0x1d736, 0x1d74e), + (0x1d750, 0x1d76e), + (0x1d770, 0x1d788), + (0x1d78a, 0x1d7a8), + (0x1d7aa, 0x1d7c2), + (0x1d7c4, 0x1d7cb), + (0x1df00, 0x1df1e), + (0x1df25, 0x1df2a), + (0x1e000, 0x1e006), + (0x1e008, 0x1e018), + (0x1e01b, 0x1e021), + (0x1e023, 0x1e024), + (0x1e026, 0x1e02a), + (0x1e030, 0x1e06d), + (0x1e08f, 0x1e08f), + (0x1e100, 0x1e12c), + (0x1e137, 0x1e13d), + (0x1e14e, 0x1e14e), + (0x1e290, 0x1e2ad), + (0x1e2c0, 0x1e2eb), + (0x1e4d0, 0x1e4eb), + (0x1e5d0, 0x1e5ed), + (0x1e5f0, 0x1e5f0), + (0x1e6c0, 0x1e6de), + (0x1e6e0, 0x1e6f5), + (0x1e6fe, 0x1e6ff), + (0x1e7e0, 0x1e7e6), + (0x1e7e8, 0x1e7eb), + (0x1e7ed, 0x1e7ee), + (0x1e7f0, 0x1e7fe), + (0x1e800, 0x1e8c4), + (0x1e900, 0x1e943), + (0x1e947, 0x1e947), + (0x1e94b, 0x1e94b), + (0x1ee00, 0x1ee03), + (0x1ee05, 0x1ee1f), + (0x1ee21, 0x1ee22), + (0x1ee24, 0x1ee24), + (0x1ee27, 0x1ee27), + (0x1ee29, 0x1ee32), + (0x1ee34, 0x1ee37), + (0x1ee39, 0x1ee39), + (0x1ee3b, 0x1ee3b), + (0x1ee42, 0x1ee42), + (0x1ee47, 0x1ee47), + (0x1ee49, 0x1ee49), + (0x1ee4b, 0x1ee4b), + (0x1ee4d, 0x1ee4f), + (0x1ee51, 0x1ee52), + (0x1ee54, 0x1ee54), + (0x1ee57, 0x1ee57), + (0x1ee59, 0x1ee59), + (0x1ee5b, 0x1ee5b), + (0x1ee5d, 0x1ee5d), + (0x1ee5f, 0x1ee5f), + (0x1ee61, 0x1ee62), + (0x1ee64, 0x1ee64), + (0x1ee67, 0x1ee6a), + (0x1ee6c, 0x1ee72), + (0x1ee74, 0x1ee77), + (0x1ee79, 0x1ee7c), + (0x1ee7e, 0x1ee7e), + (0x1ee80, 0x1ee89), + (0x1ee8b, 0x1ee9b), + (0x1eea1, 0x1eea3), + (0x1eea5, 0x1eea9), + (0x1eeab, 0x1eebb), + (0x1f130, 0x1f149), + (0x1f150, 0x1f169), + (0x1f170, 0x1f189), + (0x20000, 0x2a6df), + (0x2a700, 0x2b81d), + (0x2b820, 0x2cead), + (0x2ceb0, 0x2ebe0), + (0x2ebf0, 0x2ee5d), + (0x2f800, 0x2fa1d), + (0x30000, 0x3134a), + (0x31350, 0x33479), +]; + +pub(super) fn is_scheme_alphabetic(character: char) -> bool { + let code = character as u32; + let index = RANGES.partition_point(|&(start, _)| start <= code); + index > 0 && code <= RANGES[index - 1].1 +} diff --git a/rust/gerbil-parser-runtime/src/engine/unicode_numeric.rs b/rust/gerbil-parser-runtime/src/engine/unicode_numeric.rs new file mode 100644 index 00000000..5994c194 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/unicode_numeric.rs @@ -0,0 +1,78 @@ +// @generated by t/generate-scheme-numeric-class.ss; do not edit. +// Source authority: pinned native Scheme char-numeric? class. +pub(super) fn is_scheme_numeric(character: char) -> bool { + character.is_ascii_digit() + || matches!(character as u32, + 0x660..=0x669 | + 0x6f0..=0x6f9 | + 0x7c0..=0x7c9 | + 0x966..=0x96f | + 0x9e6..=0x9ef | + 0xa66..=0xa6f | + 0xae6..=0xaef | + 0xb66..=0xb6f | + 0xbe6..=0xbef | + 0xc66..=0xc6f | + 0xce6..=0xcef | + 0xd66..=0xd6f | + 0xde6..=0xdef | + 0xe50..=0xe59 | + 0xed0..=0xed9 | + 0xf20..=0xf29 | + 0x1040..=0x1049 | + 0x1090..=0x1099 | + 0x17e0..=0x17e9 | + 0x1810..=0x1819 | + 0x1946..=0x194f | + 0x19d0..=0x19d9 | + 0x1a80..=0x1a89 | + 0x1a90..=0x1a99 | + 0x1b50..=0x1b59 | + 0x1bb0..=0x1bb9 | + 0x1c40..=0x1c49 | + 0x1c50..=0x1c59 | + 0xa620..=0xa629 | + 0xa8d0..=0xa8d9 | + 0xa900..=0xa909 | + 0xa9d0..=0xa9d9 | + 0xa9f0..=0xa9f9 | + 0xaa50..=0xaa59 | + 0xabf0..=0xabf9 | + 0xff10..=0xff19 | + 0x104a0..=0x104a9 | + 0x10d30..=0x10d39 | + 0x10d40..=0x10d49 | + 0x11066..=0x1106f | + 0x110f0..=0x110f9 | + 0x11136..=0x1113f | + 0x111d0..=0x111d9 | + 0x112f0..=0x112f9 | + 0x11450..=0x11459 | + 0x114d0..=0x114d9 | + 0x11650..=0x11659 | + 0x116c0..=0x116c9 | + 0x116d0..=0x116e3 | + 0x11730..=0x11739 | + 0x118e0..=0x118e9 | + 0x11950..=0x11959 | + 0x11bf0..=0x11bf9 | + 0x11c50..=0x11c59 | + 0x11d50..=0x11d59 | + 0x11da0..=0x11da9 | + 0x11de0..=0x11de9 | + 0x11f50..=0x11f59 | + 0x16130..=0x16139 | + 0x16a60..=0x16a69 | + 0x16ac0..=0x16ac9 | + 0x16b50..=0x16b59 | + 0x16d70..=0x16d79 | + 0x1ccf0..=0x1ccf9 | + 0x1d7ce..=0x1d7ff | + 0x1e140..=0x1e149 | + 0x1e2f0..=0x1e2f9 | + 0x1e4f0..=0x1e4f9 | + 0x1e5f1..=0x1e5fa | + 0x1e950..=0x1e959 | + 0x1fbf0..=0x1fbf9 + ) +} diff --git a/rust/gerbil-parser-runtime/src/engine/unicode_whitespace.rs b/rust/gerbil-parser-runtime/src/engine/unicode_whitespace.rs new file mode 100644 index 00000000..282fc7fd --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/unicode_whitespace.rs @@ -0,0 +1,20 @@ +// @generated by t/generate-scheme-whitespace-class.ss; do not edit. +// Source authority: pinned native Scheme char-whitespace? class. +const RANGES: &[(u32, u32)] = &[ + (0x9, 0xd), + (0x20, 0x20), + (0x85, 0x85), + (0xa0, 0xa0), + (0x1680, 0x1680), + (0x2000, 0x200a), + (0x2028, 0x2029), + (0x202f, 0x202f), + (0x205f, 0x205f), + (0x3000, 0x3000), +]; + +pub(crate) fn is_scheme_whitespace(character: char) -> bool { + let code = character as u32; + let index = RANGES.partition_point(|&(start, _)| start <= code); + index > 0 && code <= RANGES[index - 1].1 +} diff --git a/rust/gerbil-parser-rowan/src/engine/validation.rs b/rust/gerbil-parser-runtime/src/engine/validation.rs similarity index 58% rename from rust/gerbil-parser-rowan/src/engine/validation.rs rename to rust/gerbil-parser-runtime/src/engine/validation.rs index dc17a358..516f40be 100644 --- a/rust/gerbil-parser-rowan/src/engine/validation.rs +++ b/rust/gerbil-parser-runtime/src/engine/validation.rs @@ -45,7 +45,7 @@ fn validate_spec(spec: &LanguageSpec) -> Result<(), String> { return Err("grammar digest is not a canonical SHA-256 identity".into()); } if spec.kinds.len() > usize::from(u16::MAX) + 1 { - return Err("syntax-kind catalog exceeds Rowan u16 identity space".into()); + return Err("syntax-kind catalog exceeds AOT u16 identity space".into()); } if usize::from(spec.root_kind) >= spec.kinds.len() || spec.kinds[usize::from(spec.root_kind)].category != KindCategory::Node @@ -152,12 +152,128 @@ fn validate_parser_action( } } +fn valid_text_class(class: &super::model::TextClass, depth: usize) -> bool { + use super::model::TextClass; + depth < 64 + && match class { + TextClass::Numeric | TextClass::Alphabetic | TextClass::AsciiLetter => true, + TextClass::Characters(values) => !values.is_empty(), + TextClass::Union(classes) => { + !classes.is_empty() + && classes + .iter() + .all(|class| valid_text_class(class, depth + 1)) + } + } +} + +pub(super) fn text_profile_width( + profile: &super::model::TextProfile, + depth: usize, +) -> Option { + use super::model::TextProfile; + if depth >= 64 { + return None; + } + match profile { + TextProfile::Literal(value) => (!value.is_empty()).then(|| value.chars().count()), + TextProfile::Run { + class, + minimum, + maximum, + } => (valid_text_class(class, 0) + && u32::try_from(*minimum).is_ok() + && maximum.is_none_or(|maximum| maximum >= *minimum && u32::try_from(maximum).is_ok())) + .then_some(*minimum), + TextProfile::RunContaining { + class, + required, + minimum, + maximum, + } => (valid_text_class(class, 0) + && valid_text_class(required, 0) + && u32::try_from(*minimum).is_ok() + && maximum.is_none_or(|maximum| maximum >= *minimum && u32::try_from(maximum).is_ok())) + .then_some((*minimum).max(1)), + TextProfile::EndsIn { class, body, .. } => { + let width = text_profile_width(body, depth + 1)?; + (valid_text_class(class, 0) && width > 0).then_some(width) + } + TextProfile::Sequence(steps) => { + if steps.is_empty() { + return None; + } + steps.iter().try_fold(0usize, |width, step| { + width + .checked_add(text_profile_width(step, depth + 1)?) + .filter(|width| u32::try_from(*width).is_ok()) + }) + } + TextProfile::Optional(step) => text_profile_width(step, depth + 1).map(|_| 0), + TextProfile::IfNext { + class, + body, + otherwise, + } => { + if !valid_text_class(class, 0) { + return None; + } + let yes = text_profile_width(body, depth + 1)?; + let no = otherwise.map_or(Some(0), |step| text_profile_width(step, depth + 1))?; + Some(yes.min(no)) + } + TextProfile::NotNext(class) => valid_text_class(class, 0).then_some(0), + } +} + +fn valid_module_profile(profile: &super::model::ModuleTextProfile) -> bool { + let fields = [ + profile.header_border, + profile.header_word, + profile.end_border, + profile.block_open, + profile.block_close, + profile.line_comment, + profile.name_extra, + ]; + fields + .iter() + .all(|text| !text.is_empty() && text.chars().count() <= 128) + && profile.name_extra.chars().count() == 1 + && [profile.header_border, profile.end_border] + .iter() + .all(|text| { + let first = text.chars().next(); + text.chars().all(|ch| Some(ch) == first) + }) +} + fn validate_lexical_expression(expression: &LexicalExpr) -> Result<(), String> { fn nonempty(values: &[&str]) -> bool { values.iter().all(|value| !value.is_empty()) } match expression { + LexicalExpr::HeaderDelimiter { + prefix, + count, + index, + } if prefix.is_empty() || !(1..=32).contains(count) || index >= count => { + Err("header delimiter requires a prefix, bounded count and valid index".into()) + } + LexicalExpr::HeaderData { prefix, count, .. } + if prefix.is_empty() || !(1..=32).contains(count) => + { + Err("header data requires a prefix and bounded delimiter count".into()) + } + LexicalExpr::ModuleText(profile) if !valid_module_profile(profile) => { + Err("invalid module framing profile".into()) + } + LexicalExpr::TextProfile(profile) + if text_profile_width(profile, 0).is_none_or(|width| width == 0) => + { + Err("text profile must be valid, bounded in depth, and non-nullable".into()) + } LexicalExpr::NumberLiteral { prefixes, separator, @@ -171,6 +287,13 @@ fn validate_lexical_expression(expression: &LexicalExpr) -> Result<(), String> { { Err("quoted string requires non-empty delimiters".into()) } + LexicalExpr::QuotedStringProfile { + delimiter, + unicode_width, + .. + } if delimiter.chars().count() != 1 || *unicode_width > 8 => Err( + "quoted string profile requires one delimiter and at most eight hex characters".into(), + ), LexicalExpr::LineComment(prefixes) if prefixes.is_empty() || !nonempty(prefixes) => { Err("line comment requires non-empty prefixes".into()) } @@ -190,6 +313,11 @@ fn validate_lexical_expression(expression: &LexicalExpr) -> Result<(), String> { LexicalExpr::Literals(values) if values.is_empty() || !nonempty(values) => { Err("literal expression requires non-empty spellings".into()) } + LexicalExpr::CharacterRun { character, minimum } + if character.chars().count() != 1 || *minimum == 0 => + { + Err("character run requires one character and a positive minimum".into()) + } _ => Ok(()), } } diff --git a/rust/gerbil-parser-runtime/src/engine/word_parts.rs b/rust/gerbil-parser-runtime/src/engine/word_parts.rs new file mode 100644 index 00000000..0159de23 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/engine/word_parts.rs @@ -0,0 +1,418 @@ +//! Explicit task-stack composition of declared Word/Binding results. +use super::part_profile::{PartOpcode, PreparedPartProfile, error}; +use super::{Diagnostic, ProjectedNode, ResultCapture}; +use crate::scanner::PreparedRegionSource; +use std::collections::HashMap; +use std::ops::Range; + +enum Slot { + Span(Option>), + Child, +} +enum Task { + Parts { + span: Range, + context: usize, + }, + Node { + span: Range, + rule: usize, + }, + Build { + projection: usize, + span: Range, + slots: Vec, + }, + Collect(usize), +} +struct WordSource<'a> { + regions: PreparedRegionSource<'a>, + subscripts: HashMap>, + text: &'a str, +} +fn check(source: &str, span: &Range) -> Result<(), Diagnostic> { + if span.start > span.end || source.get(span.clone()).is_none() { + Err(error(span.start, "invalid part source span")) + } else { + Ok(()) + } +} +fn advance(source: &str, span: Range, count: usize) -> Result { + let mut at = span.start; + for ch in source[span.clone()].chars().take(count) { + at += ch.len_utf8(); + } + if source[span.start..at].chars().count() == count { + Ok(at) + } else { + Err(error(at, "part opening crosses capture boundary")) + } +} +fn retreat(source: &str, span: Range, count: usize) -> Result { + let mut at = span.end; + for ch in source[span.clone()].chars().rev().take(count) { + at -= ch.len_utf8(); + } + if source[at..span.end].chars().count() == count { + Ok(at) + } else { + Err(error(at, "part closing crosses capture boundary")) + } +} +fn literal(source: &str, span: Range, vocabulary: &[&str]) -> Option { + vocabulary + .iter() + .filter(|p| source[span.clone()].starts_with(**p)) + .map(|p| span.start + p.len()) + .max() +} +fn capture(span: Range) -> Slot { + Slot::Span((span.start < span.end).then_some(span)) +} +fn leaf(projection: usize, span: Range) -> Task { + Task::Build { + projection, + slots: vec![Slot::Span(Some(span.clone()))], + span, + } +} +impl PreparedPartProfile { + /// Recognize a Word within a bounded slice of the full immutable source. + /// # Errors + /// Rejects invalid UTF-8 ranges and malformed or unterminated declared regions. + pub fn word<'a>( + &self, + source: &'a str, + span: Range, + ) -> Result, Diagnostic> { + self.components(source, span, self.word_context, self.word) + } + /// Recognize heredoc content with its declared part context. + /// # Errors + /// Rejects invalid UTF-8 ranges and malformed expansions within the capture. + pub fn here_content<'a>( + &self, + source: &'a str, + span: Range, + ) -> Result, Diagnostic> { + self.components(source, span, self.here_context, self.here) + } + /// Recognize an assignment; ordinary words return `None`. + /// # Errors + /// Rejects invalid UTF-8 ranges and malformed declared assignment values. + pub fn assignment<'a>( + &self, + source: &'a str, + span: Range, + ) -> Result>, Diagnostic> { + check(source, &span)?; + let Some(binding) = self.spec.assignment else { + return Ok(None); + }; + let Some(projection) = self.assignment else { + return Ok(None); + }; + let Some(name) = self.names[binding].match_prefix(source, span.start, span.end) else { + return Ok(None); + }; + let Some(operator) = literal( + source, + name..span.end, + self.spec.bindings[binding].operators, + ) else { + return Ok(None); + }; + let build = Task::Build { + projection, + span: span.clone(), + slots: vec![ + Slot::Span(Some(span.start..name)), + Slot::Span(Some(name..operator)), + Slot::Child, + ], + }; + self.execute( + source, + vec![ + build, + Task::Parts { + span: operator..span.end, + context: self.word_context, + }, + ], + ) + .map(Some) + } + fn components<'a>( + &self, + source: &'a str, + span: Range, + context: usize, + projection: usize, + ) -> Result, Diagnostic> { + check(source, &span)?; + self.execute( + source, + vec![ + Task::Build { + projection, + span: span.clone(), + slots: vec![Slot::Child], + }, + Task::Parts { span, context }, + ], + ) + } + fn execute<'a>( + &self, + source: &'a str, + mut tasks: Vec, + ) -> Result, Diagnostic> { + let mut raw = WordSource { + regions: self.regions.source(source), + subscripts: HashMap::new(), + text: source, + }; + let mut values = Vec::new(); + while let Some(task) = tasks.pop() { + match task { + Task::Parts { span, context } => self.parts(&mut raw, span, context, &mut tasks)?, + Task::Node { span, rule } => self.node(&mut raw, span, rule, &mut tasks)?, + Task::Collect(count) => { + let start = values + .len() + .checked_sub(count) + .ok_or_else(|| error(0, "missing part children"))?; + let nodes = values + .drain(start..) + .map(|value| match value { + ResultCapture::Node(node) => Ok(node), + _ => Err(error(0, "invalid part child value")), + }) + .collect::, _>>()?; + values.push(ResultCapture::Parts(nodes)); + } + Task::Build { + projection, + span, + slots, + } => { + let count = slots.iter().filter(|s| matches!(s, Slot::Child)).count(); + let start = values + .len() + .checked_sub(count) + .ok_or_else(|| error(span.start, "missing composite captures"))?; + let mut children = values.drain(start..); + let captures = slots + .into_iter() + .map(|slot| match slot { + Slot::Span(Some(span)) => Ok(ResultCapture::Span(span)), + Slot::Span(None) => Ok(ResultCapture::Absent), + Slot::Child => children + .next() + .ok_or_else(|| error(span.start, "missing child capture")), + }) + .collect::, _>>()?; + drop(children); + values.push(ResultCapture::Node( + self.projections[projection].project(source, span, captures)?, + )); + } + } + } + match values.pop() { + Some(ResultCapture::Node(node)) if values.is_empty() => Ok(node), + _ => Err(error(0, "invalid part publication stack")), + } + } + fn parts( + &self, + raw: &mut WordSource<'_>, + span: Range, + context: usize, + tasks: &mut Vec, + ) -> Result<(), Diagnostic> { + let source = raw.text; + let mut at = span.start; + let mut children = Vec::new(); + while at < span.end { + let rule = self.matching(source, at, span.end, context); + if let Some(index) = rule { + let rule = &self.spec.rules[index]; + let after = match rule.opcode { + PartOpcode::Quote { delimiter } | PartOpcode::QuotedBody { delimiter } => { + let quote = advance(source, at..span.end, rule.opening - 1)?; + Some(raw.regions.quote_end(quote, delimiter)?) + } + PartOpcode::Pair | PartOpcode::Parameter { .. } => { + Some(raw.regions.pair_end(at)?) + } + PartOpcode::Name { binding } => self.names[binding].match_prefix( + source, + advance(source, at..span.end, 1)?, + span.end, + ), + PartOpcode::Escape => Some( + source[at..span.end] + .chars() + .take(2) + .fold(at, |end, ch| end + ch.len_utf8()), + ), + }; + if let Some(after) = after { + if after <= at || after > span.end { + return Err(error(at, "part region crosses capture boundary")); + } + if matches!(rule.opcode, PartOpcode::Name { .. } | PartOpcode::Escape) { + children.push(leaf(self.rule_projections[index], at..after)); + } else { + children.push(Task::Node { + span: at..after, + rule: index, + }); + } + at = after; + continue; + } + } + let start = at; + at = advance(source, at..span.end, 1)?; + while at < span.end && self.matching(source, at, span.end, context).is_none() { + at = advance(source, at..span.end, 1)?; + } + children.push(leaf(self.literal, start..at)); + } + tasks.push(Task::Collect(children.len())); + tasks.extend(children.into_iter().rev()); + Ok(()) + } + fn node( + &self, + raw: &mut WordSource<'_>, + span: Range, + index: usize, + tasks: &mut Vec, + ) -> Result<(), Diagnostic> { + let rule = &self.spec.rules[index]; + let projection = self.rule_projections[index]; + let start = advance(raw.text, span.clone(), rule.opening)?; + let end = retreat(raw.text, span.clone(), rule.closing)?; + if start > end { + return Err(error(span.start, "overlapping part delimiters")); + } + match rule.opcode { + PartOpcode::Quote { .. } => { + tasks.push(Task::Build { + projection, + slots: vec![ + Slot::Span(Some(span.start..start)), + Slot::Child, + Slot::Span(Some(end..span.end)), + ], + span, + }); + tasks.push(Task::Parts { + span: start..end, + context: self.quote_contexts[index] + .ok_or_else(|| error(start, "missing quote context"))?, + }); + } + PartOpcode::Pair | PartOpcode::QuotedBody { .. } => tasks.push(Task::Build { + projection, + slots: vec![ + Slot::Span(Some(span.start..start)), + capture(start..end), + Slot::Span(Some(end..span.end)), + ], + span, + }), + PartOpcode::Parameter { binding } => { + self.parameter(raw, span, start..end, binding, projection, tasks)?; + } + _ => return Err(error(span.start, "invalid composite part operation")), + } + Ok(()) + } + fn parameter( + &self, + raw: &mut WordSource<'_>, + span: Range, + body: Range, + binding: usize, + projection: usize, + tasks: &mut Vec, + ) -> Result<(), Diagnostic> { + let source = raw.text; + let spec = &self.spec.bindings[binding]; + let prefix_limit = source[body.clone()] + .chars() + .next_back() + .map_or(body.start, |ch| body.end - ch.len_utf8()); + let name_start = + literal(source, body.start..prefix_limit, spec.prefixes).unwrap_or(body.start); + let name_end = self.names[binding] + .match_prefix(source, name_start, body.end) + .unwrap_or(name_start); + let mut children = Vec::new(); + let (after, subscript) = if let Some(sub) = spec + .subscript + .filter(|sub| source[name_end..body.end].starts_with(sub.prefix)) + { + let plan = self.subscripts[binding] + .ok_or_else(|| error(name_end, "missing subscript region plan"))?; + let regions = raw + .subscripts + .entry(binding) + .or_insert_with(|| plan.source(source)); + let after = regions.pair_end(name_end)?; + if after > body.end { + return Err(error( + name_end, + "binding subscript crosses parameter boundary", + )); + } + let inner = name_end + sub.prefix.len(); + let close = retreat(source, inner..after, 1)?; + let id = self + .subscript + .ok_or_else(|| error(name_end, "missing subscript projection"))?; + children.push(Task::Build { + projection: id, + span: name_end..after, + slots: vec![ + Slot::Span(Some(name_end..inner)), + Slot::Child, + Slot::Span(Some(close..after)), + ], + }); + children.push(Task::Parts { + span: inner..close, + context: self.word_context, + }); + (after, Slot::Child) + } else { + (name_end, Slot::Span(None)) + }; + let operator = literal(source, after..body.end, spec.operators).unwrap_or(after); + tasks.push(Task::Build { + projection, + span: span.clone(), + slots: vec![ + Slot::Span(Some(span.start..body.start)), + capture(body.start..name_start), + capture(name_start..name_end), + subscript, + capture(after..operator), + Slot::Child, + Slot::Span(Some(body.end..span.end)), + ], + }); + tasks.push(Task::Parts { + span: operator..body.end, + context: self.word_context, + }); + // Subscript result precedes the operand capture on the value stack. + tasks.extend(children); + Ok(()) + } +} diff --git a/rust/gerbil-parser-runtime/src/lib.rs b/rust/gerbil-parser-runtime/src/lib.rs new file mode 100644 index 00000000..cc6539fd --- /dev/null +++ b/rust/gerbil-parser-runtime/src/lib.rs @@ -0,0 +1,42 @@ +//! Lossless parser runtime for parser tables generated by `gerbil-parser`. +//! +//! Gerbil owns grammar declarations and LR conflict resolution at AOT time; +//! this crate executes the resulting immutable Rust tables. + +#![forbid(unsafe_code)] + +extern crate self as gerbil_parser_runtime; + +mod engine; +pub mod scanner; + +pub use engine::{ + ActionEntry, BindingSpec, BlockContents, BlockHeaderRule, BlockLineRule, BlockOpeningMode, + CaptureKind, CaptureSpec, CommandChoice, CommandEvent, CommandForm, CommandInstruction, + CommandParse, CommandProgramSpec, CommandRole, CommandSourceSpec, CommandTokenClass, + CommandTrigger, ContextualParserSpec, Diagnostic, EventCatalog, GotoEntry, GraphFieldMode, + GraphFieldRule, GraphFieldValue, GraphIndex, GraphIndexError, GraphNodeRule, + GraphProjectionSpec, GraphRecord, GraphRelation, HeadingFieldsRule, HeadingLineRule, + InlineLinkRule, KeyLineContext, KeyLineMode, KeyLineRule, KeyValueLineRule, KindCategory, + KindSpec, LanguageSpec, LexicalExpr, LexicalRule, LineStructureSpec, ListLineRule, + ModuleTextProfile, Operand, OperandAction, Parse, ParseError, ParseReceipt, ParserAction, + PartGuard, PartOpcode, PartProfileSpec, PartRule, PreparedCommandProgram, + PreparedCommandSource, PreparedPartProfile, PreparedResultNode, PreparedResultProfile, + PreparedResultProjection, PreparedTextProfile, Production, ProjectedChild, ProjectedNode, + ProjectedValue, ProjectionInstruction, ProjectionOpcode, Reduction, ResultCapture, + ResultChildCapture, ResultNodeSpec, ResultProfileSpec, ResultProjection, ScannedToken, + SelectiveGlrReceipt, SubscriptSpec, Symbol, SyntaxKind, SyntaxNode, SyntaxToken, TableLineRule, + Terminal, TerminalSpec, TextClass, TextProfile, TreeEvent, UnclosedBlockPolicy, + build_syntax_events, build_syntax_events_catalog, parse, parse_contextual, + parse_generated_events, parse_scanned, parse_structural_lines, project_syntax_graph, +}; + +#[cfg(test)] +#[path = "../tests/unit/records_contextual_fixture.rs"] +mod records_contextual_fixture; + +pub use gerbil_parser_artifact::syntax::{SyntaxTree, TextRange, TextSize}; + +#[cfg(test)] +#[path = "../tests/unit/artifact_generation.rs"] +mod artifact_generation_tests; diff --git a/rust/gerbil-parser-runtime/src/scanner/admission.rs b/rust/gerbil-parser-runtime/src/scanner/admission.rs new file mode 100644 index 00000000..577dd6a9 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/scanner/admission.rs @@ -0,0 +1,106 @@ +//! Closed opcode admission mirrors the Scheme compiler before scanner execution. +use super::{ScannerAction, ScannerMatcher, ScannerRule, ScannerSpec, error}; +use crate::Diagnostic; +fn strings(values: &[&str]) -> bool { + !values.is_empty() && values.iter().all(|v| !v.is_empty()) +} +fn matcher(value: ScannerMatcher) -> bool { + match value { + ScannerMatcher::Literal(s) => !s.is_empty(), + ScannerMatcher::Literals(ss) | ScannerMatcher::QuotedString(ss) => strings(ss), + ScannerMatcher::UnlessPrefix { + prefixes, + exceptions, + child, + } => { + strings(prefixes) + && (exceptions.is_empty() || strings(exceptions)) + && !matches!(child, ScannerMatcher::UnlessPrefix { .. }) + && matcher(*child) + } + ScannerMatcher::LinePrefix { prefix, separator } => { + !prefix.is_empty() && !prefix.contains(separator) + } + ScannerMatcher::RegionWord { + stops, + quotes, + pairs, + .. + } => super::super::region::valid_spec(stops, quotes, pairs), + ScannerMatcher::BalancedWord { + stops, + quotes, + pairs, + } => { + strings(stops) + && strings(quotes) + && quotes + .iter() + .enumerate() + .all(|(i, q)| q.chars().count() == 1 && !quotes[..i].contains(q)) + && pairs.iter().enumerate().all(|(i, p)| { + p.prefix.ends_with(p.opening) + && !pairs[..i].iter().any(|prior| prior.prefix == p.prefix) + }) + } + _ => true, + } +} +fn action(rule: &ScannerRule) -> bool { + let matcher = match rule.matcher { + ScannerMatcher::UnlessPrefix { child, .. } => *child, + other => other, + }; + match rule.action { + ScannerAction::Keep => !matches!( + matcher, + ScannerMatcher::MarkerLine | ScannerMatcher::MarkerLineAt(_) + ), + ScannerAction::ExpectMarker(_) | ScannerAction::ExpectMarkerIn { .. } => matches!( + matcher, + ScannerMatcher::Literal(_) | ScannerMatcher::Literals(_) + ), + ScannerAction::EnqueueIfExpecting(_) | ScannerAction::EnqueueMarkerIn { .. } => matches!( + matcher, + ScannerMatcher::BalancedWord { .. } + | ScannerMatcher::RegionWord { .. } + | ScannerMatcher::Identifier + | ScannerMatcher::QuotedString(_) + ), + ScannerAction::ActivateNext(_) => matches!( + matcher, + ScannerMatcher::NewlineOne | ScannerMatcher::Literal(_) + ), + ScannerAction::FinishMarker { .. } => matches!( + matcher, + ScannerMatcher::MarkerLine | ScannerMatcher::MarkerLineAt(_) + ), + } +} +pub(super) fn rules(spec: &ScannerSpec) -> Result<(), Diagnostic> { + if spec.modes.is_empty() + || spec + .modes + .iter() + .enumerate() + .any(|(i, m)| m.is_empty() || spec.modes[..i].contains(m)) + || spec + .positions + .iter() + .enumerate() + .any(|(i, p)| p.is_empty() || spec.positions[..i].contains(p)) + { + return Err(error(0, "empty or duplicate scanner modes/positions")); + } + for (i, rule) in spec.rules.iter().enumerate() { + if rule.name.is_empty() + || rule.form.is_empty() + || !matcher(rule.matcher) + || !action(rule) + || spec.rules[..i].iter().any(|r| r.name == rule.name) + { + return Err(error(0, "invalid or duplicate scanner rule")); + } + } + Ok(()) +} diff --git a/rust/gerbil-parser-runtime/src/scanner/contextual.rs b/rust/gerbil-parser-runtime/src/scanner/contextual.rs new file mode 100644 index 00000000..efb553db --- /dev/null +++ b/rust/gerbil-parser-runtime/src/scanner/contextual.rs @@ -0,0 +1,525 @@ +//! Execution of the Scheme contextual-scanner-opcodes.v1 data contract. +#[path = "admission.rs"] +mod admission; +#[path = "matching.rs"] +mod matching; +#[path = "obligations.rs"] +mod obligations; +#[path = "session.rs"] +mod session; +use crate::{Diagnostic, ScannedToken}; +use matching::matcher_end; +pub(crate) use matching::shell_delimiter; +use obligations::Obligations; +pub use session::ContextualScanSession; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +pub const SCANNER_OPCODE_CONTRACT: &str = "gerbil-parser.contextual-scanner-opcodes.v1"; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BalancedPair { + pub prefix: &'static str, + pub opening: char, + pub closing: char, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RegionQuote { + pub delimiter: char, + pub escaped: bool, + pub pairs: &'static [&'static str], +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RegionPair { + pub prefix: &'static str, + pub opening: char, + pub closing: char, + pub depth: usize, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScannerMatcher { + Literal(&'static str), + Literals(&'static [&'static str]), + UnlessPrefix { + prefixes: &'static [&'static str], + exceptions: &'static [&'static str], + child: &'static Self, + }, + LinePrefix { + prefix: &'static str, + separator: char, + }, + MarkerLineAt(char), + BodyLineAt(char), + HorizontalWhitespace, + Newline, + NewlineOne, + Identifier, + MarkerLine, + BodyLine, + QuotedString(&'static [&'static str]), + RegionWord { + stops: &'static [&'static str], + quotes: &'static [RegionQuote], + pairs: &'static [RegionPair], + consume_initial_stop: bool, + }, + BalancedWord { + stops: &'static [&'static str], + quotes: &'static [&'static str], + pairs: &'static [BalancedPair], + }, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MarkerPolicy { + Raw, + ShellQuoteRemoval, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScannerAction { + Keep, + ExpectMarker(bool), + ExpectMarkerIn { + strip_tabs: bool, + mode: &'static str, + }, + EnqueueMarkerIn { + policy: MarkerPolicy, + mode: &'static str, + }, + EnqueueIfExpecting(MarkerPolicy), + ActivateNext(&'static str), + FinishMarker { + base: &'static str, + body: &'static str, + }, +} +#[derive(Clone, Copy, Debug)] +pub struct ScannerRule { + pub name: &'static str, + pub mode: &'static str, + pub form: &'static str, + pub matcher: ScannerMatcher, + pub rank: i64, + pub action: ScannerAction, +} +#[derive(Clone, Copy, Debug)] +pub struct ScannerCell { + pub mode: &'static str, + pub position: &'static str, + pub form: &'static str, + pub terminal: &'static str, +} +#[derive(Debug)] +pub struct ScannerSpec { + pub opcode_contract: &'static str, + pub base_grammar_digest: Option<&'static str>, + pub digest: &'static str, + pub initial_mode: &'static str, + pub modes: &'static [&'static str], + pub positions: &'static [&'static str], + pub rules: &'static [ScannerRule], + pub cells: &'static [ScannerCell], +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Obligation { + pub(super) marker: String, + pub(super) strip_tabs: bool, + pub(super) quoted: bool, +} +/// Scanner indexes admitted once for each immutable generated specification. +#[derive(Debug)] +struct ScannerPlan { + rules: HashMap<&'static str, Vec<&'static ScannerRule>>, + dispatch: HashMap<(&'static str, &'static str), HashMap<&'static str, &'static str>>, +} +/// Source-local executor sharing immutable scanner indexes across parses. +#[derive(Debug)] +pub struct ContextualScanner<'source> { + spec: &'static ScannerSpec, + source: &'source str, + plan: Arc, +} +/// Immutable checkpoint bound to the prepared scanner and exact source. +#[derive(Clone, Debug)] +pub struct ScannerCheckpoint<'scanner, 'source> { + owner: &'scanner ContextualScanner<'source>, + offset: usize, + mode: &'static str, + pending: Obligations, + active: Option>, + expecting: Option, +} +impl ScannerCheckpoint<'_, '_> { + fn same_context(&self, other: &ScannerCheckpoint<'_, '_>) -> bool { + Arc::ptr_eq(&self.owner.plan, &other.owner.plan) + && self.mode == other.mode + && self.expecting == other.expecting + && self.active == other.active + && self.pending.equivalent(&other.pending) + } + /// Admit identical future scanner decisions for independently reached states. + /// Callers must supply the same future parser-position schedule. This does + /// not relocate checkpoints or establish parser/subtree equivalence. + #[must_use] + pub fn converges_with(&self, other: &ScannerCheckpoint<'_, '_>) -> bool { + self.same_context(other) + && self.owner.source[self.offset..] == other.owner.source[other.offset..] + } + + #[must_use] + pub fn byte_offset(&self) -> usize { + self.offset + } + #[must_use] + pub fn mode(&self) -> &'static str { + self.mode + } + #[must_use] + pub fn pending_markers(&self) -> usize { + self.pending.len() + } + #[must_use] + pub fn active_marker(&self) -> Option<(&str, bool, bool)> { + self.active + .as_ref() + .map(|o| (o.marker.as_str(), o.strip_tabs, o.quoted)) + } +} +pub(crate) fn error(offset: usize, message: &str) -> Diagnostic { + Diagnostic { + reason_kind: "contextual-scanner", + byte_offset: offset, + message: message.into(), + } +} +impl<'source> ContextualScanner<'source> { + pub(crate) fn source_slice(&self, start: usize, end: usize) -> &'source str { + &self.source[start..end] + } + /// # Errors + /// Invalid scanner identity, declaration or action target. + pub fn new(spec: &'static ScannerSpec, source: &'source str) -> Result { + let plan = prepare_spec_once(spec)?; + Ok(Self { spec, source, plan }) + } + #[must_use] + pub fn initial_state(&self) -> ScannerCheckpoint<'_, 'source> { + ScannerCheckpoint { + owner: self, + offset: 0, + mode: self.spec.initial_mode, + pending: Obligations::default(), + active: None, + expecting: None, + } + } + /// Re-execute a prefix on both sources under the recorded parser positions. + /// Returns a checkpoint owned by this scanner; does not relocate old state + /// or establish parser/subtree reuse. External mode changes are not replayed. + /// + /// # Errors + /// Different plans/prefixes, an incomplete schedule or changed lexical decisions. + pub fn replay_prefix<'scanner>( + &'scanner self, + checkpoint: &ScannerCheckpoint<'_, '_>, + positions: &[&str], + ) -> Result, Diagnostic> { + let old = checkpoint.owner; + if !Arc::ptr_eq(&self.plan, &old.plan) + || self.source.get(..checkpoint.offset) != Some(&old.source[..checkpoint.offset]) + { + return Err(error( + checkpoint.offset, + "scanner prefix source or plan mismatch", + )); + } + let mut before = old.initial_state(); + let mut after = self.initial_state(); + for position in positions { + let (a, next_a) = old.step(&before, position)?; + let (b, next_b) = self.step(&after, position)?; + if next_a.offset > checkpoint.offset || next_a.offset != next_b.offset || a != b { + return Err(error( + before.offset, + "scanner prefix lexical decision changed", + )); + } + before = next_a; + after = next_b; + } + if before.offset != checkpoint.offset + || !before.same_context(checkpoint) + || !before.same_context(&after) + { + return Err(error( + checkpoint.offset, + "scanner checkpoint not reproduced by position schedule", + )); + } + Ok(after) + } + /// # Errors + /// A foreign checkpoint or undeclared mode. + pub fn with_mode<'scanner>( + &'scanner self, + state: &ScannerCheckpoint<'scanner, 'source>, + mode: &'static str, + ) -> Result, Diagnostic> { + self.admit(state, self.spec.positions[0])?; + if !self.spec.modes.contains(&mode) { + return Err(error(state.offset, "undeclared scanner mode")); + } + let mut next = state.clone(); + next.mode = mode; + Ok(next) + } + fn admit( + &self, + state: &ScannerCheckpoint<'_, 'source>, + position: &str, + ) -> Result<(), Diagnostic> { + if !std::ptr::eq(state.owner, self) || !self.spec.positions.contains(&position) { + return Err(error( + state.offset, + "foreign checkpoint or undeclared parser position", + )); + } + Ok(()) + } + /// # Errors + /// An ambiguous/unmatched token, foreign checkpoint or unfinished obligation. + pub fn step<'scanner>( + &'scanner self, + state: &ScannerCheckpoint<'scanner, 'source>, + position: &str, + ) -> Result<(Option, ScannerCheckpoint<'scanner, 'source>), Diagnostic> { + self.admit(state, position)?; + if state.offset == self.source.len() { + if state.active.is_some() || !state.pending.is_empty() || state.expecting.is_some() { + return Err(error( + state.offset, + "unfinished contextual scanner obligation", + )); + } + return Ok((None, state.clone())); + } + let mut best: Option<(&ScannerRule, usize)> = None; + // Scheme index-rules reverses declaration order; equivalent ties retain that order. + for rule in self.plan.rules.get(state.mode).into_iter().flatten() { + if let Some(end) = matcher_end( + self.source, + state.offset, + rule.matcher, + state.active.as_deref(), + )? { + if end <= state.offset { + return Err(error(state.offset, "scanner did not advance")); + } + if let Some((current, current_end)) = best { + if end < current_end || (end == current_end && rule.rank < current.rank) { + continue; + } + if end == current_end && rule.rank == current.rank { + if rule.form != current.form || rule.action != current.action { + return Err(error(state.offset, "ambiguous contextual scanner match")); + } + continue; + } + } + best = Some((rule, end)); + } + } + let (rule, end) = + best.ok_or_else(|| error(state.offset, "contextual scanner has no match"))?; + let terminal = self + .plan + .dispatch + .get(&(state.mode, position)) + .and_then(|forms| forms.get(rule.form)) + .ok_or_else(|| error(state.offset, "missing scanner dispatch"))?; + let token = ScannedToken { + terminal, + start: state.offset, + end, + }; + let mut next = state.clone(); + next.offset = end; + apply_action( + &mut next, + rule.action, + &self.source[state.offset..end], + state.offset, + )?; + Ok((Some(token), next)) + } + /// Execute a declaration with one fixed parser position. Dynamic LR positions use step. + /// # Errors + /// The first scanner diagnostic; unfinished obligations also fail at EOF. + pub fn scan(&self, position: &str) -> Result, Diagnostic> { + let mut state = self.initial_state(); + let mut tokens = Vec::new(); + loop { + let (token, next) = self.step(&state, position)?; + state = next; + if let Some(token) = token { + tokens.push(token); + } else { + return Ok(tokens); + } + } + } +} + +// This cache follows the LR validate_spec_once boundary. A static specification +// has a process-long identity; input strings and checkpoints never enter it. +fn prepare_spec_once(spec: &'static ScannerSpec) -> Result, Diagnostic> { + static PLANS: OnceLock>>> = OnceLock::new(); + let key = std::ptr::from_ref(spec) as usize; + let mut plans = PLANS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .map_err(|_| error(0, "scanner plan registry poisoned"))?; + if let Some(plan) = plans.get(&key) { + return Ok(Arc::clone(plan)); + } + let plan = Arc::new(prepare_spec(spec)?); + plans.insert(key, Arc::clone(&plan)); + Ok(plan) +} + +fn prepare_spec(spec: &'static ScannerSpec) -> Result { + if spec.opcode_contract != SCANNER_OPCODE_CONTRACT + || !spec.digest.strip_prefix("sha256:").is_some_and(|s| { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) + || !spec.modes.contains(&spec.initial_mode) + || spec.positions.is_empty() + { + return Err(error(0, "invalid scanner contract or identity")); + } + admission::rules(spec)?; + let mut dispatch: HashMap<_, HashMap<_, _>> = HashMap::new(); + for cell in spec.cells { + if cell.form.is_empty() + || cell.terminal.is_empty() + || !spec.modes.contains(&cell.mode) + || !spec.positions.contains(&cell.position) + || dispatch + .entry((cell.mode, cell.position)) + .or_default() + .insert(cell.form, cell.terminal) + .is_some() + { + return Err(error(0, "invalid or duplicate scanner dispatch cell")); + } + } + for rule in spec.rules { + if !spec.modes.contains(&rule.mode) + || spec.positions.iter().any(|p| { + !dispatch + .get(&(rule.mode, *p)) + .is_some_and(|forms| forms.contains_key(rule.form)) + }) + { + return Err(error(0, "scanner rule has no dispatch")); + } + match rule.action { + ScannerAction::ActivateNext(mode) + | ScannerAction::ExpectMarkerIn { mode, .. } + | ScannerAction::EnqueueMarkerIn { mode, .. } + if !spec.modes.contains(&mode) => + { + return Err(error(0, "undeclared action mode")); + } + ScannerAction::FinishMarker { base, body } + if !spec.modes.contains(&base) || !spec.modes.contains(&body) => + { + return Err(error(0, "undeclared action mode")); + } + _ => {} + } + } + let mut rules: HashMap<_, Vec<_>> = HashMap::new(); + for rule in spec.rules.iter().rev() { + rules.entry(rule.mode).or_default().push(rule); + } + Ok(ScannerPlan { rules, dispatch }) +} + +fn enqueue_marker( + next: &mut ScannerCheckpoint<'_, '_>, + policy: MarkerPolicy, + word: &str, + at: usize, +) -> Result<(), Diagnostic> { + if let Some(strip_tabs) = next.expecting { + let (marker, quoted) = match policy { + MarkerPolicy::Raw => (word.to_owned(), false), + MarkerPolicy::ShellQuoteRemoval => shell_delimiter(word, at)?, + }; + if marker.is_empty() && !quoted { + return Err(error(at, "empty deferred delimiter")); + } + next.pending.push_back(Obligation { + marker, + strip_tabs, + quoted, + }); + next.expecting = None; + } + Ok(()) +} + +fn apply_action( + next: &mut ScannerCheckpoint<'_, '_>, + action: ScannerAction, + word: &str, + at: usize, +) -> Result<(), Diagnostic> { + match action { + ScannerAction::Keep => {} + ScannerAction::ExpectMarkerIn { strip_tabs, mode } => { + if next.expecting.is_some() { + return Err(error(at, "deferred delimiter already expected")); + } + next.expecting = Some(strip_tabs); + next.mode = mode; + } + ScannerAction::ExpectMarker(strip) => { + if next.expecting.is_some() { + return Err(error(at, "deferred delimiter already expected")); + } + next.expecting = Some(strip); + } + ScannerAction::EnqueueMarkerIn { policy, mode } => { + if next.expecting.is_none() { + return Err(error(at, "deferred delimiter not expected")); + } + enqueue_marker(next, policy, word, at)?; + next.mode = mode; + } + ScannerAction::EnqueueIfExpecting(policy) => { + enqueue_marker(next, policy, word, at)?; + } + ScannerAction::ActivateNext(body) => { + if next.expecting.is_some() { + return Err(error(at, "missing delimiter before newline")); + } + if let Some(active) = next.pending.pop_front() { + next.active = Some(active); + next.mode = body; + } + } + ScannerAction::FinishMarker { base, body } => { + if next.active.is_none() { + return Err(error(at, "marker closed without active obligation")); + } + next.active = next.pending.pop_front(); + next.mode = if next.active.is_some() { body } else { base }; + next.expecting = None; + } + } + Ok(()) +} diff --git a/rust/gerbil-parser-runtime/src/scanner/matching.rs b/rust/gerbil-parser-runtime/src/scanner/matching.rs new file mode 100644 index 00000000..85d3305a --- /dev/null +++ b/rust/gerbil-parser-runtime/src/scanner/matching.rs @@ -0,0 +1,320 @@ +//! UTF-8 recognizers for the shared scanner opcode contract. +use super::{BalancedPair, Obligation, RegionPair, RegionQuote, ScannerMatcher, error}; +use crate::Diagnostic; +fn next(source: &str, at: usize) -> usize { + at + source[at..].chars().next().map_or(0, char::len_utf8) +} +fn newline(source: &str, at: usize) -> Option { + if source[at..].starts_with("\r\n") { + Some(at + 2) + } else if source[at..].starts_with(['\r', '\n']) { + Some(at + 1) + } else { + None + } +} +fn run(source: &str, at: usize, predicate: impl Fn(char) -> bool) -> Option { + let mut end = at; + for ch in source[at..].chars() { + if !predicate(ch) { + break; + } + end += ch.len_utf8(); + } + (end > at).then_some(end) +} +pub(super) fn matcher_end( + source: &str, + at: usize, + matcher: ScannerMatcher, + active: Option<&Obligation>, +) -> Result, Diagnostic> { + let tail = &source[at..]; + Ok(match matcher { + ScannerMatcher::Literal(s) => tail.starts_with(s).then_some(at + s.len()), + ScannerMatcher::Literals(ss) => ss + .iter() + .filter(|s| tail.starts_with(**s)) + .map(|s| at + s.len()) + .max(), + ScannerMatcher::UnlessPrefix { + prefixes, + exceptions, + child, + } => { + if exceptions.iter().any(|p| tail.starts_with(p)) + || !prefixes.iter().any(|p| tail.starts_with(p)) + { + matcher_end(source, at, *child, active)? + } else { + None + } + } + ScannerMatcher::LinePrefix { prefix, separator } => tail + .starts_with(prefix) + .then(|| tail.find(separator).map_or(source.len(), |i| at + i)), + ScannerMatcher::MarkerLineAt(separator) => { + profile_line(source, at, separator, active, true) + } + ScannerMatcher::BodyLineAt(separator) => profile_line(source, at, separator, active, false), + ScannerMatcher::HorizontalWhitespace => run(source, at, |c| c == ' ' || c == '\t'), + ScannerMatcher::Newline => run(source, at, |c| c == '\r' || c == '\n'), + ScannerMatcher::NewlineOne => newline(source, at), + ScannerMatcher::Identifier => { + if tail.starts_with(|c: char| c.is_alphabetic() || c == '_') { + run(source, at, |c| c.is_alphanumeric() || c == '_' || c == '-') + } else { + None + } + } + ScannerMatcher::BodyLine | ScannerMatcher::MarkerLine => active.and_then(|a| { + let end = tail + .find(['\r', '\n']) + .map_or(source.len(), |i| newline(source, at + i).unwrap_or(at + i)); + let content = source[at..end].trim_end_matches(['\r', '\n']); + if matcher == ScannerMatcher::BodyLine + || (if a.strip_tabs { + content.trim_start_matches('\t') + } else { + content + }) == a.marker + { + Some(end) + } else { + None + } + }), + ScannerMatcher::QuotedString(delimiters) => { + let mut found = None; + for delimiter in delimiters { + if tail.starts_with(delimiter) { + let mut cursor = at + delimiter.len(); + while cursor < source.len() { + if source[cursor..].starts_with(delimiter) { + let end = cursor + delimiter.len(); + if source[end..].starts_with(delimiter) { + cursor = end + delimiter.len(); + continue; + } + found = Some(end); + break; + } + if source[cursor..].starts_with('\\') { + cursor = next(source, cursor); + if cursor < source.len() { + cursor = next(source, cursor); + } + } else { + cursor = next(source, cursor); + } + } + if found.is_some() { + break; + } + } + } + found + } + ScannerMatcher::RegionWord { + stops, + quotes, + pairs, + consume_initial_stop, + } => region_word(source, at, stops, quotes, pairs, consume_initial_stop)?, + ScannerMatcher::BalancedWord { + stops, + quotes, + pairs, + } => balanced_word(source, at, stops, quotes, pairs)?, + }) +} +#[derive(Clone, Copy)] +enum Frame { + Quote(char), + Pair(char, char, usize), +} +fn pair_at<'a>(source: &str, at: usize, pairs: &'a [BalancedPair]) -> Option<&'a BalancedPair> { + pairs + .iter() + .filter(|p| source[at..].starts_with(p.prefix)) + .max_by_key(|p| p.prefix.len()) +} +fn balanced_word( + source: &str, + at: usize, + stops: &[&str], + quotes: &[&str], + pairs: &[BalancedPair], +) -> Result, Diagnostic> { + let stopped = |cursor| { + source[cursor..].starts_with(char::is_whitespace) + || stops.iter().any(|s| source[cursor..].starts_with(s)) + }; + if stopped(at) { + return Ok(None); + } + let mut stack = Vec::new(); + let mut cursor = at; + while cursor < source.len() { + let ch = source[cursor..] + .chars() + .next() + .ok_or_else(|| error(cursor, "missing character"))?; + let quote = quotes + .iter() + .find(|q| source[cursor..].starts_with(**q)) + .and_then(|q| q.chars().next()); + let pair = pair_at(source, cursor, pairs); + match stack.last().copied() { + Some(Frame::Quote(delimiter)) => { + if ch == delimiter { + stack.pop(); + cursor = next(source, cursor); + continue; + } + if ch == '\\' && delimiter != '\'' { + cursor = next(source, cursor); + if cursor < source.len() { + cursor = next(source, cursor); + } + continue; + } + if delimiter == '"' + && let Some(p) = pair + { + stack.push(Frame::Pair(p.opening, p.closing, 1)); + cursor += p.prefix.len(); + continue; + } + } + Some(Frame::Pair(open, close, depth)) => { + if ch == '\\' { + cursor = next(source, cursor); + if cursor < source.len() { + cursor = next(source, cursor); + } + continue; + } + if let Some(q) = quote { + stack.push(Frame::Quote(q)); + cursor = next(source, cursor); + continue; + } + if let Some(p) = pair { + stack.push(Frame::Pair(p.opening, p.closing, 1)); + cursor += p.prefix.len(); + continue; + } + if ch == open { + if let Some(frame) = stack.last_mut() { + *frame = Frame::Pair(open, close, depth + 1); + } + } else if ch == close { + if depth == 1 { + stack.pop(); + } else if let Some(frame) = stack.last_mut() { + *frame = Frame::Pair(open, close, depth - 1); + } + } + } + None => { + if cursor > at && stopped(cursor) { + break; + } + if ch == '\\' { + cursor = next(source, cursor); + if cursor < source.len() { + cursor = next(source, cursor); + } + continue; + } + if let Some(q) = quote { + stack.push(Frame::Quote(q)); + cursor = next(source, cursor); + continue; + } + if let Some(p) = pair { + stack.push(Frame::Pair(p.opening, p.closing, 1)); + cursor += p.prefix.len(); + continue; + } + } + } + cursor = next(source, cursor); + } + if !stack.is_empty() { + return Err(error(at, "unterminated balanced word quote or pair")); + } + Ok(Some(cursor)) +} +pub(crate) fn shell_delimiter(word: &str, at: usize) -> Result<(String, bool), Diagnostic> { + let mut chars = word.chars().peekable(); + let mut quote = None; + let mut quoted = false; + let mut result = String::new(); + while let Some(ch) = chars.next() { + if quote != Some('\'') && ch == '\\' { + if let Some(escaped) = chars.next() { + if escaped == '\n' { + continue; + } + if quote == Some('"') && !['$', '`', '"', '\\'].contains(&escaped) { + result.push('\\'); + result.push(escaped); + } else { + quoted = true; + result.push(escaped); + } + } else { + quoted = true; + result.push(ch); + } + } else if (quote != Some('"') && ch == '\'') || (quote != Some('\'') && ch == '"') { + quote = if quote.is_some() { None } else { Some(ch) }; + quoted = true; + } else { + result.push(ch); + } + } + if quote.is_some() { + return Err(error(at, "unterminated delimiter quote")); + } + Ok((result, quoted)) +} + +fn region_word( + source: &str, + at: usize, + stops: &[&str], + quotes: &[RegionQuote], + pairs: &[RegionPair], + consume_initial_stop: bool, +) -> Result, Diagnostic> { + super::super::region::scan_word(source, at, stops, quotes, pairs, consume_initial_stop) +} + +fn profile_line( + source: &str, + at: usize, + separator: char, + active: Option<&Obligation>, + marker: bool, +) -> Option { + let active = active?; + let content_end = source[at..] + .find(separator) + .map_or(source.len(), |i| at + i); + let end = content_end + + if content_end < source.len() { + separator.len_utf8() + } else { + 0 + }; + let content = &source[at..content_end]; + let content = if active.strip_tabs { + content.trim_start_matches('\t') + } else { + content + }; + (!marker || content == active.marker).then_some(end) +} diff --git a/rust/gerbil-parser-runtime/src/scanner/mod.rs b/rust/gerbil-parser-runtime/src/scanner/mod.rs new file mode 100644 index 00000000..0b3092e1 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/scanner/mod.rs @@ -0,0 +1,23 @@ +//! Shared contextual scanner API; execution belongs to the contextual leaf. +mod contextual; +mod region; +pub use region::{PreparedRegionPlan, PreparedRegionSource, RegionScope, RegionSpec}; + +pub use contextual::{ + BalancedPair, ContextualScanSession, ContextualScanner, MarkerPolicy, RegionPair, RegionQuote, + SCANNER_OPCODE_CONTRACT, ScannerAction, ScannerCell, ScannerCheckpoint, ScannerMatcher, + ScannerRule, ScannerSpec, +}; +pub(crate) use contextual::{error, shell_delimiter}; + +#[cfg(test)] +#[path = "../../tests/unit/shared_scanner.rs"] +mod shared_scanner_tests; + +#[cfg(test)] +#[path = "../../tests/unit/contextual_scanner.rs"] +mod contextual_scanner_tests; + +#[cfg(test)] +#[path = "../../tests/unit/bash_scanner.rs"] +mod bash_scanner_tests; diff --git a/rust/gerbil-parser-runtime/src/scanner/obligations.rs b/rust/gerbil-parser-runtime/src/scanner/obligations.rs new file mode 100644 index 00000000..04b477d9 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/scanner/obligations.rs @@ -0,0 +1,71 @@ +//! Persistent two-stack FIFO: checkpoints share marker storage in O(1). +use super::Obligation; +use std::sync::Arc; +#[derive(Debug)] +struct Link { + value: Arc, + next: Option>, +} +impl Drop for Link { + fn drop(&mut self) { + // Drop long uniquely owned tails iteratively, avoiding call-stack growth. + let mut tail = self.next.take(); + while let Some(link) = tail { + let Ok(mut owned) = Arc::try_unwrap(link) else { + break; + }; + tail = owned.next.take(); + } + } +} +#[derive(Clone, Debug, Default)] +pub(super) struct Obligations { + front: Option>, + back: Option>, + length: usize, +} +impl Obligations { + pub(super) fn equivalent(&self, other: &Self) -> bool { + fn ordered(queue: &Obligations) -> impl Iterator { + let front = std::iter::successors(queue.front.as_deref(), |link| link.next.as_deref()); + let mut back: Vec<_> = + std::iter::successors(queue.back.as_deref(), |link| link.next.as_deref()).collect(); + back.reverse(); + front.chain(back).map(|link| link.value.as_ref()) + } + self.length == other.length && ordered(self).eq(ordered(other)) + } + pub(super) fn len(&self) -> usize { + self.length + } + pub(super) fn is_empty(&self) -> bool { + self.length == 0 + } + pub(super) fn push_back(&mut self, value: Obligation) { + self.back = Some(Arc::new(Link { + value: Arc::new(value), + next: self.back.take(), + })); + self.length += 1; + } + pub(super) fn pop_front(&mut self) -> Option> { + if self.front.is_none() { + let mut back = self.back.take(); + while let Some(link) = back.take() { + self.front = Some(Arc::new(Link { + value: Arc::clone(&link.value), + next: self.front.take(), + })); + back.clone_from(&link.next); + } + } + let front = self.front.take()?; + self.front.clone_from(&front.next); + self.length -= 1; + Some(Arc::clone(&front.value)) + } +} + +#[cfg(test)] +#[path = "../../tests/unit/scanner_obligations.rs"] +mod tests; diff --git a/rust/gerbil-parser-runtime/src/scanner/region.rs b/rust/gerbil-parser-runtime/src/scanner/region.rs new file mode 100644 index 00000000..b3bf9982 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/scanner/region.rs @@ -0,0 +1,367 @@ +//! Shared source-local region execution, including nested boundary memoization. +use super::{RegionPair, RegionQuote, error}; +use crate::Diagnostic; +use std::collections::HashMap; + +/// A pair scope restricts nested pair prefixes; an omitted scope permits all pairs. +#[derive(Clone, Copy, Debug)] +pub struct RegionScope { + pub prefix: &'static str, + pub pairs: &'static [&'static str], +} +/// Immutable delimiters, quote nesting and initial word-stop policy lowered by Scheme. +#[derive(Clone, Copy, Debug)] +pub struct RegionSpec { + pub stops: &'static [&'static str], + pub quotes: &'static [RegionQuote], + pub pairs: &'static [RegionPair], + pub consume_initial_stop: bool, +} +/// Immutable admitted region declarations, reusable across independent sources. +#[derive(Clone, Copy)] +pub struct PreparedRegionPlan { + spec: &'static RegionSpec, + scopes: &'static [RegionScope], +} +impl PreparedRegionPlan { + /// Admit a static region plan once. + /// # Errors + /// Rejects malformed declarations and unknown or duplicate scope owners. + pub fn new( + spec: &'static RegionSpec, + scopes: &'static [RegionScope], + ) -> Result { + if !valid_spec(spec.stops, spec.quotes, spec.pairs) + || !scopes.iter().enumerate().all(|(i, s)| { + spec.pairs.iter().any(|p| p.prefix == s.prefix) + && !scopes[..i].iter().any(|p| p.prefix == s.prefix) + && s.pairs + .iter() + .all(|n| spec.pairs.iter().any(|p| p.prefix == *n)) + }) + { + return Err(error(0, "invalid declared region specification or scopes")); + } + Ok(Self { spec, scopes }) + } + /// Borrow another full source with independent lazy boundary caches. + #[must_use] + pub fn source(self, source: &str) -> PreparedRegionSource<'_> { + PreparedRegionSource { + spec: self.spec, + source, + scopes: self.scopes, + ends: Ends { + enabled: true, + ..Ends::default() + }, + } + } +} +/// Borrowed full source with lazy caches. Construction never interprets source text. +pub struct PreparedRegionSource<'a> { + spec: &'static RegionSpec, + source: &'a str, + scopes: &'static [RegionScope], + ends: Ends, +} +#[derive(Default)] +struct Ends { + enabled: bool, + #[cfg(test)] + visits: usize, + quotes: HashMap, + pairs: HashMap<(&'static str, usize), usize>, +} +#[derive(Clone, Copy)] +enum Frame<'a> { + Quote(&'a RegionQuote, usize), + Pair(&'a RegionPair, usize, usize), +} +pub(super) fn valid_spec(stops: &[&str], quotes: &[RegionQuote], pairs: &[RegionPair]) -> bool { + !stops.is_empty() + && stops + .iter() + .enumerate() + .all(|(i, s)| !s.is_empty() && !stops[..i].contains(s)) + && pairs.iter().enumerate().all(|(i, p)| { + p.depth > 0 + && p.depth <= p.prefix.chars().count() + && p.prefix.ends_with(p.opening) + && !pairs[..i].iter().any(|q| q.prefix == p.prefix) + }) + && quotes.iter().enumerate().all(|(i, q)| { + !quotes[..i].iter().any(|r| r.delimiter == q.delimiter) + && q.pairs + .iter() + .enumerate() + .all(|(j, n)| !q.pairs[..j].contains(n) && pairs.iter().any(|p| p.prefix == *n)) + }) +} +impl<'a> PreparedRegionSource<'a> { + /// Prepare an admitted static plan and lazy source-local caches. + /// # Errors + /// Rejects malformed declarations and unknown or duplicate scope owners. + pub fn new( + spec: &'static RegionSpec, + source: &'a str, + scopes: &'static [RegionScope], + ) -> Result { + Ok(PreparedRegionPlan::new(spec, scopes)?.source(source)) + } + + fn entry(&self, at: usize) -> Result<(), Diagnostic> { + if at >= self.source.len() || !self.source.is_char_boundary(at) { + Err(error(at, "invalid declared region entry")) + } else { + Ok(()) + } + } + /// Recognize a word starting at an absolute UTF-8 byte offset. + /// # Errors + /// Rejects invalid offsets and unterminated nested regions. + pub fn word_end(&mut self, at: usize) -> Result, Diagnostic> { + self.entry(at)?; + self.scan(at, at, Vec::new(), true) + } + /// Return the exclusive byte end of a declared quote. + /// # Errors + /// Rejects invalid offsets, mismatched delimiters and unterminated regions. + pub fn quote_end(&mut self, at: usize, delimiter: char) -> Result { + self.entry(at)?; + let q = self + .spec + .quotes + .iter() + .find(|q| q.delimiter == delimiter) + .filter(|_| self.source[at..].starts_with(delimiter)) + .ok_or_else(|| error(at, "invalid declared quote entry"))?; + if let Some(end) = self.ends.quotes.get(&at) { + return Ok(*end); + } + self.scan( + at, + at + delimiter.len_utf8(), + vec![Frame::Quote(q, at)], + false, + )? + .ok_or_else(|| error(at, "missing declared quote end")) + } + /// Return the exclusive byte end of the longest declared opener. + /// # Errors + /// Rejects invalid offsets, missing openers and unterminated regions. + pub fn pair_end(&mut self, at: usize) -> Result { + self.entry(at)?; + let p = pair_at(self.source, at, self.spec.pairs, None) + .ok_or_else(|| error(at, "missing declared region opener"))?; + if let Some(end) = self.ends.pairs.get(&(p.prefix, at)) { + return Ok(*end); + } + self.scan( + at, + at + p.prefix.len(), + vec![Frame::Pair(p, p.depth, at)], + false, + )? + .ok_or_else(|| error(at, "missing declared region end")) + } + fn scan( + &mut self, + start: usize, + at: usize, + stack: Vec>, + word: bool, + ) -> Result, Diagnostic> { + scan( + self.source, + start, + at, + stack, + word, + self.spec.stops, + self.spec.quotes, + self.spec.pairs, + self.spec.consume_initial_stop, + self.scopes, + &mut self.ends, + ) + } +} +fn next(source: &str, at: usize) -> usize { + at + source[at..].chars().next().map_or(0, char::len_utf8) +} +fn escape(source: &str, at: usize) -> usize { + let after = next(source, at); + if after < source.len() { + next(source, after) + } else { + after + } +} +fn pair_at<'a>( + source: &str, + at: usize, + pairs: &'a [RegionPair], + allowed: Option<&[&str]>, +) -> Option<&'a RegionPair> { + pairs + .iter() + .filter(|p| { + allowed.is_none_or(|names| names.contains(&p.prefix)) + && source[at..].starts_with(p.prefix) + }) + .max_by_key(|p| p.prefix.len()) +} +fn close(frame: Frame<'_>, end: usize, ends: &mut Ends) { + if !ends.enabled { + return; + } + match frame { + Frame::Quote(_, start) => { + ends.quotes.insert(start, end); + } + Frame::Pair(p, _, start) => { + ends.pairs.insert((p.prefix, start), end); + } + } +} +// One explicit stack is shared by scanner words and lazy quote/pair decomposition. +#[allow(clippy::too_many_arguments)] +fn scan<'a>( + source: &str, + start: usize, + mut at: usize, + mut stack: Vec>, + word: bool, + stops: &[&str], + quotes: &'a [RegionQuote], + pairs: &'a [RegionPair], + consume_initial_stop: bool, + scopes: &[RegionScope], + ends: &mut Ends, +) -> Result, Diagnostic> { + while at < source.len() { + #[cfg(test)] + { + ends.visits += 1; + } + let ch = source[at..] + .chars() + .next() + .ok_or_else(|| error(at, "missing region character"))?; + let frame = stack.last().copied(); + let allowed = match frame { + Some(Frame::Quote(q, _)) => Some(q.pairs), + Some(Frame::Pair(p, _, _)) => scopes + .iter() + .find(|s| s.prefix == p.prefix) + .map(|s| s.pairs), + None => None, + }; + let nested = pair_at(source, at, pairs, allowed); + if let Some(frame) = frame { + let quote = match frame { + Frame::Quote(q, _) => Some(q), + Frame::Pair(..) => None, + }; + if ch == '\\' && quote.is_none_or(|q| q.escaped) { + at = escape(source, at); + continue; + } + if quote.is_some_and(|q| q.delimiter == ch) { + close(frame, next(source, at), ends); + stack.pop(); + } else if quote.is_none() && quotes.iter().any(|q| q.delimiter == ch) { + let q = quotes + .iter() + .find(|q| q.delimiter == ch) + .expect("declared quote"); + stack.push(Frame::Quote(q, at)); + } else if let Some(p) = nested { + if let Some(end) = ends.pairs.get(&(p.prefix, at)) { + at = *end; + continue; + } + stack.push(Frame::Pair(p, p.depth, at)); + at += p.prefix.len(); + continue; + } else if let Frame::Pair(p, depth, origin) = frame { + if ch == p.opening { + *stack.last_mut().expect("active frame") = Frame::Pair( + p, + depth + .checked_add(1) + .ok_or_else(|| error(at, "region depth overflow"))?, + origin, + ); + } else if ch == p.closing { + if depth == 1 { + close(frame, next(source, at), ends); + stack.pop(); + } else { + *stack.last_mut().expect("active frame") = + Frame::Pair(p, depth - 1, origin); + } + } + } + at = next(source, at); + if stack.is_empty() && !word { + return Ok(Some(at)); + } + } else { + if !word { + return Ok(Some(at)); + } + if let Some(p) = nested { + stack.push(Frame::Pair(p, p.depth, at)); + at += p.prefix.len(); + continue; + } + if (at > start || !consume_initial_stop) + && (crate::engine::is_scheme_whitespace(ch) + || stops.iter().any(|s| source[at..].starts_with(s))) + { + return Ok((at > start).then_some(at)); + } + if ch == '\\' { + at = escape(source, at); + continue; + } + if let Some(q) = quotes.iter().find(|q| q.delimiter == ch) { + stack.push(Frame::Quote(q, at)); + } + at = next(source, at); + } + } + if stack.is_empty() { + Ok((at > start).then_some(at)) + } else { + Err(error(start, "unterminated declared region")) + } +} +pub(super) fn scan_word( + source: &str, + at: usize, + stops: &[&str], + quotes: &[RegionQuote], + pairs: &[RegionPair], + consume_initial_stop: bool, +) -> Result, Diagnostic> { + scan( + source, + at, + at, + Vec::new(), + true, + stops, + quotes, + pairs, + consume_initial_stop, + &[], + &mut Ends::default(), + ) +} + +#[cfg(test)] +#[path = "../../tests/unit/region_boundaries.rs"] +mod tests; diff --git a/rust/gerbil-parser-runtime/src/scanner/session.rs b/rust/gerbil-parser-runtime/src/scanner/session.rs new file mode 100644 index 00000000..b95b5d16 --- /dev/null +++ b/rust/gerbil-parser-runtime/src/scanner/session.rs @@ -0,0 +1,158 @@ +//! Owned fixed-position scanning history; no caller-supplied checkpoints. +use super::obligations::Obligations; +use super::{ContextualScanner, Obligation, ScannerCheckpoint, ScannerPlan, ScannerSpec}; +use crate::{Diagnostic, ScannedToken}; +use std::sync::Arc; + +#[derive(Debug)] +struct RecordedState { + offset: usize, + mode: &'static str, + pending: Obligations, + active: Option>, + expecting: Option, +} +impl RecordedState { + fn capture(state: &ScannerCheckpoint<'_, '_>) -> Self { + Self { + offset: state.offset, + mode: state.mode, + pending: state.pending.clone(), + active: state.active.clone(), + expecting: state.expecting, + } + } + fn bind<'a, 's>(&self, owner: &'a ContextualScanner<'s>) -> ScannerCheckpoint<'a, 's> { + ScannerCheckpoint { + owner, + offset: self.offset, + mode: self.mode, + pending: self.pending.clone(), + active: self.active.clone(), + expecting: self.expecting, + } + } +} + +/// A complete scan owning its source and private token/context history. +/// Only the same immutable plan and fixed parser position can reuse its suffix. +/// Recognition and artifact publication still execute on the complete new source. +#[derive(Debug)] +pub struct ContextualScanSession { + spec: &'static ScannerSpec, + plan: Arc, + source: String, + position: String, + tokens: Vec, + states: Vec, + scanned: usize, + reused: usize, +} +impl ContextualScanSession { + /// Scan new owned input, optionally using an earlier successful scan. + /// A different plan or position causes a full scan. Failed scans publish no session. + /// # Errors + /// Invalid specifications, positions or lexical obligations. + pub fn new( + spec: &'static ScannerSpec, + source: String, + position: &str, + previous: Option<&Self>, + ) -> Result { + let scanner = ContextualScanner::new(spec, &source)?; + let previous = previous + .filter(|old| Arc::ptr_eq(&old.plan, &scanner.plan) && old.position == position); + let old_scanner = previous.map(|old| ContextualScanner { + spec: old.spec, + source: &old.source, + plan: old.plan.clone(), + }); + let suffix_start = previous.map(|old| common_suffix_start(&old.source, &source)); + let mut state = scanner.initial_state(); + let mut tokens = Vec::new(); + let mut states = vec![RecordedState::capture(&state)]; + let mut executed = 0; + let mut reused = 0; + loop { + if let (Some(old), Some(old_scanner), Some(start)) = + (previous, old_scanner.as_ref(), suffix_start) + && state.offset >= start + { + let old_offset = old.source.len() - (source.len() - state.offset); + if let Ok(index) = old.states.binary_search_by_key(&old_offset, |s| s.offset) + && state.same_context(&old.states[index].bind(old_scanner)) + { + // Common suffix bytes and complete contexts certify this recorded + // continuation. Rebind private history, never foreign checkpoints. + let relocate = |offset| source.len() - (old.source.len() - offset); + tokens.extend(old.tokens[index..].iter().map(|token| ScannedToken { + terminal: token.terminal, + start: relocate(token.start), + end: relocate(token.end), + })); + states.extend(old.states[index + 1..].iter().map(|record| { + let mut rebound = record.bind(&scanner); + rebound.offset = relocate(record.offset); + RecordedState::capture(&rebound) + })); + reused = old.tokens.len() - index; + break; + } + } + let (token, next) = scanner.step(&state, position)?; + state = next; + if let Some(token) = token { + tokens.push(token); + states.push(RecordedState::capture(&state)); + executed += 1; + } else { + break; + } + } + let plan = scanner.plan.clone(); + Ok(Self { + spec, + plan, + source, + position: position.to_owned(), + tokens, + states, + scanned: executed, + reused, + }) + } + #[must_use] + pub fn source(&self) -> &str { + &self.source + } + #[must_use] + pub fn tokens(&self) -> &[ScannedToken] { + &self.tokens + } + #[must_use] + pub fn scanned_token_count(&self) -> usize { + self.scanned + } + #[must_use] + pub fn reused_token_count(&self) -> usize { + self.reused + } + pub(crate) fn admits(&self, spec: &'static ScannerSpec, position: &str) -> bool { + std::ptr::eq(self.spec, spec) && self.position == position + } +} + +fn common_suffix_start(old: &str, new: &str) -> usize { + let count = old + .bytes() + .rev() + .zip(new.bytes().rev()) + .take_while(|(a, b)| a == b) + .count(); + let mut start = new.len() - count; + // An equal byte suffix can start within a UTF-8 scalar; use complete characters. + while !new.is_char_boundary(start) || !old.is_char_boundary(old.len() - (new.len() - start)) { + start += 1; + } + start +} diff --git a/rust/gerbil-parser-runtime/tests/fixtures/artifact_generated.rs b/rust/gerbil-parser-runtime/tests/fixtures/artifact_generated.rs new file mode 100644 index 00000000..a6760fcc --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/artifact_generated.rs @@ -0,0 +1,363 @@ +// @generated by the Scheme GPA1 producer; do not edit. +pub const SCOPES_DESCRIPTOR: &[u8] = &[ + 40, 111, 98, 106, 101, 99, 116, 32, 40, 34, 102, 105, 101, 108, 100, 115, 34, 32, 40, 108, 105, + 115, 116, 32, 34, 105, 116, 101, 109, 34, 32, 34, 109, 97, 114, 107, 101, 114, 34, 32, 34, 97, + 98, 115, 101, 110, 116, 34, 32, 34, 116, 101, 120, 116, 34, 41, 41, 32, 40, 34, 103, 114, 97, + 109, 109, 97, 114, 68, 105, 103, 101, 115, 116, 34, 32, 34, 115, 104, 97, 50, 53, 54, 58, 55, + 52, 53, 98, 102, 53, 50, 97, 52, 101, 98, 100, 48, 98, 100, 99, 52, 54, 53, 57, 53, 56, 50, + 100, 54, 98, 51, 99, 100, 57, 55, 57, 97, 51, 102, 56, 98, 52, 48, 55, 102, 56, 57, 54, 100, + 48, 54, 54, 56, 48, 100, 48, 57, 50, 50, 54, 97, 54, 100, 98, 52, 49, 100, 99, 34, 41, 32, 40, + 34, 108, 97, 110, 103, 117, 97, 103, 101, 34, 32, 34, 102, 105, 101, 108, 100, 45, 115, 99, + 111, 112, 101, 115, 34, 41, 32, 40, 34, 114, 111, 111, 116, 75, 105, 110, 100, 34, 32, 34, 68, + 111, 99, 117, 109, 101, 110, 116, 34, 41, 32, 40, 34, 115, 99, 104, 101, 109, 97, 34, 32, 34, + 103, 101, 114, 98, 105, 108, 45, 112, 97, 114, 115, 101, 114, 46, 110, 97, 116, 105, 118, 101, + 45, 100, 101, 115, 99, 114, 105, 112, 116, 111, 114, 46, 118, 49, 34, 41, 32, 40, 34, 115, 121, + 110, 116, 97, 120, 75, 105, 110, 100, 115, 34, 32, 40, 108, 105, 115, 116, 32, 40, 108, 105, + 115, 116, 32, 34, 68, 111, 99, 117, 109, 101, 110, 116, 34, 32, 34, 110, 111, 100, 101, 34, 32, + 40, 108, 105, 115, 116, 32, 34, 105, 116, 101, 109, 34, 32, 34, 109, 97, 114, 107, 101, 114, + 34, 32, 34, 97, 98, 115, 101, 110, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 77, 97, + 114, 107, 101, 114, 34, 32, 34, 110, 111, 100, 101, 34, 32, 40, 108, 105, 115, 116, 41, 41, 32, + 40, 108, 105, 115, 116, 32, 34, 73, 100, 101, 110, 116, 105, 102, 105, 101, 114, 34, 32, 34, + 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, 116, 101, 120, 116, 34, 41, + 41, 32, 40, 108, 105, 115, 116, 32, 34, 85, 110, 107, 110, 111, 119, 110, 34, 32, 34, 116, 111, + 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, 116, 101, 120, 116, 34, 41, 41, 41, 41, + 32, 40, 34, 116, 101, 114, 109, 105, 110, 97, 108, 115, 34, 32, 40, 108, 105, 115, 116, 32, 40, + 108, 105, 115, 116, 32, 34, 105, 116, 101, 109, 45, 116, 111, 107, 101, 110, 34, 32, 34, 73, + 100, 101, 110, 116, 105, 102, 105, 101, 114, 34, 41, 32, 40, 108, 105, 115, 116, 32, 34, 117, + 110, 107, 110, 111, 119, 110, 34, 32, 34, 85, 110, 107, 110, 111, 119, 110, 34, 41, 41, 41, 41, +]; +pub static SCOPES_CASES: &[(&[u8], &[u8])] = &[ + ( + &[], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 116, 91, 245, 42, 78, 189, 11, 220, + 70, 89, 88, 45, 107, 60, 217, 121, 163, 248, 180, 7, 248, 150, 208, 102, 128, 208, 146, + 38, 166, 219, 65, 220, 227, 176, 196, 66, 152, 252, 28, 20, 154, 251, 244, 200, 153, + 111, 185, 36, 39, 174, 65, 228, 100, 155, 147, 76, 164, 149, 153, 27, 120, 82, 184, 85, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + ), + ( + &[120, 120], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 116, 91, 245, 42, 78, 189, 11, + 220, 70, 89, 88, 45, 107, 60, 217, 121, 163, 248, 180, 7, 248, 150, 208, 102, 128, 208, + 146, 38, 166, 219, 65, 220, 93, 222, 137, 104, 135, 246, 117, 76, 155, 21, 191, 227, + 164, 65, 174, 72, 6, 223, 47, 222, 148, 0, 19, 17, 224, 139, 241, 16, 98, 46, 11, 190, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, + 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, + ], + ), +]; +pub const DESCRIPTOR: &[u8] = &[ + 40, 111, 98, 106, 101, 99, 116, 32, 40, 34, 102, 105, 101, 108, 100, 115, 34, 32, 40, 108, 105, + 115, 116, 32, 34, 110, 97, 109, 101, 34, 32, 34, 118, 97, 108, 117, 101, 34, 32, 34, 116, 101, + 120, 116, 34, 41, 41, 32, 40, 34, 103, 114, 97, 109, 109, 97, 114, 68, 105, 103, 101, 115, 116, + 34, 32, 34, 115, 104, 97, 50, 53, 54, 58, 57, 99, 55, 101, 56, 98, 48, 102, 56, 48, 55, 102, + 98, 49, 98, 102, 49, 51, 49, 57, 56, 99, 52, 49, 55, 99, 101, 53, 98, 55, 52, 98, 57, 48, 102, + 54, 53, 100, 54, 55, 102, 48, 53, 52, 98, 52, 56, 49, 56, 56, 99, 102, 53, 50, 54, 56, 100, 98, + 97, 52, 100, 49, 100, 52, 34, 41, 32, 40, 34, 108, 97, 110, 103, 117, 97, 103, 101, 34, 32, 34, + 114, 101, 99, 111, 114, 100, 45, 97, 115, 115, 105, 103, 110, 109, 101, 110, 116, 115, 34, 41, + 32, 40, 34, 114, 111, 111, 116, 75, 105, 110, 100, 34, 32, 34, 68, 111, 99, 117, 109, 101, 110, + 116, 34, 41, 32, 40, 34, 115, 99, 104, 101, 109, 97, 34, 32, 34, 103, 101, 114, 98, 105, 108, + 45, 112, 97, 114, 115, 101, 114, 46, 110, 97, 116, 105, 118, 101, 45, 100, 101, 115, 99, 114, + 105, 112, 116, 111, 114, 46, 118, 49, 34, 41, 32, 40, 34, 115, 121, 110, 116, 97, 120, 75, 105, + 110, 100, 115, 34, 32, 40, 108, 105, 115, 116, 32, 40, 108, 105, 115, 116, 32, 34, 68, 111, 99, + 117, 109, 101, 110, 116, 34, 32, 34, 110, 111, 100, 101, 34, 32, 40, 108, 105, 115, 116, 41, + 41, 32, 40, 108, 105, 115, 116, 32, 34, 65, 115, 115, 105, 103, 110, 109, 101, 110, 116, 34, + 32, 34, 110, 111, 100, 101, 34, 32, 40, 108, 105, 115, 116, 32, 34, 110, 97, 109, 101, 34, 32, + 34, 118, 97, 108, 117, 101, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 87, 104, 105, 116, + 101, 115, 112, 97, 99, 101, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, + 116, 32, 34, 116, 101, 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 78, 101, 119, + 108, 105, 110, 101, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, + 34, 116, 101, 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 78, 117, 109, 98, 101, + 114, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, 116, 101, + 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 73, 100, 101, 110, 116, 105, 102, + 105, 101, 114, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, + 116, 101, 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 83, 116, 114, 105, 110, + 103, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, 116, 101, + 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 80, 117, 110, 99, 116, 117, 97, 116, + 105, 111, 110, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, + 116, 101, 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 85, 110, 107, 110, 111, + 119, 110, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, 116, + 101, 120, 116, 34, 41, 41, 41, 41, 32, 40, 34, 116, 101, 114, 109, 105, 110, 97, 108, 115, 34, + 32, 40, 108, 105, 115, 116, 32, 40, 108, 105, 115, 116, 32, 34, 119, 104, 105, 116, 101, 115, + 112, 97, 99, 101, 34, 32, 34, 87, 104, 105, 116, 101, 115, 112, 97, 99, 101, 34, 41, 32, 40, + 108, 105, 115, 116, 32, 34, 110, 101, 119, 108, 105, 110, 101, 34, 32, 34, 78, 101, 119, 108, + 105, 110, 101, 34, 41, 32, 40, 108, 105, 115, 116, 32, 34, 110, 117, 109, 98, 101, 114, 34, 32, + 34, 78, 117, 109, 98, 101, 114, 34, 41, 32, 40, 108, 105, 115, 116, 32, 34, 105, 100, 101, 110, + 116, 105, 102, 105, 101, 114, 34, 32, 34, 73, 100, 101, 110, 116, 105, 102, 105, 101, 114, 34, + 41, 32, 40, 108, 105, 115, 116, 32, 34, 115, 116, 114, 105, 110, 103, 34, 32, 34, 83, 116, 114, + 105, 110, 103, 34, 41, 32, 40, 108, 105, 115, 116, 32, 34, 112, 117, 110, 99, 116, 117, 97, + 116, 105, 111, 110, 34, 32, 34, 80, 117, 110, 99, 116, 117, 97, 116, 105, 111, 110, 34, 41, 32, + 40, 108, 105, 115, 116, 32, 34, 117, 110, 107, 110, 111, 119, 110, 34, 32, 34, 85, 110, 107, + 110, 111, 119, 110, 34, 41, 41, 41, 41, +]; +pub const CONTEXTUAL_DESCRIPTOR: &[u8] = &[ + 40, 111, 98, 106, 101, 99, 116, 32, 40, 34, 102, 105, 101, 108, 100, 115, 34, 32, 40, 108, 105, + 115, 116, 32, 34, 110, 97, 109, 101, 34, 32, 34, 118, 97, 108, 117, 101, 34, 32, 34, 116, 101, + 120, 116, 34, 41, 41, 32, 40, 34, 103, 114, 97, 109, 109, 97, 114, 68, 105, 103, 101, 115, 116, + 34, 32, 34, 115, 104, 97, 50, 53, 54, 58, 102, 54, 99, 49, 48, 48, 99, 48, 98, 57, 97, 54, 55, + 57, 51, 51, 100, 99, 101, 56, 50, 48, 51, 48, 48, 48, 100, 57, 55, 53, 57, 53, 52, 53, 98, 102, + 48, 52, 57, 53, 56, 56, 101, 50, 56, 52, 57, 51, 101, 99, 101, 53, 54, 54, 54, 54, 54, 48, 98, + 102, 53, 49, 53, 57, 34, 41, 32, 40, 34, 108, 97, 110, 103, 117, 97, 103, 101, 34, 32, 34, 114, + 101, 99, 111, 114, 100, 45, 97, 115, 115, 105, 103, 110, 109, 101, 110, 116, 115, 34, 41, 32, + 40, 34, 114, 111, 111, 116, 75, 105, 110, 100, 34, 32, 34, 68, 111, 99, 117, 109, 101, 110, + 116, 34, 41, 32, 40, 34, 115, 99, 104, 101, 109, 97, 34, 32, 34, 103, 101, 114, 98, 105, 108, + 45, 112, 97, 114, 115, 101, 114, 46, 110, 97, 116, 105, 118, 101, 45, 100, 101, 115, 99, 114, + 105, 112, 116, 111, 114, 46, 118, 49, 34, 41, 32, 40, 34, 115, 121, 110, 116, 97, 120, 75, 105, + 110, 100, 115, 34, 32, 40, 108, 105, 115, 116, 32, 40, 108, 105, 115, 116, 32, 34, 68, 111, 99, + 117, 109, 101, 110, 116, 34, 32, 34, 110, 111, 100, 101, 34, 32, 40, 108, 105, 115, 116, 41, + 41, 32, 40, 108, 105, 115, 116, 32, 34, 65, 115, 115, 105, 103, 110, 109, 101, 110, 116, 34, + 32, 34, 110, 111, 100, 101, 34, 32, 40, 108, 105, 115, 116, 32, 34, 110, 97, 109, 101, 34, 32, + 34, 118, 97, 108, 117, 101, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 87, 104, 105, 116, + 101, 115, 112, 97, 99, 101, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, + 116, 32, 34, 116, 101, 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 78, 101, 119, + 108, 105, 110, 101, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, + 34, 116, 101, 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 78, 117, 109, 98, 101, + 114, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, 116, 101, + 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 73, 100, 101, 110, 116, 105, 102, + 105, 101, 114, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, + 116, 101, 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 83, 116, 114, 105, 110, + 103, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, 116, 101, + 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 80, 117, 110, 99, 116, 117, 97, 116, + 105, 111, 110, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, + 116, 101, 120, 116, 34, 41, 41, 32, 40, 108, 105, 115, 116, 32, 34, 85, 110, 107, 110, 111, + 119, 110, 34, 32, 34, 116, 111, 107, 101, 110, 34, 32, 40, 108, 105, 115, 116, 32, 34, 116, + 101, 120, 116, 34, 41, 41, 41, 41, 32, 40, 34, 116, 101, 114, 109, 105, 110, 97, 108, 115, 34, + 32, 40, 108, 105, 115, 116, 32, 40, 108, 105, 115, 116, 32, 34, 119, 104, 105, 116, 101, 115, + 112, 97, 99, 101, 34, 32, 34, 87, 104, 105, 116, 101, 115, 112, 97, 99, 101, 34, 41, 32, 40, + 108, 105, 115, 116, 32, 34, 110, 101, 119, 108, 105, 110, 101, 34, 32, 34, 78, 101, 119, 108, + 105, 110, 101, 34, 41, 32, 40, 108, 105, 115, 116, 32, 34, 110, 117, 109, 98, 101, 114, 34, 32, + 34, 78, 117, 109, 98, 101, 114, 34, 41, 32, 40, 108, 105, 115, 116, 32, 34, 105, 100, 101, 110, + 116, 105, 102, 105, 101, 114, 34, 32, 34, 73, 100, 101, 110, 116, 105, 102, 105, 101, 114, 34, + 41, 32, 40, 108, 105, 115, 116, 32, 34, 115, 116, 114, 105, 110, 103, 34, 32, 34, 83, 116, 114, + 105, 110, 103, 34, 41, 32, 40, 108, 105, 115, 116, 32, 34, 112, 117, 110, 99, 116, 117, 97, + 116, 105, 111, 110, 34, 32, 34, 80, 117, 110, 99, 116, 117, 97, 116, 105, 111, 110, 34, 41, 32, + 40, 108, 105, 115, 116, 32, 34, 117, 110, 107, 110, 111, 119, 110, 34, 32, 34, 85, 110, 107, + 110, 111, 119, 110, 34, 41, 41, 41, 41, +]; +pub const FIELD_COUNT: usize = 3; +pub static CASES: &[(&[u8], &[u8])] = &[ + ( + &[], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 156, 126, 139, 15, 128, 127, 177, + 191, 19, 25, 140, 65, 124, 229, 183, 75, 144, 246, 93, 103, 240, 84, 180, 129, 136, + 207, 82, 104, 219, 164, 209, 212, 227, 176, 196, 66, 152, 252, 28, 20, 154, 251, 244, + 200, 153, 111, 185, 36, 39, 174, 65, 228, 100, 155, 147, 76, 164, 149, 153, 27, 120, + 82, 184, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + ), + ( + &[206, 177, 61, 49, 10], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 156, 126, 139, 15, 128, 127, 177, + 191, 19, 25, 140, 65, 124, 229, 183, 75, 144, 246, 93, 103, 240, 84, 180, 129, 136, + 207, 82, 104, 219, 164, 209, 212, 168, 53, 105, 126, 80, 116, 251, 193, 35, 236, 192, + 68, 136, 151, 2, 78, 181, 83, 212, 168, 47, 88, 176, 64, 172, 226, 5, 146, 39, 165, + 165, 217, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, + 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, + 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, + 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, + 0, + ], + ), + ( + &[97, 32, 61, 32, 49, 13, 10, 98, 61, 34, 206, 178, 34, 10], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 156, 126, 139, 15, 128, 127, 177, + 191, 19, 25, 140, 65, 124, 229, 183, 75, 144, 246, 93, 103, 240, 84, 180, 129, 136, + 207, 82, 104, 219, 164, 209, 212, 120, 69, 61, 254, 139, 205, 52, 101, 89, 255, 140, + 213, 248, 52, 217, 219, 45, 127, 247, 21, 79, 14, 118, 23, 252, 105, 217, 163, 229, + 160, 83, 227, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 2, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 5, + 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, + 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 9, 0, + 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, + 0, 4, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 13, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, + 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 14, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, + 0, 0, 0, + ], + ), + ( + &[110, 97, 109, 101, 61, 118, 97, 108, 117, 101], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 156, 126, 139, 15, 128, 127, 177, + 191, 19, 25, 140, 65, 124, 229, 183, 75, 144, 246, 93, 103, 240, 84, 180, 129, 136, + 207, 82, 104, 219, 164, 209, 212, 90, 191, 17, 14, 149, 143, 180, 168, 241, 33, 252, + 162, 208, 207, 185, 182, 180, 220, 236, 68, 201, 4, 146, 14, 25, 229, 20, 144, 244, 34, + 164, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 0, + 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, + 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 10, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10, 0, 0, 0, + ], + ), + ( + &[97, 61], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 156, 126, 139, 15, 128, 127, 177, + 191, 19, 25, 140, 65, 124, 229, 183, 75, 144, 246, 93, 103, 240, 84, 180, 129, 136, + 207, 82, 104, 219, 164, 209, 212, 150, 163, 108, 234, 108, 94, 84, 36, 44, 49, 157, 11, + 53, 63, 118, 222, 203, 3, 186, 147, 88, 182, 204, 223, 145, 222, 163, 125, 130, 131, + 87, 167, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, + 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, + ], + ), + ( + &[97, 61, 49, 0, 33], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 156, 126, 139, 15, 128, 127, 177, + 191, 19, 25, 140, 65, 124, 229, 183, 75, 144, 246, 93, 103, 240, 84, 180, 129, 136, + 207, 82, 104, 219, 164, 209, 212, 56, 85, 17, 104, 145, 142, 99, 210, 105, 215, 76, + 107, 11, 188, 118, 248, 141, 24, 82, 7, 231, 244, 23, 232, 134, 95, 223, 79, 255, 37, + 239, 180, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, + 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, + 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, + 0, 0, 0, 5, 0, 0, 0, + ], + ), +]; +pub static CONTEXTUAL_CASES: &[(&[u8], &[u8])] = &[ + ( + &[], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 246, 193, 0, 192, 185, 166, 121, + 51, 220, 232, 32, 48, 0, 217, 117, 149, 69, 191, 4, 149, 136, 226, 132, 147, 236, 229, + 102, 102, 96, 191, 81, 89, 227, 176, 196, 66, 152, 252, 28, 20, 154, 251, 244, 200, + 153, 111, 185, 36, 39, 174, 65, 228, 100, 155, 147, 76, 164, 149, 153, 27, 120, 82, + 184, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + ), + ( + &[206, 177, 61, 49, 10], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 246, 193, 0, 192, 185, 166, 121, + 51, 220, 232, 32, 48, 0, 217, 117, 149, 69, 191, 4, 149, 136, 226, 132, 147, 236, 229, + 102, 102, 96, 191, 81, 89, 168, 53, 105, 126, 80, 116, 251, 193, 35, 236, 192, 68, 136, + 151, 2, 78, 181, 83, 212, 168, 47, 88, 176, 64, 172, 226, 5, 146, 39, 165, 165, 217, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, + 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, + 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, + 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, + 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, + ], + ), + ( + &[97, 32, 61, 32, 49, 13, 10, 98, 61, 34, 206, 178, 34, 10], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 246, 193, 0, 192, 185, 166, 121, + 51, 220, 232, 32, 48, 0, 217, 117, 149, 69, 191, 4, 149, 136, 226, 132, 147, 236, 229, + 102, 102, 96, 191, 81, 89, 120, 69, 61, 254, 139, 205, 52, 101, 89, 255, 140, 213, 248, + 52, 217, 219, 45, 127, 247, 21, 79, 14, 118, 23, 252, 105, 217, 163, 229, 160, 83, 227, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 5, + 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 0, 0, + 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 5, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, + 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0, 2, 0, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, + 0, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7, + 0, 0, 0, 8, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, + 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 3, 0, 0, + 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 13, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 13, + 0, 0, 0, 14, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, + 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, + ], + ), + ( + &[110, 97, 109, 101, 61, 118, 97, 108, 117, 101], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 246, 193, 0, 192, 185, 166, 121, + 51, 220, 232, 32, 48, 0, 217, 117, 149, 69, 191, 4, 149, 136, 226, 132, 147, 236, 229, + 102, 102, 96, 191, 81, 89, 90, 191, 17, 14, 149, 143, 180, 168, 241, 33, 252, 162, 208, + 207, 185, 182, 180, 220, 236, 68, 201, 4, 146, 14, 25, 229, 20, 144, 244, 34, 164, 37, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 5, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 3, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, + 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 10, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, + ], + ), + ( + &[97, 61], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 246, 193, 0, 192, 185, 166, 121, + 51, 220, 232, 32, 48, 0, 217, 117, 149, 69, 191, 4, 149, 136, 226, 132, 147, 236, 229, + 102, 102, 96, 191, 81, 89, 150, 163, 108, 234, 108, 94, 84, 36, 44, 49, 157, 11, 53, + 63, 118, 222, 203, 3, 186, 147, 88, 182, 204, 223, 145, 222, 163, 125, 130, 131, 87, + 167, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, + 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, + ], + ), + ( + &[97, 61, 49, 0, 33], + &[ + 71, 80, 65, 49, 1, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 246, 193, 0, 192, 185, 166, 121, + 51, 220, 232, 32, 48, 0, 217, 117, 149, 69, 191, 4, 149, 136, 226, 132, 147, 236, 229, + 102, 102, 96, 191, 81, 89, 56, 85, 17, 104, 145, 142, 99, 210, 105, 215, 76, 107, 11, + 188, 118, 248, 141, 24, 82, 7, 231, 244, 23, 232, 134, 95, 223, 79, 255, 37, 239, 180, + 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 5, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, + 0, 3, 0, 0, 0, 5, 0, 0, 0, + ], + ), +]; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/bash_scanner.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/bash_scanner.rs new file mode 100644 index 00000000..97c6acfb --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/bash_scanner.rs @@ -0,0 +1,1620 @@ +// @generated from Scheme contextual Scanner IR; do not edit. +#[allow(unused_imports)] +use gerbil_parser_runtime::scanner::{ + BalancedPair, MarkerPolicy, ScannerAction, ScannerCell, ScannerMatcher, ScannerRule, + ScannerSpec, +}; +pub static SCANNER: ScannerSpec = ScannerSpec { + opcode_contract: "gerbil-parser.contextual-scanner-opcodes.v1", + digest: "sha256:350c02574541fb278cf653584eb79c1eb2b5f326ad06dc92a43f8daffbb099a3", + base_grammar_digest: None, + initial_mode: "command", + modes: &["command", "marker", "body"], + positions: &["source"], + rules: &[ + ScannerRule { + name: "space", + mode: "command", + form: "horizontal-whitespace", + matcher: ScannerMatcher::HorizontalWhitespace, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "newline", + mode: "command", + form: "newline", + matcher: ScannerMatcher::Literal("\n"), + rank: 0, + action: ScannerAction::ActivateNext("body"), + }, + ScannerRule { + name: "continuation", + mode: "command", + form: "line-continuation", + matcher: ScannerMatcher::Literal("\\\n"), + rank: 10, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "comment", + mode: "command", + form: "comment", + matcher: ScannerMatcher::LinePrefix { + prefix: "#", + separator: '\u{a}', + }, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "strip", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literal("<<-"), + rank: 30, + action: ScannerAction::ExpectMarkerIn { + strip_tabs: true, + mode: "marker", + }, + }, + ScannerRule { + name: "plain", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literal("<<"), + rank: 30, + action: ScannerAction::ExpectMarkerIn { + strip_tabs: false, + mode: "marker", + }, + }, + ScannerRule { + name: "operator", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literals(&[ + ";;&", "&>>", "<<<", "&&", "||", "|&", ";;", ";&", ">>", "<>", "<&", ">&", ">|", + "&>", ";", "&", "|", "(", ")", "<", ">", + ]), + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "word", + mode: "command", + form: "word", + matcher: ScannerMatcher::UnlessPrefix { + prefixes: &[ + " ", "\t", "\n", "#", "\\\n", ";", "&", "|", "(", ")", "<", ">", + ], + exceptions: &["<(", ">("], + child: &ScannerMatcher::RegionWord { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }, + }, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-space", + mode: "marker", + form: "horizontal-whitespace", + matcher: ScannerMatcher::HorizontalWhitespace, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-continuation", + mode: "marker", + form: "line-continuation", + matcher: ScannerMatcher::Literal("\\\n"), + rank: 10, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-newline", + mode: "marker", + form: "newline", + matcher: ScannerMatcher::Literal("\n"), + rank: 10, + action: ScannerAction::ActivateNext("body"), + }, + ScannerRule { + name: "marker-word", + mode: "marker", + form: "heredoc-marker", + matcher: ScannerMatcher::UnlessPrefix { + prefixes: &[" ", "\t", "\n", "\\\n"], + exceptions: &[], + child: &ScannerMatcher::RegionWord { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }, + }, + rank: 0, + action: ScannerAction::EnqueueMarkerIn { + policy: MarkerPolicy::ShellQuoteRemoval, + mode: "command", + }, + }, + ScannerRule { + name: "end", + mode: "body", + form: "heredoc-end", + matcher: ScannerMatcher::MarkerLineAt('\u{a}'), + rank: 10, + action: ScannerAction::FinishMarker { + base: "command", + body: "body", + }, + }, + ScannerRule { + name: "body", + mode: "body", + form: "heredoc-content", + matcher: ScannerMatcher::BodyLineAt('\u{a}'), + rank: 0, + action: ScannerAction::Keep, + }, + ], + cells: &[ + ScannerCell { + mode: "command", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "command", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "command", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "command", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "command", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "command", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ScannerCell { + mode: "body", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "body", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "body", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "body", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "body", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "body", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ], +}; +pub static TRACES: &[(&str, &[gerbil_parser_runtime::ScannedToken])] = &[ + ("", &[]), + ( + "echo α", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "horizontal-whitespace", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 5, + end: 7, + }, + ], + ), + ( + "# unterminated '\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "comment", + start: 0, + end: 16, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 16, + end: 17, + }, + ], + ), + ( + "# $(unterminated\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "comment", + start: 0, + end: 16, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 16, + end: 17, + }, + ], + ), + ( + "echo \\\nα", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "horizontal-whitespace", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "line-continuation", + start: 5, + end: 7, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 7, + end: 9, + }, + ], + ), + ( + "echo <(printf α) >(cat)", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "horizontal-whitespace", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 5, + end: 17, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "horizontal-whitespace", + start: 17, + end: 18, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 18, + end: 24, + }, + ], + ), + ( + "echo \rα", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "horizontal-whitespace", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 5, + end: 8, + }, + ], + ), + ( + "echo \r\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "horizontal-whitespace", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 5, + end: 6, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 6, + end: 7, + }, + ], + ), + ( + "\rfoo", + &[gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }], + ), + ( + "…α", + &[gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }], + ), + ( + "cat <>y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 4, + end: 5, + }, + ], + ), + ( + "x<<>y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 3, + end: 4, + }, + ], + ), + ( + "x<>y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 3, + end: 4, + }, + ], + ), + ( + "x<&y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 3, + end: 4, + }, + ], + ), + ( + "x>&y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 3, + end: 4, + }, + ], + ), + ( + "x>|y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 3, + end: 4, + }, + ], + ), + ( + "x&>y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 3, + end: 4, + }, + ], + ), + ( + "x;y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 2, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 2, + end: 3, + }, + ], + ), + ( + "x&y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 2, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 2, + end: 3, + }, + ], + ), + ( + "x|y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 2, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 2, + end: 3, + }, + ], + ), + ( + "x(y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 2, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 2, + end: 3, + }, + ], + ), + ( + "x)y", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 2, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 2, + end: 3, + }, + ], + ), + ( + "xy", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 1, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "operator", + start: 1, + end: 2, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 2, + end: 3, + }, + ], + ), +]; +pub static REJECTED: &[&str] = &["cat <)] = &[ + (0, "前::tail", 4, 4, None), + (0, "前:α_123:tail", 4, 4, None), + (0, "前:α_123:tail", 4, 6, Some(6)), + (0, "前:α_123:tail", 4, 7, Some(7)), + (0, "前:α_123:tail", 4, 8, Some(8)), + (0, "前:α_123:tail", 4, 9, Some(9)), + (0, "前:α_123:tail", 4, 10, Some(10)), + (0, "前:123x:tail", 4, 4, None), + (0, "前:123x:tail", 4, 5, Some(5)), + (0, "前:123x:tail", 4, 6, Some(5)), + (0, "前:123x:tail", 4, 7, Some(5)), + (0, "前:123x:tail", 4, 8, Some(5)), + (0, "前:@rest:tail", 4, 4, None), + (0, "前:@rest:tail", 4, 5, Some(5)), + (0, "前:@rest:tail", 4, 6, Some(5)), + (0, "前:@rest:tail", 4, 7, Some(5)), + (0, "前:@rest:tail", 4, 8, Some(5)), + (0, "前:@rest:tail", 4, 9, Some(5)), + (0, "前:_rest:tail", 4, 4, None), + (0, "前:_rest:tail", 4, 5, Some(5)), + (0, "前:_rest:tail", 4, 6, Some(5)), + (0, "前:_rest:tail", 4, 7, Some(5)), + (0, "前:_rest:tail", 4, 8, Some(5)), + (0, "前:_rest:tail", 4, 9, Some(5)), + (0, "前:α-β:tail", 4, 4, None), + (0, "前:α-β:tail", 4, 6, Some(6)), + (0, "前:α-β:tail", 4, 7, Some(6)), + (0, "前:α-β:tail", 4, 9, Some(6)), + (0, "前:λ中😀:tail", 4, 4, None), + (0, "前:λ中😀:tail", 4, 6, Some(6)), + (0, "前:λ中😀:tail", 4, 9, Some(9)), + (0, "前:λ中😀:tail", 4, 13, Some(9)), + (0, "前:1α:tail", 4, 4, None), + (0, "前:1α:tail", 4, 5, Some(5)), + (0, "前:1α:tail", 4, 7, Some(5)), + (0, "前:!α:tail", 4, 4, None), + (0, "前:!α:tail", 4, 5, Some(5)), + (0, "前:!α:tail", 4, 7, Some(5)), + (0, "前:?α:tail", 4, 4, None), + (0, "前:?α:tail", 4, 5, Some(5)), + (0, "前:?α:tail", 4, 7, Some(5)), + (0, "前:aβγ:tail", 4, 4, None), + (0, "前:aβγ:tail", 4, 5, Some(5)), + (0, "前:aβγ:tail", 4, 7, Some(7)), + (0, "前:aβγ:tail", 4, 9, Some(9)), + (1, "前::tail", 4, 4, None), + (1, "前:α_123:tail", 4, 4, None), + (1, "前:α_123:tail", 4, 6, Some(6)), + (1, "前:α_123:tail", 4, 7, Some(7)), + (1, "前:α_123:tail", 4, 8, Some(8)), + (1, "前:α_123:tail", 4, 9, Some(9)), + (1, "前:α_123:tail", 4, 10, Some(10)), + (1, "前:123x:tail", 4, 4, None), + (1, "前:123x:tail", 4, 5, Some(5)), + (1, "前:123x:tail", 4, 6, Some(6)), + (1, "前:123x:tail", 4, 7, Some(7)), + (1, "前:123x:tail", 4, 8, Some(7)), + (1, "前:@rest:tail", 4, 4, None), + (1, "前:@rest:tail", 4, 5, Some(5)), + (1, "前:@rest:tail", 4, 6, Some(5)), + (1, "前:@rest:tail", 4, 7, Some(5)), + (1, "前:@rest:tail", 4, 8, Some(5)), + (1, "前:@rest:tail", 4, 9, Some(5)), + (1, "前:_rest:tail", 4, 4, None), + (1, "前:_rest:tail", 4, 5, Some(5)), + (1, "前:_rest:tail", 4, 6, Some(6)), + (1, "前:_rest:tail", 4, 7, Some(7)), + (1, "前:_rest:tail", 4, 8, Some(8)), + (1, "前:_rest:tail", 4, 9, Some(9)), + (1, "前:α-β:tail", 4, 4, None), + (1, "前:α-β:tail", 4, 6, Some(6)), + (1, "前:α-β:tail", 4, 7, Some(6)), + (1, "前:α-β:tail", 4, 9, Some(6)), + (1, "前:λ中😀:tail", 4, 4, None), + (1, "前:λ中😀:tail", 4, 6, Some(6)), + (1, "前:λ中😀:tail", 4, 9, Some(9)), + (1, "前:λ中😀:tail", 4, 13, Some(9)), + (1, "前:1α:tail", 4, 4, None), + (1, "前:1α:tail", 4, 5, Some(5)), + (1, "前:1α:tail", 4, 7, Some(5)), + (1, "前:!α:tail", 4, 4, None), + (1, "前:!α:tail", 4, 5, Some(5)), + (1, "前:!α:tail", 4, 7, Some(5)), + (1, "前:?α:tail", 4, 4, None), + (1, "前:?α:tail", 4, 5, Some(5)), + (1, "前:?α:tail", 4, 7, Some(5)), + (1, "前:aβγ:tail", 4, 4, None), + (1, "前:aβγ:tail", 4, 5, Some(5)), + (1, "前:aβγ:tail", 4, 7, Some(7)), + (1, "前:aβγ:tail", 4, 9, Some(9)), + (2, "前::tail", 4, 4, None), + (2, "前:α_123:tail", 4, 4, None), + (2, "前:α_123:tail", 4, 6, Some(6)), + (2, "前:α_123:tail", 4, 7, Some(7)), + (2, "前:α_123:tail", 4, 8, Some(8)), + (2, "前:α_123:tail", 4, 9, Some(9)), + (2, "前:α_123:tail", 4, 10, Some(10)), + (2, "前:123x:tail", 4, 4, None), + (2, "前:123x:tail", 4, 5, None), + (2, "前:123x:tail", 4, 6, None), + (2, "前:123x:tail", 4, 7, None), + (2, "前:123x:tail", 4, 8, None), + (2, "前:@rest:tail", 4, 4, None), + (2, "前:@rest:tail", 4, 5, None), + (2, "前:@rest:tail", 4, 6, None), + (2, "前:@rest:tail", 4, 7, None), + (2, "前:@rest:tail", 4, 8, None), + (2, "前:@rest:tail", 4, 9, None), + (2, "前:_rest:tail", 4, 4, None), + (2, "前:_rest:tail", 4, 5, Some(5)), + (2, "前:_rest:tail", 4, 6, Some(6)), + (2, "前:_rest:tail", 4, 7, Some(7)), + (2, "前:_rest:tail", 4, 8, Some(8)), + (2, "前:_rest:tail", 4, 9, Some(9)), + (2, "前:α-β:tail", 4, 4, None), + (2, "前:α-β:tail", 4, 6, Some(6)), + (2, "前:α-β:tail", 4, 7, Some(6)), + (2, "前:α-β:tail", 4, 9, Some(6)), + (2, "前:λ中😀:tail", 4, 4, None), + (2, "前:λ中😀:tail", 4, 6, Some(6)), + (2, "前:λ中😀:tail", 4, 9, Some(9)), + (2, "前:λ中😀:tail", 4, 13, Some(9)), + (2, "前:1α:tail", 4, 4, None), + (2, "前:1α:tail", 4, 5, None), + (2, "前:1α:tail", 4, 7, None), + (2, "前:!α:tail", 4, 4, None), + (2, "前:!α:tail", 4, 5, None), + (2, "前:!α:tail", 4, 7, None), + (2, "前:?α:tail", 4, 4, None), + (2, "前:?α:tail", 4, 5, None), + (2, "前:?α:tail", 4, 7, None), + (2, "前:aβγ:tail", 4, 4, None), + (2, "前:aβγ:tail", 4, 5, Some(5)), + (2, "前:aβγ:tail", 4, 7, Some(7)), + (2, "前:aβγ:tail", 4, 9, Some(9)), +]; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/command_source.bin b/rust/gerbil-parser-runtime/tests/fixtures/generated/command_source.bin new file mode 100644 index 00000000..726b8e77 Binary files /dev/null and b/rust/gerbil-parser-runtime/tests/fixtures/generated/command_source.bin differ diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/command_source.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/command_source.rs new file mode 100644 index 00000000..e56dbf08 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/command_source.rs @@ -0,0 +1,9284 @@ +// @generated by actual Scheme Source recognition; do not edit. +pub struct Control { + pub source: &'static str, + pub accepted: bool, + pub events: &'static [gerbil_parser_runtime::TreeEvent], + pub fields: &'static [&'static str], + pub links: &'static [(usize, usize)], +} +pub mod bash { + use gerbil_parser_runtime::TreeEvent; + // @generated from the admitted Scheme ResultProfile IR. + use gerbil_parser_runtime::{ + CaptureKind, CaptureSpec, KindCategory, KindSpec, ProjectionInstruction, ProjectionOpcode, + ResultNodeSpec, ResultProfileSpec, ResultProjection, + }; + pub static RESULT_PROFILE: ResultProfileSpec = ResultProfileSpec { + kinds: &[ + KindSpec { + name: "Redirection", + category: KindCategory::Node, + }, + KindSpec { + name: "ArrayAssignment", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandList", + category: KindCategory::Node, + }, + KindSpec { + name: "IfCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "WhileCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "UntilCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "SelectCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseClause", + category: KindCategory::Node, + }, + KindSpec { + name: "FunctionDefinition", + category: KindCategory::Node, + }, + KindSpec { + name: "ConditionalCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "RedirectedCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "BraceGroup", + category: KindCategory::Node, + }, + KindSpec { + name: "Subshell", + category: KindCategory::Node, + }, + KindSpec { + name: "Pipeline", + category: KindCategory::Node, + }, + KindSpec { + name: "AndOrList", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocument", + category: KindCategory::Node, + }, + KindSpec { + name: "BashFile", + category: KindCategory::Node, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Node, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Node, + }, + KindSpec { + name: "Word", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocumentLine", + category: KindCategory::Node, + }, + KindSpec { + name: "SingleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "DoubleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "AnsiCString", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ProcessSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ArraySubscript", + category: KindCategory::Node, + }, + KindSpec { + name: "ParameterExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "Assignment", + category: KindCategory::Node, + }, + KindSpec { + name: "unparsed-source", + category: KindCategory::Token, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Token, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Token, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-open", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-close", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-open", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-body", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-close", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-open", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-close", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-open", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-prefix", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-name", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-close", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-name", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "horizontal-whitespace", + category: KindCategory::Token, + }, + KindSpec { + name: "newline", + category: KindCategory::Token, + }, + KindSpec { + name: "line-continuation", + category: KindCategory::Token, + }, + KindSpec { + name: "comment", + category: KindCategory::Token, + }, + KindSpec { + name: "word", + category: KindCategory::Token, + }, + KindSpec { + name: "operator", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-marker", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-content", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-end", + category: KindCategory::Token, + }, + ], + nodes: &[ + ResultNodeSpec { + kind: 0, + fields: &["descriptor", "operator", "target"], + }, + ResultNodeSpec { + kind: 1, + fields: &["assignment", "open", "close", "separator", "element"], + }, + ResultNodeSpec { + kind: 2, + fields: &["assignment", "name", "argument", "redirect"], + }, + ResultNodeSpec { + kind: 3, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 4, + fields: &["keyword", "condition", "body", "else-body"], + }, + ResultNodeSpec { + kind: 5, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 6, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 7, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 8, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 9, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 10, + fields: &["keyword", "subject", "separator", "clause"], + }, + ResultNodeSpec { + kind: 11, + fields: &[ + "open", + "pattern", + "alternate", + "close", + "body", + "terminator", + ], + }, + ResultNodeSpec { + kind: 12, + fields: &["keyword", "name", "open", "close", "body"], + }, + ResultNodeSpec { + kind: 13, + fields: &["open", "close", "operator", "operand"], + }, + ResultNodeSpec { + kind: 14, + fields: &["open", "close", "expression", "operator"], + }, + ResultNodeSpec { + kind: 15, + fields: &["command", "redirect"], + }, + ResultNodeSpec { + kind: 16, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 17, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 18, + fields: &["keyword", "option", "negate", "command", "operator"], + }, + ResultNodeSpec { + kind: 19, + fields: &["command", "operator"], + }, + ResultNodeSpec { + kind: 20, + fields: &["delimiter", "content"], + }, + ResultNodeSpec { + kind: 21, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 22, + fields: &["text"], + }, + ResultNodeSpec { + kind: 23, + fields: &["text"], + }, + ResultNodeSpec { + kind: 24, + fields: &["text"], + }, + ResultNodeSpec { + kind: 25, + fields: &["part"], + }, + ResultNodeSpec { + kind: 26, + fields: &["part"], + }, + ResultNodeSpec { + kind: 27, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 28, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 29, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 30, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 31, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 32, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 33, + fields: &["open", "index", "close"], + }, + ResultNodeSpec { + kind: 34, + fields: &[ + "open", + "prefix", + "name", + "subscript", + "operator", + "operand", + "close", + ], + }, + ResultNodeSpec { + kind: 35, + fields: &["name", "operator", "value"], + }, + ], + projections: &[ + ResultProjection { + id: "LiteralPart", + kind: 22, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 37, + required: true, + }, + }], + }, + ResultProjection { + id: "EscapeSequence", + kind: 23, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 38, + required: true, + }, + }], + }, + ResultProjection { + id: "SimpleParameter", + kind: 24, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 39, + required: true, + }, + }], + }, + ResultProjection { + id: "Word", + kind: 25, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "HereDocumentLine", + kind: 26, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "SingleQuoted", + kind: 27, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "DoubleQuoted", + kind: 28, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "AnsiCString", + kind: 29, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArithmeticExpansion", + kind: 30, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "CommandSubstitution", + kind: 31, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ProcessSubstitution", + kind: 32, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArraySubscript", + kind: 33, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 45, + required: true, + }, + }, + ProjectionInstruction { + field: "index", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 46, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ParameterExpansion", + kind: 34, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "prefix", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "subscript", + kind: CaptureKind::Node, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 47, + required: true, + }, + }, + ProjectionInstruction { + field: "prefix", + opcode: ProjectionOpcode::Token { + kind: 48, + required: false, + }, + }, + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 49, + required: false, + }, + }, + ProjectionInstruction { + field: "subscript", + opcode: ProjectionOpcode::One { required: false }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 50, + required: false, + }, + }, + ProjectionInstruction { + field: "operand", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 51, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "Assignment", + kind: 35, + captures: &[ + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 52, + required: true, + }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 53, + required: true, + }, + }, + ProjectionInstruction { + field: "value", + opcode: ProjectionOpcode::Many, + }, + ], + }, + ], + }; + use gerbil_parser_runtime::{ + BindingSpec, PartGuard, PartOpcode, PartProfileSpec, PartRule, SubscriptSpec, + }; + pub mod regions { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[]; + } + mod subscript_1 { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "[", + opening: '\u{5b}', + closing: '\u{5d}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[ + gerbil_parser_runtime::scanner::RegionScope { + prefix: "${", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "$(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "$((", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "<(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: ">(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "[", + pairs: &["${", "$(", "$((", "<(", ">(", "["], + }, + ]; + } + pub static PART_PROFILE: PartProfileSpec = PartProfileSpec { + contexts: &[ + "word", + "SingleQuoted", + "DoubleQuoted", + "AnsiCString", + "HereDocument", + ], + literal: "LiteralPart", + assignment: Some(2), + bindings: &[ + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("?@*#$!-_0"), + ]), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("?@*#$!-_0"), + ]), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + }, + prefixes: &[], + operators: &[], + subscript: None, + }, + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("@*?#$!-"), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("@*?#$!-"), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Numeric, + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + }), + }, + prefixes: &["#", "!"], + operators: &[ + ":-", ":=", ":+", ":?", "##", "%%", "//", "^^", ",,", "~~", ":", "-", "=", "+", + "?", "#", "%", "/", "@", "^", ",", "~", + ], + subscript: Some(SubscriptSpec { + prefix: "[", + closing: '\u{5d}', + regions: &subscript_1::REGION, + scopes: subscript_1::SCOPES, + }), + }, + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: Some(1), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 0, + maximum: None, + }, + ]), + prefixes: &[], + operators: &["=", "+="], + subscript: None, + }, + ], + rules: &[ + PartRule { + contexts: &["word"], + prefix: "$'", + guard: PartGuard::Any, + projection: "AnsiCString", + opening: 2, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{27}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "'", + guard: PartGuard::Any, + projection: "SingleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{27}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "\"", + guard: PartGuard::Any, + projection: "DoubleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{22}', + }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "${", + guard: PartGuard::Any, + projection: "ParameterExpansion", + opening: 2, + closing: 1, + opcode: PartOpcode::Parameter { binding: 1 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$((", + guard: PartGuard::Any, + projection: "ArithmeticExpansion", + opening: 3, + closing: 2, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$(", + guard: PartGuard::Any, + projection: "CommandSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word"], + prefix: "<(", + guard: PartGuard::Any, + projection: "ProcessSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word"], + prefix: ">(", + guard: PartGuard::Any, + projection: "ProcessSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "`", + guard: PartGuard::Any, + projection: "CommandSubstitution", + opening: 1, + closing: 1, + opcode: PartOpcode::QuotedBody { + delimiter: '\u{60}', + }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$", + guard: PartGuard::Next, + projection: "SimpleParameter", + opening: 1, + closing: 0, + opcode: PartOpcode::Name { binding: 0 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "AnsiCString"], + prefix: "\\", + guard: PartGuard::Any, + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + PartRule { + contexts: &["HereDocument"], + prefix: "\\", + guard: PartGuard::Characters("$`\\\n"), + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + ], + }; + use gerbil_parser_runtime::{ + CommandChoice, CommandForm, CommandInstruction, CommandProgramSpec, CommandRole, + CommandTokenClass, CommandTrigger, + }; + pub static COMMAND_PROFILE: CommandProgramSpec = CommandProgramSpec { + assignment_tail: "array-tail", + pipeline_head: "pipeline-prefix", + nodes: &[ + ("Redirection", "Redirection"), + ("ArrayAssignment", "ArrayAssignment"), + ("SimpleCommand", "SimpleCommand"), + ("CommandList", "CommandList"), + ("IfCommand", "IfCommand"), + ("WhileCommand", "WhileCommand"), + ("UntilCommand", "UntilCommand"), + ("ForCommand", "ForCommand"), + ("SelectCommand", "SelectCommand"), + ("ArithmeticForCommand", "ArithmeticForCommand"), + ("CaseCommand", "CaseCommand"), + ("CaseClause", "CaseClause"), + ("FunctionDefinition", "FunctionDefinition"), + ("ConditionalCommand", "ConditionalCommand"), + ("ArithmeticCommand", "ArithmeticCommand"), + ("RedirectedCommand", "RedirectedCommand"), + ("BraceGroup", "BraceGroup"), + ("Subshell", "Subshell"), + ("Pipeline", "Pipeline"), + ("AndOrList", "AndOrList"), + ("HereDocument", "HereDocument"), + ("BashFile", "BashFile"), + ], + roles: &[ + CommandRole { + name: "trivia", + classes: &[ + CommandTokenClass { + kind: "horizontal-whitespace", + literals: &[], + }, + CommandTokenClass { + kind: "comment", + literals: &[], + }, + CommandTokenClass { + kind: "line-continuation", + literals: &[], + }, + ], + }, + CommandRole { + name: "separator", + classes: &[ + CommandTokenClass { + kind: "newline", + literals: &[], + }, + CommandTokenClass { + kind: "operator", + literals: &[";", "&"], + }, + ], + }, + CommandRole { + name: "redirect", + classes: &[CommandTokenClass { + kind: "operator", + literals: &[ + "<", ">", ">>", "<>", "<<", "<<-", "<<<", "<&", ">&", ">|", "&>", "&>>", + ], + }], + }, + CommandRole { + name: "here-redirect", + classes: &[CommandTokenClass { + kind: "operator", + literals: &["<<", "<<-"], + }], + }, + CommandRole { + name: "strip-redirect", + classes: &[CommandTokenClass { + kind: "operator", + literals: &["<<-"], + }], + }, + CommandRole { + name: "reserved", + classes: &[CommandTokenClass { + kind: "word", + literals: &[ + "if", "then", "elif", "else", "fi", "while", "until", "do", "done", "for", + "select", "case", "in", "esac", "function", "{", "}", + ], + }], + }, + CommandRole { + name: "conditional-operator", + classes: &[CommandTokenClass { + kind: "word", + literals: &[ + "==", "=", "!=", "=~", "-eq", "-ne", "-lt", "-le", "-gt", "-ge", "-z", + "-n", "!", + ], + }], + }, + CommandRole { + name: "pipeline", + classes: &[CommandTokenClass { + kind: "operator", + literals: &["|", "|&"], + }], + }, + CommandRole { + name: "and-or", + classes: &[CommandTokenClass { + kind: "operator", + literals: &["&&", "||"], + }], + }, + CommandRole { + name: "case-end", + classes: &[CommandTokenClass { + kind: "operator", + literals: &[";;", ";&", ";;&"], + }], + }, + ], + descriptor: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Numeric, + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("{"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: Some(1), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 0, + maximum: None, + }, + gerbil_parser_runtime::TextProfile::Literal("}"), + ])), + }, + forms: &[ + CommandForm { + id: "conditional-branch", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["if"], + }, + priority: 34, + kind: "IfCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["if"], + }, + }, + CommandInstruction::List { + field: "condition", + until: CommandTrigger::Token { + kind: "word", + literals: &["then"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["then"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["elif", "else", "fi"], + }, + allow_empty: false, + }, + CommandInstruction::Many { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["elif"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["elif"], + }, + }, + CommandInstruction::List { + field: "condition", + until: CommandTrigger::Token { + kind: "word", + literals: &["then"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["then"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["elif", "else", "fi"], + }, + allow_empty: false, + }, + ], + }, + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["else"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["else"], + }, + }, + CommandInstruction::List { + field: "else-body", + until: CommandTrigger::Token { + kind: "word", + literals: &["fi"], + }, + allow_empty: false, + }, + ], + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["fi"], + }, + }, + ], + }, + CommandForm { + id: "while-loop", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["while"], + }, + priority: 34, + kind: "WhileCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["while"], + }, + }, + CommandInstruction::List { + field: "condition", + until: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + }, + ], + }, + CommandForm { + id: "until-loop", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["until"], + }, + priority: 34, + kind: "UntilCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["until"], + }, + }, + CommandInstruction::List { + field: "condition", + until: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + }, + ], + }, + CommandForm { + id: "for-loop", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["for"], + }, + priority: 34, + kind: "ForCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["for"], + }, + }, + CommandInstruction::Branch { + trigger: CommandTrigger::Adjacent(&[ + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + ]), + yes: &[ + CommandInstruction::As("ArithmeticForCommand"), + CommandInstruction::Call { + field: "header", + form: "arithmetic", + }, + ], + no: &[ + CommandInstruction::Word("variable"), + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + }, + CommandInstruction::Until { + trigger: CommandTrigger::Role("separator"), + program: &[CommandInstruction::Word("item")], + }, + ], + }, + ], + }, + CommandInstruction::Take { + field: "separator", + trigger: CommandTrigger::Role("separator"), + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + }, + ], + }, + CommandForm { + id: "select-loop", + trigger: CommandTrigger::Lookahead(&[ + CommandTrigger::Token { + kind: "word", + literals: &["select"], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + ]), + priority: 35, + kind: "SelectCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["select"], + }, + }, + CommandInstruction::Word("variable"), + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + }, + CommandInstruction::Until { + trigger: CommandTrigger::Role("separator"), + program: &[CommandInstruction::Word("item")], + }, + ], + }, + CommandInstruction::Take { + field: "separator", + trigger: CommandTrigger::Role("separator"), + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + }, + ], + }, + CommandForm { + id: "case-selection", + trigger: CommandTrigger::Lookahead(&[ + CommandTrigger::Token { + kind: "word", + literals: &["case"], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + ]), + priority: 36, + kind: "CaseCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["case"], + }, + }, + CommandInstruction::Word("subject"), + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + }, + CommandInstruction::Until { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["esac"], + }, + program: &[CommandInstruction::Choose(&[ + CommandChoice { + trigger: CommandTrigger::Role("separator"), + program: &[CommandInstruction::Take { + field: "separator", + trigger: CommandTrigger::Role("separator"), + }], + }, + CommandChoice { + trigger: CommandTrigger::Or(&[ + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + ]), + program: &[CommandInstruction::Node { + field: "clause", + kind: "CaseClause", + program: &[ + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + program: &[CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }], + }, + CommandInstruction::Word("pattern"), + CommandInstruction::Many { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["|"], + }, + program: &[ + CommandInstruction::Take { + field: "alternate", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["|"], + }, + }, + CommandInstruction::Word("pattern"), + ], + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Or(&[ + CommandTrigger::Role("case-end"), + CommandTrigger::Token { + kind: "word", + literals: &["esac"], + }, + ]), + allow_empty: true, + }, + CommandInstruction::Optional { + trigger: CommandTrigger::Role("case-end"), + program: &[CommandInstruction::Take { + field: "terminator", + trigger: CommandTrigger::Role("case-end"), + }], + }, + ], + }], + }, + ])], + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["esac"], + }, + }, + ], + }, + CommandForm { + id: "named-function", + trigger: CommandTrigger::Lookahead(&[ + CommandTrigger::Token { + kind: "word", + literals: &["function"], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + ]), + priority: 35, + kind: "FunctionDefinition", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["function"], + }, + }, + CommandInstruction::Word("name"), + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + ], + }, + CommandInstruction::Command("body"), + ], + }, + CommandForm { + id: "function-header", + trigger: CommandTrigger::Lookahead(&[ + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + ]), + priority: 3, + kind: "FunctionDefinition", + program: &[ + CommandInstruction::Word("name"), + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + CommandInstruction::Command("body"), + ], + }, + CommandForm { + id: "conditional", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["[["], + }, + priority: 34, + kind: "ConditionalCommand", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["[["], + }, + }, + CommandInstruction::Until { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["]]"], + }, + program: &[CommandInstruction::Choose(&[ + CommandChoice { + trigger: CommandTrigger::Role("conditional-operator"), + program: &[CommandInstruction::Take { + field: "operator", + trigger: CommandTrigger::Role("conditional-operator"), + }], + }, + CommandChoice { + trigger: CommandTrigger::Token { + kind: "word", + literals: &[], + }, + program: &[CommandInstruction::Word("operand")], + }, + CommandChoice { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[], + }, + program: &[CommandInstruction::Raw("operator")], + }, + ])], + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["]]"], + }, + }, + ], + }, + CommandForm { + id: "arithmetic", + trigger: CommandTrigger::Adjacent(&[ + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + ]), + priority: 35, + kind: "ArithmeticCommand", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Balance { + open: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + close: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + depth: 2, + open_field: "open", + close_field: "close", + program: &[CommandInstruction::Choose(&[ + CommandChoice { + trigger: CommandTrigger::Token { + kind: "word", + literals: &[], + }, + program: &[CommandInstruction::Word("expression")], + }, + CommandChoice { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[], + }, + program: &[CommandInstruction::Raw("operator")], + }, + ])], + }, + ], + }, + CommandForm { + id: "brace-group", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["{"], + }, + priority: 34, + kind: "BraceGroup", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["{"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["}"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["}"], + }, + }, + ], + }, + CommandForm { + id: "subshell", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + priority: 34, + kind: "Subshell", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + ], + }, + CommandForm { + id: "array-tail", + trigger: CommandTrigger::Manual(Some(&CommandTrigger::Token { + kind: "operator", + literals: &["("], + })), + priority: 0, + kind: "ArrayAssignment", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Many { + trigger: CommandTrigger::Or(&[ + CommandTrigger::Token { + kind: "newline", + literals: &[], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + ]), + program: &[CommandInstruction::Choose(&[ + CommandChoice { + trigger: CommandTrigger::Token { + kind: "newline", + literals: &[], + }, + program: &[CommandInstruction::Raw("separator")], + }, + CommandChoice { + trigger: CommandTrigger::Token { + kind: "word", + literals: &[], + }, + program: &[CommandInstruction::Word("element")], + }, + ])], + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + ], + }, + CommandForm { + id: "pipeline-prefix", + trigger: CommandTrigger::Manual(None), + priority: 0, + kind: "Pipeline", + program: &[ + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["time"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["time"], + }, + }, + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["-p"], + }, + program: &[CommandInstruction::Take { + field: "option", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["-p"], + }, + }], + }, + ], + }, + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["!"], + }, + program: &[CommandInstruction::Take { + field: "negate", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["!"], + }, + }], + }, + ], + }, + ], + }; + pub mod scanner { + // @generated from Scheme contextual Scanner IR; do not edit. + #[allow(unused_imports)] + use gerbil_parser_runtime::scanner::{ + BalancedPair, MarkerPolicy, ScannerAction, ScannerCell, ScannerMatcher, ScannerRule, + ScannerSpec, + }; + pub static SCANNER: ScannerSpec = ScannerSpec { + opcode_contract: "gerbil-parser.contextual-scanner-opcodes.v1", + digest: "sha256:350c02574541fb278cf653584eb79c1eb2b5f326ad06dc92a43f8daffbb099a3", + base_grammar_digest: None, + initial_mode: "command", + modes: &["command", "marker", "body"], + positions: &["source"], + rules: &[ + ScannerRule { + name: "space", + mode: "command", + form: "horizontal-whitespace", + matcher: ScannerMatcher::HorizontalWhitespace, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "newline", + mode: "command", + form: "newline", + matcher: ScannerMatcher::Literal("\n"), + rank: 0, + action: ScannerAction::ActivateNext("body"), + }, + ScannerRule { + name: "continuation", + mode: "command", + form: "line-continuation", + matcher: ScannerMatcher::Literal("\\\n"), + rank: 10, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "comment", + mode: "command", + form: "comment", + matcher: ScannerMatcher::LinePrefix { + prefix: "#", + separator: '\u{a}', + }, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "strip", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literal("<<-"), + rank: 30, + action: ScannerAction::ExpectMarkerIn { + strip_tabs: true, + mode: "marker", + }, + }, + ScannerRule { + name: "plain", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literal("<<"), + rank: 30, + action: ScannerAction::ExpectMarkerIn { + strip_tabs: false, + mode: "marker", + }, + }, + ScannerRule { + name: "operator", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literals(&[ + ";;&", "&>>", "<<<", "&&", "||", "|&", ";;", ";&", ">>", "<>", "<&", ">&", + ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ]), + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "word", + mode: "command", + form: "word", + matcher: ScannerMatcher::UnlessPrefix { + prefixes: &[ + " ", "\t", "\n", "#", "\\\n", ";", "&", "|", "(", ")", "<", ">", + ], + exceptions: &["<(", ">("], + child: &ScannerMatcher::RegionWord { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", + ">>", "<>", "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", + ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }, + }, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-space", + mode: "marker", + form: "horizontal-whitespace", + matcher: ScannerMatcher::HorizontalWhitespace, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-continuation", + mode: "marker", + form: "line-continuation", + matcher: ScannerMatcher::Literal("\\\n"), + rank: 10, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-newline", + mode: "marker", + form: "newline", + matcher: ScannerMatcher::Literal("\n"), + rank: 10, + action: ScannerAction::ActivateNext("body"), + }, + ScannerRule { + name: "marker-word", + mode: "marker", + form: "heredoc-marker", + matcher: ScannerMatcher::UnlessPrefix { + prefixes: &[" ", "\t", "\n", "\\\n"], + exceptions: &[], + child: &ScannerMatcher::RegionWord { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", + ">>", "<>", "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", + ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }, + }, + rank: 0, + action: ScannerAction::EnqueueMarkerIn { + policy: MarkerPolicy::ShellQuoteRemoval, + mode: "command", + }, + }, + ScannerRule { + name: "end", + mode: "body", + form: "heredoc-end", + matcher: ScannerMatcher::MarkerLineAt('\u{a}'), + rank: 10, + action: ScannerAction::FinishMarker { + base: "command", + body: "body", + }, + }, + ScannerRule { + name: "body", + mode: "body", + form: "heredoc-content", + matcher: ScannerMatcher::BodyLineAt('\u{a}'), + rank: 0, + action: ScannerAction::Keep, + }, + ], + cells: &[ + ScannerCell { + mode: "command", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "command", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "command", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "command", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "command", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "command", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ScannerCell { + mode: "body", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "body", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "body", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "body", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "body", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "body", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ], + }; + } + pub static SOURCE: gerbil_parser_runtime::CommandSourceSpec = + gerbil_parser_runtime::CommandSourceSpec { + scanner: &scanner::SCANNER, + parts: &PART_PROFILE, + regions: ®ions::REGION, + results: &RESULT_PROFILE, + commands: &COMMAND_PROFILE, + }; + pub static CONTROLS: &[super::Control] = &[ + super::Control { + source: "", + accepted: true, + events: &[TreeEvent::StartNode(21), TreeEvent::FinishNode], + fields: &[], + links: &[], + }, + super::Control { + source: " # hi\n", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::Token { + kind: 54, + start: 0, + end: 2, + }, + TreeEvent::Token { + kind: 57, + start: 2, + end: 6, + }, + TreeEvent::Token { + kind: 55, + start: 6, + end: 7, + }, + TreeEvent::FinishNode, + ], + fields: &["separator"], + links: &[], + }, + super::Control { + source: "echo α😀", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 0, + end: 4, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 4, + end: 5, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", "name", "part", "text", "argument", "part", "text", + ], + links: &[], + }, + super::Control { + source: "a=中 x+=λ echo \"${x:-中}\" 2>out tail", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(2), + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 0, + end: 1, + }, + TreeEvent::Token { + kind: 53, + start: 1, + end: 2, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 2, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 5, + end: 6, + }, + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 6, + end: 7, + }, + TreeEvent::Token { + kind: 53, + start: 7, + end: 9, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 11, + end: 12, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 12, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 16, + end: 17, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 17, + end: 18, + }, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 18, + end: 20, + }, + TreeEvent::Token { + kind: 49, + start: 20, + end: 21, + }, + TreeEvent::Token { + kind: 50, + start: 21, + end: 23, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 23, + end: 26, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 26, + end: 27, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 27, + end: 28, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 28, + end: 29, + }, + TreeEvent::StartNode(0), + TreeEvent::Token { + kind: 58, + start: 29, + end: 30, + }, + TreeEvent::Token { + kind: 59, + start: 30, + end: 31, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 31, + end: 34, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 34, + end: 35, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 35, + end: 39, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "assignment", + "name", + "operator", + "value", + "text", + "assignment", + "name", + "operator", + "value", + "text", + "name", + "part", + "text", + "argument", + "part", + "open", + "part", + "open", + "name", + "operator", + "operand", + "text", + "close", + "close", + "redirect", + "descriptor", + "operator", + "target", + "part", + "text", + "argument", + "part", + "text", + ], + links: &[], + }, + super::Control { + source: "arr=(α \"中\"\nλ) echo ok", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(2), + TreeEvent::StartNode(1), + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 0, + end: 3, + }, + TreeEvent::Token { + kind: 53, + start: 3, + end: 4, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 4, + end: 5, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 7, + end: 8, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 8, + end: 9, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 55, + start: 13, + end: 14, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 14, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 16, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 17, + end: 18, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 18, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 22, + end: 23, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 23, + end: 25, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "assignment", + "assignment", + "name", + "operator", + "open", + "element", + "part", + "text", + "element", + "part", + "open", + "part", + "text", + "close", + "separator", + "element", + "part", + "text", + "close", + "name", + "part", + "text", + "argument", + "part", + "text", + ], + links: &[], + }, + super::Control { + source: "echo α|&cat && echo 中 || echo λ", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(19), + TreeEvent::StartNode(18), + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 0, + end: 4, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 4, + end: 5, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 7, + end: 9, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 13, + end: 15, + }, + TreeEvent::Token { + kind: 54, + start: 15, + end: 16, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 16, + end: 20, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 20, + end: 21, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 21, + end: 24, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 24, + end: 25, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 25, + end: 27, + }, + TreeEvent::Token { + kind: 54, + start: 27, + end: 28, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 28, + end: 32, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 32, + end: 33, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 33, + end: 35, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", "command", "command", "name", "part", "text", "argument", "part", + "text", "operator", "command", "name", "part", "text", "operator", "command", + "name", "part", "text", "argument", "part", "text", "operator", "command", "name", + "part", "text", "argument", "part", "text", + ], + links: &[], + }, + super::Control { + source: "time -p ! echo α | cat", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(18), + TreeEvent::Token { + kind: 58, + start: 0, + end: 4, + }, + TreeEvent::Token { + kind: 54, + start: 4, + end: 5, + }, + TreeEvent::Token { + kind: 58, + start: 5, + end: 7, + }, + TreeEvent::Token { + kind: 54, + start: 7, + end: 8, + }, + TreeEvent::Token { + kind: 58, + start: 8, + end: 9, + }, + TreeEvent::Token { + kind: 54, + start: 9, + end: 10, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 14, + end: 15, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 17, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 18, + end: 19, + }, + TreeEvent::Token { + kind: 54, + start: 19, + end: 20, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 20, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", "keyword", "option", "negate", "command", "name", "part", "text", + "argument", "part", "text", "operator", "command", "name", "part", "text", + ], + links: &[], + }, + super::Control { + source: "! echo α", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(18), + TreeEvent::Token { + kind: 58, + start: 0, + end: 1, + }, + TreeEvent::Token { + kind: 54, + start: 1, + end: 2, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 2, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 6, + end: 7, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 7, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", "negate", "command", "name", "part", "text", "argument", "part", "text", + ], + links: &[], + }, + super::Control { + source: "time echo α", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(18), + TreeEvent::Token { + kind: 58, + start: 0, + end: 4, + }, + TreeEvent::Token { + kind: 54, + start: 4, + end: 5, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 9, + end: 10, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", "keyword", "command", "name", "part", "text", "argument", "part", "text", + ], + links: &[], + }, + super::Control { + source: "if echo α; then echo 中; elif echo λ; then echo β; else echo 😀; fi", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(4), + TreeEvent::Token { + kind: 58, + start: 0, + end: 2, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 2, + end: 3, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 3, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 7, + end: 8, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 8, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 10, + end: 11, + }, + TreeEvent::Token { + kind: 54, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 12, + end: 16, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 16, + end: 17, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 17, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 21, + end: 22, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 22, + end: 25, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 25, + end: 26, + }, + TreeEvent::Token { + kind: 54, + start: 26, + end: 27, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 27, + end: 31, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 31, + end: 32, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 32, + end: 36, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 36, + end: 37, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 37, + end: 39, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 39, + end: 40, + }, + TreeEvent::Token { + kind: 54, + start: 40, + end: 41, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 41, + end: 45, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 45, + end: 46, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 46, + end: 50, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 50, + end: 51, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 51, + end: 53, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 53, + end: 54, + }, + TreeEvent::Token { + kind: 54, + start: 54, + end: 55, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 55, + end: 59, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 59, + end: 60, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 60, + end: 64, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 64, + end: 65, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 65, + end: 69, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 69, + end: 70, + }, + TreeEvent::Token { + kind: 54, + start: 70, + end: 71, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 71, + end: 73, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "keyword", + "condition", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + "condition", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + "else-body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + ], + links: &[], + }, + super::Control { + source: "while echo α; do echo 中; done", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(5), + TreeEvent::Token { + kind: 58, + start: 0, + end: 5, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 5, + end: 6, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 10, + end: 11, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 13, + end: 14, + }, + TreeEvent::Token { + kind: 54, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 15, + end: 17, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 17, + end: 18, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 18, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 22, + end: 23, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 23, + end: 26, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 26, + end: 27, + }, + TreeEvent::Token { + kind: 54, + start: 27, + end: 28, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 28, + end: 32, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "keyword", + "condition", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + ], + links: &[], + }, + super::Control { + source: "until echo α; do echo 中; done", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(6), + TreeEvent::Token { + kind: 58, + start: 0, + end: 5, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 5, + end: 6, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 10, + end: 11, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 13, + end: 14, + }, + TreeEvent::Token { + kind: 54, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 15, + end: 17, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 17, + end: 18, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 18, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 22, + end: 23, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 23, + end: 26, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 26, + end: 27, + }, + TreeEvent::Token { + kind: 54, + start: 27, + end: 28, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 28, + end: 32, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "keyword", + "condition", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + ], + links: &[], + }, + super::Control { + source: "for x in α 中; do echo $x; done", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(7), + TreeEvent::Token { + kind: 58, + start: 0, + end: 3, + }, + TreeEvent::Token { + kind: 54, + start: 3, + end: 4, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 5, + end: 6, + }, + TreeEvent::Token { + kind: 58, + start: 6, + end: 8, + }, + TreeEvent::Token { + kind: 54, + start: 8, + end: 9, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 11, + end: 12, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 12, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 15, + end: 16, + }, + TreeEvent::Token { + kind: 54, + start: 16, + end: 17, + }, + TreeEvent::Token { + kind: 58, + start: 17, + end: 19, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 19, + end: 20, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 20, + end: 24, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 24, + end: 25, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 25, + end: 27, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 27, + end: 28, + }, + TreeEvent::Token { + kind: 54, + start: 28, + end: 29, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 29, + end: 33, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "keyword", + "variable", + "part", + "text", + "keyword", + "item", + "part", + "text", + "item", + "part", + "text", + "separator", + "keyword", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + ], + links: &[], + }, + super::Control { + source: "for ((i=0;i<2;i++)); do echo $i; done", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(9), + TreeEvent::Token { + kind: 58, + start: 0, + end: 3, + }, + TreeEvent::Token { + kind: 54, + start: 3, + end: 4, + }, + TreeEvent::StartNode(14), + TreeEvent::Token { + kind: 59, + start: 4, + end: 5, + }, + TreeEvent::Token { + kind: 59, + start: 5, + end: 6, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 9, + end: 10, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 11, + end: 12, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 13, + end: 14, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 14, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 17, + end: 18, + }, + TreeEvent::Token { + kind: 59, + start: 18, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 19, + end: 20, + }, + TreeEvent::Token { + kind: 54, + start: 20, + end: 21, + }, + TreeEvent::Token { + kind: 58, + start: 21, + end: 23, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 23, + end: 24, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 24, + end: 28, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 28, + end: 29, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 29, + end: 31, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 31, + end: 32, + }, + TreeEvent::Token { + kind: 54, + start: 32, + end: 33, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 33, + end: 37, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "keyword", + "header", + "open", + "open", + "expression", + "part", + "text", + "operator", + "expression", + "part", + "text", + "operator", + "expression", + "part", + "text", + "operator", + "expression", + "part", + "text", + "close", + "close", + "separator", + "keyword", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + ], + links: &[], + }, + super::Control { + source: "select x in α 中; do echo $x; done", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(8), + TreeEvent::Token { + kind: 58, + start: 0, + end: 6, + }, + TreeEvent::Token { + kind: 54, + start: 6, + end: 7, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 7, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 8, + end: 9, + }, + TreeEvent::Token { + kind: 58, + start: 9, + end: 11, + }, + TreeEvent::Token { + kind: 54, + start: 11, + end: 12, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 12, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 14, + end: 15, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 18, + end: 19, + }, + TreeEvent::Token { + kind: 54, + start: 19, + end: 20, + }, + TreeEvent::Token { + kind: 58, + start: 20, + end: 22, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 22, + end: 23, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 23, + end: 27, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 27, + end: 28, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 28, + end: 30, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 30, + end: 31, + }, + TreeEvent::Token { + kind: 54, + start: 31, + end: 32, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 32, + end: 36, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "keyword", + "variable", + "part", + "text", + "keyword", + "item", + "part", + "text", + "item", + "part", + "text", + "separator", + "keyword", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "keyword", + ], + links: &[], + }, + super::Control { + source: "case α in α|β) echo 中;; (λ) ;; esac", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(10), + TreeEvent::Token { + kind: 58, + start: 0, + end: 4, + }, + TreeEvent::Token { + kind: 54, + start: 4, + end: 5, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 7, + end: 8, + }, + TreeEvent::Token { + kind: 58, + start: 8, + end: 10, + }, + TreeEvent::Token { + kind: 54, + start: 10, + end: 11, + }, + TreeEvent::StartNode(11), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 13, + end: 14, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 14, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 16, + end: 17, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 17, + end: 18, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 18, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 22, + end: 23, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 23, + end: 26, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 26, + end: 28, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 28, + end: 29, + }, + TreeEvent::StartNode(11), + TreeEvent::Token { + kind: 59, + start: 29, + end: 30, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 30, + end: 32, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 32, + end: 33, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 33, + end: 34, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 34, + end: 36, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 36, + end: 37, + }, + TreeEvent::Token { + kind: 58, + start: 37, + end: 41, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "keyword", + "subject", + "part", + "text", + "keyword", + "clause", + "pattern", + "part", + "text", + "alternate", + "pattern", + "part", + "text", + "close", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "terminator", + "clause", + "open", + "pattern", + "part", + "text", + "close", + "body", + "terminator", + "keyword", + ], + links: &[], + }, + super::Control { + source: "function f() { echo α; } >out", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(12), + TreeEvent::Token { + kind: 58, + start: 0, + end: 8, + }, + TreeEvent::Token { + kind: 54, + start: 8, + end: 9, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 10, + end: 11, + }, + TreeEvent::Token { + kind: 59, + start: 11, + end: 12, + }, + TreeEvent::Token { + kind: 54, + start: 12, + end: 13, + }, + TreeEvent::StartNode(15), + TreeEvent::StartNode(16), + TreeEvent::Token { + kind: 58, + start: 13, + end: 14, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 14, + end: 15, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 19, + end: 20, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 20, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 22, + end: 23, + }, + TreeEvent::Token { + kind: 54, + start: 23, + end: 24, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 24, + end: 25, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 25, + end: 26, + }, + TreeEvent::StartNode(0), + TreeEvent::Token { + kind: 59, + start: 26, + end: 27, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 27, + end: 30, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "keyword", + "name", + "part", + "text", + "open", + "close", + "body", + "command", + "open", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "close", + "redirect", + "operator", + "target", + "part", + "text", + ], + links: &[], + }, + super::Control { + source: "f ( ) { echo 中; }", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(12), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 0, + end: 1, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 1, + end: 2, + }, + TreeEvent::Token { + kind: 59, + start: 2, + end: 3, + }, + TreeEvent::Token { + kind: 54, + start: 3, + end: 4, + }, + TreeEvent::Token { + kind: 59, + start: 4, + end: 5, + }, + TreeEvent::Token { + kind: 54, + start: 5, + end: 6, + }, + TreeEvent::StartNode(16), + TreeEvent::Token { + kind: 58, + start: 6, + end: 7, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 7, + end: 8, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 8, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 12, + end: 13, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 13, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 16, + end: 17, + }, + TreeEvent::Token { + kind: 54, + start: 17, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 18, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "name", + "part", + "text", + "open", + "close", + "body", + "open", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "close", + ], + links: &[], + }, + super::Control { + source: "[[ α == 中 && -n $x ]]", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(13), + TreeEvent::Token { + kind: 58, + start: 0, + end: 2, + }, + TreeEvent::Token { + kind: 54, + start: 2, + end: 3, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 3, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 5, + end: 6, + }, + TreeEvent::Token { + kind: 58, + start: 6, + end: 8, + }, + TreeEvent::Token { + kind: 54, + start: 8, + end: 9, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 12, + end: 13, + }, + TreeEvent::Token { + kind: 59, + start: 13, + end: 15, + }, + TreeEvent::Token { + kind: 54, + start: 15, + end: 16, + }, + TreeEvent::Token { + kind: 58, + start: 16, + end: 18, + }, + TreeEvent::Token { + kind: 54, + start: 18, + end: 19, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 19, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 21, + end: 22, + }, + TreeEvent::Token { + kind: 58, + start: 22, + end: 24, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", "open", "operand", "part", "text", "operator", "operand", "part", + "text", "operator", "operator", "operand", "part", "text", "close", + ], + links: &[], + }, + super::Control { + source: "((1+(2*3)))", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(14), + TreeEvent::Token { + kind: 59, + start: 0, + end: 1, + }, + TreeEvent::Token { + kind: 59, + start: 1, + end: 2, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 2, + end: 4, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 4, + end: 5, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 8, + end: 9, + }, + TreeEvent::Token { + kind: 59, + start: 9, + end: 10, + }, + TreeEvent::Token { + kind: 59, + start: 10, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "open", + "open", + "expression", + "part", + "text", + "open", + "expression", + "part", + "text", + "close", + "close", + "close", + ], + links: &[], + }, + super::Control { + source: "{ echo α; } 2>out", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(15), + TreeEvent::StartNode(16), + TreeEvent::Token { + kind: 58, + start: 0, + end: 1, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 1, + end: 2, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 2, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 6, + end: 7, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 7, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 9, + end: 10, + }, + TreeEvent::Token { + kind: 54, + start: 10, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 12, + end: 13, + }, + TreeEvent::StartNode(0), + TreeEvent::Token { + kind: 58, + start: 13, + end: 14, + }, + TreeEvent::Token { + kind: 59, + start: 14, + end: 15, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "command", + "open", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "close", + "redirect", + "descriptor", + "operator", + "target", + "part", + "text", + ], + links: &[], + }, + super::Control { + source: "(echo α; echo 中)", + accepted: true, + events: &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(17), + TreeEvent::Token { + kind: 59, + start: 0, + end: 1, + }, + TreeEvent::StartNode(3), + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 1, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 5, + end: 6, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 8, + end: 9, + }, + TreeEvent::Token { + kind: 54, + start: 9, + end: 10, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 14, + end: 15, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 18, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "open", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "close", + ], + links: &[], + }, + super::Control { + source: "cat <", + accepted: false, + events: &[], + fields: &[], + links: &[], + }, + ]; +} +pub mod extended { + use gerbil_parser_runtime::TreeEvent; + // @generated from the admitted Scheme ResultProfile IR. + use gerbil_parser_runtime::{ + CaptureKind, CaptureSpec, KindCategory, KindSpec, ProjectionInstruction, ProjectionOpcode, + ResultNodeSpec, ResultProfileSpec, ResultProjection, + }; + pub static RESULT_PROFILE: ResultProfileSpec = ResultProfileSpec { + kinds: &[ + KindSpec { + name: "Redirection", + category: KindCategory::Node, + }, + KindSpec { + name: "ArrayAssignment", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandList", + category: KindCategory::Node, + }, + KindSpec { + name: "IfCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "WhileCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "UntilCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "SelectCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseClause", + category: KindCategory::Node, + }, + KindSpec { + name: "FunctionDefinition", + category: KindCategory::Node, + }, + KindSpec { + name: "ConditionalCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "RedirectedCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "BraceGroup", + category: KindCategory::Node, + }, + KindSpec { + name: "Subshell", + category: KindCategory::Node, + }, + KindSpec { + name: "Pipeline", + category: KindCategory::Node, + }, + KindSpec { + name: "AndOrList", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocument", + category: KindCategory::Node, + }, + KindSpec { + name: "BashFile", + category: KindCategory::Node, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Node, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Node, + }, + KindSpec { + name: "Word", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocumentLine", + category: KindCategory::Node, + }, + KindSpec { + name: "SingleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "DoubleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "AnsiCString", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ProcessSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ArraySubscript", + category: KindCategory::Node, + }, + KindSpec { + name: "ParameterExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "Assignment", + category: KindCategory::Node, + }, + KindSpec { + name: "unparsed-source", + category: KindCategory::Token, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Token, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Token, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-open", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-close", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-open", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-body", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-close", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-open", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-close", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-open", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-prefix", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-name", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-close", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-name", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "horizontal-whitespace", + category: KindCategory::Token, + }, + KindSpec { + name: "newline", + category: KindCategory::Token, + }, + KindSpec { + name: "line-continuation", + category: KindCategory::Token, + }, + KindSpec { + name: "comment", + category: KindCategory::Token, + }, + KindSpec { + name: "word", + category: KindCategory::Token, + }, + KindSpec { + name: "operator", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-marker", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-content", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-end", + category: KindCategory::Token, + }, + ], + nodes: &[ + ResultNodeSpec { + kind: 0, + fields: &["descriptor", "operator", "target"], + }, + ResultNodeSpec { + kind: 1, + fields: &["assignment", "open", "close", "separator", "element"], + }, + ResultNodeSpec { + kind: 2, + fields: &["assignment", "name", "argument", "redirect"], + }, + ResultNodeSpec { + kind: 3, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 4, + fields: &["keyword", "condition", "body", "else-body"], + }, + ResultNodeSpec { + kind: 5, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 6, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 7, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 8, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 9, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 10, + fields: &["keyword", "subject", "separator", "clause"], + }, + ResultNodeSpec { + kind: 11, + fields: &[ + "open", + "pattern", + "alternate", + "close", + "body", + "terminator", + ], + }, + ResultNodeSpec { + kind: 12, + fields: &["keyword", "name", "open", "close", "body"], + }, + ResultNodeSpec { + kind: 13, + fields: &["open", "close", "operator", "operand"], + }, + ResultNodeSpec { + kind: 14, + fields: &["open", "close", "expression", "operator"], + }, + ResultNodeSpec { + kind: 15, + fields: &["command", "redirect"], + }, + ResultNodeSpec { + kind: 16, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 17, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 18, + fields: &["keyword", "option", "negate", "command", "operator"], + }, + ResultNodeSpec { + kind: 19, + fields: &["command", "operator"], + }, + ResultNodeSpec { + kind: 20, + fields: &["delimiter", "content"], + }, + ResultNodeSpec { + kind: 21, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 22, + fields: &["text"], + }, + ResultNodeSpec { + kind: 23, + fields: &["text"], + }, + ResultNodeSpec { + kind: 24, + fields: &["text"], + }, + ResultNodeSpec { + kind: 25, + fields: &["part"], + }, + ResultNodeSpec { + kind: 26, + fields: &["part"], + }, + ResultNodeSpec { + kind: 27, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 28, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 29, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 30, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 31, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 32, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 33, + fields: &["open", "index", "close"], + }, + ResultNodeSpec { + kind: 34, + fields: &[ + "open", + "prefix", + "name", + "subscript", + "operator", + "operand", + "close", + ], + }, + ResultNodeSpec { + kind: 35, + fields: &["name", "operator", "value"], + }, + ], + projections: &[ + ResultProjection { + id: "LiteralPart", + kind: 22, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 37, + required: true, + }, + }], + }, + ResultProjection { + id: "EscapeSequence", + kind: 23, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 38, + required: true, + }, + }], + }, + ResultProjection { + id: "SimpleParameter", + kind: 24, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 39, + required: true, + }, + }], + }, + ResultProjection { + id: "Word", + kind: 25, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "HereDocumentLine", + kind: 26, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "SingleQuoted", + kind: 27, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "DoubleQuoted", + kind: 28, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "AnsiCString", + kind: 29, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArithmeticExpansion", + kind: 30, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "CommandSubstitution", + kind: 31, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ProcessSubstitution", + kind: 32, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArraySubscript", + kind: 33, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 45, + required: true, + }, + }, + ProjectionInstruction { + field: "index", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 46, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ParameterExpansion", + kind: 34, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "prefix", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "subscript", + kind: CaptureKind::Node, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 47, + required: true, + }, + }, + ProjectionInstruction { + field: "prefix", + opcode: ProjectionOpcode::Token { + kind: 48, + required: false, + }, + }, + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 49, + required: false, + }, + }, + ProjectionInstruction { + field: "subscript", + opcode: ProjectionOpcode::One { required: false }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 50, + required: false, + }, + }, + ProjectionInstruction { + field: "operand", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 51, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "Assignment", + kind: 35, + captures: &[ + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 52, + required: true, + }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 53, + required: true, + }, + }, + ProjectionInstruction { + field: "value", + opcode: ProjectionOpcode::Many, + }, + ], + }, + ], + }; + use gerbil_parser_runtime::{ + BindingSpec, PartGuard, PartOpcode, PartProfileSpec, PartRule, SubscriptSpec, + }; + pub mod regions { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[]; + } + mod subscript_1 { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "[", + opening: '\u{5b}', + closing: '\u{5d}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[ + gerbil_parser_runtime::scanner::RegionScope { + prefix: "${", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "$(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "$((", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "<(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: ">(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "[", + pairs: &["${", "$(", "$((", "<(", ">(", "["], + }, + ]; + } + pub static PART_PROFILE: PartProfileSpec = PartProfileSpec { + contexts: &[ + "word", + "SingleQuoted", + "DoubleQuoted", + "AnsiCString", + "HereDocument", + ], + literal: "LiteralPart", + assignment: Some(2), + bindings: &[ + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("?@*#$!-_0"), + ]), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("?@*#$!-_0"), + ]), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + }, + prefixes: &[], + operators: &[], + subscript: None, + }, + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("@*?#$!-"), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("@*?#$!-"), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Numeric, + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + }), + }, + prefixes: &["#", "!"], + operators: &[ + ":-", ":=", ":+", ":?", "##", "%%", "//", "^^", ",,", "~~", ":", "-", "=", "+", + "?", "#", "%", "/", "@", "^", ",", "~", + ], + subscript: Some(SubscriptSpec { + prefix: "[", + closing: '\u{5d}', + regions: &subscript_1::REGION, + scopes: subscript_1::SCOPES, + }), + }, + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: Some(1), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 0, + maximum: None, + }, + ]), + prefixes: &[], + operators: &["=", "+="], + subscript: None, + }, + ], + rules: &[ + PartRule { + contexts: &["word"], + prefix: "$'", + guard: PartGuard::Any, + projection: "AnsiCString", + opening: 2, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{27}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "'", + guard: PartGuard::Any, + projection: "SingleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{27}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "\"", + guard: PartGuard::Any, + projection: "DoubleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{22}', + }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "${", + guard: PartGuard::Any, + projection: "ParameterExpansion", + opening: 2, + closing: 1, + opcode: PartOpcode::Parameter { binding: 1 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$((", + guard: PartGuard::Any, + projection: "ArithmeticExpansion", + opening: 3, + closing: 2, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$(", + guard: PartGuard::Any, + projection: "CommandSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word"], + prefix: "<(", + guard: PartGuard::Any, + projection: "ProcessSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word"], + prefix: ">(", + guard: PartGuard::Any, + projection: "ProcessSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "`", + guard: PartGuard::Any, + projection: "CommandSubstitution", + opening: 1, + closing: 1, + opcode: PartOpcode::QuotedBody { + delimiter: '\u{60}', + }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$", + guard: PartGuard::Next, + projection: "SimpleParameter", + opening: 1, + closing: 0, + opcode: PartOpcode::Name { binding: 0 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "AnsiCString"], + prefix: "\\", + guard: PartGuard::Any, + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + PartRule { + contexts: &["HereDocument"], + prefix: "\\", + guard: PartGuard::Characters("$`\\\n"), + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + ], + }; + use gerbil_parser_runtime::{ + CommandChoice, CommandForm, CommandInstruction, CommandProgramSpec, CommandRole, + CommandTokenClass, CommandTrigger, + }; + pub static COMMAND_PROFILE: CommandProgramSpec = CommandProgramSpec { + assignment_tail: "array-tail", + pipeline_head: "pipeline-prefix", + nodes: &[ + ("Redirection", "Redirection"), + ("ArrayAssignment", "ArrayAssignment"), + ("SimpleCommand", "SimpleCommand"), + ("CommandList", "CommandList"), + ("IfCommand", "IfCommand"), + ("WhileCommand", "WhileCommand"), + ("UntilCommand", "UntilCommand"), + ("ForCommand", "ForCommand"), + ("SelectCommand", "SelectCommand"), + ("ArithmeticForCommand", "ArithmeticForCommand"), + ("CaseCommand", "CaseCommand"), + ("CaseClause", "CaseClause"), + ("FunctionDefinition", "FunctionDefinition"), + ("ConditionalCommand", "ConditionalCommand"), + ("ArithmeticCommand", "ArithmeticCommand"), + ("RedirectedCommand", "RedirectedCommand"), + ("BraceGroup", "BraceGroup"), + ("Subshell", "Subshell"), + ("Pipeline", "Pipeline"), + ("AndOrList", "AndOrList"), + ("HereDocument", "HereDocument"), + ("BashFile", "CommandList"), + ], + roles: &[ + CommandRole { + name: "trivia", + classes: &[ + CommandTokenClass { + kind: "horizontal-whitespace", + literals: &[], + }, + CommandTokenClass { + kind: "comment", + literals: &[], + }, + CommandTokenClass { + kind: "line-continuation", + literals: &[], + }, + ], + }, + CommandRole { + name: "separator", + classes: &[ + CommandTokenClass { + kind: "newline", + literals: &[], + }, + CommandTokenClass { + kind: "operator", + literals: &[";", "&"], + }, + ], + }, + CommandRole { + name: "redirect", + classes: &[CommandTokenClass { + kind: "operator", + literals: &[ + "<", ">", ">>", "<>", "<<", "<<-", "<<<", "<&", ">&", ">|", "&>", "&>>", + ], + }], + }, + CommandRole { + name: "here-redirect", + classes: &[CommandTokenClass { + kind: "operator", + literals: &["<<", "<<-"], + }], + }, + CommandRole { + name: "strip-redirect", + classes: &[CommandTokenClass { + kind: "operator", + literals: &["<<-"], + }], + }, + CommandRole { + name: "reserved", + classes: &[CommandTokenClass { + kind: "word", + literals: &[ + "if", "then", "elif", "else", "fi", "while", "until", "do", "done", "for", + "select", "case", "in", "esac", "function", "{", "}", + ], + }], + }, + CommandRole { + name: "conditional-operator", + classes: &[CommandTokenClass { + kind: "word", + literals: &[ + "==", "=", "!=", "=~", "-eq", "-ne", "-lt", "-le", "-gt", "-ge", "-z", + "-n", "!", + ], + }], + }, + CommandRole { + name: "pipeline", + classes: &[CommandTokenClass { + kind: "operator", + literals: &["&&"], + }], + }, + CommandRole { + name: "and-or", + classes: &[CommandTokenClass { + kind: "operator", + literals: &["&&", "||"], + }], + }, + CommandRole { + name: "case-end", + classes: &[CommandTokenClass { + kind: "operator", + literals: &[";;", ";&", ";;&"], + }], + }, + ], + descriptor: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Numeric, + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("{"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: Some(1), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 0, + maximum: None, + }, + gerbil_parser_runtime::TextProfile::Literal("}"), + ])), + }, + forms: &[ + CommandForm { + id: "conditional-branch", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["if"], + }, + priority: 34, + kind: "IfCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["if"], + }, + }, + CommandInstruction::List { + field: "condition", + until: CommandTrigger::Token { + kind: "word", + literals: &["then"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["then"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["elif", "else", "fi"], + }, + allow_empty: false, + }, + CommandInstruction::Many { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["elif"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["elif"], + }, + }, + CommandInstruction::List { + field: "condition", + until: CommandTrigger::Token { + kind: "word", + literals: &["then"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["then"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["elif", "else", "fi"], + }, + allow_empty: false, + }, + ], + }, + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["else"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["else"], + }, + }, + CommandInstruction::List { + field: "else-body", + until: CommandTrigger::Token { + kind: "word", + literals: &["fi"], + }, + allow_empty: false, + }, + ], + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["fi"], + }, + }, + ], + }, + CommandForm { + id: "while-loop", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["while"], + }, + priority: 34, + kind: "WhileCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["while"], + }, + }, + CommandInstruction::List { + field: "condition", + until: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + }, + ], + }, + CommandForm { + id: "until-loop", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["until"], + }, + priority: 34, + kind: "UntilCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["until"], + }, + }, + CommandInstruction::List { + field: "condition", + until: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + }, + ], + }, + CommandForm { + id: "for-loop", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["for"], + }, + priority: 34, + kind: "ForCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["for"], + }, + }, + CommandInstruction::Branch { + trigger: CommandTrigger::Adjacent(&[ + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + ]), + yes: &[ + CommandInstruction::As("ArithmeticForCommand"), + CommandInstruction::Call { + field: "header", + form: "arithmetic", + }, + ], + no: &[ + CommandInstruction::Word("variable"), + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + }, + CommandInstruction::Until { + trigger: CommandTrigger::Role("separator"), + program: &[CommandInstruction::Word("item")], + }, + ], + }, + ], + }, + CommandInstruction::Take { + field: "separator", + trigger: CommandTrigger::Role("separator"), + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + }, + ], + }, + CommandForm { + id: "select-loop", + trigger: CommandTrigger::Lookahead(&[ + CommandTrigger::Token { + kind: "word", + literals: &["select"], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + ]), + priority: 35, + kind: "SelectCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["select"], + }, + }, + CommandInstruction::Word("variable"), + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + }, + CommandInstruction::Until { + trigger: CommandTrigger::Role("separator"), + program: &[CommandInstruction::Word("item")], + }, + ], + }, + CommandInstruction::Take { + field: "separator", + trigger: CommandTrigger::Role("separator"), + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["do"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["done"], + }, + }, + ], + }, + CommandForm { + id: "case-selection", + trigger: CommandTrigger::Lookahead(&[ + CommandTrigger::Token { + kind: "word", + literals: &["case"], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + ]), + priority: 36, + kind: "CaseCommand", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["case"], + }, + }, + CommandInstruction::Word("subject"), + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["in"], + }, + }, + CommandInstruction::Until { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["esac"], + }, + program: &[CommandInstruction::Choose(&[ + CommandChoice { + trigger: CommandTrigger::Role("separator"), + program: &[CommandInstruction::Take { + field: "separator", + trigger: CommandTrigger::Role("separator"), + }], + }, + CommandChoice { + trigger: CommandTrigger::Or(&[ + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + ]), + program: &[CommandInstruction::Node { + field: "clause", + kind: "CaseClause", + program: &[ + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + program: &[CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }], + }, + CommandInstruction::Word("pattern"), + CommandInstruction::Many { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["|"], + }, + program: &[ + CommandInstruction::Take { + field: "alternate", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["|"], + }, + }, + CommandInstruction::Word("pattern"), + ], + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Or(&[ + CommandTrigger::Role("case-end"), + CommandTrigger::Token { + kind: "word", + literals: &["esac"], + }, + ]), + allow_empty: true, + }, + CommandInstruction::Optional { + trigger: CommandTrigger::Role("case-end"), + program: &[CommandInstruction::Take { + field: "terminator", + trigger: CommandTrigger::Role("case-end"), + }], + }, + ], + }], + }, + ])], + }, + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["esac"], + }, + }, + ], + }, + CommandForm { + id: "named-function", + trigger: CommandTrigger::Lookahead(&[ + CommandTrigger::Token { + kind: "word", + literals: &["function"], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + ]), + priority: 35, + kind: "FunctionDefinition", + program: &[ + CommandInstruction::Take { + field: "keyword", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["function"], + }, + }, + CommandInstruction::Word("name"), + CommandInstruction::Optional { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + ], + }, + CommandInstruction::Command("body"), + ], + }, + CommandForm { + id: "function-header", + trigger: CommandTrigger::Lookahead(&[ + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + ]), + priority: 3, + kind: "FunctionDefinition", + program: &[ + CommandInstruction::Word("name"), + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + CommandInstruction::Command("body"), + ], + }, + CommandForm { + id: "conditional", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["[["], + }, + priority: 34, + kind: "ConditionalCommand", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["[["], + }, + }, + CommandInstruction::Until { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["]]"], + }, + program: &[CommandInstruction::Choose(&[ + CommandChoice { + trigger: CommandTrigger::Role("conditional-operator"), + program: &[CommandInstruction::Take { + field: "operator", + trigger: CommandTrigger::Role("conditional-operator"), + }], + }, + CommandChoice { + trigger: CommandTrigger::Token { + kind: "word", + literals: &[], + }, + program: &[CommandInstruction::Word("operand")], + }, + CommandChoice { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[], + }, + program: &[CommandInstruction::Raw("operator")], + }, + ])], + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["]]"], + }, + }, + ], + }, + CommandForm { + id: "arithmetic", + trigger: CommandTrigger::Adjacent(&[ + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + ]), + priority: 35, + kind: "ArithmeticCommand", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Balance { + open: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + close: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + depth: 2, + open_field: "open", + close_field: "close", + program: &[CommandInstruction::Choose(&[ + CommandChoice { + trigger: CommandTrigger::Token { + kind: "word", + literals: &[], + }, + program: &[CommandInstruction::Word("expression")], + }, + CommandChoice { + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[], + }, + program: &[CommandInstruction::Raw("operator")], + }, + ])], + }, + ], + }, + CommandForm { + id: "brace-group", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["开始"], + }, + priority: 0, + kind: "BraceGroup", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["开始"], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "word", + literals: &["结束"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "word", + literals: &["结束"], + }, + }, + ], + }, + CommandForm { + id: "subshell", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + priority: 34, + kind: "Subshell", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::List { + field: "body", + until: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + allow_empty: false, + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + ], + }, + CommandForm { + id: "array-tail", + trigger: CommandTrigger::Manual(Some(&CommandTrigger::Token { + kind: "operator", + literals: &["("], + })), + priority: 0, + kind: "ArrayAssignment", + program: &[ + CommandInstruction::Take { + field: "open", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &["("], + }, + }, + CommandInstruction::Many { + trigger: CommandTrigger::Or(&[ + CommandTrigger::Token { + kind: "newline", + literals: &[], + }, + CommandTrigger::Token { + kind: "word", + literals: &[], + }, + ]), + program: &[CommandInstruction::Choose(&[ + CommandChoice { + trigger: CommandTrigger::Token { + kind: "newline", + literals: &[], + }, + program: &[CommandInstruction::Raw("separator")], + }, + CommandChoice { + trigger: CommandTrigger::Token { + kind: "word", + literals: &[], + }, + program: &[CommandInstruction::Word("element")], + }, + ])], + }, + CommandInstruction::Take { + field: "close", + trigger: CommandTrigger::Token { + kind: "operator", + literals: &[")"], + }, + }, + ], + }, + CommandForm { + id: "pipeline-prefix", + trigger: CommandTrigger::Manual(None), + priority: 0, + kind: "Pipeline", + program: &[], + }, + ], + }; + pub mod scanner { + // @generated from Scheme contextual Scanner IR; do not edit. + #[allow(unused_imports)] + use gerbil_parser_runtime::scanner::{ + BalancedPair, MarkerPolicy, ScannerAction, ScannerCell, ScannerMatcher, ScannerRule, + ScannerSpec, + }; + pub static SCANNER: ScannerSpec = ScannerSpec { + opcode_contract: "gerbil-parser.contextual-scanner-opcodes.v1", + digest: "sha256:350c02574541fb278cf653584eb79c1eb2b5f326ad06dc92a43f8daffbb099a3", + base_grammar_digest: None, + initial_mode: "command", + modes: &["command", "marker", "body"], + positions: &["source"], + rules: &[ + ScannerRule { + name: "space", + mode: "command", + form: "horizontal-whitespace", + matcher: ScannerMatcher::HorizontalWhitespace, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "newline", + mode: "command", + form: "newline", + matcher: ScannerMatcher::Literal("\n"), + rank: 0, + action: ScannerAction::ActivateNext("body"), + }, + ScannerRule { + name: "continuation", + mode: "command", + form: "line-continuation", + matcher: ScannerMatcher::Literal("\\\n"), + rank: 10, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "comment", + mode: "command", + form: "comment", + matcher: ScannerMatcher::LinePrefix { + prefix: "#", + separator: '\u{a}', + }, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "strip", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literal("<<-"), + rank: 30, + action: ScannerAction::ExpectMarkerIn { + strip_tabs: true, + mode: "marker", + }, + }, + ScannerRule { + name: "plain", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literal("<<"), + rank: 30, + action: ScannerAction::ExpectMarkerIn { + strip_tabs: false, + mode: "marker", + }, + }, + ScannerRule { + name: "operator", + mode: "command", + form: "operator", + matcher: ScannerMatcher::Literals(&[ + ";;&", "&>>", "<<<", "&&", "||", "|&", ";;", ";&", ">>", "<>", "<&", ">&", + ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ]), + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "word", + mode: "command", + form: "word", + matcher: ScannerMatcher::UnlessPrefix { + prefixes: &[ + " ", "\t", "\n", "#", "\\\n", ";", "&", "|", "(", ")", "<", ">", + ], + exceptions: &["<(", ">("], + child: &ScannerMatcher::RegionWord { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", + ">>", "<>", "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", + ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }, + }, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-space", + mode: "marker", + form: "horizontal-whitespace", + matcher: ScannerMatcher::HorizontalWhitespace, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-continuation", + mode: "marker", + form: "line-continuation", + matcher: ScannerMatcher::Literal("\\\n"), + rank: 10, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "marker-newline", + mode: "marker", + form: "newline", + matcher: ScannerMatcher::Literal("\n"), + rank: 10, + action: ScannerAction::ActivateNext("body"), + }, + ScannerRule { + name: "marker-word", + mode: "marker", + form: "heredoc-marker", + matcher: ScannerMatcher::UnlessPrefix { + prefixes: &[" ", "\t", "\n", "\\\n"], + exceptions: &[], + child: &ScannerMatcher::RegionWord { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", + ">>", "<>", "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", + ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }, + }, + rank: 0, + action: ScannerAction::EnqueueMarkerIn { + policy: MarkerPolicy::ShellQuoteRemoval, + mode: "command", + }, + }, + ScannerRule { + name: "end", + mode: "body", + form: "heredoc-end", + matcher: ScannerMatcher::MarkerLineAt('\u{a}'), + rank: 10, + action: ScannerAction::FinishMarker { + base: "command", + body: "body", + }, + }, + ScannerRule { + name: "body", + mode: "body", + form: "heredoc-content", + matcher: ScannerMatcher::BodyLineAt('\u{a}'), + rank: 0, + action: ScannerAction::Keep, + }, + ], + cells: &[ + ScannerCell { + mode: "command", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "command", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "command", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "command", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "command", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "command", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "command", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "marker", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ScannerCell { + mode: "body", + position: "source", + form: "horizontal-whitespace", + terminal: "horizontal-whitespace", + }, + ScannerCell { + mode: "body", + position: "source", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "body", + position: "source", + form: "line-continuation", + terminal: "line-continuation", + }, + ScannerCell { + mode: "body", + position: "source", + form: "comment", + terminal: "comment", + }, + ScannerCell { + mode: "body", + position: "source", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "body", + position: "source", + form: "operator", + terminal: "operator", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-marker", + terminal: "heredoc-marker", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-content", + terminal: "heredoc-content", + }, + ScannerCell { + mode: "body", + position: "source", + form: "heredoc-end", + terminal: "heredoc-end", + }, + ], + }; + } + pub static SOURCE: gerbil_parser_runtime::CommandSourceSpec = + gerbil_parser_runtime::CommandSourceSpec { + scanner: &scanner::SCANNER, + parts: &PART_PROFILE, + regions: ®ions::REGION, + results: &RESULT_PROFILE, + commands: &COMMAND_PROFILE, + }; + pub static CONTROLS: &[super::Control] = &[ + super::Control { + source: "开始 echo α; 结束", + accepted: true, + events: &[ + TreeEvent::StartNode(3), + TreeEvent::StartNode(16), + TreeEvent::Token { + kind: 58, + start: 0, + end: 6, + }, + TreeEvent::StartNode(3), + TreeEvent::Token { + kind: 54, + start: 6, + end: 7, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 7, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 11, + end: 12, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 12, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 14, + end: 15, + }, + TreeEvent::Token { + kind: 54, + start: 15, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 58, + start: 16, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", + "open", + "body", + "command", + "name", + "part", + "text", + "argument", + "part", + "text", + "separator", + "close", + ], + links: &[], + }, + super::Control { + source: "echo α && echo 中", + accepted: true, + events: &[ + TreeEvent::StartNode(3), + TreeEvent::StartNode(18), + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 0, + end: 4, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 4, + end: 5, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 7, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 8, + end: 10, + }, + TreeEvent::Token { + kind: 54, + start: 10, + end: 11, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 15, + end: 16, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 16, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", "command", "name", "part", "text", "argument", "part", "text", + "operator", "command", "name", "part", "text", "argument", "part", "text", + ], + links: &[], + }, + super::Control { + source: "{ echo α; }", + accepted: false, + events: &[], + fields: &[], + links: &[], + }, + super::Control { + source: "开始 结束", + accepted: false, + events: &[], + fields: &[], + links: &[], + }, + super::Control { + source: "time echo α", + accepted: true, + events: &[ + TreeEvent::StartNode(3), + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 0, + end: 4, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 4, + end: 5, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 54, + start: 9, + end: 10, + }, + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + "command", "name", "part", "text", "argument", "part", "text", "argument", "part", + "text", + ], + links: &[], + }, + ]; +} diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/fhirpath_profile.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/fhirpath_profile.rs new file mode 100644 index 00000000..28c4349d --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/fhirpath_profile.rs @@ -0,0 +1,19730 @@ +// @generated by gerbil-parser/src/compiler/rust-runtime.ss +// Source authority: canonical Grammar IR v1; do not edit. + +use gerbil_parser_runtime::{ + ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, + Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, +}; + +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "Expression", + category: KindCategory::Node, + }, + KindSpec { + name: "Term", + category: KindCategory::Node, + }, + KindSpec { + name: "Literal", + category: KindCategory::Node, + }, + KindSpec { + name: "ExternalConstant", + category: KindCategory::Node, + }, + KindSpec { + name: "Invocation", + category: KindCategory::Node, + }, + KindSpec { + name: "Function", + category: KindCategory::Node, + }, + KindSpec { + name: "ParamList", + category: KindCategory::Node, + }, + KindSpec { + name: "Quantity", + category: KindCategory::Node, + }, + KindSpec { + name: "Unit", + category: KindCategory::Node, + }, + KindSpec { + name: "DateTimePrecision", + category: KindCategory::Node, + }, + KindSpec { + name: "PluralDateTimePrecision", + category: KindCategory::Node, + }, + KindSpec { + name: "TypeSpecifier", + category: KindCategory::Node, + }, + KindSpec { + name: "QualifiedIdentifier", + category: KindCategory::Node, + }, + KindSpec { + name: "Identifier", + category: KindCategory::Node, + }, + KindSpec { + name: "WhitespaceTrivia", + category: KindCategory::Token, + }, + KindSpec { + name: "CommentTrivia", + category: KindCategory::Token, + }, + KindSpec { + name: "DateTimeToken", + category: KindCategory::Token, + }, + KindSpec { + name: "TimeToken", + category: KindCategory::Token, + }, + KindSpec { + name: "DateToken", + category: KindCategory::Token, + }, + KindSpec { + name: "DelimitedIdentifierToken", + category: KindCategory::Token, + }, + KindSpec { + name: "StringToken", + category: KindCategory::Token, + }, + KindSpec { + name: "NumberToken", + category: KindCategory::Token, + }, + KindSpec { + name: "IdentifierToken", + category: KindCategory::Token, + }, + KindSpec { + name: "PunctuationToken", + category: KindCategory::Token, + }, +]; + +static TERMINALS: &[TerminalSpec] = &[ + TerminalSpec { + name: "whitespace", + syntax_kind: 14, + }, + TerminalSpec { + name: "comment", + syntax_kind: 15, + }, + TerminalSpec { + name: "datetime", + syntax_kind: 16, + }, + TerminalSpec { + name: "time", + syntax_kind: 17, + }, + TerminalSpec { + name: "date", + syntax_kind: 18, + }, + TerminalSpec { + name: "delimited-identifier", + syntax_kind: 19, + }, + TerminalSpec { + name: "string", + syntax_kind: 20, + }, + TerminalSpec { + name: "number", + syntax_kind: 21, + }, + TerminalSpec { + name: "identifier", + syntax_kind: 22, + }, + TerminalSpec { + name: "punctuation", + syntax_kind: 23, + }, +]; + +static LEXICAL_RULES: &[LexicalRule] = &[ + LexicalRule { + terminal: "whitespace", + expression: LexicalExpr::Whitespace, + precedence: 0, + extra: true, + }, + LexicalRule { + terminal: "comment", + expression: LexicalExpr::Choice(&[ + LexicalExpr::LineComment(&["//"]), + LexicalExpr::BlockComment { + opening: "/*", + closing: "*/", + }, + ]), + precedence: 0, + extra: true, + }, + LexicalRule { + terminal: "datetime", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("@"), + gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 4, + maximum: Some(4), + }, + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("-"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("-"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("-"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("-"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + ]), + otherwise: None, + }, + ]), + otherwise: None, + }, + ]), + gerbil_parser_runtime::TextProfile::Literal("T"), + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Numeric, + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters(":"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal(":"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters(":"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal(":"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::Optional( + &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("."), + gerbil_parser_runtime::TextProfile::Run { + class: + gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + ]), + ), + ]), + otherwise: None, + }, + ]), + otherwise: None, + }, + ]), + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("Z"), + body: &gerbil_parser_runtime::TextProfile::Literal("Z"), + otherwise: Some(&gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("+-"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("+-"), + minimum: 1, + maximum: Some(1), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::Literal(":"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + ]), + otherwise: None, + }), + }, + ]), + otherwise: None, + }, + ])), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "time", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("@T"), + gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters(":"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal(":"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters(":"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal(":"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::Optional( + &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("."), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + ]), + ), + ]), + otherwise: None, + }, + ]), + otherwise: None, + }, + ]), + ])), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "date", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("@"), + gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 4, + maximum: Some(4), + }, + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("-"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("-"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("-"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("-"), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 2, + maximum: Some(2), + }, + ]), + otherwise: None, + }, + ]), + otherwise: None, + }, + ]), + gerbil_parser_runtime::TextProfile::NotNext( + gerbil_parser_runtime::TextClass::Characters("T"), + ), + ])), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "delimited-identifier", + expression: LexicalExpr::QuotedStringProfile { + delimiter: "`", + escapes: "`'\\/fnrt", + unicode_width: 4, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "string", + expression: LexicalExpr::QuotedStringProfile { + delimiter: "'", + escapes: "`'\\/fnrt", + unicode_width: 4, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "number", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + gerbil_parser_runtime::TextProfile::Optional( + &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("."), + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + ]), + ), + ])), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "identifier", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::AsciiLetter, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: Some(1), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::AsciiLetter, + gerbil_parser_runtime::TextClass::Characters("_"), + gerbil_parser_runtime::TextClass::Numeric, + ]), + minimum: 0, + maximum: None, + }, + ])), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "punctuation", + expression: LexicalExpr::Literals(&[ + "<=", ">=", "!=", "!~", "$this", "$index", "$total", ".", "[", "]", "+", "-", "*", "/", + "&", "|", "<", ">", "=", "~", "%", "(", ")", "{", "}", ",", + ]), + precedence: 0, + extra: false, + }, +]; + +static ACTION_0: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_1: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Accept, +}]; +static ACTION_2: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(1), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(1), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(1), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(1), + }, +]; +static ACTION_3: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(2), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(2), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(2), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(2), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Shift(62), + }, +]; +static ACTION_4: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(6), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(6), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(6), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(6), + }, +]; +static ACTION_5: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(7), + }, +]; +static ACTION_6: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(13), + }, +]; +static ACTION_7: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(14), + }, +]; +static ACTION_8: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(18), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(18), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(18), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(18), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(18), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(18), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(18), + }, +]; +static ACTION_9: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(19), + }, +]; +static ACTION_10: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(25), + }, +]; +static ACTION_11: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(26), + }, +]; +static ACTION_12: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(34), + }, +]; +static ACTION_13: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(35), + }, +]; +static ACTION_14: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(43), + }, +]; +static ACTION_15: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(44), + }, +]; +static ACTION_16: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(48), + }, +]; +static ACTION_17: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(49), + }, +]; +static ACTION_18: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(55), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(55), + }, +]; +static ACTION_19: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(56), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(56), + }, +]; +static ACTION_20: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(63), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(63), + }, +]; +static ACTION_21: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(64), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(64), + }, +]; +static ACTION_22: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(72), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(72), + }, +]; +static ACTION_23: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(73), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(73), + }, +]; +static ACTION_24: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(74), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(74), + }, +]; +static ACTION_25: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_26: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(76), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(76), + }, +]; +static ACTION_27: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(77), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(77), + }, +]; +static ACTION_28: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(78), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(78), + }, +]; +static ACTION_29: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(83), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(83), + }, +]; +static ACTION_30: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(84), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(84), + }, +]; +static ACTION_31: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(85), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(85), + }, +]; +static ACTION_32: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(86), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(86), + }, +]; +static ACTION_33: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_34: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(88), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(88), + }, +]; +static ACTION_35: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal("}"), + action: ParserAction::Shift(77), +}]; +static ACTION_36: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(90), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(90), + }, +]; +static ACTION_37: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(91), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(91), + }, +]; +static ACTION_38: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(92), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(92), + }, +]; +static ACTION_39: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(93), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(81), + }, + ActionEntry { + terminal: Terminal::Literal("year"), + action: ParserAction::Shift(83), + }, + ActionEntry { + terminal: Terminal::Literal("month"), + action: ParserAction::Shift(84), + }, + ActionEntry { + terminal: Terminal::Literal("week"), + action: ParserAction::Shift(85), + }, + ActionEntry { + terminal: Terminal::Literal("day"), + action: ParserAction::Shift(86), + }, + ActionEntry { + terminal: Terminal::Literal("hour"), + action: ParserAction::Shift(87), + }, + ActionEntry { + terminal: Terminal::Literal("minute"), + action: ParserAction::Shift(88), + }, + ActionEntry { + terminal: Terminal::Literal("second"), + action: ParserAction::Shift(89), + }, + ActionEntry { + terminal: Terminal::Literal("millisecond"), + action: ParserAction::Shift(90), + }, + ActionEntry { + terminal: Terminal::Literal("years"), + action: ParserAction::Shift(92), + }, + ActionEntry { + terminal: Terminal::Literal("months"), + action: ParserAction::Shift(93), + }, + ActionEntry { + terminal: Terminal::Literal("weeks"), + action: ParserAction::Shift(94), + }, + ActionEntry { + terminal: Terminal::Literal("days"), + action: ParserAction::Shift(95), + }, + ActionEntry { + terminal: Terminal::Literal("hours"), + action: ParserAction::Shift(96), + }, + ActionEntry { + terminal: Terminal::Literal("minutes"), + action: ParserAction::Shift(97), + }, + ActionEntry { + terminal: Terminal::Literal("seconds"), + action: ParserAction::Shift(98), + }, + ActionEntry { + terminal: Terminal::Literal("milliseconds"), + action: ParserAction::Shift(99), + }, +]; +static ACTION_40: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(94), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(94), + }, +]; +static ACTION_41: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(95), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(95), + }, +]; +static ACTION_42: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(96), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(96), + }, +]; +static ACTION_43: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(97), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(97), + }, +]; +static ACTION_44: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(98), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(98), + }, +]; +static ACTION_45: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(102), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_46: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(102), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(102), + }, +]; +static ACTION_47: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(103), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(103), + }, +]; +static ACTION_48: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(104), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(104), + }, +]; +static ACTION_49: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(105), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(105), + }, +]; +static ACTION_50: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(106), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(106), + }, +]; +static ACTION_51: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(107), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(107), + }, +]; +static ACTION_52: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(108), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(108), + }, +]; +static ACTION_53: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(112), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(112), + }, +]; +static ACTION_54: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(119), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(119), + }, +]; +static ACTION_55: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(148), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(148), + }, +]; +static ACTION_56: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(149), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(149), + }, +]; +static ACTION_57: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(150), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(150), + }, +]; +static ACTION_58: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(151), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(151), + }, +]; +static ACTION_59: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(152), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(152), + }, +]; +static ACTION_60: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(153), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(153), + }, +]; +static ACTION_61: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(154), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(154), + }, +]; +static ACTION_62: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_63: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(4), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(4), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(4), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(4), + }, +]; +static ACTION_64: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(5), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(5), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(5), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(5), + }, +]; +static ACTION_65: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Shift(106), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Shift(107), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(12), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(12), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(12), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(12), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(12), + }, +]; +static ACTION_66: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Shift(110), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(17), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(17), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(17), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(17), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(17), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(17), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(17), + }, +]; +static ACTION_67: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(112), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(113), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(24), + }, +]; +static ACTION_68: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Shift(116), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Shift(117), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Shift(118), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Shift(119), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(33), + }, +]; +static ACTION_69: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Shift(122), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Shift(123), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Shift(124), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Shift(125), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(42), + }, +]; +static ACTION_70: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Shift(128), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(47), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(47), + }, +]; +static ACTION_71: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(130), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(131), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(54), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(54), + }, +]; +static ACTION_72: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(134), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(135), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Shift(136), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(62), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(62), + }, +]; +static ACTION_73: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Shift(139), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Shift(140), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Shift(141), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Shift(142), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(71), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(71), + }, +]; +static ACTION_74: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(75), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(75), + }, +]; +static ACTION_75: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Shift(145), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Shift(146), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(82), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(82), + }, +]; +static ACTION_76: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Shift(148), +}]; +static ACTION_77: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(89), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(89), + }, +]; +static ACTION_78: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(118), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(118), + }, +]; +static ACTION_79: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(120), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(120), + }, +]; +static ACTION_80: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(121), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(121), + }, +]; +static ACTION_81: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(122), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(122), + }, +]; +static ACTION_82: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(123), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(123), + }, +]; +static ACTION_83: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(124), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(124), + }, +]; +static ACTION_84: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(125), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(125), + }, +]; +static ACTION_85: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(126), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(126), + }, +]; +static ACTION_86: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(127), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(127), + }, +]; +static ACTION_87: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(128), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(128), + }, +]; +static ACTION_88: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(129), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(129), + }, +]; +static ACTION_89: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(130), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(130), + }, +]; +static ACTION_90: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(131), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(131), + }, +]; +static ACTION_91: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(132), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(132), + }, +]; +static ACTION_92: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(133), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(133), + }, +]; +static ACTION_93: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(134), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(134), + }, +]; +static ACTION_94: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(135), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(135), + }, +]; +static ACTION_95: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(136), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(136), + }, +]; +static ACTION_96: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(137), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(137), + }, +]; +static ACTION_97: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(138), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(138), + }, +]; +static ACTION_98: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(139), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(139), + }, +]; +static ACTION_99: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(140), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(140), + }, +]; +static ACTION_100: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(141), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(141), + }, +]; +static ACTION_101: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(99), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(99), + }, +]; +static ACTION_102: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(100), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(100), + }, +]; +static ACTION_103: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(101), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(101), + }, +]; +static ACTION_104: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(109), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_105: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(3), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(3), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(3), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(3), + }, +]; +static ACTION_106: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(8), + }, +]; +static ACTION_107: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(9), + }, +]; +static ACTION_108: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_109: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(11), + }, +]; +static ACTION_110: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_111: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(16), + }, +]; +static ACTION_112: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(20), + }, +]; +static ACTION_113: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(21), + }, +]; +static ACTION_114: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_115: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(23), + }, +]; +static ACTION_116: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(27), + }, +]; +static ACTION_117: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(28), + }, +]; +static ACTION_118: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(29), + }, +]; +static ACTION_119: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(30), + }, +]; +static ACTION_120: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_121: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(32), + }, +]; +static ACTION_122: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(36), + }, +]; +static ACTION_123: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(37), + }, +]; +static ACTION_124: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(38), + }, +]; +static ACTION_125: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(39), + }, +]; +static ACTION_126: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_127: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(41), + }, +]; +static ACTION_128: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_129: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(46), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(46), + }, +]; +static ACTION_130: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(50), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(50), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(50), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(50), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(50), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(50), + }, +]; +static ACTION_131: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(51), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(51), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(51), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(51), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(51), + }, +]; +static ACTION_132: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_133: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(53), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(53), + }, +]; +static ACTION_134: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(57), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(57), + }, +]; +static ACTION_135: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(58), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(58), + }, +]; +static ACTION_136: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(59), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(59), + }, +]; +static ACTION_137: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_138: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(61), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(61), + }, +]; +static ACTION_139: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(65), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(65), + }, +]; +static ACTION_140: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(66), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(66), + }, +]; +static ACTION_141: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(67), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(67), + }, +]; +static ACTION_142: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(68), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Reduce(68), + }, +]; +static ACTION_143: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_144: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(70), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(70), + }, +]; +static ACTION_145: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_146: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_147: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(81), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(81), + }, +]; +static ACTION_148: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(87), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(87), + }, +]; +static ACTION_149: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(110), +}]; +static ACTION_150: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Shift(167), +}]; +static ACTION_151: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(113), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(113), + }, +]; +static ACTION_152: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(117), +}]; +static ACTION_153: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(10), + }, +]; +static ACTION_154: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(15), + }, +]; +static ACTION_155: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(22), + }, +]; +static ACTION_156: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(31), + }, +]; +static ACTION_157: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(40), + }, +]; +static ACTION_158: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(45), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(45), + }, +]; +static ACTION_159: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(52), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(52), + }, +]; +static ACTION_160: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(142), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(142), + }, +]; +static ACTION_161: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(143), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(143), + }, +]; +static ACTION_162: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(147), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(147), + }, +]; +static ACTION_163: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(60), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(60), + }, +]; +static ACTION_164: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(69), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(69), + }, +]; +static ACTION_165: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(79), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(79), + }, +]; +static ACTION_166: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Shift(170), +}]; +static ACTION_167: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(111), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(111), + }, +]; +static ACTION_168: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Shift(171), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(116), + }, +]; +static ACTION_169: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Shift(173), + }, + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(146), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(146), + }, +]; +static ACTION_170: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("&"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("*"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("/"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("div"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("mod"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("["), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(80), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(80), + }, +]; +static ACTION_171: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal("+"), + action: ParserAction::Shift(23), + }, + ActionEntry { + terminal: Terminal::Literal("-"), + action: ParserAction::Shift(24), + }, + ActionEntry { + terminal: Terminal::Literal("("), + action: ParserAction::Shift(33), + }, + ActionEntry { + terminal: Terminal::Literal("{"), + action: ParserAction::Shift(35), + }, + ActionEntry { + terminal: Terminal::Literal("true"), + action: ParserAction::Shift(36), + }, + ActionEntry { + terminal: Terminal::Literal("false"), + action: ParserAction::Shift(37), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(39), + }, + ActionEntry { + terminal: Terminal::Token("date"), + action: ParserAction::Shift(40), + }, + ActionEntry { + terminal: Terminal::Token("datetime"), + action: ParserAction::Shift(41), + }, + ActionEntry { + terminal: Terminal::Token("time"), + action: ParserAction::Shift(42), + }, + ActionEntry { + terminal: Terminal::Literal("%"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Literal("$this"), + action: ParserAction::Shift(49), + }, + ActionEntry { + terminal: Terminal::Literal("$index"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Literal("$total"), + action: ParserAction::Shift(51), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_172: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(115), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(115), + }, +]; +static ACTION_173: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(55), + }, + ActionEntry { + terminal: Terminal::Token("delimited-identifier"), + action: ParserAction::Shift(56), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Shift(57), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Shift(58), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Shift(59), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Shift(60), + }, +]; +static ACTION_174: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(145), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(145), + }, +]; +static ACTION_175: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(114), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(114), + }, +]; +static ACTION_176: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("implies"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("or"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("xor"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("and"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("in"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("contains"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("~"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("!="), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("!~"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("<="), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("<"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal(">"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal(">="), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("|"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("is"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("as"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("."), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal("]"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal(")"), + action: ParserAction::Reduce(144), + }, + ActionEntry { + terminal: Terminal::Literal(","), + action: ParserAction::Reduce(144), + }, +]; +static ACTIONS: &[&[ActionEntry]] = &[ + ACTION_0, ACTION_1, ACTION_2, ACTION_3, ACTION_4, ACTION_5, ACTION_6, ACTION_7, ACTION_8, + ACTION_9, ACTION_10, ACTION_11, ACTION_12, ACTION_13, ACTION_14, ACTION_15, ACTION_16, + ACTION_17, ACTION_18, ACTION_19, ACTION_20, ACTION_21, ACTION_22, ACTION_23, ACTION_24, + ACTION_25, ACTION_26, ACTION_27, ACTION_28, ACTION_29, ACTION_30, ACTION_31, ACTION_32, + ACTION_33, ACTION_34, ACTION_35, ACTION_36, ACTION_37, ACTION_38, ACTION_39, ACTION_40, + ACTION_41, ACTION_42, ACTION_43, ACTION_44, ACTION_45, ACTION_46, ACTION_47, ACTION_48, + ACTION_49, ACTION_50, ACTION_51, ACTION_52, ACTION_53, ACTION_54, ACTION_55, ACTION_56, + ACTION_57, ACTION_58, ACTION_59, ACTION_60, ACTION_61, ACTION_62, ACTION_63, ACTION_64, + ACTION_65, ACTION_66, ACTION_67, ACTION_68, ACTION_69, ACTION_70, ACTION_71, ACTION_72, + ACTION_73, ACTION_74, ACTION_75, ACTION_76, ACTION_77, ACTION_78, ACTION_79, ACTION_80, + ACTION_81, ACTION_82, ACTION_83, ACTION_84, ACTION_85, ACTION_86, ACTION_87, ACTION_88, + ACTION_89, ACTION_90, ACTION_91, ACTION_92, ACTION_93, ACTION_94, ACTION_95, ACTION_96, + ACTION_97, ACTION_98, ACTION_99, ACTION_100, ACTION_101, ACTION_102, ACTION_103, ACTION_104, + ACTION_105, ACTION_106, ACTION_107, ACTION_108, ACTION_109, ACTION_110, ACTION_111, ACTION_112, + ACTION_113, ACTION_114, ACTION_115, ACTION_116, ACTION_117, ACTION_118, ACTION_119, ACTION_120, + ACTION_121, ACTION_122, ACTION_123, ACTION_124, ACTION_125, ACTION_126, ACTION_127, ACTION_128, + ACTION_129, ACTION_130, ACTION_131, ACTION_132, ACTION_133, ACTION_134, ACTION_135, ACTION_136, + ACTION_137, ACTION_138, ACTION_139, ACTION_140, ACTION_141, ACTION_142, ACTION_143, ACTION_144, + ACTION_145, ACTION_146, ACTION_147, ACTION_148, ACTION_149, ACTION_150, ACTION_151, ACTION_152, + ACTION_153, ACTION_154, ACTION_155, ACTION_156, ACTION_157, ACTION_158, ACTION_159, ACTION_160, + ACTION_161, ACTION_162, ACTION_163, ACTION_164, ACTION_165, ACTION_166, ACTION_167, ACTION_168, + ACTION_169, ACTION_170, ACTION_171, ACTION_172, ACTION_173, ACTION_174, ACTION_175, ACTION_176, +]; + +static GOTO_0: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "expression", + state: 1, + }, + GotoEntry { + nonterminal: "implies-expression", + state: 2, + }, + GotoEntry { + nonterminal: "or-expression", + state: 3, + }, + GotoEntry { + nonterminal: "$implies-expression.0", + state: 4, + }, + GotoEntry { + nonterminal: "and-expression", + state: 5, + }, + GotoEntry { + nonterminal: "$or-expression.3", + state: 6, + }, + GotoEntry { + nonterminal: "membership-expression", + state: 7, + }, + GotoEntry { + nonterminal: "$and-expression.7", + state: 8, + }, + GotoEntry { + nonterminal: "equality-expression", + state: 9, + }, + GotoEntry { + nonterminal: "$membership-expression.10", + state: 10, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_1: &[GotoEntry] = &[]; +static GOTO_2: &[GotoEntry] = &[]; +static GOTO_3: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$implies-expression.2", + state: 63, + }, + GotoEntry { + nonterminal: "$implies-expression.1", + state: 64, + }, +]; +static GOTO_4: &[GotoEntry] = &[]; +static GOTO_5: &[GotoEntry] = &[GotoEntry { + nonterminal: "$or-expression.4", + state: 65, +}]; +static GOTO_6: &[GotoEntry] = &[]; +static GOTO_7: &[GotoEntry] = &[GotoEntry { + nonterminal: "$and-expression.8", + state: 66, +}]; +static GOTO_8: &[GotoEntry] = &[]; +static GOTO_9: &[GotoEntry] = &[GotoEntry { + nonterminal: "$membership-expression.11", + state: 67, +}]; +static GOTO_10: &[GotoEntry] = &[]; +static GOTO_11: &[GotoEntry] = &[GotoEntry { + nonterminal: "$equality-expression.15", + state: 68, +}]; +static GOTO_12: &[GotoEntry] = &[]; +static GOTO_13: &[GotoEntry] = &[GotoEntry { + nonterminal: "$inequality-expression.19", + state: 69, +}]; +static GOTO_14: &[GotoEntry] = &[]; +static GOTO_15: &[GotoEntry] = &[GotoEntry { + nonterminal: "$union-expression.23", + state: 70, +}]; +static GOTO_16: &[GotoEntry] = &[]; +static GOTO_17: &[GotoEntry] = &[GotoEntry { + nonterminal: "$type-expression.26", + state: 71, +}]; +static GOTO_18: &[GotoEntry] = &[]; +static GOTO_19: &[GotoEntry] = &[GotoEntry { + nonterminal: "$additive-expression.30", + state: 72, +}]; +static GOTO_20: &[GotoEntry] = &[]; +static GOTO_21: &[GotoEntry] = &[GotoEntry { + nonterminal: "$multiplicative-expression.34", + state: 73, +}]; +static GOTO_22: &[GotoEntry] = &[]; +static GOTO_23: &[GotoEntry] = &[]; +static GOTO_24: &[GotoEntry] = &[]; +static GOTO_25: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 74, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_26: &[GotoEntry] = &[]; +static GOTO_27: &[GotoEntry] = &[]; +static GOTO_28: &[GotoEntry] = &[GotoEntry { + nonterminal: "$postfix-expression.40", + state: 75, +}]; +static GOTO_29: &[GotoEntry] = &[]; +static GOTO_30: &[GotoEntry] = &[]; +static GOTO_31: &[GotoEntry] = &[]; +static GOTO_32: &[GotoEntry] = &[]; +static GOTO_33: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "implies-expression", + state: 2, + }, + GotoEntry { + nonterminal: "or-expression", + state: 3, + }, + GotoEntry { + nonterminal: "$implies-expression.0", + state: 4, + }, + GotoEntry { + nonterminal: "and-expression", + state: 5, + }, + GotoEntry { + nonterminal: "$or-expression.3", + state: 6, + }, + GotoEntry { + nonterminal: "membership-expression", + state: 7, + }, + GotoEntry { + nonterminal: "$and-expression.7", + state: 8, + }, + GotoEntry { + nonterminal: "equality-expression", + state: 9, + }, + GotoEntry { + nonterminal: "$membership-expression.10", + state: 10, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "expression", + state: 76, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_34: &[GotoEntry] = &[]; +static GOTO_35: &[GotoEntry] = &[]; +static GOTO_36: &[GotoEntry] = &[]; +static GOTO_37: &[GotoEntry] = &[]; +static GOTO_38: &[GotoEntry] = &[]; +static GOTO_39: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "unit", + state: 78, + }, + GotoEntry { + nonterminal: "date-time-precision", + state: 79, + }, + GotoEntry { + nonterminal: "plural-date-time-precision", + state: 80, + }, + GotoEntry { + nonterminal: "$unit.53", + state: 82, + }, + GotoEntry { + nonterminal: "$date-time-precision.54", + state: 91, + }, + GotoEntry { + nonterminal: "$plural-date-time-precision.55", + state: 100, + }, +]; +static GOTO_40: &[GotoEntry] = &[]; +static GOTO_41: &[GotoEntry] = &[]; +static GOTO_42: &[GotoEntry] = &[]; +static GOTO_43: &[GotoEntry] = &[]; +static GOTO_44: &[GotoEntry] = &[]; +static GOTO_45: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "identifier", + state: 101, + }, + GotoEntry { + nonterminal: "$external-constant.45", + state: 103, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_46: &[GotoEntry] = &[]; +static GOTO_47: &[GotoEntry] = &[]; +static GOTO_48: &[GotoEntry] = &[]; +static GOTO_49: &[GotoEntry] = &[]; +static GOTO_50: &[GotoEntry] = &[]; +static GOTO_51: &[GotoEntry] = &[]; +static GOTO_52: &[GotoEntry] = &[]; +static GOTO_53: &[GotoEntry] = &[]; +static GOTO_54: &[GotoEntry] = &[]; +static GOTO_55: &[GotoEntry] = &[]; +static GOTO_56: &[GotoEntry] = &[]; +static GOTO_57: &[GotoEntry] = &[]; +static GOTO_58: &[GotoEntry] = &[]; +static GOTO_59: &[GotoEntry] = &[]; +static GOTO_60: &[GotoEntry] = &[]; +static GOTO_61: &[GotoEntry] = &[]; +static GOTO_62: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "implies-expression", + state: 105, + }, + GotoEntry { + nonterminal: "or-expression", + state: 3, + }, + GotoEntry { + nonterminal: "$implies-expression.0", + state: 4, + }, + GotoEntry { + nonterminal: "and-expression", + state: 5, + }, + GotoEntry { + nonterminal: "$or-expression.3", + state: 6, + }, + GotoEntry { + nonterminal: "membership-expression", + state: 7, + }, + GotoEntry { + nonterminal: "$and-expression.7", + state: 8, + }, + GotoEntry { + nonterminal: "equality-expression", + state: 9, + }, + GotoEntry { + nonterminal: "$membership-expression.10", + state: 10, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_63: &[GotoEntry] = &[]; +static GOTO_64: &[GotoEntry] = &[]; +static GOTO_65: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$or-expression.6", + state: 108, + }, + GotoEntry { + nonterminal: "$or-expression.5", + state: 109, + }, +]; +static GOTO_66: &[GotoEntry] = &[GotoEntry { + nonterminal: "$and-expression.9", + state: 111, +}]; +static GOTO_67: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$membership-expression.13", + state: 114, + }, + GotoEntry { + nonterminal: "$membership-expression.12", + state: 115, + }, +]; +static GOTO_68: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$equality-expression.17", + state: 120, + }, + GotoEntry { + nonterminal: "$equality-expression.16", + state: 121, + }, +]; +static GOTO_69: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$inequality-expression.21", + state: 126, + }, + GotoEntry { + nonterminal: "$inequality-expression.20", + state: 127, + }, +]; +static GOTO_70: &[GotoEntry] = &[GotoEntry { + nonterminal: "$union-expression.24", + state: 129, +}]; +static GOTO_71: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$type-expression.28", + state: 132, + }, + GotoEntry { + nonterminal: "$type-expression.27", + state: 133, + }, +]; +static GOTO_72: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$additive-expression.32", + state: 137, + }, + GotoEntry { + nonterminal: "$additive-expression.31", + state: 138, + }, +]; +static GOTO_73: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$multiplicative-expression.36", + state: 143, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.35", + state: 144, + }, +]; +static GOTO_74: &[GotoEntry] = &[]; +static GOTO_75: &[GotoEntry] = &[GotoEntry { + nonterminal: "$postfix-expression.41", + state: 147, +}]; +static GOTO_76: &[GotoEntry] = &[]; +static GOTO_77: &[GotoEntry] = &[]; +static GOTO_78: &[GotoEntry] = &[]; +static GOTO_79: &[GotoEntry] = &[]; +static GOTO_80: &[GotoEntry] = &[]; +static GOTO_81: &[GotoEntry] = &[]; +static GOTO_82: &[GotoEntry] = &[]; +static GOTO_83: &[GotoEntry] = &[]; +static GOTO_84: &[GotoEntry] = &[]; +static GOTO_85: &[GotoEntry] = &[]; +static GOTO_86: &[GotoEntry] = &[]; +static GOTO_87: &[GotoEntry] = &[]; +static GOTO_88: &[GotoEntry] = &[]; +static GOTO_89: &[GotoEntry] = &[]; +static GOTO_90: &[GotoEntry] = &[]; +static GOTO_91: &[GotoEntry] = &[]; +static GOTO_92: &[GotoEntry] = &[]; +static GOTO_93: &[GotoEntry] = &[]; +static GOTO_94: &[GotoEntry] = &[]; +static GOTO_95: &[GotoEntry] = &[]; +static GOTO_96: &[GotoEntry] = &[]; +static GOTO_97: &[GotoEntry] = &[]; +static GOTO_98: &[GotoEntry] = &[]; +static GOTO_99: &[GotoEntry] = &[]; +static GOTO_100: &[GotoEntry] = &[]; +static GOTO_101: &[GotoEntry] = &[]; +static GOTO_102: &[GotoEntry] = &[]; +static GOTO_103: &[GotoEntry] = &[]; +static GOTO_104: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "implies-expression", + state: 2, + }, + GotoEntry { + nonterminal: "or-expression", + state: 3, + }, + GotoEntry { + nonterminal: "$implies-expression.0", + state: 4, + }, + GotoEntry { + nonterminal: "and-expression", + state: 5, + }, + GotoEntry { + nonterminal: "$or-expression.3", + state: 6, + }, + GotoEntry { + nonterminal: "membership-expression", + state: 7, + }, + GotoEntry { + nonterminal: "$and-expression.7", + state: 8, + }, + GotoEntry { + nonterminal: "equality-expression", + state: 9, + }, + GotoEntry { + nonterminal: "$membership-expression.10", + state: 10, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "param-list", + state: 149, + }, + GotoEntry { + nonterminal: "$function.48", + state: 150, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "expression", + state: 151, + }, + GotoEntry { + nonterminal: "$param-list.49", + state: 152, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_105: &[GotoEntry] = &[]; +static GOTO_106: &[GotoEntry] = &[]; +static GOTO_107: &[GotoEntry] = &[]; +static GOTO_108: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "and-expression", + state: 153, + }, + GotoEntry { + nonterminal: "membership-expression", + state: 7, + }, + GotoEntry { + nonterminal: "$and-expression.7", + state: 8, + }, + GotoEntry { + nonterminal: "equality-expression", + state: 9, + }, + GotoEntry { + nonterminal: "$membership-expression.10", + state: 10, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_109: &[GotoEntry] = &[]; +static GOTO_110: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "membership-expression", + state: 154, + }, + GotoEntry { + nonterminal: "equality-expression", + state: 9, + }, + GotoEntry { + nonterminal: "$membership-expression.10", + state: 10, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_111: &[GotoEntry] = &[]; +static GOTO_112: &[GotoEntry] = &[]; +static GOTO_113: &[GotoEntry] = &[]; +static GOTO_114: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "equality-expression", + state: 155, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_115: &[GotoEntry] = &[]; +static GOTO_116: &[GotoEntry] = &[]; +static GOTO_117: &[GotoEntry] = &[]; +static GOTO_118: &[GotoEntry] = &[]; +static GOTO_119: &[GotoEntry] = &[]; +static GOTO_120: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "inequality-expression", + state: 156, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_121: &[GotoEntry] = &[]; +static GOTO_122: &[GotoEntry] = &[]; +static GOTO_123: &[GotoEntry] = &[]; +static GOTO_124: &[GotoEntry] = &[]; +static GOTO_125: &[GotoEntry] = &[]; +static GOTO_126: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "union-expression", + state: 157, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_127: &[GotoEntry] = &[]; +static GOTO_128: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "type-expression", + state: 158, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_129: &[GotoEntry] = &[]; +static GOTO_130: &[GotoEntry] = &[]; +static GOTO_131: &[GotoEntry] = &[]; +static GOTO_132: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "type-specifier", + state: 159, + }, + GotoEntry { + nonterminal: "qualified-identifier", + state: 160, + }, + GotoEntry { + nonterminal: "identifier", + state: 161, + }, + GotoEntry { + nonterminal: "$qualified-identifier.56", + state: 162, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_133: &[GotoEntry] = &[]; +static GOTO_134: &[GotoEntry] = &[]; +static GOTO_135: &[GotoEntry] = &[]; +static GOTO_136: &[GotoEntry] = &[]; +static GOTO_137: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "multiplicative-expression", + state: 163, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_138: &[GotoEntry] = &[]; +static GOTO_139: &[GotoEntry] = &[]; +static GOTO_140: &[GotoEntry] = &[]; +static GOTO_141: &[GotoEntry] = &[]; +static GOTO_142: &[GotoEntry] = &[]; +static GOTO_143: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "polarity-expression", + state: 164, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_144: &[GotoEntry] = &[]; +static GOTO_145: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "invocation", + state: 165, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_146: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "implies-expression", + state: 2, + }, + GotoEntry { + nonterminal: "or-expression", + state: 3, + }, + GotoEntry { + nonterminal: "$implies-expression.0", + state: 4, + }, + GotoEntry { + nonterminal: "and-expression", + state: 5, + }, + GotoEntry { + nonterminal: "$or-expression.3", + state: 6, + }, + GotoEntry { + nonterminal: "membership-expression", + state: 7, + }, + GotoEntry { + nonterminal: "$and-expression.7", + state: 8, + }, + GotoEntry { + nonterminal: "equality-expression", + state: 9, + }, + GotoEntry { + nonterminal: "$membership-expression.10", + state: 10, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "expression", + state: 166, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_147: &[GotoEntry] = &[]; +static GOTO_148: &[GotoEntry] = &[]; +static GOTO_149: &[GotoEntry] = &[]; +static GOTO_150: &[GotoEntry] = &[]; +static GOTO_151: &[GotoEntry] = &[GotoEntry { + nonterminal: "$param-list.50", + state: 168, +}]; +static GOTO_152: &[GotoEntry] = &[]; +static GOTO_153: &[GotoEntry] = &[]; +static GOTO_154: &[GotoEntry] = &[]; +static GOTO_155: &[GotoEntry] = &[]; +static GOTO_156: &[GotoEntry] = &[]; +static GOTO_157: &[GotoEntry] = &[]; +static GOTO_158: &[GotoEntry] = &[]; +static GOTO_159: &[GotoEntry] = &[]; +static GOTO_160: &[GotoEntry] = &[]; +static GOTO_161: &[GotoEntry] = &[GotoEntry { + nonterminal: "$qualified-identifier.57", + state: 169, +}]; +static GOTO_162: &[GotoEntry] = &[]; +static GOTO_163: &[GotoEntry] = &[]; +static GOTO_164: &[GotoEntry] = &[]; +static GOTO_165: &[GotoEntry] = &[]; +static GOTO_166: &[GotoEntry] = &[]; +static GOTO_167: &[GotoEntry] = &[]; +static GOTO_168: &[GotoEntry] = &[GotoEntry { + nonterminal: "$param-list.51", + state: 172, +}]; +static GOTO_169: &[GotoEntry] = &[GotoEntry { + nonterminal: "$qualified-identifier.58", + state: 174, +}]; +static GOTO_170: &[GotoEntry] = &[]; +static GOTO_171: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "implies-expression", + state: 2, + }, + GotoEntry { + nonterminal: "or-expression", + state: 3, + }, + GotoEntry { + nonterminal: "$implies-expression.0", + state: 4, + }, + GotoEntry { + nonterminal: "and-expression", + state: 5, + }, + GotoEntry { + nonterminal: "$or-expression.3", + state: 6, + }, + GotoEntry { + nonterminal: "membership-expression", + state: 7, + }, + GotoEntry { + nonterminal: "$and-expression.7", + state: 8, + }, + GotoEntry { + nonterminal: "equality-expression", + state: 9, + }, + GotoEntry { + nonterminal: "$membership-expression.10", + state: 10, + }, + GotoEntry { + nonterminal: "inequality-expression", + state: 11, + }, + GotoEntry { + nonterminal: "$equality-expression.14", + state: 12, + }, + GotoEntry { + nonterminal: "union-expression", + state: 13, + }, + GotoEntry { + nonterminal: "$inequality-expression.18", + state: 14, + }, + GotoEntry { + nonterminal: "type-expression", + state: 15, + }, + GotoEntry { + nonterminal: "$union-expression.22", + state: 16, + }, + GotoEntry { + nonterminal: "additive-expression", + state: 17, + }, + GotoEntry { + nonterminal: "$type-expression.25", + state: 18, + }, + GotoEntry { + nonterminal: "multiplicative-expression", + state: 19, + }, + GotoEntry { + nonterminal: "$additive-expression.29", + state: 20, + }, + GotoEntry { + nonterminal: "polarity-expression", + state: 21, + }, + GotoEntry { + nonterminal: "$multiplicative-expression.33", + state: 22, + }, + GotoEntry { + nonterminal: "$polarity-expression.38", + state: 25, + }, + GotoEntry { + nonterminal: "postfix-expression", + state: 26, + }, + GotoEntry { + nonterminal: "$polarity-expression.37", + state: 27, + }, + GotoEntry { + nonterminal: "term", + state: 28, + }, + GotoEntry { + nonterminal: "$postfix-expression.39", + state: 29, + }, + GotoEntry { + nonterminal: "invocation", + state: 30, + }, + GotoEntry { + nonterminal: "literal", + state: 31, + }, + GotoEntry { + nonterminal: "external-constant", + state: 32, + }, + GotoEntry { + nonterminal: "$term.42", + state: 34, + }, + GotoEntry { + nonterminal: "quantity", + state: 43, + }, + GotoEntry { + nonterminal: "$literal.43", + state: 44, + }, + GotoEntry { + nonterminal: "$external-constant.44", + state: 46, + }, + GotoEntry { + nonterminal: "function", + state: 47, + }, + GotoEntry { + nonterminal: "identifier", + state: 48, + }, + GotoEntry { + nonterminal: "$invocation.46", + state: 52, + }, + GotoEntry { + nonterminal: "$function.47", + state: 53, + }, + GotoEntry { + nonterminal: "expression", + state: 175, + }, + GotoEntry { + nonterminal: "$quantity.52", + state: 54, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_172: &[GotoEntry] = &[]; +static GOTO_173: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "identifier", + state: 176, + }, + GotoEntry { + nonterminal: "$identifier.59", + state: 61, + }, +]; +static GOTO_174: &[GotoEntry] = &[]; +static GOTO_175: &[GotoEntry] = &[]; +static GOTO_176: &[GotoEntry] = &[]; +static GOTOS: &[&[GotoEntry]] = &[ + GOTO_0, GOTO_1, GOTO_2, GOTO_3, GOTO_4, GOTO_5, GOTO_6, GOTO_7, GOTO_8, GOTO_9, GOTO_10, + GOTO_11, GOTO_12, GOTO_13, GOTO_14, GOTO_15, GOTO_16, GOTO_17, GOTO_18, GOTO_19, GOTO_20, + GOTO_21, GOTO_22, GOTO_23, GOTO_24, GOTO_25, GOTO_26, GOTO_27, GOTO_28, GOTO_29, GOTO_30, + GOTO_31, GOTO_32, GOTO_33, GOTO_34, GOTO_35, GOTO_36, GOTO_37, GOTO_38, GOTO_39, GOTO_40, + GOTO_41, GOTO_42, GOTO_43, GOTO_44, GOTO_45, GOTO_46, GOTO_47, GOTO_48, GOTO_49, GOTO_50, + GOTO_51, GOTO_52, GOTO_53, GOTO_54, GOTO_55, GOTO_56, GOTO_57, GOTO_58, GOTO_59, GOTO_60, + GOTO_61, GOTO_62, GOTO_63, GOTO_64, GOTO_65, GOTO_66, GOTO_67, GOTO_68, GOTO_69, GOTO_70, + GOTO_71, GOTO_72, GOTO_73, GOTO_74, GOTO_75, GOTO_76, GOTO_77, GOTO_78, GOTO_79, GOTO_80, + GOTO_81, GOTO_82, GOTO_83, GOTO_84, GOTO_85, GOTO_86, GOTO_87, GOTO_88, GOTO_89, GOTO_90, + GOTO_91, GOTO_92, GOTO_93, GOTO_94, GOTO_95, GOTO_96, GOTO_97, GOTO_98, GOTO_99, GOTO_100, + GOTO_101, GOTO_102, GOTO_103, GOTO_104, GOTO_105, GOTO_106, GOTO_107, GOTO_108, GOTO_109, + GOTO_110, GOTO_111, GOTO_112, GOTO_113, GOTO_114, GOTO_115, GOTO_116, GOTO_117, GOTO_118, + GOTO_119, GOTO_120, GOTO_121, GOTO_122, GOTO_123, GOTO_124, GOTO_125, GOTO_126, GOTO_127, + GOTO_128, GOTO_129, GOTO_130, GOTO_131, GOTO_132, GOTO_133, GOTO_134, GOTO_135, GOTO_136, + GOTO_137, GOTO_138, GOTO_139, GOTO_140, GOTO_141, GOTO_142, GOTO_143, GOTO_144, GOTO_145, + GOTO_146, GOTO_147, GOTO_148, GOTO_149, GOTO_150, GOTO_151, GOTO_152, GOTO_153, GOTO_154, + GOTO_155, GOTO_156, GOTO_157, GOTO_158, GOTO_159, GOTO_160, GOTO_161, GOTO_162, GOTO_163, + GOTO_164, GOTO_165, GOTO_166, GOTO_167, GOTO_168, GOTO_169, GOTO_170, GOTO_171, GOTO_172, + GOTO_173, GOTO_174, GOTO_175, GOTO_176, +]; + +static PRODUCTIONS: &[Production] = &[ + Production { + lhs: "$accept", + rhs: &[Operand { + symbol: Symbol::Nonterminal("expression"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("implies-expression"), + actions: &[OperandAction::Field("operand"), OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$implies-expression.1", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$implies-expression.2", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("implies")), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("implies-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$implies-expression.1", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$implies-expression.2"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$implies-expression.0", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("or-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$implies-expression.1"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "implies-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$implies-expression.0"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$or-expression.4", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$or-expression.6", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("or")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$or-expression.6", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("xor")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$or-expression.5", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$or-expression.6"), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("and-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$or-expression.4", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$or-expression.4"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$or-expression.5"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$or-expression.3", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("and-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$or-expression.4"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "or-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$or-expression.3"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$and-expression.8", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$and-expression.9", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("and")), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("membership-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$and-expression.8", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$and-expression.8"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$and-expression.9"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$and-expression.7", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("membership-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$and-expression.8"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "and-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$and-expression.7"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$membership-expression.11", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$membership-expression.13", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("in")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$membership-expression.13", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("contains")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$membership-expression.12", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$membership-expression.13"), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("equality-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$membership-expression.11", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$membership-expression.11"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$membership-expression.12"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$membership-expression.10", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("equality-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$membership-expression.11"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "membership-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$membership-expression.10"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$equality-expression.15", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$equality-expression.17", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("=")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$equality-expression.17", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("~")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$equality-expression.17", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("!=")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$equality-expression.17", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("!~")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$equality-expression.16", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$equality-expression.17"), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("inequality-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$equality-expression.15", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$equality-expression.15"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$equality-expression.16"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$equality-expression.14", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("inequality-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$equality-expression.15"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "equality-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$equality-expression.14"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$inequality-expression.19", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$inequality-expression.21", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("<=")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$inequality-expression.21", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("<")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$inequality-expression.21", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal(">")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$inequality-expression.21", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal(">=")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$inequality-expression.20", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$inequality-expression.21"), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("union-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$inequality-expression.19", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$inequality-expression.19"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$inequality-expression.20"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$inequality-expression.18", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("union-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$inequality-expression.19"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "inequality-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$inequality-expression.18"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$union-expression.23", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$union-expression.24", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("|")), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("type-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$union-expression.23", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$union-expression.23"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$union-expression.24"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$union-expression.22", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("type-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$union-expression.23"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "union-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$union-expression.22"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$type-expression.26", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$type-expression.28", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("is")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$type-expression.28", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("as")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$type-expression.27", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$type-expression.28"), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("type-specifier"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$type-expression.26", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$type-expression.26"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$type-expression.27"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$type-expression.25", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("additive-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$type-expression.26"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "type-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$type-expression.25"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$additive-expression.30", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$additive-expression.32", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("+")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$additive-expression.32", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("-")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$additive-expression.32", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("&")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$additive-expression.31", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$additive-expression.32"), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("multiplicative-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$additive-expression.30", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$additive-expression.30"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$additive-expression.31"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$additive-expression.29", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("multiplicative-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$additive-expression.30"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "additive-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$additive-expression.29"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$multiplicative-expression.34", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$multiplicative-expression.36", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("*")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$multiplicative-expression.36", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("/")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$multiplicative-expression.36", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("div")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$multiplicative-expression.36", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("mod")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$multiplicative-expression.35", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$multiplicative-expression.36"), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("polarity-expression"), + actions: &[OperandAction::Field("right")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$multiplicative-expression.34", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$multiplicative-expression.34"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$multiplicative-expression.35"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$multiplicative-expression.33", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("polarity-expression"), + actions: &[OperandAction::Field("left")], + }, + Operand { + symbol: Symbol::Nonterminal("$multiplicative-expression.34"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "multiplicative-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$multiplicative-expression.33"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$polarity-expression.38", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("+")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$polarity-expression.38", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("-")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$polarity-expression.37", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$polarity-expression.38"), + actions: &[OperandAction::Field("operator")], + }, + Operand { + symbol: Symbol::Nonterminal("polarity-expression"), + actions: &[OperandAction::Field("operand")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$polarity-expression.37", + rhs: &[Operand { + symbol: Symbol::Nonterminal("postfix-expression"), + actions: &[OperandAction::Field("operand")], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "polarity-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$polarity-expression.37"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$postfix-expression.40", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$postfix-expression.41", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal(".")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("invocation"), + actions: &[OperandAction::Field("invocation")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$postfix-expression.41", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("[")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("expression"), + actions: &[OperandAction::Field("index")], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Literal("]")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$postfix-expression.40", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$postfix-expression.40"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$postfix-expression.41"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$postfix-expression.39", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("term"), + actions: &[OperandAction::Field("operand")], + }, + Operand { + symbol: Symbol::Nonterminal("$postfix-expression.40"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "postfix-expression", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$postfix-expression.39"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$term.42", + rhs: &[Operand { + symbol: Symbol::Nonterminal("invocation"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$term.42", + rhs: &[Operand { + symbol: Symbol::Nonterminal("literal"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$term.42", + rhs: &[Operand { + symbol: Symbol::Nonterminal("external-constant"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$term.42", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("(")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("expression"), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Literal(")")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "term", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$term.42"), + actions: &[OperandAction::Field("value"), OperandAction::Alias(1)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("{")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Literal("}")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("true")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("false")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("string")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("number")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("date")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("datetime")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("time")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$literal.43", + rhs: &[Operand { + symbol: Symbol::Nonterminal("quantity"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "literal", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$literal.43"), + actions: &[OperandAction::Field("value"), OperandAction::Alias(2)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$external-constant.45", + rhs: &[Operand { + symbol: Symbol::Nonterminal("identifier"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$external-constant.45", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("string")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$external-constant.44", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("%")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$external-constant.45"), + actions: &[OperandAction::Field("name")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "external-constant", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$external-constant.44"), + actions: &[OperandAction::Alias(3)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$invocation.46", + rhs: &[Operand { + symbol: Symbol::Nonterminal("function"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$invocation.46", + rhs: &[Operand { + symbol: Symbol::Nonterminal("identifier"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$invocation.46", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("$this")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$invocation.46", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("$index")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$invocation.46", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("$total")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "invocation", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$invocation.46"), + actions: &[OperandAction::Field("value"), OperandAction::Alias(4)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$function.48", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$function.48", + rhs: &[Operand { + symbol: Symbol::Nonterminal("param-list"), + actions: &[OperandAction::Field("parameter")], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$function.47", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("identifier"), + actions: &[OperandAction::Field("name")], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Literal("(")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$function.48"), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Literal(")")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "function", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$function.47"), + actions: &[OperandAction::Alias(5)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$param-list.50", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$param-list.51", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal(",")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("expression"), + actions: &[OperandAction::Field("parameter")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$param-list.50", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$param-list.50"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$param-list.51"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$param-list.49", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("expression"), + actions: &[OperandAction::Field("parameter")], + }, + Operand { + symbol: Symbol::Nonterminal("$param-list.50"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "param-list", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$param-list.49"), + actions: &[OperandAction::Alias(6)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$quantity.52", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("number")), + actions: &[OperandAction::Field("value")], + }, + Operand { + symbol: Symbol::Nonterminal("unit"), + actions: &[OperandAction::Field("unit")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "quantity", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$quantity.52"), + actions: &[OperandAction::Alias(7)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.53", + rhs: &[Operand { + symbol: Symbol::Nonterminal("date-time-precision"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.53", + rhs: &[Operand { + symbol: Symbol::Nonterminal("plural-date-time-precision"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.53", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("string")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "unit", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$unit.53"), + actions: &[OperandAction::Field("value"), OperandAction::Alias(8)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$date-time-precision.54", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("year")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$date-time-precision.54", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("month")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$date-time-precision.54", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("week")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$date-time-precision.54", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("day")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$date-time-precision.54", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("hour")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$date-time-precision.54", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("minute")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$date-time-precision.54", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("second")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$date-time-precision.54", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("millisecond")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "date-time-precision", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$date-time-precision.54"), + actions: &[OperandAction::Field("value"), OperandAction::Alias(9)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$plural-date-time-precision.55", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("years")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$plural-date-time-precision.55", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("months")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$plural-date-time-precision.55", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("weeks")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$plural-date-time-precision.55", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("days")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$plural-date-time-precision.55", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("hours")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$plural-date-time-precision.55", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("minutes")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$plural-date-time-precision.55", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("seconds")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$plural-date-time-precision.55", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("milliseconds")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "plural-date-time-precision", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$plural-date-time-precision.55"), + actions: &[OperandAction::Field("value"), OperandAction::Alias(10)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "type-specifier", + rhs: &[Operand { + symbol: Symbol::Nonterminal("qualified-identifier"), + actions: &[OperandAction::Field("name"), OperandAction::Alias(11)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$qualified-identifier.57", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$qualified-identifier.58", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal(".")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("identifier"), + actions: &[OperandAction::Field("name")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$qualified-identifier.57", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$qualified-identifier.57"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$qualified-identifier.58"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$qualified-identifier.56", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("identifier"), + actions: &[OperandAction::Field("name")], + }, + Operand { + symbol: Symbol::Nonterminal("$qualified-identifier.57"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "qualified-identifier", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$qualified-identifier.56"), + actions: &[OperandAction::Alias(12)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$identifier.59", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("identifier")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$identifier.59", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("delimited-identifier")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$identifier.59", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("as")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$identifier.59", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("contains")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$identifier.59", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("in")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$identifier.59", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Literal("is")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "identifier", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$identifier.59"), + actions: &[OperandAction::Field("value"), OperandAction::Alias(13)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, +]; + +pub static LANGUAGE: LanguageSpec = LanguageSpec { + language: "fhirpath", + version: "2.0.0", + contract: "fhirpath-normative-2.0.0-syntax.v1", + grammar_digest: "sha256:4cf0e808878f324196c3d4f6b3f2ea1fb05d5afd2a6fe8acc806369408c6a0ab", + case_insensitive: false, + root_kind: 0, + kinds: KINDS, + terminals: TERMINALS, + lexical_rules: LEXICAL_RULES, + actions: ACTIONS, + gotos: GOTOS, + productions: PRODUCTIONS, +}; + +pub const PROFILE_CASES: &[(&str, &str, Option)] = &[ + ("identifier", "Patient٣.more", Some(8)), + ("identifier", "_a9", Some(3)), + ("identifier", "α", None), + ("identifier", "A²", Some(1)), + ("identifier", "A４", Some(2)), + ("number", "1.", Some(1)), + ("number", "12.34rest", Some(5)), + ("number", "١٢.٣٤", Some(5)), + ("number", "²", None), + ("number", "1..2", Some(1)), + ("number", ".1", None), + ("date", "@2024", Some(5)), + ("date", "@2024-", None), + ("date", "@2024-2", None), + ("date", "@2024-12", Some(8)), + ("date", "@2024-12-", None), + ("date", "@2024-12-31", Some(11)), + ("date", "@2024-02-29T", None), + ("date", "@20245", Some(5)), + ("datetime", "@2024T", Some(6)), + ("datetime", "@2024Tx", Some(6)), + ("datetime", "@2024T1", None), + ("datetime", "@2024T12:3", None), + ("datetime", "@2024T12Z+01:00", Some(9)), + ("datetime", "@2024T12+01:00", Some(14)), + ("datetime", "@2024T12+0:00", None), + ("datetime", "@2024T12+01", None), + ("datetime", "@2024T12:00:01.", Some(14)), + ("datetime", "@2024T12:00:01.2", Some(16)), + ("time", "@T12", Some(4)), + ("time", "@T1", None), + ("time", "@T12:", None), + ("time", "@T12:34", Some(7)), + ("time", "@T12:34:5", None), + ("time", "@T12:34:56.", Some(10)), + ("time", "@T12:34:56.１２", Some(13)), + ("time", "@T12Z", Some(4)), +]; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/header_record.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/header_record.rs new file mode 100644 index 00000000..279b33d1 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/header_record.rs @@ -0,0 +1,194 @@ +// @generated by gerbil-parser/src/compiler/rust-runtime.ss +// Source authority: canonical Grammar IR v1; do not edit. + +use gerbil_parser_runtime::{ + ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, + Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, +}; + +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "SourceFile", + category: KindCategory::Node, + }, + KindSpec { + name: "Separator", + category: KindCategory::Token, + }, + KindSpec { + name: "ComponentSeparator", + category: KindCategory::Token, + }, + KindSpec { + name: "Data", + category: KindCategory::Token, + }, +]; + +static TERMINALS: &[TerminalSpec] = &[ + TerminalSpec { + name: "field", + syntax_kind: 1, + }, + TerminalSpec { + name: "component", + syntax_kind: 2, + }, + TerminalSpec { + name: "data", + syntax_kind: 3, + }, +]; + +static LEXICAL_RULES: &[LexicalRule] = &[ + LexicalRule { + terminal: "field", + expression: LexicalExpr::HeaderDelimiter { + prefix: "記", + count: 2, + index: 0, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "component", + expression: LexicalExpr::HeaderDelimiter { + prefix: "記", + count: 2, + index: 1, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "data", + expression: LexicalExpr::HeaderData { + prefix: "記", + count: 2, + stops: ";", + }, + precedence: 0, + extra: false, + }, +]; + +static ACTION_0: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal("記"), + action: ParserAction::Shift(2), +}]; +static ACTION_1: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Accept, +}]; +static ACTION_2: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("field"), + action: ParserAction::Shift(4), +}]; +static ACTION_3: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(2), +}]; +static ACTION_4: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("component"), + action: ParserAction::Shift(5), +}]; +static ACTION_5: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Shift(6), +}]; +static ACTION_6: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("field"), + action: ParserAction::Shift(7), +}]; +static ACTION_7: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(1), +}]; +static ACTIONS: &[&[ActionEntry]] = &[ + ACTION_0, ACTION_1, ACTION_2, ACTION_3, ACTION_4, ACTION_5, ACTION_6, ACTION_7, +]; + +static GOTO_0: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "document", + state: 1, + }, + GotoEntry { + nonterminal: "$document.0", + state: 3, + }, +]; +static GOTO_1: &[GotoEntry] = &[]; +static GOTO_2: &[GotoEntry] = &[]; +static GOTO_3: &[GotoEntry] = &[]; +static GOTO_4: &[GotoEntry] = &[]; +static GOTO_5: &[GotoEntry] = &[]; +static GOTO_6: &[GotoEntry] = &[]; +static GOTO_7: &[GotoEntry] = &[]; +static GOTOS: &[&[GotoEntry]] = &[ + GOTO_0, GOTO_1, GOTO_2, GOTO_3, GOTO_4, GOTO_5, GOTO_6, GOTO_7, +]; + +static PRODUCTIONS: &[Production] = &[ + Production { + lhs: "$accept", + rhs: &[Operand { + symbol: Symbol::Nonterminal("document"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.0", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("記")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("field")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("component")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("data")), + actions: &[OperandAction::Field("value")], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("field")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "document", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$document.0"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, +]; + +pub static LANGUAGE: LanguageSpec = LanguageSpec { + language: "header-record", + version: "v1", + contract: "header-record.v1", + grammar_digest: "sha256:873cd15f92b782d853532bed6c962a766b2de258026e42b5ed7055b678a62536", + case_insensitive: false, + root_kind: 0, + kinds: KINDS, + terminals: TERMINALS, + lexical_rules: LEXICAL_RULES, + actions: ACTIONS, + gotos: GOTOS, + productions: PRODUCTIONS, +}; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/hl7_header.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/hl7_header.rs new file mode 100644 index 00000000..48b15971 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/hl7_header.rs @@ -0,0 +1,2067 @@ +// @generated by gerbil-parser/src/compiler/rust-runtime.ss +// Source authority: canonical Grammar IR v1; do not edit. + +use gerbil_parser_runtime::{ + ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, + Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, +}; + +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "Message", + category: KindCategory::Node, + }, + KindSpec { + name: "HeaderSegment", + category: KindCategory::Node, + }, + KindSpec { + name: "Segment", + category: KindCategory::Node, + }, + KindSpec { + name: "Field", + category: KindCategory::Node, + }, + KindSpec { + name: "Repetition", + category: KindCategory::Node, + }, + KindSpec { + name: "Component", + category: KindCategory::Node, + }, + KindSpec { + name: "Subcomponent", + category: KindCategory::Node, + }, + KindSpec { + name: "Escape", + category: KindCategory::Node, + }, + KindSpec { + name: "SegmentId", + category: KindCategory::Token, + }, + KindSpec { + name: "Data", + category: KindCategory::Token, + }, + KindSpec { + name: "SegmentTerminator", + category: KindCategory::Token, + }, + KindSpec { + name: "FieldSeparator", + category: KindCategory::Token, + }, + KindSpec { + name: "ComponentSeparator", + category: KindCategory::Token, + }, + KindSpec { + name: "RepetitionSeparator", + category: KindCategory::Token, + }, + KindSpec { + name: "EscapeCharacter", + category: KindCategory::Token, + }, + KindSpec { + name: "SubcomponentSeparator", + category: KindCategory::Token, + }, +]; + +static TERMINALS: &[TerminalSpec] = &[ + TerminalSpec { + name: "segment-id", + syntax_kind: 8, + }, + TerminalSpec { + name: "data", + syntax_kind: 9, + }, + TerminalSpec { + name: "segment-terminator", + syntax_kind: 10, + }, + TerminalSpec { + name: "field-separator", + syntax_kind: 11, + }, + TerminalSpec { + name: "component-separator", + syntax_kind: 12, + }, + TerminalSpec { + name: "repetition-separator", + syntax_kind: 13, + }, + TerminalSpec { + name: "escape-character", + syntax_kind: 14, + }, + TerminalSpec { + name: "subcomponent-separator", + syntax_kind: 15, + }, +]; + +static LEXICAL_RULES: &[LexicalRule] = &[ + LexicalRule { + terminal: "segment-id", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Characters("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), + gerbil_parser_runtime::TextClass::Numeric, + ]), + minimum: 3, + maximum: Some(3), + }, + gerbil_parser_runtime::TextProfile::NotNext(gerbil_parser_runtime::TextClass::Union( + &[ + gerbil_parser_runtime::TextClass::Characters("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), + gerbil_parser_runtime::TextClass::Numeric, + ], + )), + ])), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "data", + expression: LexicalExpr::HeaderData { + prefix: "MSH", + count: 5, + stops: "\r\n", + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "segment-terminator", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("\r"), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("\r"), + gerbil_parser_runtime::TextProfile::Optional( + &gerbil_parser_runtime::TextProfile::Literal("\n"), + ), + ]), + otherwise: Some(&gerbil_parser_runtime::TextProfile::Literal("\n")), + }), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "field-separator", + expression: LexicalExpr::HeaderDelimiter { + prefix: "MSH", + count: 5, + index: 0, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "component-separator", + expression: LexicalExpr::HeaderDelimiter { + prefix: "MSH", + count: 5, + index: 1, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "repetition-separator", + expression: LexicalExpr::HeaderDelimiter { + prefix: "MSH", + count: 5, + index: 2, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "escape-character", + expression: LexicalExpr::HeaderDelimiter { + prefix: "MSH", + count: 5, + index: 3, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "subcomponent-separator", + expression: LexicalExpr::HeaderDelimiter { + prefix: "MSH", + count: 5, + index: 4, + }, + precedence: 0, + extra: false, + }, +]; + +static ACTION_0: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal("MSH"), + action: ParserAction::Shift(4), +}]; +static ACTION_1: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Accept, +}]; +static ACTION_2: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(1), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(1), + }, +]; +static ACTION_3: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(4), +}]; +static ACTION_4: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Shift(7), +}]; +static ACTION_5: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(11), + }, +]; +static ACTION_6: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(3), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Shift(9), + }, +]; +static ACTION_7: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Shift(11), +}]; +static ACTION_8: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(2), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(2), + }, +]; +static ACTION_9: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(12), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(12), + }, +]; +static ACTION_10: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(18), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(18), + }, +]; +static ACTION_11: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Shift(13), +}]; +static ACTION_12: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Shift(14), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Shift(16), + }, +]; +static ACTION_13: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Shift(17), +}]; +static ACTION_14: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Shift(26), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Shift(27), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Shift(31), + }, +]; +static ACTION_15: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(16), + }, +]; +static ACTION_16: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(17), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(17), + }, +]; +static ACTION_17: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Shift(33), +}]; +static ACTION_18: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(14), + }, +]; +static ACTION_19: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(15), + }, +]; +static ACTION_20: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(19), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(19), + }, +]; +static ACTION_21: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(23), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(23), + }, +]; +static ACTION_22: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(24), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(24), + }, +]; +static ACTION_23: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(30), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(30), + }, +]; +static ACTION_24: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(31), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(31), + }, +]; +static ACTION_25: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(37), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(37), + }, +]; +static ACTION_26: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(38), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Reduce(38), + }, +]; +static ACTION_27: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(39), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Reduce(39), + }, +]; +static ACTION_28: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(40), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Reduce(40), + }, +]; +static ACTION_29: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(41), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Reduce(41), + }, +]; +static ACTION_30: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Shift(26), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Shift(27), + }, + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(43), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Shift(31), + }, +]; +static ACTION_31: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(44), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Shift(38), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Shift(39), + }, +]; +static ACTION_32: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(49), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Reduce(49), + }, +]; +static ACTION_33: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(5), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(5), + }, +]; +static ACTION_34: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Shift(43), + }, + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(22), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(22), + }, +]; +static ACTION_35: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Shift(45), + }, + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(29), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(29), + }, +]; +static ACTION_36: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Shift(47), + }, + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(36), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(36), + }, +]; +static ACTION_37: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(42), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Reduce(42), + }, +]; +static ACTION_38: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(45), +}]; +static ACTION_39: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(46), +}]; +static ACTION_40: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(47), +}]; +static ACTION_41: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Shift(49), +}]; +static ACTION_42: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Shift(50), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Shift(52), + }, +]; +static ACTION_43: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Shift(26), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Shift(27), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Shift(31), + }, +]; +static ACTION_44: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(21), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(21), + }, +]; +static ACTION_45: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(25), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Shift(26), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Shift(27), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Shift(31), + }, +]; +static ACTION_46: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(28), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(28), + }, +]; +static ACTION_47: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(32), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Shift(26), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Shift(27), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Shift(31), + }, +]; +static ACTION_48: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(35), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(35), + }, +]; +static ACTION_49: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(48), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Reduce(48), + }, +]; +static ACTION_50: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(6), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(6), + }, + ActionEntry { + terminal: Terminal::Token("data"), + action: ParserAction::Shift(26), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Shift(27), + }, + ActionEntry { + terminal: Terminal::Token("escape-character"), + action: ParserAction::Shift(31), + }, +]; +static ACTION_51: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(9), + }, +]; +static ACTION_52: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Token("segment-id"), + action: ParserAction::Reduce(10), + }, +]; +static ACTION_53: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(20), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(20), + }, +]; +static ACTION_54: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(26), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(26), + }, +]; +static ACTION_55: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(27), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(27), + }, +]; +static ACTION_56: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(33), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(33), + }, +]; +static ACTION_57: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Token("component-separator"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Token("repetition-separator"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Token("subcomponent-separator"), + action: ParserAction::Reduce(34), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(34), + }, +]; +static ACTION_58: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(7), + }, +]; +static ACTION_59: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("field-separator"), + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("segment-terminator"), + action: ParserAction::Reduce(8), + }, +]; +static ACTIONS: &[&[ActionEntry]] = &[ + ACTION_0, ACTION_1, ACTION_2, ACTION_3, ACTION_4, ACTION_5, ACTION_6, ACTION_7, ACTION_8, + ACTION_9, ACTION_10, ACTION_11, ACTION_12, ACTION_13, ACTION_14, ACTION_15, ACTION_16, + ACTION_17, ACTION_18, ACTION_19, ACTION_20, ACTION_21, ACTION_22, ACTION_23, ACTION_24, + ACTION_25, ACTION_26, ACTION_27, ACTION_28, ACTION_29, ACTION_30, ACTION_31, ACTION_32, + ACTION_33, ACTION_34, ACTION_35, ACTION_36, ACTION_37, ACTION_38, ACTION_39, ACTION_40, + ACTION_41, ACTION_42, ACTION_43, ACTION_44, ACTION_45, ACTION_46, ACTION_47, ACTION_48, + ACTION_49, ACTION_50, ACTION_51, ACTION_52, ACTION_53, ACTION_54, ACTION_55, ACTION_56, + ACTION_57, ACTION_58, ACTION_59, +]; + +static GOTO_0: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "message", + state: 1, + }, + GotoEntry { + nonterminal: "header-segment", + state: 2, + }, + GotoEntry { + nonterminal: "$message.0", + state: 3, + }, + GotoEntry { + nonterminal: "$header-segment.2", + state: 5, + }, +]; +static GOTO_1: &[GotoEntry] = &[]; +static GOTO_2: &[GotoEntry] = &[GotoEntry { + nonterminal: "$message.1", + state: 6, +}]; +static GOTO_3: &[GotoEntry] = &[]; +static GOTO_4: &[GotoEntry] = &[]; +static GOTO_5: &[GotoEntry] = &[]; +static GOTO_6: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "segment", + state: 8, + }, + GotoEntry { + nonterminal: "$segment.6", + state: 10, + }, +]; +static GOTO_7: &[GotoEntry] = &[]; +static GOTO_8: &[GotoEntry] = &[]; +static GOTO_9: &[GotoEntry] = &[GotoEntry { + nonterminal: "$segment.7", + state: 12, +}]; +static GOTO_10: &[GotoEntry] = &[]; +static GOTO_11: &[GotoEntry] = &[]; +static GOTO_12: &[GotoEntry] = &[GotoEntry { + nonterminal: "$segment.8", + state: 15, +}]; +static GOTO_13: &[GotoEntry] = &[]; +static GOTO_14: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "field", + state: 18, + }, + GotoEntry { + nonterminal: "$segment.9", + state: 19, + }, + GotoEntry { + nonterminal: "repetition", + state: 20, + }, + GotoEntry { + nonterminal: "$field.10", + state: 21, + }, + GotoEntry { + nonterminal: "component", + state: 22, + }, + GotoEntry { + nonterminal: "$repetition.13", + state: 23, + }, + GotoEntry { + nonterminal: "subcomponent", + state: 24, + }, + GotoEntry { + nonterminal: "$component.17", + state: 25, + }, + GotoEntry { + nonterminal: "escape", + state: 28, + }, + GotoEntry { + nonterminal: "$subcomponent.22", + state: 29, + }, + GotoEntry { + nonterminal: "$subcomponent.21", + state: 30, + }, + GotoEntry { + nonterminal: "$escape.23", + state: 32, + }, +]; +static GOTO_15: &[GotoEntry] = &[]; +static GOTO_16: &[GotoEntry] = &[]; +static GOTO_17: &[GotoEntry] = &[]; +static GOTO_18: &[GotoEntry] = &[]; +static GOTO_19: &[GotoEntry] = &[]; +static GOTO_20: &[GotoEntry] = &[GotoEntry { + nonterminal: "$field.11", + state: 34, +}]; +static GOTO_21: &[GotoEntry] = &[]; +static GOTO_22: &[GotoEntry] = &[GotoEntry { + nonterminal: "$repetition.14", + state: 35, +}]; +static GOTO_23: &[GotoEntry] = &[]; +static GOTO_24: &[GotoEntry] = &[GotoEntry { + nonterminal: "$component.18", + state: 36, +}]; +static GOTO_25: &[GotoEntry] = &[]; +static GOTO_26: &[GotoEntry] = &[]; +static GOTO_27: &[GotoEntry] = &[]; +static GOTO_28: &[GotoEntry] = &[]; +static GOTO_29: &[GotoEntry] = &[]; +static GOTO_30: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "escape", + state: 28, + }, + GotoEntry { + nonterminal: "$subcomponent.22", + state: 37, + }, + GotoEntry { + nonterminal: "$escape.23", + state: 32, + }, +]; +static GOTO_31: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "$escape.25", + state: 40, + }, + GotoEntry { + nonterminal: "$escape.24", + state: 41, + }, +]; +static GOTO_32: &[GotoEntry] = &[]; +static GOTO_33: &[GotoEntry] = &[GotoEntry { + nonterminal: "$header-segment.3", + state: 42, +}]; +static GOTO_34: &[GotoEntry] = &[GotoEntry { + nonterminal: "$field.12", + state: 44, +}]; +static GOTO_35: &[GotoEntry] = &[GotoEntry { + nonterminal: "$repetition.15", + state: 46, +}]; +static GOTO_36: &[GotoEntry] = &[GotoEntry { + nonterminal: "$component.19", + state: 48, +}]; +static GOTO_37: &[GotoEntry] = &[]; +static GOTO_38: &[GotoEntry] = &[]; +static GOTO_39: &[GotoEntry] = &[]; +static GOTO_40: &[GotoEntry] = &[]; +static GOTO_41: &[GotoEntry] = &[]; +static GOTO_42: &[GotoEntry] = &[GotoEntry { + nonterminal: "$header-segment.4", + state: 51, +}]; +static GOTO_43: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "repetition", + state: 53, + }, + GotoEntry { + nonterminal: "component", + state: 22, + }, + GotoEntry { + nonterminal: "$repetition.13", + state: 23, + }, + GotoEntry { + nonterminal: "subcomponent", + state: 24, + }, + GotoEntry { + nonterminal: "$component.17", + state: 25, + }, + GotoEntry { + nonterminal: "escape", + state: 28, + }, + GotoEntry { + nonterminal: "$subcomponent.22", + state: 29, + }, + GotoEntry { + nonterminal: "$subcomponent.21", + state: 30, + }, + GotoEntry { + nonterminal: "$escape.23", + state: 32, + }, +]; +static GOTO_44: &[GotoEntry] = &[]; +static GOTO_45: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "component", + state: 54, + }, + GotoEntry { + nonterminal: "$repetition.16", + state: 55, + }, + GotoEntry { + nonterminal: "subcomponent", + state: 24, + }, + GotoEntry { + nonterminal: "$component.17", + state: 25, + }, + GotoEntry { + nonterminal: "escape", + state: 28, + }, + GotoEntry { + nonterminal: "$subcomponent.22", + state: 29, + }, + GotoEntry { + nonterminal: "$subcomponent.21", + state: 30, + }, + GotoEntry { + nonterminal: "$escape.23", + state: 32, + }, +]; +static GOTO_46: &[GotoEntry] = &[]; +static GOTO_47: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "subcomponent", + state: 56, + }, + GotoEntry { + nonterminal: "$component.20", + state: 57, + }, + GotoEntry { + nonterminal: "escape", + state: 28, + }, + GotoEntry { + nonterminal: "$subcomponent.22", + state: 29, + }, + GotoEntry { + nonterminal: "$subcomponent.21", + state: 30, + }, + GotoEntry { + nonterminal: "$escape.23", + state: 32, + }, +]; +static GOTO_48: &[GotoEntry] = &[]; +static GOTO_49: &[GotoEntry] = &[]; +static GOTO_50: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "field", + state: 58, + }, + GotoEntry { + nonterminal: "$header-segment.5", + state: 59, + }, + GotoEntry { + nonterminal: "repetition", + state: 20, + }, + GotoEntry { + nonterminal: "$field.10", + state: 21, + }, + GotoEntry { + nonterminal: "component", + state: 22, + }, + GotoEntry { + nonterminal: "$repetition.13", + state: 23, + }, + GotoEntry { + nonterminal: "subcomponent", + state: 24, + }, + GotoEntry { + nonterminal: "$component.17", + state: 25, + }, + GotoEntry { + nonterminal: "escape", + state: 28, + }, + GotoEntry { + nonterminal: "$subcomponent.22", + state: 29, + }, + GotoEntry { + nonterminal: "$subcomponent.21", + state: 30, + }, + GotoEntry { + nonterminal: "$escape.23", + state: 32, + }, +]; +static GOTO_51: &[GotoEntry] = &[]; +static GOTO_52: &[GotoEntry] = &[]; +static GOTO_53: &[GotoEntry] = &[]; +static GOTO_54: &[GotoEntry] = &[]; +static GOTO_55: &[GotoEntry] = &[]; +static GOTO_56: &[GotoEntry] = &[]; +static GOTO_57: &[GotoEntry] = &[]; +static GOTO_58: &[GotoEntry] = &[]; +static GOTO_59: &[GotoEntry] = &[]; +static GOTOS: &[&[GotoEntry]] = &[ + GOTO_0, GOTO_1, GOTO_2, GOTO_3, GOTO_4, GOTO_5, GOTO_6, GOTO_7, GOTO_8, GOTO_9, GOTO_10, + GOTO_11, GOTO_12, GOTO_13, GOTO_14, GOTO_15, GOTO_16, GOTO_17, GOTO_18, GOTO_19, GOTO_20, + GOTO_21, GOTO_22, GOTO_23, GOTO_24, GOTO_25, GOTO_26, GOTO_27, GOTO_28, GOTO_29, GOTO_30, + GOTO_31, GOTO_32, GOTO_33, GOTO_34, GOTO_35, GOTO_36, GOTO_37, GOTO_38, GOTO_39, GOTO_40, + GOTO_41, GOTO_42, GOTO_43, GOTO_44, GOTO_45, GOTO_46, GOTO_47, GOTO_48, GOTO_49, GOTO_50, + GOTO_51, GOTO_52, GOTO_53, GOTO_54, GOTO_55, GOTO_56, GOTO_57, GOTO_58, GOTO_59, +]; + +static PRODUCTIONS: &[Production] = &[ + Production { + lhs: "$accept", + rhs: &[Operand { + symbol: Symbol::Nonterminal("message"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$message.1", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$message.1", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$message.1"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("segment"), + actions: &[OperandAction::Field("segment")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$message.0", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("header-segment"), + actions: &[OperandAction::Field("header")], + }, + Operand { + symbol: Symbol::Nonterminal("$message.1"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "message", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$message.0"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$header-segment.3", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$header-segment.5", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$header-segment.5", + rhs: &[Operand { + symbol: Symbol::Nonterminal("field"), + actions: &[OperandAction::Field("field")], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$header-segment.4", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("field-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$header-segment.5"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$header-segment.3", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$header-segment.3"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$header-segment.4"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$header-segment.2", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Literal("MSH")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("field-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("component-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("repetition-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("escape-character")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("subcomponent-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$header-segment.3"), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("segment-terminator")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "header-segment", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$header-segment.2"), + actions: &[OperandAction::Alias(1)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$segment.7", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$segment.9", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$segment.9", + rhs: &[Operand { + symbol: Symbol::Nonterminal("field"), + actions: &[OperandAction::Field("field")], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$segment.8", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("field-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$segment.9"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$segment.7", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$segment.7"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$segment.8"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$segment.6", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("segment-id")), + actions: &[OperandAction::Field("name")], + }, + Operand { + symbol: Symbol::Nonterminal("$segment.7"), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("segment-terminator")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "segment", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$segment.6"), + actions: &[OperandAction::Alias(2)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$field.11", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$field.12", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("repetition-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("repetition"), + actions: &[OperandAction::Field("repetition")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$field.11", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$field.11"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$field.12"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$field.10", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("repetition"), + actions: &[OperandAction::Field("repetition")], + }, + Operand { + symbol: Symbol::Nonterminal("$field.11"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "field", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$field.10"), + actions: &[OperandAction::Alias(3)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$repetition.14", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$repetition.16", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$repetition.16", + rhs: &[Operand { + symbol: Symbol::Nonterminal("component"), + actions: &[OperandAction::Field("component")], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$repetition.15", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("component-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$repetition.16"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$repetition.14", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$repetition.14"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$repetition.15"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$repetition.13", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("component"), + actions: &[OperandAction::Field("component")], + }, + Operand { + symbol: Symbol::Nonterminal("$repetition.14"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "repetition", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$repetition.13"), + actions: &[OperandAction::Alias(4)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$component.18", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$component.20", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$component.20", + rhs: &[Operand { + symbol: Symbol::Nonterminal("subcomponent"), + actions: &[OperandAction::Field("subcomponent")], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$component.19", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("subcomponent-separator")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$component.20"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$component.18", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$component.18"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$component.19"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$component.17", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("subcomponent"), + actions: &[OperandAction::Field("subcomponent")], + }, + Operand { + symbol: Symbol::Nonterminal("$component.18"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "component", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$component.17"), + actions: &[OperandAction::Alias(5)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$subcomponent.22", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("data")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$subcomponent.22", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("segment-id")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$subcomponent.22", + rhs: &[Operand { + symbol: Symbol::Nonterminal("escape"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$subcomponent.21", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$subcomponent.22"), + actions: &[OperandAction::Field("value")], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$subcomponent.21", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$subcomponent.21"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$subcomponent.22"), + actions: &[OperandAction::Field("value")], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "subcomponent", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$subcomponent.21"), + actions: &[OperandAction::Alias(6)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$escape.24", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$escape.25", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("data")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$escape.25", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("segment-id")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$escape.24", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$escape.25"), + actions: &[OperandAction::Field("value")], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$escape.23", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("escape-character")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$escape.24"), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("escape-character")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "escape", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$escape.23"), + actions: &[OperandAction::Alias(7)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, +]; + +pub static LANGUAGE: LanguageSpec = LanguageSpec { + language: "hl7v2", + version: "2.5.1", + contract: "hl7v2-er7.v1", + grammar_digest: "sha256:adaee8fca79e880bcde40625e79278722af4520c7fc5b5fdca3d7dba3b972cc7", + case_insensitive: false, + root_kind: 0, + kinds: KINDS, + terminals: TERMINALS, + lexical_rules: LEXICAL_RULES, + actions: ACTIONS, + gotos: GOTOS, + productions: PRODUCTIONS, +}; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/module_source.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/module_source.rs new file mode 100644 index 00000000..8fd167bd --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/module_source.rs @@ -0,0 +1,734 @@ +// @generated by gerbil-parser/src/compiler/rust-runtime.ss +// Source authority: canonical Grammar IR v1; do not edit. + +use gerbil_parser_runtime::{ + ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, + Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, +}; + +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "Document", + category: KindCategory::Node, + }, + KindSpec { + name: "Unit", + category: KindCategory::Node, + }, + KindSpec { + name: "ModuleText", + category: KindCategory::Token, + }, + KindSpec { + name: "Border", + category: KindCategory::Token, + }, + KindSpec { + name: "End", + category: KindCategory::Token, + }, + KindSpec { + name: "Name", + category: KindCategory::Token, + }, + KindSpec { + name: "String", + category: KindCategory::Token, + }, + KindSpec { + name: "Space", + category: KindCategory::Token, + }, + KindSpec { + name: "Comment", + category: KindCategory::Token, + }, +]; + +static TERMINALS: &[TerminalSpec] = &[ + TerminalSpec { + name: "text", + syntax_kind: 2, + }, + TerminalSpec { + name: "border", + syntax_kind: 3, + }, + TerminalSpec { + name: "end", + syntax_kind: 4, + }, + TerminalSpec { + name: "name", + syntax_kind: 5, + }, + TerminalSpec { + name: "string", + syntax_kind: 6, + }, + TerminalSpec { + name: "space", + syntax_kind: 7, + }, + TerminalSpec { + name: "comment", + syntax_kind: 8, + }, +]; + +static LEXICAL_RULES: &[LexicalRule] = &[ + LexicalRule { + terminal: "text", + expression: LexicalExpr::ModuleText(&gerbil_parser_runtime::ModuleTextProfile { + header_border: "----", + header_word: "MODULE", + end_border: "====", + block_open: "(*", + block_close: "*)", + line_comment: "\\*", + name_extra: "_", + }), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "border", + expression: LexicalExpr::CharacterRun { + character: "-", + minimum: 4, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "end", + expression: LexicalExpr::CharacterRun { + character: "=", + minimum: 4, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "name", + expression: LexicalExpr::Identifier, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "string", + expression: LexicalExpr::QuotedString(&["\""]), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "space", + expression: LexicalExpr::Whitespace, + precedence: 0, + extra: true, + }, + LexicalRule { + terminal: "comment", + expression: LexicalExpr::Choice(&[ + LexicalExpr::LineComment(&["\\*"]), + LexicalExpr::NestedBlockComment { + opening: "(*", + closing: "*)", + }, + ]), + precedence: 0, + extra: true, + }, +]; + +static ACTION_0: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(1), + }, + ActionEntry { + terminal: Terminal::Token("text"), + action: ParserAction::Shift(2), + }, +]; +static ACTION_1: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Accept, +}]; +static ACTION_2: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(2), +}]; +static ACTION_3: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Shift(8), +}]; +static ACTION_4: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(9), +}]; +static ACTION_5: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(3), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(3), + }, + ActionEntry { + terminal: Terminal::Token("text"), + action: ParserAction::Shift(10), + }, +]; +static ACTION_6: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(6), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(6), + }, +]; +static ACTION_7: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Shift(8), + }, +]; +static ACTION_8: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal("MODULE"), + action: ParserAction::Shift(13), +}]; +static ACTION_9: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Token("text"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(16), + }, + ActionEntry { + terminal: Terminal::Token("end"), + action: ParserAction::Reduce(16), + }, +]; +static ACTION_10: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(4), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(4), + }, +]; +static ACTION_11: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(5), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(5), + }, +]; +static ACTION_12: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(7), + }, +]; +static ACTION_13: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Shift(14), +}]; +static ACTION_14: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Shift(15), +}]; +static ACTION_15: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Token("end"), + action: ParserAction::Reduce(10), + }, +]; +static ACTION_16: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Shift(17), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(18), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Shift(8), + }, + ActionEntry { + terminal: Terminal::Token("end"), + action: ParserAction::Shift(21), + }, +]; +static ACTION_17: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(11), + }, + ActionEntry { + terminal: Terminal::Token("end"), + action: ParserAction::Reduce(11), + }, +]; +static ACTION_18: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Reduce(12), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(12), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(12), + }, + ActionEntry { + terminal: Terminal::Token("end"), + action: ParserAction::Reduce(12), + }, +]; +static ACTION_19: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(13), + }, + ActionEntry { + terminal: Terminal::Token("end"), + action: ParserAction::Reduce(13), + }, +]; +static ACTION_20: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(14), + }, + ActionEntry { + terminal: Terminal::Token("end"), + action: ParserAction::Reduce(14), + }, +]; +static ACTION_21: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Token("text"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Token("border"), + action: ParserAction::Reduce(15), + }, + ActionEntry { + terminal: Terminal::Token("end"), + action: ParserAction::Reduce(15), + }, +]; +static ACTIONS: &[&[ActionEntry]] = &[ + ACTION_0, ACTION_1, ACTION_2, ACTION_3, ACTION_4, ACTION_5, ACTION_6, ACTION_7, ACTION_8, + ACTION_9, ACTION_10, ACTION_11, ACTION_12, ACTION_13, ACTION_14, ACTION_15, ACTION_16, + ACTION_17, ACTION_18, ACTION_19, ACTION_20, ACTION_21, +]; + +static GOTO_0: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "document", + state: 1, + }, + GotoEntry { + nonterminal: "$document.1", + state: 3, + }, + GotoEntry { + nonterminal: "$document.0", + state: 4, + }, +]; +static GOTO_1: &[GotoEntry] = &[]; +static GOTO_2: &[GotoEntry] = &[]; +static GOTO_3: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "unit", + state: 5, + }, + GotoEntry { + nonterminal: "$document.3", + state: 6, + }, + GotoEntry { + nonterminal: "$document.2", + state: 7, + }, + GotoEntry { + nonterminal: "$unit.5", + state: 9, + }, +]; +static GOTO_4: &[GotoEntry] = &[]; +static GOTO_5: &[GotoEntry] = &[GotoEntry { + nonterminal: "$document.4", + state: 11, +}]; +static GOTO_6: &[GotoEntry] = &[]; +static GOTO_7: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "unit", + state: 5, + }, + GotoEntry { + nonterminal: "$document.3", + state: 12, + }, + GotoEntry { + nonterminal: "$unit.5", + state: 9, + }, +]; +static GOTO_8: &[GotoEntry] = &[]; +static GOTO_9: &[GotoEntry] = &[]; +static GOTO_10: &[GotoEntry] = &[]; +static GOTO_11: &[GotoEntry] = &[]; +static GOTO_12: &[GotoEntry] = &[]; +static GOTO_13: &[GotoEntry] = &[]; +static GOTO_14: &[GotoEntry] = &[]; +static GOTO_15: &[GotoEntry] = &[GotoEntry { + nonterminal: "$unit.6", + state: 16, +}]; +static GOTO_16: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "unit", + state: 19, + }, + GotoEntry { + nonterminal: "$unit.7", + state: 20, + }, + GotoEntry { + nonterminal: "$unit.5", + state: 9, + }, +]; +static GOTO_17: &[GotoEntry] = &[]; +static GOTO_18: &[GotoEntry] = &[]; +static GOTO_19: &[GotoEntry] = &[]; +static GOTO_20: &[GotoEntry] = &[]; +static GOTO_21: &[GotoEntry] = &[]; +static GOTOS: &[&[GotoEntry]] = &[ + GOTO_0, GOTO_1, GOTO_2, GOTO_3, GOTO_4, GOTO_5, GOTO_6, GOTO_7, GOTO_8, GOTO_9, GOTO_10, + GOTO_11, GOTO_12, GOTO_13, GOTO_14, GOTO_15, GOTO_16, GOTO_17, GOTO_18, GOTO_19, GOTO_20, + GOTO_21, +]; + +static PRODUCTIONS: &[Production] = &[ + Production { + lhs: "$accept", + rhs: &[Operand { + symbol: Symbol::Nonterminal("document"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.1", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.1", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("text")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.4", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.4", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("text")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.3", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("unit"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$document.4"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.2", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$document.3"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.2", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$document.2"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$document.3"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.0", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$document.1"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$document.2"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "document", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$document.0"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.6", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.7", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("name")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.7", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("string")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.7", + rhs: &[Operand { + symbol: Symbol::Nonterminal("unit"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.6", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$unit.6"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$unit.7"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$unit.5", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("border")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Literal("MODULE")), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("name")), + actions: &[OperandAction::Field("name")], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("border")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$unit.6"), + actions: &[], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("end")), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "unit", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$unit.5"), + actions: &[OperandAction::Alias(1)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, +]; + +pub static LANGUAGE: LanguageSpec = LanguageSpec { + language: "module-framing", + version: "v1", + contract: "module-framing.v1", + grammar_digest: "sha256:07c96670f5167601c9fa6d36d70f85d9da28fedf6fa51958d475274e59ee57e9", + case_insensitive: false, + root_kind: 0, + kinds: KINDS, + terminals: TERMINALS, + lexical_rules: LEXICAL_RULES, + actions: ACTIONS, + gotos: GOTOS, + productions: PRODUCTIONS, +}; + +pub const MODULE_CASES: &[(&str, usize, Option)] = &[ + ("preamble\n---- MODULE Demo ----", 0, Some(9)), + ("---- MODULE A ----\n====\nfooter", 0, None), + ("---- MODULE A ----\n====\nfooter", 23, Some(30)), + ("---- MODULE A ----\n====\nfooter", 24, Some(30)), + ("---- MODULE 字 ----\n====\nλ", 24, Some(25)), + ( + "---- MODULE A ----\n---- MODULE B ----\n====\ninner\n====\nfooter", + 43, + None, + ), + ( + "---- MODULE A ----\n---- MODULE B ----\n====\ninner\n====\nfooter", + 54, + Some(60), + ), + ("---- MODULE A ----\n\"====\"\n====\nfooter", 25, None), + ( + "---- MODULE A ----\n====\nfooter\n---- MODULE B ----", + 24, + Some(31), + ), + ("preamble", 0, Some(8)), + ("preamble", 8, None), +]; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/ordered_results.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/ordered_results.rs new file mode 100644 index 00000000..edc3e794 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/ordered_results.rs @@ -0,0 +1,1150 @@ +// @generated from the admitted Scheme ResultProfile IR. +use gerbil_parser_runtime::{ + CaptureKind, CaptureSpec, KindCategory, KindSpec, ProjectionInstruction, ProjectionOpcode, + ResultNodeSpec, ResultProfileSpec, ResultProjection, +}; +pub static RESULT_PROFILE: ResultProfileSpec = ResultProfileSpec { + kinds: &[ + KindSpec { + name: "Redirection", + category: KindCategory::Node, + }, + KindSpec { + name: "ArrayAssignment", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandList", + category: KindCategory::Node, + }, + KindSpec { + name: "IfCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "WhileCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "UntilCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "SelectCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseClause", + category: KindCategory::Node, + }, + KindSpec { + name: "FunctionDefinition", + category: KindCategory::Node, + }, + KindSpec { + name: "ConditionalCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "RedirectedCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "BraceGroup", + category: KindCategory::Node, + }, + KindSpec { + name: "Subshell", + category: KindCategory::Node, + }, + KindSpec { + name: "Pipeline", + category: KindCategory::Node, + }, + KindSpec { + name: "AndOrList", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocument", + category: KindCategory::Node, + }, + KindSpec { + name: "BashFile", + category: KindCategory::Node, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Node, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Node, + }, + KindSpec { + name: "Word", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocumentLine", + category: KindCategory::Node, + }, + KindSpec { + name: "SingleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "DoubleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "AnsiCString", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ProcessSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ArraySubscript", + category: KindCategory::Node, + }, + KindSpec { + name: "ParameterExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "Assignment", + category: KindCategory::Node, + }, + KindSpec { + name: "unparsed-source", + category: KindCategory::Token, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Token, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Token, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-open", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-close", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-open", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-body", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-close", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-open", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-close", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-open", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-prefix", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-name", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-close", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-name", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "horizontal-whitespace", + category: KindCategory::Token, + }, + KindSpec { + name: "newline", + category: KindCategory::Token, + }, + KindSpec { + name: "line-continuation", + category: KindCategory::Token, + }, + KindSpec { + name: "comment", + category: KindCategory::Token, + }, + KindSpec { + name: "word", + category: KindCategory::Token, + }, + KindSpec { + name: "operator", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-marker", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-content", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-end", + category: KindCategory::Token, + }, + ], + nodes: &[ + ResultNodeSpec { + kind: 0, + fields: &["descriptor", "operator", "target"], + }, + ResultNodeSpec { + kind: 1, + fields: &["assignment", "open", "close", "separator", "element"], + }, + ResultNodeSpec { + kind: 2, + fields: &["assignment", "name", "argument", "redirect"], + }, + ResultNodeSpec { + kind: 3, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 4, + fields: &["keyword", "condition", "body", "else-body"], + }, + ResultNodeSpec { + kind: 5, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 6, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 7, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 8, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 9, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 10, + fields: &["keyword", "subject", "separator", "clause"], + }, + ResultNodeSpec { + kind: 11, + fields: &[ + "open", + "pattern", + "alternate", + "close", + "body", + "terminator", + ], + }, + ResultNodeSpec { + kind: 12, + fields: &["keyword", "name", "open", "close", "body"], + }, + ResultNodeSpec { + kind: 13, + fields: &["open", "close", "operator", "operand"], + }, + ResultNodeSpec { + kind: 14, + fields: &["open", "close", "expression", "operator"], + }, + ResultNodeSpec { + kind: 15, + fields: &["command", "redirect"], + }, + ResultNodeSpec { + kind: 16, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 17, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 18, + fields: &["keyword", "option", "negate", "command", "operator"], + }, + ResultNodeSpec { + kind: 19, + fields: &["command", "operator"], + }, + ResultNodeSpec { + kind: 20, + fields: &["delimiter", "content"], + }, + ResultNodeSpec { + kind: 21, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 22, + fields: &["text"], + }, + ResultNodeSpec { + kind: 23, + fields: &["text"], + }, + ResultNodeSpec { + kind: 24, + fields: &["text"], + }, + ResultNodeSpec { + kind: 25, + fields: &["part"], + }, + ResultNodeSpec { + kind: 26, + fields: &["part"], + }, + ResultNodeSpec { + kind: 27, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 28, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 29, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 30, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 31, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 32, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 33, + fields: &["open", "index", "close"], + }, + ResultNodeSpec { + kind: 34, + fields: &[ + "open", + "prefix", + "name", + "subscript", + "operator", + "operand", + "close", + ], + }, + ResultNodeSpec { + kind: 35, + fields: &["name", "operator", "value"], + }, + ], + projections: &[ + ResultProjection { + id: "LiteralPart", + kind: 22, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 37, + required: true, + }, + }], + }, + ResultProjection { + id: "EscapeSequence", + kind: 23, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 38, + required: true, + }, + }], + }, + ResultProjection { + id: "SimpleParameter", + kind: 24, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 39, + required: true, + }, + }], + }, + ResultProjection { + id: "Word", + kind: 25, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "HereDocumentLine", + kind: 26, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "SingleQuoted", + kind: 27, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "DoubleQuoted", + kind: 28, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "AnsiCString", + kind: 29, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArithmeticExpansion", + kind: 30, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "CommandSubstitution", + kind: 31, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ProcessSubstitution", + kind: 32, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArraySubscript", + kind: 33, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 45, + required: true, + }, + }, + ProjectionInstruction { + field: "index", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 46, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ParameterExpansion", + kind: 34, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "prefix", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "subscript", + kind: CaptureKind::Node, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 47, + required: true, + }, + }, + ProjectionInstruction { + field: "prefix", + opcode: ProjectionOpcode::Token { + kind: 48, + required: false, + }, + }, + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 49, + required: false, + }, + }, + ProjectionInstruction { + field: "subscript", + opcode: ProjectionOpcode::One { required: false }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 50, + required: false, + }, + }, + ProjectionInstruction { + field: "operand", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 51, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "Assignment", + kind: 35, + captures: &[ + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 52, + required: true, + }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 53, + required: true, + }, + }, + ProjectionInstruction { + field: "value", + opcode: ProjectionOpcode::Many, + }, + ], + }, + ], +}; +use gerbil_parser_runtime::{ + Diagnostic, PreparedResultProfile, ProjectedNode, ProjectedValue, ResultChildCapture, TreeEvent, +}; +pub fn replay_0<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.bind_node("SimpleCommand")?.build( + source, + 0..3, + vec![ + ResultChildCapture { + field: "name", + value: ProjectedValue::Node(profile.bind_node("Word")?.build( + source, + 0..1, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: 37, + span: 0..1, + }, + }], + )?), + }, + ResultChildCapture { + field: "redirect", + value: ProjectedValue::Node(profile.bind_node("Redirection")?.build( + source, + 1..2, + vec![ResultChildCapture { + field: "operator", + value: ProjectedValue::Token { + kind: 59, + span: 1..2, + }, + }], + )?), + }, + ResultChildCapture { + field: "argument", + value: ProjectedValue::Node(profile.bind_node("Word")?.build( + source, + 2..3, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: 37, + span: 2..3, + }, + }], + )?), + }, + ], + ) +} +pub fn replay_1<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.bind_node("Pipeline")?.build( + source, + 0..3, + vec![ + ResultChildCapture { + field: "command", + value: ProjectedValue::Node(profile.bind_node("SimpleCommand")?.build( + source, + 0..1, + vec![ResultChildCapture { + field: "name", + value: ProjectedValue::Node(profile.bind_node("Word")?.build( + source, + 0..1, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: 37, + span: 0..1, + }, + }], + )?), + }], + )?), + }, + ResultChildCapture { + field: "operator", + value: ProjectedValue::Token { + kind: 59, + span: 1..2, + }, + }, + ResultChildCapture { + field: "command", + value: ProjectedValue::Node(profile.bind_node("SimpleCommand")?.build( + source, + 2..3, + vec![ResultChildCapture { + field: "name", + value: ProjectedValue::Node(profile.bind_node("Word")?.build( + source, + 2..3, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: 37, + span: 2..3, + }, + }], + )?), + }], + )?), + }, + ], + ) +} +pub fn replay_2<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.bind_node("BashFile")?.build( + source, + 0..8, + vec![ResultChildCapture { + field: "command", + value: ProjectedValue::Node(profile.bind_node("SimpleCommand")?.build( + source, + 0..8, + vec![ + ResultChildCapture { + field: "argument", + value: ProjectedValue::Node(profile.bind_node("Word")?.build( + source, + 0..2, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: 37, + span: 0..2, + }, + }], + )?), + }, + ResultChildCapture { + field: "name", + value: ProjectedValue::Node(profile.bind_node("Word")?.build( + source, + 2..4, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: 37, + span: 2..4, + }, + }], + )?), + }, + ResultChildCapture { + field: "argument", + value: ProjectedValue::Node(profile.bind_node("Word")?.build( + source, + 4..8, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: 37, + span: 4..8, + }, + }], + )?), + }, + ], + )?), + }], + ) +} +pub fn replay_3<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.bind_node("BashFile")?.build(source, 0..0, vec![]) +} +pub static SOURCES: &[&str] = &["abc", "a|b", "αβ😀", ""]; +pub static EVENTS: &[&[TreeEvent]] = &[ + &[ + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::Token { + kind: 37, + start: 0, + end: 1, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(0), + TreeEvent::Token { + kind: 59, + start: 1, + end: 2, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(25), + TreeEvent::Token { + kind: 37, + start: 2, + end: 3, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(18), + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::Token { + kind: 37, + start: 0, + end: 1, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 59, + start: 1, + end: 2, + }, + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::Token { + kind: 37, + start: 2, + end: 3, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(21), + TreeEvent::StartNode(2), + TreeEvent::StartNode(25), + TreeEvent::Token { + kind: 37, + start: 0, + end: 2, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(25), + TreeEvent::Token { + kind: 37, + start: 2, + end: 4, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(25), + TreeEvent::Token { + kind: 37, + start: 4, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[TreeEvent::StartNode(21), TreeEvent::FinishNode], +]; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/region_boundaries.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/region_boundaries.rs new file mode 100644 index 00000000..0c688149 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/region_boundaries.rs @@ -0,0 +1,152 @@ +// @generated by Scheme region execution; do not edit. +pub mod bash { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", "<&", + ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[]; + type Request = (u32, usize, i64); + pub static TRACES: &[(&str, &[Request])] = &[ + ("α😀", &[(0, 0, 6)]), + ("'α😀'", &[(0, 0, 8), (39, 0, 8), (39, 7, -1)]), + ( + "\"α${x:-中}\"", + &[(0, 0, 13), (34, 0, 13), (1, 3, 12), (34, 12, -1)], + ), + ("${arr[${i:-1}]:-λ}", &[(0, 0, 19), (1, 0, 19), (1, 6, 13)]), + ("$((1+$(echo α)))", &[(0, 0, 17), (1, 0, 17), (1, 5, 15)]), + ( + "<(echo \"中\")>stop", + &[(0, 0, 13), (1, 0, 13), (34, 7, 12), (34, 11, -1)], + ), + ("`echo α`", &[(0, 0, 9), (96, 0, 9), (96, 8, -1)]), + ("x\\α;tail", &[(0, 0, 4)]), + ("x tail", &[(0, 0, 1)]), + ("'unterminated", &[(0, 0, -1), (39, 0, -1)]), + ("${x:-\"bad}", &[(0, 0, -1), (1, 0, -1), (34, 5, -1)]), + ("${x:-${y:-z}}", &[(0, 0, 13), (1, 0, 13), (1, 5, 12)]), + ("λ{[}tail", &[(0, 0, 9)]), + ("[λ{[}λ{中}]", &[(0, 0, 14)]), + ("«λ{中}»;tail", &[(0, 0, 11)]), + ("«\\»中»", &[(0, 0, 10)]), + ("[λ{中}", &[(0, 0, 8)]), + (";tail", &[(0, 0, 5)]), + ("${x}", &[(0, 0, 4), (1, 0, 4)]), + ("λ{λ{中}}", &[(0, 0, 11)]), + ]; +} +pub mod generic { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[";"], + quotes: &[gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{ab}', + escaped: true, + pairs: &["λ{"], + }], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "λ{", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "[", + opening: '\u{5b}', + closing: '\u{5d}', + depth: 1, + }, + ], + consume_initial_stop: false, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[ + gerbil_parser_runtime::scanner::RegionScope { + prefix: "λ{", + pairs: &["λ{"], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "[", + pairs: &["λ{", "["], + }, + ]; + type Request = (u32, usize, i64); + pub static TRACES: &[(&str, &[Request])] = &[ + ("α😀", &[(0, 0, 6)]), + ("'α😀'", &[(0, 0, 8)]), + ("\"α${x:-中}\"", &[(0, 0, 13)]), + ("${arr[${i:-1}]:-λ}", &[(0, 0, 19), (1, 5, 14)]), + ("$((1+$(echo α)))", &[(0, 0, 11)]), + ("<(echo \"中\")>stop", &[(0, 0, 6)]), + ("`echo α`", &[(0, 0, 5)]), + ("x\\α;tail", &[(0, 0, 4)]), + ("x tail", &[(0, 0, 1)]), + ("'unterminated", &[(0, 0, 13)]), + ("${x:-\"bad}", &[(0, 0, 10)]), + ("${x:-${y:-z}}", &[(0, 0, 13)]), + ("λ{[}tail", &[(0, 0, -1), (1, 0, 5), (1, 3, -1)]), + ( + "[λ{[}λ{中}]", + &[(0, 0, -1), (1, 0, 14), (1, 1, 6), (1, 4, 14), (1, 6, 13)], + ), + ("«λ{中}»;tail", &[(0, 0, -1), (171, 0, -1), (1, 2, 9)]), + ("«\\»中»", &[(0, 0, -1), (171, 0, -1)]), + ("[λ{中}", &[(0, 0, -1), (1, 0, -1), (1, 1, 8)]), + (";tail", &[(0, 0, -2)]), + ("${x}", &[(0, 0, 4)]), + ("λ{λ{中}}", &[(0, 0, 11), (1, 0, 11), (1, 3, 10)]), + ]; +} diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/result_projections.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/result_projections.rs new file mode 100644 index 00000000..2a15b4f6 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/result_projections.rs @@ -0,0 +1,1815 @@ +// @generated from the admitted Scheme ResultProfile IR. +use gerbil_parser_runtime::{ + CaptureKind, CaptureSpec, KindCategory, KindSpec, ProjectionInstruction, ProjectionOpcode, + ResultNodeSpec, ResultProfileSpec, ResultProjection, +}; +pub static RESULT_PROFILE: ResultProfileSpec = ResultProfileSpec { + kinds: &[ + KindSpec { + name: "Redirection", + category: KindCategory::Node, + }, + KindSpec { + name: "ArrayAssignment", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandList", + category: KindCategory::Node, + }, + KindSpec { + name: "IfCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "WhileCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "UntilCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "SelectCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseClause", + category: KindCategory::Node, + }, + KindSpec { + name: "FunctionDefinition", + category: KindCategory::Node, + }, + KindSpec { + name: "ConditionalCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "RedirectedCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "BraceGroup", + category: KindCategory::Node, + }, + KindSpec { + name: "Subshell", + category: KindCategory::Node, + }, + KindSpec { + name: "Pipeline", + category: KindCategory::Node, + }, + KindSpec { + name: "AndOrList", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocument", + category: KindCategory::Node, + }, + KindSpec { + name: "BashFile", + category: KindCategory::Node, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Node, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Node, + }, + KindSpec { + name: "Word", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocumentLine", + category: KindCategory::Node, + }, + KindSpec { + name: "SingleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "DoubleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "AnsiCString", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ProcessSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ArraySubscript", + category: KindCategory::Node, + }, + KindSpec { + name: "ParameterExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "Assignment", + category: KindCategory::Node, + }, + KindSpec { + name: "unparsed-source", + category: KindCategory::Token, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Token, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Token, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-open", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-close", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-open", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-body", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-close", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-open", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-close", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-open", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-prefix", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-name", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-close", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-name", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "horizontal-whitespace", + category: KindCategory::Token, + }, + KindSpec { + name: "newline", + category: KindCategory::Token, + }, + KindSpec { + name: "line-continuation", + category: KindCategory::Token, + }, + KindSpec { + name: "comment", + category: KindCategory::Token, + }, + KindSpec { + name: "word", + category: KindCategory::Token, + }, + KindSpec { + name: "operator", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-marker", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-content", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-end", + category: KindCategory::Token, + }, + ], + nodes: &[ + ResultNodeSpec { + kind: 0, + fields: &["descriptor", "operator", "target"], + }, + ResultNodeSpec { + kind: 1, + fields: &["assignment", "open", "close", "separator", "element"], + }, + ResultNodeSpec { + kind: 2, + fields: &["assignment", "name", "argument", "redirect"], + }, + ResultNodeSpec { + kind: 3, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 4, + fields: &["keyword", "condition", "body", "else-body"], + }, + ResultNodeSpec { + kind: 5, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 6, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 7, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 8, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 9, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 10, + fields: &["keyword", "subject", "separator", "clause"], + }, + ResultNodeSpec { + kind: 11, + fields: &[ + "open", + "pattern", + "alternate", + "close", + "body", + "terminator", + ], + }, + ResultNodeSpec { + kind: 12, + fields: &["keyword", "name", "open", "close", "body"], + }, + ResultNodeSpec { + kind: 13, + fields: &["open", "close", "operator", "operand"], + }, + ResultNodeSpec { + kind: 14, + fields: &["open", "close", "expression", "operator"], + }, + ResultNodeSpec { + kind: 15, + fields: &["command", "redirect"], + }, + ResultNodeSpec { + kind: 16, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 17, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 18, + fields: &["keyword", "option", "negate", "command", "operator"], + }, + ResultNodeSpec { + kind: 19, + fields: &["command", "operator"], + }, + ResultNodeSpec { + kind: 20, + fields: &["delimiter", "content"], + }, + ResultNodeSpec { + kind: 21, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 22, + fields: &["text"], + }, + ResultNodeSpec { + kind: 23, + fields: &["text"], + }, + ResultNodeSpec { + kind: 24, + fields: &["text"], + }, + ResultNodeSpec { + kind: 25, + fields: &["part"], + }, + ResultNodeSpec { + kind: 26, + fields: &["part"], + }, + ResultNodeSpec { + kind: 27, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 28, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 29, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 30, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 31, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 32, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 33, + fields: &["open", "index", "close"], + }, + ResultNodeSpec { + kind: 34, + fields: &[ + "open", + "prefix", + "name", + "subscript", + "operator", + "operand", + "close", + ], + }, + ResultNodeSpec { + kind: 35, + fields: &["name", "operator", "value"], + }, + ], + projections: &[ + ResultProjection { + id: "LiteralPart", + kind: 22, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 37, + required: true, + }, + }], + }, + ResultProjection { + id: "EscapeSequence", + kind: 23, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 38, + required: true, + }, + }], + }, + ResultProjection { + id: "SimpleParameter", + kind: 24, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 39, + required: true, + }, + }], + }, + ResultProjection { + id: "Word", + kind: 25, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "HereDocumentLine", + kind: 26, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "SingleQuoted", + kind: 27, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "DoubleQuoted", + kind: 28, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "AnsiCString", + kind: 29, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArithmeticExpansion", + kind: 30, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "CommandSubstitution", + kind: 31, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ProcessSubstitution", + kind: 32, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArraySubscript", + kind: 33, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 45, + required: true, + }, + }, + ProjectionInstruction { + field: "index", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 46, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ParameterExpansion", + kind: 34, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "prefix", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "subscript", + kind: CaptureKind::Node, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 47, + required: true, + }, + }, + ProjectionInstruction { + field: "prefix", + opcode: ProjectionOpcode::Token { + kind: 48, + required: false, + }, + }, + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 49, + required: false, + }, + }, + ProjectionInstruction { + field: "subscript", + opcode: ProjectionOpcode::One { required: false }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 50, + required: false, + }, + }, + ProjectionInstruction { + field: "operand", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 51, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "Assignment", + kind: 35, + captures: &[ + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 52, + required: true, + }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 53, + required: true, + }, + }, + ProjectionInstruction { + field: "value", + opcode: ProjectionOpcode::Many, + }, + ], + }, + ], +}; +use gerbil_parser_runtime::{ + Diagnostic, PreparedResultProfile, ProjectedNode, ResultCapture, TreeEvent, +}; +pub static SOURCES: &[&str] = &[ + "", + "α😀", + "'α😀'", + "\"α${x:-中}\"", + "$'a\\n'", + "${arr[${i:-1}]:-λ}", + "$(echo α)", + "$((1+2))", + "<(echo 中)", + "`echo α`", + "$x\\α", + "name+=\"α${x:-中}\"", + "α${x:-中}\\\n", +]; +fn replay_0<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project("Word", source, 0..0, vec![ResultCapture::Parts(vec![])]) +} +fn replay_1<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..6, + vec![ResultCapture::Parts(vec![profile.project( + "LiteralPart", + source, + 0..6, + vec![ResultCapture::Span(0..6)], + )?])], + ) +} +fn replay_2<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..8, + vec![ResultCapture::Parts(vec![profile.project( + "SingleQuoted", + source, + 0..8, + vec![ + ResultCapture::Span(0..1), + ResultCapture::Parts(vec![profile.project( + "LiteralPart", + source, + 1..7, + vec![ResultCapture::Span(1..7)], + )?]), + ResultCapture::Span(7..8), + ], + )?])], + ) +} +fn replay_3<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..13, + vec![ResultCapture::Parts(vec![profile.project( + "DoubleQuoted", + source, + 0..13, + vec![ + ResultCapture::Span(0..1), + ResultCapture::Parts(vec![ + profile.project( + "LiteralPart", + source, + 1..3, + vec![ResultCapture::Span(1..3)], + )?, + profile.project( + "ParameterExpansion", + source, + 3..12, + vec![ + ResultCapture::Span(3..5), + ResultCapture::Absent, + ResultCapture::Span(5..6), + ResultCapture::Absent, + ResultCapture::Span(6..8), + ResultCapture::Parts(vec![profile.project( + "LiteralPart", + source, + 8..11, + vec![ResultCapture::Span(8..11)], + )?]), + ResultCapture::Span(11..12), + ], + )?, + ]), + ResultCapture::Span(12..13), + ], + )?])], + ) +} +fn replay_4<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..6, + vec![ResultCapture::Parts(vec![profile.project( + "AnsiCString", + source, + 0..6, + vec![ + ResultCapture::Span(0..2), + ResultCapture::Parts(vec![ + profile.project( + "LiteralPart", + source, + 2..3, + vec![ResultCapture::Span(2..3)], + )?, + profile.project( + "EscapeSequence", + source, + 3..5, + vec![ResultCapture::Span(3..5)], + )?, + ]), + ResultCapture::Span(5..6), + ], + )?])], + ) +} +fn replay_5<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..19, + vec![ResultCapture::Parts(vec![profile.project( + "ParameterExpansion", + source, + 0..19, + vec![ + ResultCapture::Span(0..2), + ResultCapture::Absent, + ResultCapture::Span(2..5), + ResultCapture::Node(profile.project( + "ArraySubscript", + source, + 5..14, + vec![ + ResultCapture::Span(5..6), + ResultCapture::Parts(vec![profile.project( + "ParameterExpansion", + source, + 6..13, + vec![ + ResultCapture::Span(6..8), + ResultCapture::Absent, + ResultCapture::Span(8..9), + ResultCapture::Absent, + ResultCapture::Span(9..11), + ResultCapture::Parts(vec![profile.project( + "LiteralPart", + source, + 11..12, + vec![ResultCapture::Span(11..12)], + )?]), + ResultCapture::Span(12..13), + ], + )?]), + ResultCapture::Span(13..14), + ], + )?), + ResultCapture::Span(14..16), + ResultCapture::Parts(vec![profile.project( + "LiteralPart", + source, + 16..18, + vec![ResultCapture::Span(16..18)], + )?]), + ResultCapture::Span(18..19), + ], + )?])], + ) +} +fn replay_6<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..10, + vec![ResultCapture::Parts(vec![profile.project( + "CommandSubstitution", + source, + 0..10, + vec![ + ResultCapture::Span(0..2), + ResultCapture::Span(2..9), + ResultCapture::Span(9..10), + ], + )?])], + ) +} +fn replay_7<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..8, + vec![ResultCapture::Parts(vec![profile.project( + "ArithmeticExpansion", + source, + 0..8, + vec![ + ResultCapture::Span(0..3), + ResultCapture::Span(3..6), + ResultCapture::Span(6..8), + ], + )?])], + ) +} +fn replay_8<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..11, + vec![ResultCapture::Parts(vec![profile.project( + "ProcessSubstitution", + source, + 0..11, + vec![ + ResultCapture::Span(0..2), + ResultCapture::Span(2..10), + ResultCapture::Span(10..11), + ], + )?])], + ) +} +fn replay_9<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..9, + vec![ResultCapture::Parts(vec![profile.project( + "CommandSubstitution", + source, + 0..9, + vec![ + ResultCapture::Span(0..1), + ResultCapture::Span(1..8), + ResultCapture::Span(8..9), + ], + )?])], + ) +} +fn replay_10<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Word", + source, + 0..5, + vec![ResultCapture::Parts(vec![ + profile.project( + "SimpleParameter", + source, + 0..2, + vec![ResultCapture::Span(0..2)], + )?, + profile.project( + "EscapeSequence", + source, + 2..5, + vec![ResultCapture::Span(2..5)], + )?, + ])], + ) +} +fn replay_11<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "Assignment", + source, + 0..19, + vec![ + ResultCapture::Span(0..4), + ResultCapture::Span(4..6), + ResultCapture::Parts(vec![profile.project( + "DoubleQuoted", + source, + 6..19, + vec![ + ResultCapture::Span(6..7), + ResultCapture::Parts(vec![ + profile.project( + "LiteralPart", + source, + 7..9, + vec![ResultCapture::Span(7..9)], + )?, + profile.project( + "ParameterExpansion", + source, + 9..18, + vec![ + ResultCapture::Span(9..11), + ResultCapture::Absent, + ResultCapture::Span(11..12), + ResultCapture::Absent, + ResultCapture::Span(12..14), + ResultCapture::Parts(vec![profile.project( + "LiteralPart", + source, + 14..17, + vec![ResultCapture::Span(14..17)], + )?]), + ResultCapture::Span(17..18), + ], + )?, + ]), + ResultCapture::Span(18..19), + ], + )?]), + ], + ) +} +fn replay_12<'s>( + profile: &PreparedResultProfile, + source: &'s str, +) -> Result, Diagnostic> { + profile.project( + "HereDocumentLine", + source, + 0..13, + vec![ResultCapture::Parts(vec![ + profile.project("LiteralPart", source, 0..2, vec![ResultCapture::Span(0..2)])?, + profile.project( + "ParameterExpansion", + source, + 2..11, + vec![ + ResultCapture::Span(2..4), + ResultCapture::Absent, + ResultCapture::Span(4..5), + ResultCapture::Absent, + ResultCapture::Span(5..7), + ResultCapture::Parts(vec![profile.project( + "LiteralPart", + source, + 7..10, + vec![ResultCapture::Span(7..10)], + )?]), + ResultCapture::Span(10..11), + ], + )?, + profile.project( + "EscapeSequence", + source, + 11..13, + vec![ResultCapture::Span(11..13)], + )?, + ])], + ) +} +pub fn replay<'s>( + profile: &PreparedResultProfile, + source: &'s str, + index: usize, +) -> Result, Diagnostic> { + match index { + 0 => replay_0(profile, source), + 1 => replay_1(profile, source), + 2 => replay_2(profile, source), + 3 => replay_3(profile, source), + 4 => replay_4(profile, source), + 5 => replay_5(profile, source), + 6 => replay_6(profile, source), + 7 => replay_7(profile, source), + 8 => replay_8(profile, source), + 9 => replay_9(profile, source), + 10 => replay_10(profile, source), + 11 => replay_11(profile, source), + 12 => replay_12(profile, source), + _ => panic!("unknown generated control"), + } +} +pub static EXPECTED: &[&[TreeEvent]] = &[ + &[TreeEvent::StartNode(25), TreeEvent::FinishNode], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 0, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(27), + TreeEvent::Token { + kind: 40, + start: 0, + end: 1, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 1, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 7, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 0, + end: 1, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 1, + end: 3, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 3, + end: 5, + }, + TreeEvent::Token { + kind: 49, + start: 5, + end: 6, + }, + TreeEvent::Token { + kind: 50, + start: 6, + end: 8, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 8, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(29), + TreeEvent::Token { + kind: 40, + start: 0, + end: 2, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 2, + end: 3, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 3, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 5, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 0, + end: 2, + }, + TreeEvent::Token { + kind: 49, + start: 2, + end: 5, + }, + TreeEvent::StartNode(33), + TreeEvent::Token { + kind: 45, + start: 5, + end: 6, + }, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 6, + end: 8, + }, + TreeEvent::Token { + kind: 49, + start: 8, + end: 9, + }, + TreeEvent::Token { + kind: 50, + start: 9, + end: 11, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 46, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 50, + start: 14, + end: 16, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 16, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 18, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(31), + TreeEvent::Token { + kind: 42, + start: 0, + end: 2, + }, + TreeEvent::Token { + kind: 43, + start: 2, + end: 9, + }, + TreeEvent::Token { + kind: 44, + start: 9, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(30), + TreeEvent::Token { + kind: 42, + start: 0, + end: 3, + }, + TreeEvent::Token { + kind: 43, + start: 3, + end: 6, + }, + TreeEvent::Token { + kind: 44, + start: 6, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(32), + TreeEvent::Token { + kind: 42, + start: 0, + end: 2, + }, + TreeEvent::Token { + kind: 43, + start: 2, + end: 10, + }, + TreeEvent::Token { + kind: 44, + start: 10, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(31), + TreeEvent::Token { + kind: 42, + start: 0, + end: 1, + }, + TreeEvent::Token { + kind: 43, + start: 1, + end: 8, + }, + TreeEvent::Token { + kind: 44, + start: 8, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 0, + end: 2, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 2, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 0, + end: 4, + }, + TreeEvent::Token { + kind: 53, + start: 4, + end: 6, + }, + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 6, + end: 7, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 7, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 9, + end: 11, + }, + TreeEvent::Token { + kind: 49, + start: 11, + end: 12, + }, + TreeEvent::Token { + kind: 50, + start: 12, + end: 14, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 14, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 17, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 18, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 0, + end: 2, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 2, + end: 4, + }, + TreeEvent::Token { + kind: 49, + start: 4, + end: 5, + }, + TreeEvent::Token { + kind: 50, + start: 5, + end: 7, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 7, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 10, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 11, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], +]; +pub static FIELDS: &[&[(&str, &str)]] = &[ + &[], + &[("Word", "part"), ("LiteralPart", "text")], + &[ + ("Word", "part"), + ("SingleQuoted", "open"), + ("SingleQuoted", "part"), + ("LiteralPart", "text"), + ("SingleQuoted", "close"), + ], + &[ + ("Word", "part"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("DoubleQuoted", "close"), + ], + &[ + ("Word", "part"), + ("AnsiCString", "open"), + ("AnsiCString", "part"), + ("LiteralPart", "text"), + ("AnsiCString", "part"), + ("EscapeSequence", "text"), + ("AnsiCString", "close"), + ], + &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "subscript"), + ("ArraySubscript", "open"), + ("ArraySubscript", "index"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("ArraySubscript", "close"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + &[ + ("Word", "part"), + ("CommandSubstitution", "open"), + ("CommandSubstitution", "body"), + ("CommandSubstitution", "close"), + ], + &[ + ("Word", "part"), + ("ArithmeticExpansion", "open"), + ("ArithmeticExpansion", "body"), + ("ArithmeticExpansion", "close"), + ], + &[ + ("Word", "part"), + ("ProcessSubstitution", "open"), + ("ProcessSubstitution", "body"), + ("ProcessSubstitution", "close"), + ], + &[ + ("Word", "part"), + ("CommandSubstitution", "open"), + ("CommandSubstitution", "body"), + ("CommandSubstitution", "close"), + ], + &[ + ("Word", "part"), + ("SimpleParameter", "text"), + ("Word", "part"), + ("EscapeSequence", "text"), + ], + &[ + ("Assignment", "name"), + ("Assignment", "operator"), + ("Assignment", "value"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("DoubleQuoted", "close"), + ], + &[ + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ], +]; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/structured_lexical.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/structured_lexical.rs new file mode 100644 index 00000000..b6e5a126 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/structured_lexical.rs @@ -0,0 +1,337 @@ +// @generated by gerbil-parser/src/compiler/rust-runtime.ss +// Source authority: canonical Grammar IR v1; do not edit. + +use gerbil_parser_runtime::{ + ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, + Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, +}; + +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "Entry", + category: KindCategory::Node, + }, + KindSpec { + name: "Space", + category: KindCategory::Token, + }, + KindSpec { + name: "Identifier", + category: KindCategory::Token, + }, + KindSpec { + name: "ProofStep", + category: KindCategory::Token, + }, + KindSpec { + name: "ProofReference", + category: KindCategory::Token, + }, +]; + +static TERMINALS: &[TerminalSpec] = &[ + TerminalSpec { + name: "space", + syntax_kind: 1, + }, + TerminalSpec { + name: "identifier", + syntax_kind: 2, + }, + TerminalSpec { + name: "proof-step-name", + syntax_kind: 3, + }, + TerminalSpec { + name: "proof-reference", + syntax_kind: 4, + }, +]; + +static LEXICAL_RULES: &[LexicalRule] = &[ + LexicalRule { + terminal: "space", + expression: LexicalExpr::Whitespace, + precedence: 0, + extra: true, + }, + LexicalRule { + terminal: "identifier", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::RunContaining { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + required: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "proof-step-name", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::EndsIn { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("<"), + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("+*"), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("+*"), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }), + }, + gerbil_parser_runtime::TextProfile::Literal(">"), + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Optional( + &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("*-"), + minimum: 1, + maximum: Some(1), + }, + )), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("."), + minimum: 0, + maximum: None, + }, + ]), + positive: false, + }), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "proof-reference", + expression: LexicalExpr::TextProfile(&gerbil_parser_runtime::TextProfile::EndsIn { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + body: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Literal("<"), + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("+*"), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("+*"), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }), + }, + gerbil_parser_runtime::TextProfile::Literal(">"), + gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Optional( + &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("*-"), + minimum: 1, + maximum: Some(1), + }, + )), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("."), + minimum: 0, + maximum: None, + }, + ]), + positive: true, + }), + precedence: 0, + extra: false, + }, +]; + +static ACTION_0: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(2), + }, + ActionEntry { + terminal: Terminal::Token("proof-step-name"), + action: ParserAction::Shift(3), + }, + ActionEntry { + terminal: Terminal::Token("proof-reference"), + action: ParserAction::Shift(4), + }, +]; +static ACTION_1: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Accept, +}]; +static ACTION_2: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(1), +}]; +static ACTION_3: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(2), +}]; +static ACTION_4: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(3), +}]; +static ACTION_5: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(4), +}]; +static ACTIONS: &[&[ActionEntry]] = &[ACTION_0, ACTION_1, ACTION_2, ACTION_3, ACTION_4, ACTION_5]; + +static GOTO_0: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "entry", + state: 1, + }, + GotoEntry { + nonterminal: "$entry.0", + state: 5, + }, +]; +static GOTO_1: &[GotoEntry] = &[]; +static GOTO_2: &[GotoEntry] = &[]; +static GOTO_3: &[GotoEntry] = &[]; +static GOTO_4: &[GotoEntry] = &[]; +static GOTO_5: &[GotoEntry] = &[]; +static GOTOS: &[&[GotoEntry]] = &[GOTO_0, GOTO_1, GOTO_2, GOTO_3, GOTO_4, GOTO_5]; + +static PRODUCTIONS: &[Production] = &[ + Production { + lhs: "$accept", + rhs: &[Operand { + symbol: Symbol::Nonterminal("entry"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$entry.0", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("identifier")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$entry.0", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("proof-step-name")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$entry.0", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("proof-reference")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "entry", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$entry.0"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, +]; + +pub static LANGUAGE: LanguageSpec = LanguageSpec { + language: "structured-lexical", + version: "v1", + contract: "structured-lexical.v1", + grammar_digest: "sha256:9a75716b88708baf309d7a5bb2821f6579cf31bfb9472aab04abccb9005d8835", + case_insensitive: false, + root_kind: 0, + kinds: KINDS, + terminals: TERMINALS, + lexical_rules: LEXICAL_RULES, + actions: ACTIONS, + gotos: GOTOS, + productions: PRODUCTIONS, +}; + +pub const PROFILE_CASES: &[(&str, &str, Option)] = &[ + ("identifier", "123", None), + ("identifier", "1_name", Some(6)), + ("identifier", "α٣_字!", Some(4)), + ("identifier", "²", None), + ("identifier", "α²", Some(1)), + ("identifier", "_", Some(1)), + ("identifier", "４2", None), + ("identifier", "４a", Some(2)), + ("identifier", "", None), + ("proof-step-name", "<1>1. QED", Some(5)), + ("proof-step-name", "<+>*.", Some(5)), + ("proof-step-name", "<*>-.", Some(5)), + ("proof-step-name", "<1>", Some(3)), + ("proof-step-name", "<1>name", None), + ("proof-step-name", "<1>α٣.", Some(6)), + ("proof-step-name", "<١>字...", Some(7)), + ("proof-step-name", "<+1>x.", None), + ("proof-step-name", "<1", None), + ("proof-step-name", "<>.", None), + ("proof-reference", "<1>name", Some(7)), + ("proof-reference", "<1>α٣", Some(5)), + ("proof-reference", "<١>字", Some(4)), + ("proof-reference", "<1>1.", None), + ("proof-reference", "<1>", None), + ("proof-reference", "<*>*", None), + ("proof-reference", "<+>_", Some(4)), +]; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/generated/word_parts.rs b/rust/gerbil-parser-runtime/tests/fixtures/generated/word_parts.rs new file mode 100644 index 00000000..16747846 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/generated/word_parts.rs @@ -0,0 +1,6404 @@ +// @generated by Scheme Word recognition; do not edit. +pub struct Control { + pub mode: &'static str, + pub source: &'static str, + pub start: usize, + pub end: usize, + pub status: u8, + pub events: &'static [gerbil_parser_runtime::TreeEvent], + pub fields: &'static [(&'static str, &'static str)], +} +pub mod bash { + use gerbil_parser_runtime::TreeEvent; + // @generated from the admitted Scheme ResultProfile IR. + use gerbil_parser_runtime::{ + CaptureKind, CaptureSpec, KindCategory, KindSpec, ProjectionInstruction, ProjectionOpcode, + ResultNodeSpec, ResultProfileSpec, ResultProjection, + }; + pub static RESULT_PROFILE: ResultProfileSpec = ResultProfileSpec { + kinds: &[ + KindSpec { + name: "Redirection", + category: KindCategory::Node, + }, + KindSpec { + name: "ArrayAssignment", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandList", + category: KindCategory::Node, + }, + KindSpec { + name: "IfCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "WhileCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "UntilCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "SelectCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseClause", + category: KindCategory::Node, + }, + KindSpec { + name: "FunctionDefinition", + category: KindCategory::Node, + }, + KindSpec { + name: "ConditionalCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "RedirectedCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "BraceGroup", + category: KindCategory::Node, + }, + KindSpec { + name: "Subshell", + category: KindCategory::Node, + }, + KindSpec { + name: "Pipeline", + category: KindCategory::Node, + }, + KindSpec { + name: "AndOrList", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocument", + category: KindCategory::Node, + }, + KindSpec { + name: "BashFile", + category: KindCategory::Node, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Node, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Node, + }, + KindSpec { + name: "Word", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocumentLine", + category: KindCategory::Node, + }, + KindSpec { + name: "SingleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "DoubleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "AnsiCString", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ProcessSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ArraySubscript", + category: KindCategory::Node, + }, + KindSpec { + name: "ParameterExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "Assignment", + category: KindCategory::Node, + }, + KindSpec { + name: "unparsed-source", + category: KindCategory::Token, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Token, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Token, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-open", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-close", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-open", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-body", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-close", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-open", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-close", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-open", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-prefix", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-name", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-close", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-name", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "horizontal-whitespace", + category: KindCategory::Token, + }, + KindSpec { + name: "newline", + category: KindCategory::Token, + }, + KindSpec { + name: "line-continuation", + category: KindCategory::Token, + }, + KindSpec { + name: "comment", + category: KindCategory::Token, + }, + KindSpec { + name: "word", + category: KindCategory::Token, + }, + KindSpec { + name: "operator", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-marker", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-content", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-end", + category: KindCategory::Token, + }, + ], + nodes: &[ + ResultNodeSpec { + kind: 0, + fields: &["descriptor", "operator", "target"], + }, + ResultNodeSpec { + kind: 1, + fields: &["assignment", "open", "close", "separator", "element"], + }, + ResultNodeSpec { + kind: 2, + fields: &["assignment", "name", "argument", "redirect"], + }, + ResultNodeSpec { + kind: 3, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 4, + fields: &["keyword", "condition", "body", "else-body"], + }, + ResultNodeSpec { + kind: 5, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 6, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 7, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 8, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 9, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 10, + fields: &["keyword", "subject", "separator", "clause"], + }, + ResultNodeSpec { + kind: 11, + fields: &[ + "open", + "pattern", + "alternate", + "close", + "body", + "terminator", + ], + }, + ResultNodeSpec { + kind: 12, + fields: &["keyword", "name", "open", "close", "body"], + }, + ResultNodeSpec { + kind: 13, + fields: &["open", "close", "operator", "operand"], + }, + ResultNodeSpec { + kind: 14, + fields: &["open", "close", "expression", "operator"], + }, + ResultNodeSpec { + kind: 15, + fields: &["command", "redirect"], + }, + ResultNodeSpec { + kind: 16, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 17, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 18, + fields: &["keyword", "option", "negate", "command", "operator"], + }, + ResultNodeSpec { + kind: 19, + fields: &["command", "operator"], + }, + ResultNodeSpec { + kind: 20, + fields: &["delimiter", "content"], + }, + ResultNodeSpec { + kind: 21, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 22, + fields: &["text"], + }, + ResultNodeSpec { + kind: 23, + fields: &["text"], + }, + ResultNodeSpec { + kind: 24, + fields: &["text"], + }, + ResultNodeSpec { + kind: 25, + fields: &["part"], + }, + ResultNodeSpec { + kind: 26, + fields: &["part"], + }, + ResultNodeSpec { + kind: 27, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 28, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 29, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 30, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 31, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 32, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 33, + fields: &["open", "index", "close"], + }, + ResultNodeSpec { + kind: 34, + fields: &[ + "open", + "prefix", + "name", + "subscript", + "operator", + "operand", + "close", + ], + }, + ResultNodeSpec { + kind: 35, + fields: &["name", "operator", "value"], + }, + ], + projections: &[ + ResultProjection { + id: "LiteralPart", + kind: 22, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 37, + required: true, + }, + }], + }, + ResultProjection { + id: "EscapeSequence", + kind: 23, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 38, + required: true, + }, + }], + }, + ResultProjection { + id: "SimpleParameter", + kind: 24, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 39, + required: true, + }, + }], + }, + ResultProjection { + id: "Word", + kind: 25, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "HereDocumentLine", + kind: 26, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "SingleQuoted", + kind: 27, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "DoubleQuoted", + kind: 28, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "AnsiCString", + kind: 29, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArithmeticExpansion", + kind: 30, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "CommandSubstitution", + kind: 31, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ProcessSubstitution", + kind: 32, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArraySubscript", + kind: 33, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 45, + required: true, + }, + }, + ProjectionInstruction { + field: "index", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 46, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ParameterExpansion", + kind: 34, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "prefix", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "subscript", + kind: CaptureKind::Node, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 47, + required: true, + }, + }, + ProjectionInstruction { + field: "prefix", + opcode: ProjectionOpcode::Token { + kind: 48, + required: false, + }, + }, + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 49, + required: false, + }, + }, + ProjectionInstruction { + field: "subscript", + opcode: ProjectionOpcode::One { required: false }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 50, + required: false, + }, + }, + ProjectionInstruction { + field: "operand", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 51, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "Assignment", + kind: 35, + captures: &[ + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 52, + required: true, + }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 53, + required: true, + }, + }, + ProjectionInstruction { + field: "value", + opcode: ProjectionOpcode::Many, + }, + ], + }, + ], + }; + use gerbil_parser_runtime::{ + BindingSpec, PartGuard, PartOpcode, PartProfileSpec, PartRule, SubscriptSpec, + }; + pub mod regions { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[]; + } + mod subscript_1 { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "[", + opening: '\u{5b}', + closing: '\u{5d}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[ + gerbil_parser_runtime::scanner::RegionScope { + prefix: "${", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "$(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "$((", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "<(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: ">(", + pairs: &["${", "$(", "$((", "<(", ">("], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "[", + pairs: &["${", "$(", "$((", "<(", ">(", "["], + }, + ]; + } + pub static PART_PROFILE: PartProfileSpec = PartProfileSpec { + contexts: &[ + "word", + "SingleQuoted", + "DoubleQuoted", + "AnsiCString", + "HereDocument", + ], + literal: "LiteralPart", + assignment: Some(2), + bindings: &[ + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("?@*#$!-_0"), + ]), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("?@*#$!-_0"), + ]), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + }, + prefixes: &[], + operators: &[], + subscript: None, + }, + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("@*?#$!-"), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("@*?#$!-"), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Numeric, + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + }), + }, + prefixes: &["#", "!"], + operators: &[ + ":-", ":=", ":+", ":?", "##", "%%", "//", "^^", ",,", "~~", ":", "-", "=", "+", + "?", "#", "%", "/", "@", "^", ",", "~", + ], + subscript: Some(SubscriptSpec { + prefix: "[", + closing: '\u{5d}', + regions: &subscript_1::REGION, + scopes: subscript_1::SCOPES, + }), + }, + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: Some(1), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 0, + maximum: None, + }, + ]), + prefixes: &[], + operators: &["=", "+="], + subscript: None, + }, + ], + rules: &[ + PartRule { + contexts: &["word"], + prefix: "$'", + guard: PartGuard::Any, + projection: "AnsiCString", + opening: 2, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{27}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "'", + guard: PartGuard::Any, + projection: "SingleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{27}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "\"", + guard: PartGuard::Any, + projection: "DoubleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{22}', + }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "${", + guard: PartGuard::Any, + projection: "ParameterExpansion", + opening: 2, + closing: 1, + opcode: PartOpcode::Parameter { binding: 1 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$((", + guard: PartGuard::Any, + projection: "ArithmeticExpansion", + opening: 3, + closing: 2, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$(", + guard: PartGuard::Any, + projection: "CommandSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word"], + prefix: "<(", + guard: PartGuard::Any, + projection: "ProcessSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word"], + prefix: ">(", + guard: PartGuard::Any, + projection: "ProcessSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "`", + guard: PartGuard::Any, + projection: "CommandSubstitution", + opening: 1, + closing: 1, + opcode: PartOpcode::QuotedBody { + delimiter: '\u{60}', + }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$", + guard: PartGuard::Next, + projection: "SimpleParameter", + opening: 1, + closing: 0, + opcode: PartOpcode::Name { binding: 0 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "AnsiCString"], + prefix: "\\", + guard: PartGuard::Any, + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + PartRule { + contexts: &["HereDocument"], + prefix: "\\", + guard: PartGuard::Characters("$`\\\n"), + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + ], + }; + pub static CONTROLS: &[super::Control] = &[ + super::Control { + mode: "word", + source: "前::尾", + start: 4, + end: 4, + status: 1, + events: &[TreeEvent::StartNode(25), TreeEvent::FinishNode], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:α😀:尾", + start: 4, + end: 10, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:'α😀':尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(27), + TreeEvent::Token { + kind: 40, + start: 4, + end: 5, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("SingleQuoted", "open"), + ("SingleQuoted", "part"), + ("LiteralPart", "text"), + ("SingleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:\"α${x:-中}\":尾", + start: 4, + end: 17, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 4, + end: 5, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 7, + end: 9, + }, + TreeEvent::Token { + kind: 49, + start: 9, + end: 10, + }, + TreeEvent::Token { + kind: 50, + start: 10, + end: 12, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 12, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 15, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 16, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("DoubleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:$'a\\n':尾", + start: 4, + end: 10, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(29), + TreeEvent::Token { + kind: 40, + start: 4, + end: 6, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 7, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 9, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("AnsiCString", "open"), + ("AnsiCString", "part"), + ("LiteralPart", "text"), + ("AnsiCString", "part"), + ("EscapeSequence", "text"), + ("AnsiCString", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:${arr[${i:-1}]:-λ}:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 49, + start: 6, + end: 9, + }, + TreeEvent::StartNode(33), + TreeEvent::Token { + kind: 45, + start: 9, + end: 10, + }, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 10, + end: 12, + }, + TreeEvent::Token { + kind: 49, + start: 12, + end: 13, + }, + TreeEvent::Token { + kind: 50, + start: 13, + end: 15, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 16, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 46, + start: 17, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 50, + start: 18, + end: 20, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 20, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 22, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "subscript"), + ("ArraySubscript", "open"), + ("ArraySubscript", "index"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("ArraySubscript", "close"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:$(echo α):尾", + start: 4, + end: 14, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(31), + TreeEvent::Token { + kind: 42, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 43, + start: 6, + end: 13, + }, + TreeEvent::Token { + kind: 44, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("CommandSubstitution", "open"), + ("CommandSubstitution", "body"), + ("CommandSubstitution", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:$((1+2)):尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(30), + TreeEvent::Token { + kind: 42, + start: 4, + end: 7, + }, + TreeEvent::Token { + kind: 43, + start: 7, + end: 10, + }, + TreeEvent::Token { + kind: 44, + start: 10, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ArithmeticExpansion", "open"), + ("ArithmeticExpansion", "body"), + ("ArithmeticExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:<(echo 中):尾", + start: 4, + end: 15, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(32), + TreeEvent::Token { + kind: 42, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 43, + start: 6, + end: 14, + }, + TreeEvent::Token { + kind: 44, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ProcessSubstitution", "open"), + ("ProcessSubstitution", "body"), + ("ProcessSubstitution", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:`echo α`:尾", + start: 4, + end: 13, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(31), + TreeEvent::Token { + kind: 42, + start: 4, + end: 5, + }, + TreeEvent::Token { + kind: 43, + start: 5, + end: 12, + }, + TreeEvent::Token { + kind: 44, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("CommandSubstitution", "open"), + ("CommandSubstitution", "body"), + ("CommandSubstitution", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:$x\\α:尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 6, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("SimpleParameter", "text"), + ("Word", "part"), + ("EscapeSequence", "text"), + ], + }, + super::Control { + mode: "assignment", + source: "前:name+=\"α${x:-中}\":尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 4, + end: 8, + }, + TreeEvent::Token { + kind: 53, + start: 8, + end: 10, + }, + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 10, + end: 11, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 13, + end: 15, + }, + TreeEvent::Token { + kind: 49, + start: 15, + end: 16, + }, + TreeEvent::Token { + kind: 50, + start: 16, + end: 18, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 18, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 21, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 22, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Assignment", "name"), + ("Assignment", "operator"), + ("Assignment", "value"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("DoubleQuoted", "close"), + ], + }, + super::Control { + mode: "here", + source: "前:α${x:-中}\\\n:尾", + start: 4, + end: 17, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 6, + end: 8, + }, + TreeEvent::Token { + kind: 49, + start: 8, + end: 9, + }, + TreeEvent::Token { + kind: 50, + start: 9, + end: 11, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 15, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ], + }, + super::Control { + mode: "word", + source: "前:$😀:尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:$?:尾", + start: 4, + end: 6, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("SimpleParameter", "text")], + }, + super::Control { + mode: "word", + source: "前:${#}:尾", + start: 4, + end: 8, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 49, + start: 6, + end: 7, + }, + TreeEvent::Token { + kind: 51, + start: 7, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:${!arr[@]}:尾", + start: 4, + end: 14, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 48, + start: 6, + end: 7, + }, + TreeEvent::Token { + kind: 49, + start: 7, + end: 10, + }, + TreeEvent::StartNode(33), + TreeEvent::Token { + kind: 45, + start: 10, + end: 11, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 46, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "prefix"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "subscript"), + ("ArraySubscript", "open"), + ("ArraySubscript", "index"), + ("LiteralPart", "text"), + ("ArraySubscript", "close"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:${x:-${y:-λ}}:尾", + start: 4, + end: 18, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 49, + start: 6, + end: 7, + }, + TreeEvent::Token { + kind: 50, + start: 7, + end: 9, + }, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 9, + end: 11, + }, + TreeEvent::Token { + kind: 49, + start: 11, + end: 12, + }, + TreeEvent::Token { + kind: 50, + start: 12, + end: 14, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 14, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 16, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 17, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:${x[1[2]]:=中}:尾", + start: 4, + end: 19, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 49, + start: 6, + end: 7, + }, + TreeEvent::StartNode(33), + TreeEvent::Token { + kind: 45, + start: 7, + end: 8, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 8, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 46, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 50, + start: 13, + end: 15, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 18, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "subscript"), + ("ArraySubscript", "open"), + ("ArraySubscript", "index"), + ("LiteralPart", "text"), + ("ArraySubscript", "close"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:${x[${y:-[}]:-λ}:尾", + start: 4, + end: 21, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 49, + start: 6, + end: 7, + }, + TreeEvent::StartNode(33), + TreeEvent::Token { + kind: 45, + start: 7, + end: 8, + }, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 8, + end: 10, + }, + TreeEvent::Token { + kind: 49, + start: 10, + end: 11, + }, + TreeEvent::Token { + kind: 50, + start: 11, + end: 13, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 46, + start: 15, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 50, + start: 16, + end: 18, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 18, + end: 20, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 20, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "subscript"), + ("ArraySubscript", "open"), + ("ArraySubscript", "index"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("ArraySubscript", "close"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:${x//α/β}:尾", + start: 4, + end: 15, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 49, + start: 6, + end: 7, + }, + TreeEvent::Token { + kind: 50, + start: 7, + end: 9, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:${123:-α}:尾", + start: 4, + end: 14, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 49, + start: 6, + end: 9, + }, + TreeEvent::Token { + kind: 50, + start: 9, + end: 11, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:\\:尾", + start: 4, + end: 5, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("EscapeSequence", "text")], + }, + super::Control { + mode: "word", + source: "前:a\\😀b:尾", + start: 4, + end: 11, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 5, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("EscapeSequence", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "word", + source: "前:\"a\\$x\\q\":尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 4, + end: 5, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 6, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 8, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 9, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("EscapeSequence", "text"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("EscapeSequence", "text"), + ("DoubleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:$'a\\😀b':尾", + start: 4, + end: 14, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(29), + TreeEvent::Token { + kind: 40, + start: 4, + end: 6, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 7, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("AnsiCString", "open"), + ("AnsiCString", "part"), + ("LiteralPart", "text"), + ("AnsiCString", "part"), + ("EscapeSequence", "text"), + ("AnsiCString", "part"), + ("LiteralPart", "text"), + ("AnsiCString", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:a$(echo 中)b${x}c:尾", + start: 4, + end: 22, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(31), + TreeEvent::Token { + kind: 42, + start: 5, + end: 7, + }, + TreeEvent::Token { + kind: 43, + start: 7, + end: 15, + }, + TreeEvent::Token { + kind: 44, + start: 15, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 16, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 17, + end: 19, + }, + TreeEvent::Token { + kind: 49, + start: 19, + end: 20, + }, + TreeEvent::Token { + kind: 51, + start: 20, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 21, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("CommandSubstitution", "open"), + ("CommandSubstitution", "body"), + ("CommandSubstitution", "close"), + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "close"), + ("Word", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "here", + source: "前:'raw\"text:尾", + start: 4, + end: 13, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("HereDocumentLine", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "here", + source: "前:\\x\\α\\$x:尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 9, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "here", + source: "前:$(echo α)\n${x:-λ}:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(31), + TreeEvent::Token { + kind: 42, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 43, + start: 6, + end: 13, + }, + TreeEvent::Token { + kind: 44, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 15, + end: 17, + }, + TreeEvent::Token { + kind: 49, + start: 17, + end: 18, + }, + TreeEvent::Token { + kind: 50, + start: 18, + end: 20, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 20, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 22, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("HereDocumentLine", "part"), + ("CommandSubstitution", "open"), + ("CommandSubstitution", "body"), + ("CommandSubstitution", "close"), + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "here", + source: "前:\\`x\\\n:尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 7, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ], + }, + super::Control { + mode: "assignment", + source: "前:α_2=😀:尾", + start: 4, + end: 13, + status: 1, + events: &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 4, + end: 8, + }, + TreeEvent::Token { + kind: 53, + start: 8, + end: 9, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Assignment", "name"), + ("Assignment", "operator"), + ("Assignment", "value"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "assignment", + source: "前:name=α${x[0]:-中}:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 4, + end: 8, + }, + TreeEvent::Token { + kind: 53, + start: 8, + end: 9, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 11, + end: 13, + }, + TreeEvent::Token { + kind: 49, + start: 13, + end: 14, + }, + TreeEvent::StartNode(33), + TreeEvent::Token { + kind: 45, + start: 14, + end: 15, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 46, + start: 16, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 50, + start: 17, + end: 19, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 19, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 22, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Assignment", "name"), + ("Assignment", "operator"), + ("Assignment", "value"), + ("LiteralPart", "text"), + ("Assignment", "value"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "subscript"), + ("ArraySubscript", "open"), + ("ArraySubscript", "index"), + ("LiteralPart", "text"), + ("ArraySubscript", "close"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "assignment", + source: "前:name=:尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 4, + end: 8, + }, + TreeEvent::Token { + kind: 53, + start: 8, + end: 9, + }, + TreeEvent::FinishNode, + ], + fields: &[("Assignment", "name"), ("Assignment", "operator")], + }, + super::Control { + mode: "assignment", + source: "前:name:尾", + start: 4, + end: 8, + status: 2, + events: &[], + fields: &[], + }, + super::Control { + mode: "assignment", + source: "前:1name=α:尾", + start: 4, + end: 12, + status: 2, + events: &[], + fields: &[], + }, + super::Control { + mode: "assignment", + source: "前:α-β=中:尾", + start: 4, + end: 13, + status: 2, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:':尾", + start: 4, + end: 5, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:\"unterminated:尾", + start: 4, + end: 17, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:${:尾", + start: 4, + end: 6, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:$(:尾", + start: 4, + end: 6, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:$((1):尾", + start: 4, + end: 9, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:${x[0}:尾", + start: 4, + end: 10, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "assignment", + source: "前:name='bad:尾", + start: 4, + end: 13, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "here", + source: "前:${x:尾", + start: 4, + end: 7, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:λ{！arr［λ{i→中}]→😀}:尾", + start: 4, + end: 36, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 36, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:«αλ{x→中}«:尾", + start: 4, + end: 21, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:※😀tail:尾", + start: 4, + end: 15, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:λ{x→λ{y→中}}:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:λ{x［0]→中}:尾", + start: 4, + end: 20, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 20, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:λ{x［0}:尾", + start: 4, + end: 13, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:a$😀$x:尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 10, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("SimpleParameter", "text"), + ], + }, + super::Control { + mode: "here", + source: "前:literal※😀«raw:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("HereDocumentLine", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:λ{！}:尾", + start: 4, + end: 11, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:λ{！x→α}:尾", + start: 4, + end: 17, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + ]; +} +pub mod generic { + use gerbil_parser_runtime::TreeEvent; + // @generated from the admitted Scheme ResultProfile IR. + use gerbil_parser_runtime::{ + CaptureKind, CaptureSpec, KindCategory, KindSpec, ProjectionInstruction, ProjectionOpcode, + ResultNodeSpec, ResultProfileSpec, ResultProjection, + }; + pub static RESULT_PROFILE: ResultProfileSpec = ResultProfileSpec { + kinds: &[ + KindSpec { + name: "Redirection", + category: KindCategory::Node, + }, + KindSpec { + name: "ArrayAssignment", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandList", + category: KindCategory::Node, + }, + KindSpec { + name: "IfCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "WhileCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "UntilCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "SelectCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticForCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "CaseClause", + category: KindCategory::Node, + }, + KindSpec { + name: "FunctionDefinition", + category: KindCategory::Node, + }, + KindSpec { + name: "ConditionalCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "RedirectedCommand", + category: KindCategory::Node, + }, + KindSpec { + name: "BraceGroup", + category: KindCategory::Node, + }, + KindSpec { + name: "Subshell", + category: KindCategory::Node, + }, + KindSpec { + name: "Pipeline", + category: KindCategory::Node, + }, + KindSpec { + name: "AndOrList", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocument", + category: KindCategory::Node, + }, + KindSpec { + name: "BashFile", + category: KindCategory::Node, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Node, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Node, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Node, + }, + KindSpec { + name: "Word", + category: KindCategory::Node, + }, + KindSpec { + name: "HereDocumentLine", + category: KindCategory::Node, + }, + KindSpec { + name: "SingleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "DoubleQuoted", + category: KindCategory::Node, + }, + KindSpec { + name: "AnsiCString", + category: KindCategory::Node, + }, + KindSpec { + name: "ArithmeticExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "CommandSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ProcessSubstitution", + category: KindCategory::Node, + }, + KindSpec { + name: "ArraySubscript", + category: KindCategory::Node, + }, + KindSpec { + name: "ParameterExpansion", + category: KindCategory::Node, + }, + KindSpec { + name: "Assignment", + category: KindCategory::Node, + }, + KindSpec { + name: "unparsed-source", + category: KindCategory::Token, + }, + KindSpec { + name: "LiteralPart", + category: KindCategory::Token, + }, + KindSpec { + name: "EscapeSequence", + category: KindCategory::Token, + }, + KindSpec { + name: "SimpleParameter", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-open", + category: KindCategory::Token, + }, + KindSpec { + name: "quote-close", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-open", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-body", + category: KindCategory::Token, + }, + KindSpec { + name: "substitution-close", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-open", + category: KindCategory::Token, + }, + KindSpec { + name: "subscript-close", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-open", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-prefix", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-name", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "parameter-close", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-name", + category: KindCategory::Token, + }, + KindSpec { + name: "assignment-operator", + category: KindCategory::Token, + }, + KindSpec { + name: "horizontal-whitespace", + category: KindCategory::Token, + }, + KindSpec { + name: "newline", + category: KindCategory::Token, + }, + KindSpec { + name: "line-continuation", + category: KindCategory::Token, + }, + KindSpec { + name: "comment", + category: KindCategory::Token, + }, + KindSpec { + name: "word", + category: KindCategory::Token, + }, + KindSpec { + name: "operator", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-marker", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-content", + category: KindCategory::Token, + }, + KindSpec { + name: "heredoc-end", + category: KindCategory::Token, + }, + ], + nodes: &[ + ResultNodeSpec { + kind: 0, + fields: &["descriptor", "operator", "target"], + }, + ResultNodeSpec { + kind: 1, + fields: &["assignment", "open", "close", "separator", "element"], + }, + ResultNodeSpec { + kind: 2, + fields: &["assignment", "name", "argument", "redirect"], + }, + ResultNodeSpec { + kind: 3, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 4, + fields: &["keyword", "condition", "body", "else-body"], + }, + ResultNodeSpec { + kind: 5, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 6, + fields: &["keyword", "condition", "body"], + }, + ResultNodeSpec { + kind: 7, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 8, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 9, + fields: &["keyword", "header", "variable", "item", "separator", "body"], + }, + ResultNodeSpec { + kind: 10, + fields: &["keyword", "subject", "separator", "clause"], + }, + ResultNodeSpec { + kind: 11, + fields: &[ + "open", + "pattern", + "alternate", + "close", + "body", + "terminator", + ], + }, + ResultNodeSpec { + kind: 12, + fields: &["keyword", "name", "open", "close", "body"], + }, + ResultNodeSpec { + kind: 13, + fields: &["open", "close", "operator", "operand"], + }, + ResultNodeSpec { + kind: 14, + fields: &["open", "close", "expression", "operator"], + }, + ResultNodeSpec { + kind: 15, + fields: &["command", "redirect"], + }, + ResultNodeSpec { + kind: 16, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 17, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 18, + fields: &["keyword", "option", "negate", "command", "operator"], + }, + ResultNodeSpec { + kind: 19, + fields: &["command", "operator"], + }, + ResultNodeSpec { + kind: 20, + fields: &["delimiter", "content"], + }, + ResultNodeSpec { + kind: 21, + fields: &["command", "separator", "here-document"], + }, + ResultNodeSpec { + kind: 22, + fields: &["text"], + }, + ResultNodeSpec { + kind: 23, + fields: &["text"], + }, + ResultNodeSpec { + kind: 24, + fields: &["text"], + }, + ResultNodeSpec { + kind: 25, + fields: &["part"], + }, + ResultNodeSpec { + kind: 26, + fields: &["part"], + }, + ResultNodeSpec { + kind: 27, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 28, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 29, + fields: &["open", "part", "close"], + }, + ResultNodeSpec { + kind: 30, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 31, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 32, + fields: &["open", "body", "close"], + }, + ResultNodeSpec { + kind: 33, + fields: &["open", "index", "close"], + }, + ResultNodeSpec { + kind: 34, + fields: &[ + "open", + "prefix", + "name", + "subscript", + "operator", + "operand", + "close", + ], + }, + ResultNodeSpec { + kind: 35, + fields: &["name", "operator", "value"], + }, + ], + projections: &[ + ResultProjection { + id: "LiteralPart", + kind: 22, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 37, + required: true, + }, + }], + }, + ResultProjection { + id: "EscapeSequence", + kind: 23, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 38, + required: true, + }, + }], + }, + ResultProjection { + id: "SimpleParameter", + kind: 24, + captures: &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }], + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 39, + required: true, + }, + }], + }, + ResultProjection { + id: "Word", + kind: 25, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "HereDocumentLine", + kind: 26, + captures: &[CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }], + instructions: &[ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }], + }, + ResultProjection { + id: "SingleQuoted", + kind: 27, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "DoubleQuoted", + kind: 28, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "AnsiCString", + kind: 29, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 40, + required: true, + }, + }, + ProjectionInstruction { + field: "part", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 41, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArithmeticExpansion", + kind: 30, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "CommandSubstitution", + kind: 31, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ProcessSubstitution", + kind: 32, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "body", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 42, + required: true, + }, + }, + ProjectionInstruction { + field: "body", + opcode: ProjectionOpcode::Token { + kind: 43, + required: false, + }, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 44, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ArraySubscript", + kind: 33, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 45, + required: true, + }, + }, + ProjectionInstruction { + field: "index", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 46, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "ParameterExpansion", + kind: 34, + captures: &[ + CaptureSpec { + name: "open", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "prefix", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "subscript", + kind: CaptureKind::Node, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + CaptureSpec { + name: "close", + kind: CaptureKind::Span, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "open", + opcode: ProjectionOpcode::Token { + kind: 47, + required: true, + }, + }, + ProjectionInstruction { + field: "prefix", + opcode: ProjectionOpcode::Token { + kind: 48, + required: false, + }, + }, + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 49, + required: false, + }, + }, + ProjectionInstruction { + field: "subscript", + opcode: ProjectionOpcode::One { required: false }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 50, + required: false, + }, + }, + ProjectionInstruction { + field: "operand", + opcode: ProjectionOpcode::Many, + }, + ProjectionInstruction { + field: "close", + opcode: ProjectionOpcode::Token { + kind: 51, + required: true, + }, + }, + ], + }, + ResultProjection { + id: "Assignment", + kind: 35, + captures: &[ + CaptureSpec { + name: "name", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "operator", + kind: CaptureKind::Span, + }, + CaptureSpec { + name: "parts", + kind: CaptureKind::Parts, + }, + ], + instructions: &[ + ProjectionInstruction { + field: "name", + opcode: ProjectionOpcode::Token { + kind: 52, + required: true, + }, + }, + ProjectionInstruction { + field: "operator", + opcode: ProjectionOpcode::Token { + kind: 53, + required: true, + }, + }, + ProjectionInstruction { + field: "value", + opcode: ProjectionOpcode::Many, + }, + ], + }, + ], + }; + use gerbil_parser_runtime::{ + BindingSpec, PartGuard, PartOpcode, PartProfileSpec, PartRule, SubscriptSpec, + }; + pub mod regions { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{ab}', + escaped: true, + pairs: &["λ{"], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "λ{", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[]; + } + mod subscript_1 { + pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = + gerbil_parser_runtime::scanner::RegionSpec { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{ab}', + escaped: true, + pairs: &["λ{"], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "λ{", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "［", + opening: '\u{ff3b}', + closing: '\u{5d}', + depth: 1, + }, + ], + consume_initial_stop: true, + }; + pub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[ + gerbil_parser_runtime::scanner::RegionScope { + prefix: "${", + pairs: &["${", "$(", "$((", "<(", ">(", "λ{"], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "$(", + pairs: &["${", "$(", "$((", "<(", ">(", "λ{"], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "$((", + pairs: &["${", "$(", "$((", "<(", ">(", "λ{"], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "<(", + pairs: &["${", "$(", "$((", "<(", ">(", "λ{"], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: ">(", + pairs: &["${", "$(", "$((", "<(", ">(", "λ{"], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "λ{", + pairs: &["${", "$(", "$((", "<(", ">(", "λ{"], + }, + gerbil_parser_runtime::scanner::RegionScope { + prefix: "［", + pairs: &["${", "$(", "$((", "<(", ">(", "λ{", "［"], + }, + ]; + } + pub static PART_PROFILE: PartProfileSpec = PartProfileSpec { + contexts: &[ + "word", + "SingleQuoted", + "DoubleQuoted", + "AnsiCString", + "HereDocument", + ], + literal: "LiteralPart", + assignment: Some(2), + bindings: &[ + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("?@*#$!-_0"), + ]), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("?@*#$!-_0"), + ]), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + }, + prefixes: &[], + operators: &[], + subscript: None, + }, + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Characters("@*?#$!-"), + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Characters("@*?#$!-"), + minimum: 1, + maximum: Some(1), + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::IfNext { + class: gerbil_parser_runtime::TextClass::Numeric, + body: &gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Numeric, + minimum: 1, + maximum: None, + }, + otherwise: Some(&gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: None, + }), + }), + }, + prefixes: &["！"], + operators: &["→"], + subscript: Some(SubscriptSpec { + prefix: "［", + closing: '\u{5d}', + regions: &subscript_1::REGION, + scopes: subscript_1::SCOPES, + }), + }, + BindingSpec { + name: &gerbil_parser_runtime::TextProfile::Sequence(&[ + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 1, + maximum: Some(1), + }, + gerbil_parser_runtime::TextProfile::Run { + class: gerbil_parser_runtime::TextClass::Union(&[ + gerbil_parser_runtime::TextClass::Alphabetic, + gerbil_parser_runtime::TextClass::Numeric, + gerbil_parser_runtime::TextClass::Characters("_"), + ]), + minimum: 0, + maximum: None, + }, + ]), + prefixes: &[], + operators: &["=", "+="], + subscript: None, + }, + ], + rules: &[ + PartRule { + contexts: &["word"], + prefix: "$", + guard: PartGuard::Next, + projection: "SimpleParameter", + opening: 1, + closing: 0, + opcode: PartOpcode::Name { binding: 0 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "λ{", + guard: PartGuard::Any, + projection: "ParameterExpansion", + opening: 2, + closing: 1, + opcode: PartOpcode::Parameter { binding: 1 }, + }, + PartRule { + contexts: &["word"], + prefix: "«", + guard: PartGuard::Any, + projection: "DoubleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{ab}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "※", + guard: PartGuard::Any, + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + PartRule { + contexts: &["word"], + prefix: "$'", + guard: PartGuard::Any, + projection: "AnsiCString", + opening: 2, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{27}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "'", + guard: PartGuard::Any, + projection: "SingleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{27}', + }, + }, + PartRule { + contexts: &["word"], + prefix: "\"", + guard: PartGuard::Any, + projection: "DoubleQuoted", + opening: 1, + closing: 1, + opcode: PartOpcode::Quote { + delimiter: '\u{22}', + }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "${", + guard: PartGuard::Any, + projection: "ParameterExpansion", + opening: 2, + closing: 1, + opcode: PartOpcode::Parameter { binding: 1 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$((", + guard: PartGuard::Any, + projection: "ArithmeticExpansion", + opening: 3, + closing: 2, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "$(", + guard: PartGuard::Any, + projection: "CommandSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word"], + prefix: "<(", + guard: PartGuard::Any, + projection: "ProcessSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word"], + prefix: ">(", + guard: PartGuard::Any, + projection: "ProcessSubstitution", + opening: 2, + closing: 1, + opcode: PartOpcode::Pair, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "HereDocument"], + prefix: "`", + guard: PartGuard::Any, + projection: "CommandSubstitution", + opening: 1, + closing: 1, + opcode: PartOpcode::QuotedBody { + delimiter: '\u{60}', + }, + }, + PartRule { + contexts: &["DoubleQuoted", "HereDocument"], + prefix: "$", + guard: PartGuard::Next, + projection: "SimpleParameter", + opening: 1, + closing: 0, + opcode: PartOpcode::Name { binding: 0 }, + }, + PartRule { + contexts: &["word", "DoubleQuoted", "AnsiCString"], + prefix: "\\", + guard: PartGuard::Any, + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + PartRule { + contexts: &["HereDocument"], + prefix: "\\", + guard: PartGuard::Characters("$`\\\n"), + projection: "EscapeSequence", + opening: 1, + closing: 0, + opcode: PartOpcode::Escape, + }, + ], + }; + pub static CONTROLS: &[super::Control] = &[ + super::Control { + mode: "word", + source: "前::尾", + start: 4, + end: 4, + status: 1, + events: &[TreeEvent::StartNode(25), TreeEvent::FinishNode], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:α😀:尾", + start: 4, + end: 10, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:'α😀':尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(27), + TreeEvent::Token { + kind: 40, + start: 4, + end: 5, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("SingleQuoted", "open"), + ("SingleQuoted", "part"), + ("LiteralPart", "text"), + ("SingleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:\"α${x:-中}\":尾", + start: 4, + end: 17, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 4, + end: 5, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 7, + end: 9, + }, + TreeEvent::Token { + kind: 49, + start: 9, + end: 10, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 15, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 16, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("DoubleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:$'a\\n':尾", + start: 4, + end: 10, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(27), + TreeEvent::Token { + kind: 40, + start: 5, + end: 6, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 9, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("SingleQuoted", "open"), + ("SingleQuoted", "part"), + ("LiteralPart", "text"), + ("SingleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:${arr[${i:-1}]:-λ}:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "word", + source: "前:$(echo α):尾", + start: 4, + end: 14, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:$((1+2)):尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:<(echo 中):尾", + start: 4, + end: 15, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(32), + TreeEvent::Token { + kind: 42, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 43, + start: 6, + end: 14, + }, + TreeEvent::Token { + kind: 44, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ProcessSubstitution", "open"), + ("ProcessSubstitution", "body"), + ("ProcessSubstitution", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:`echo α`:尾", + start: 4, + end: 13, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(31), + TreeEvent::Token { + kind: 42, + start: 4, + end: 5, + }, + TreeEvent::Token { + kind: 43, + start: 5, + end: 12, + }, + TreeEvent::Token { + kind: 44, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("CommandSubstitution", "open"), + ("CommandSubstitution", "body"), + ("CommandSubstitution", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:$x\\α:尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 6, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("SimpleParameter", "text"), + ("Word", "part"), + ("EscapeSequence", "text"), + ], + }, + super::Control { + mode: "assignment", + source: "前:name+=\"α${x:-中}\":尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 4, + end: 8, + }, + TreeEvent::Token { + kind: 53, + start: 8, + end: 10, + }, + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 10, + end: 11, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 13, + end: 15, + }, + TreeEvent::Token { + kind: 49, + start: 15, + end: 16, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 16, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 21, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 22, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Assignment", "name"), + ("Assignment", "operator"), + ("Assignment", "value"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("DoubleQuoted", "close"), + ], + }, + super::Control { + mode: "here", + source: "前:α${x:-中}\\\n:尾", + start: 4, + end: 17, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 6, + end: 8, + }, + TreeEvent::Token { + kind: 49, + start: 8, + end: 9, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 15, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ], + }, + super::Control { + mode: "word", + source: "前:$😀:尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:$?:尾", + start: 4, + end: 6, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("SimpleParameter", "text")], + }, + super::Control { + mode: "word", + source: "前:${#}:尾", + start: 4, + end: 8, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:${!arr[@]}:尾", + start: 4, + end: 14, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:${x:-${y:-λ}}:尾", + start: 4, + end: 18, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "word", + source: "前:${x[1[2]]:=中}:尾", + start: 4, + end: 19, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:${x[${y:-[}]:-λ}:尾", + start: 4, + end: 21, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 8, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "word", + source: "前:${x//α/β}:尾", + start: 4, + end: 15, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:${123:-α}:尾", + start: 4, + end: 14, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:\\:尾", + start: 4, + end: 5, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("EscapeSequence", "text")], + }, + super::Control { + mode: "word", + source: "前:a\\😀b:尾", + start: 4, + end: 11, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 5, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 10, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("EscapeSequence", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "word", + source: "前:\"a\\$x\\q\":尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 4, + end: 5, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 6, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 8, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 9, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("EscapeSequence", "text"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("EscapeSequence", "text"), + ("DoubleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:$'a\\😀b':尾", + start: 4, + end: 14, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(27), + TreeEvent::Token { + kind: 40, + start: 5, + end: 6, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("SingleQuoted", "open"), + ("SingleQuoted", "part"), + ("LiteralPart", "text"), + ("SingleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:a$(echo 中)b${x}c:尾", + start: 4, + end: 22, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 17, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "here", + source: "前:'raw\"text:尾", + start: 4, + end: 13, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("HereDocumentLine", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "here", + source: "前:\\x\\α\\$x:尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 9, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "here", + source: "前:$(echo α)\n${x:-λ}:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(31), + TreeEvent::Token { + kind: 42, + start: 4, + end: 6, + }, + TreeEvent::Token { + kind: 43, + start: 6, + end: 13, + }, + TreeEvent::Token { + kind: 44, + start: 13, + end: 14, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 14, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 15, + end: 17, + }, + TreeEvent::Token { + kind: 49, + start: 17, + end: 18, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 18, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 22, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("HereDocumentLine", "part"), + ("CommandSubstitution", "open"), + ("CommandSubstitution", "body"), + ("CommandSubstitution", "close"), + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "here", + source: "前:\\`x\\\n:尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 7, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 7, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ("HereDocumentLine", "part"), + ("LiteralPart", "text"), + ("HereDocumentLine", "part"), + ("EscapeSequence", "text"), + ], + }, + super::Control { + mode: "assignment", + source: "前:α_2=😀:尾", + start: 4, + end: 13, + status: 1, + events: &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 4, + end: 8, + }, + TreeEvent::Token { + kind: 53, + start: 8, + end: 9, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Assignment", "name"), + ("Assignment", "operator"), + ("Assignment", "value"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "assignment", + source: "前:name=α${x[0]:-中}:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 4, + end: 8, + }, + TreeEvent::Token { + kind: 53, + start: 8, + end: 9, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 9, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Assignment", "name"), + ("Assignment", "operator"), + ("Assignment", "value"), + ("LiteralPart", "text"), + ("Assignment", "value"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "assignment", + source: "前:name=:尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(35), + TreeEvent::Token { + kind: 52, + start: 4, + end: 8, + }, + TreeEvent::Token { + kind: 53, + start: 8, + end: 9, + }, + TreeEvent::FinishNode, + ], + fields: &[("Assignment", "name"), ("Assignment", "operator")], + }, + super::Control { + mode: "assignment", + source: "前:name:尾", + start: 4, + end: 8, + status: 2, + events: &[], + fields: &[], + }, + super::Control { + mode: "assignment", + source: "前:1name=α:尾", + start: 4, + end: 12, + status: 2, + events: &[], + fields: &[], + }, + super::Control { + mode: "assignment", + source: "前:α-β=中:尾", + start: 4, + end: 13, + status: 2, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:':尾", + start: 4, + end: 5, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:\"unterminated:尾", + start: 4, + end: 17, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:${:尾", + start: 4, + end: 6, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:$(:尾", + start: 4, + end: 6, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 6, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:$((1):尾", + start: 4, + end: 9, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 9, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:${x[0}:尾", + start: 4, + end: 10, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("Word", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "assignment", + source: "前:name='bad:尾", + start: 4, + end: 13, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "here", + source: "前:${x:尾", + start: 4, + end: 7, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:λ{！arr［λ{i→中}]→😀}:尾", + start: 4, + end: 36, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 7, + }, + TreeEvent::Token { + kind: 48, + start: 7, + end: 10, + }, + TreeEvent::Token { + kind: 49, + start: 10, + end: 13, + }, + TreeEvent::StartNode(33), + TreeEvent::Token { + kind: 45, + start: 13, + end: 16, + }, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 16, + end: 19, + }, + TreeEvent::Token { + kind: 49, + start: 19, + end: 20, + }, + TreeEvent::Token { + kind: 50, + start: 20, + end: 23, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 23, + end: 26, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 26, + end: 27, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 46, + start: 27, + end: 28, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 50, + start: 28, + end: 31, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 31, + end: 35, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 35, + end: 36, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "prefix"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "subscript"), + ("ArraySubscript", "open"), + ("ArraySubscript", "index"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("ArraySubscript", "close"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:«αλ{x→中}«:尾", + start: 4, + end: 21, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(28), + TreeEvent::Token { + kind: 40, + start: 4, + end: 6, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 6, + end: 8, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 8, + end: 11, + }, + TreeEvent::Token { + kind: 49, + start: 11, + end: 12, + }, + TreeEvent::Token { + kind: 50, + start: 12, + end: 15, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 15, + end: 18, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 18, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 41, + start: 19, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("DoubleQuoted", "open"), + ("DoubleQuoted", "part"), + ("LiteralPart", "text"), + ("DoubleQuoted", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("DoubleQuoted", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:※😀tail:尾", + start: 4, + end: 15, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(23), + TreeEvent::Token { + kind: 38, + start: 4, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 15, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("EscapeSequence", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ], + }, + super::Control { + mode: "word", + source: "前:λ{x→λ{y→中}}:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 7, + }, + TreeEvent::Token { + kind: 49, + start: 7, + end: 8, + }, + TreeEvent::Token { + kind: 50, + start: 8, + end: 11, + }, + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 11, + end: 14, + }, + TreeEvent::Token { + kind: 49, + start: 14, + end: 15, + }, + TreeEvent::Token { + kind: 50, + start: 15, + end: 18, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 18, + end: 21, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 21, + end: 22, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 22, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:λ{x［0]→中}:尾", + start: 4, + end: 20, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 7, + }, + TreeEvent::Token { + kind: 49, + start: 7, + end: 8, + }, + TreeEvent::StartNode(33), + TreeEvent::Token { + kind: 45, + start: 8, + end: 11, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 11, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 46, + start: 12, + end: 13, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 50, + start: 13, + end: 16, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 16, + end: 19, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 19, + end: 20, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "subscript"), + ("ArraySubscript", "open"), + ("ArraySubscript", "index"), + ("LiteralPart", "text"), + ("ArraySubscript", "close"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:λ{x［0}:尾", + start: 4, + end: 13, + status: 0, + events: &[], + fields: &[], + }, + super::Control { + mode: "word", + source: "前:a$😀$x:尾", + start: 4, + end: 12, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 5, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 5, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::StartNode(24), + TreeEvent::Token { + kind: 39, + start: 10, + end: 12, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("LiteralPart", "text"), + ("Word", "part"), + ("SimpleParameter", "text"), + ], + }, + super::Control { + mode: "here", + source: "前:literal※😀«raw:尾", + start: 4, + end: 23, + status: 1, + events: &[ + TreeEvent::StartNode(26), + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 4, + end: 23, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[("HereDocumentLine", "part"), ("LiteralPart", "text")], + }, + super::Control { + mode: "word", + source: "前:λ{！}:尾", + start: 4, + end: 11, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 7, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 7, + end: 10, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 10, + end: 11, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + super::Control { + mode: "word", + source: "前:λ{！x→α}:尾", + start: 4, + end: 17, + status: 1, + events: &[ + TreeEvent::StartNode(25), + TreeEvent::StartNode(34), + TreeEvent::Token { + kind: 47, + start: 4, + end: 7, + }, + TreeEvent::Token { + kind: 48, + start: 7, + end: 10, + }, + TreeEvent::Token { + kind: 49, + start: 10, + end: 11, + }, + TreeEvent::Token { + kind: 50, + start: 11, + end: 14, + }, + TreeEvent::StartNode(22), + TreeEvent::Token { + kind: 37, + start: 14, + end: 16, + }, + TreeEvent::FinishNode, + TreeEvent::Token { + kind: 51, + start: 16, + end: 17, + }, + TreeEvent::FinishNode, + TreeEvent::FinishNode, + ], + fields: &[ + ("Word", "part"), + ("ParameterExpansion", "open"), + ("ParameterExpansion", "prefix"), + ("ParameterExpansion", "name"), + ("ParameterExpansion", "operator"), + ("ParameterExpansion", "operand"), + ("LiteralPart", "text"), + ("ParameterExpansion", "close"), + ], + }, + ]; +} diff --git a/rust/gerbil-parser-runtime/tests/fixtures/quoted_profile_generated.rs b/rust/gerbil-parser-runtime/tests/fixtures/quoted_profile_generated.rs new file mode 100644 index 00000000..96d01455 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/quoted_profile_generated.rs @@ -0,0 +1,168 @@ +// @generated by gerbil-parser/src/compiler/rust-runtime.ss +// Source authority: canonical Grammar IR v1; do not edit. + +use gerbil_parser_runtime::{ + ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, + Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, +}; + +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "SourceFile", + category: KindCategory::Node, + }, + KindSpec { + name: "String", + category: KindCategory::Token, + }, + KindSpec { + name: "Name", + category: KindCategory::Token, + }, + KindSpec { + name: "Unknown", + category: KindCategory::Token, + }, +]; + +static TERMINALS: &[TerminalSpec] = &[ + TerminalSpec { + name: "string", + syntax_kind: 1, + }, + TerminalSpec { + name: "name", + syntax_kind: 2, + }, + TerminalSpec { + name: "unknown", + syntax_kind: 3, + }, +]; + +static LEXICAL_RULES: &[LexicalRule] = &[ + LexicalRule { + terminal: "string", + expression: LexicalExpr::QuotedStringProfile { + delimiter: "'", + escapes: "`'\\/fnrt", + unicode_width: 4, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "name", + expression: LexicalExpr::QuotedStringProfile { + delimiter: "`", + escapes: "`'\\/fnrt", + unicode_width: 4, + }, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "unknown", + expression: LexicalExpr::Fallback, + precedence: 0, + extra: false, + }, +]; + +static ACTION_0: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(2), + }, + ActionEntry { + terminal: Terminal::Token("name"), + action: ParserAction::Shift(3), + }, +]; +static ACTION_1: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Accept, +}]; +static ACTION_2: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(1), +}]; +static ACTION_3: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(2), +}]; +static ACTION_4: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(3), +}]; +static ACTIONS: &[&[ActionEntry]] = &[ACTION_0, ACTION_1, ACTION_2, ACTION_3, ACTION_4]; + +static GOTO_0: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "document", + state: 1, + }, + GotoEntry { + nonterminal: "$document.0", + state: 4, + }, +]; +static GOTO_1: &[GotoEntry] = &[]; +static GOTO_2: &[GotoEntry] = &[]; +static GOTO_3: &[GotoEntry] = &[]; +static GOTO_4: &[GotoEntry] = &[]; +static GOTOS: &[&[GotoEntry]] = &[GOTO_0, GOTO_1, GOTO_2, GOTO_3, GOTO_4]; + +static PRODUCTIONS: &[Production] = &[ + Production { + lhs: "$accept", + rhs: &[Operand { + symbol: Symbol::Nonterminal("document"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.0", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("string")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.0", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("name")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "document", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$document.0"), + actions: &[OperandAction::Field("item"), OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, +]; + +pub static LANGUAGE: LanguageSpec = LanguageSpec { + language: "quoted-profile", + version: "v1", + contract: "quoted-profile.v1", + grammar_digest: "sha256:6cb81d1784076aa0915de79ea9f37088227324a5c9191f3f2dbf5add1935733c", + case_insensitive: false, + root_kind: 0, + kinds: KINDS, + terminals: TERMINALS, + lexical_rules: LEXICAL_RULES, + actions: ACTIONS, + gotos: GOTOS, + productions: PRODUCTIONS, +}; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/records_contextual_generated.rs b/rust/gerbil-parser-runtime/tests/fixtures/records_contextual_generated.rs new file mode 100644 index 00000000..b5891605 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/records_contextual_generated.rs @@ -0,0 +1,590 @@ +// @generated by gerbil-parser/src/compiler/rust-runtime.ss +// Source authority: canonical Grammar IR v1; do not edit. + +use gerbil_parser_runtime::{ + ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, + Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, +}; + +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "Document", + category: KindCategory::Node, + }, + KindSpec { + name: "Assignment", + category: KindCategory::Node, + }, + KindSpec { + name: "Whitespace", + category: KindCategory::Token, + }, + KindSpec { + name: "Newline", + category: KindCategory::Token, + }, + KindSpec { + name: "Number", + category: KindCategory::Token, + }, + KindSpec { + name: "Identifier", + category: KindCategory::Token, + }, + KindSpec { + name: "String", + category: KindCategory::Token, + }, + KindSpec { + name: "Punctuation", + category: KindCategory::Token, + }, + KindSpec { + name: "Unknown", + category: KindCategory::Token, + }, +]; + +static TERMINALS: &[TerminalSpec] = &[ + TerminalSpec { + name: "whitespace", + syntax_kind: 2, + }, + TerminalSpec { + name: "newline", + syntax_kind: 3, + }, + TerminalSpec { + name: "number", + syntax_kind: 4, + }, + TerminalSpec { + name: "identifier", + syntax_kind: 5, + }, + TerminalSpec { + name: "string", + syntax_kind: 6, + }, + TerminalSpec { + name: "punctuation", + syntax_kind: 7, + }, + TerminalSpec { + name: "unknown", + syntax_kind: 8, + }, +]; + +static LEXICAL_RULES: &[LexicalRule] = &[ + LexicalRule { + terminal: "whitespace", + expression: LexicalExpr::HorizontalWhitespace, + precedence: 0, + extra: true, + }, + LexicalRule { + terminal: "newline", + expression: LexicalExpr::Newline, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "number", + expression: LexicalExpr::DecimalDigits, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "identifier", + expression: LexicalExpr::Identifier, + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "string", + expression: LexicalExpr::QuotedString(&["\""]), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "punctuation", + expression: LexicalExpr::Literals(&["="]), + precedence: 0, + extra: false, + }, + LexicalRule { + terminal: "unknown", + expression: LexicalExpr::Fallback, + precedence: 0, + extra: false, + }, +]; + +static ACTION_0: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(1), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(1), + }, +]; +static ACTION_1: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Accept, +}]; +static ACTION_2: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(3), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(4), + }, +]; +static ACTION_3: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(2), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(2), + }, +]; +static ACTION_4: &[ActionEntry] = &[ActionEntry { + terminal: Terminal::Literal("="), + action: ParserAction::Shift(6), +}]; +static ACTION_5: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(10), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(10), + }, +]; +static ACTION_6: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Token("number"), + action: ParserAction::Shift(7), + }, + ActionEntry { + terminal: Terminal::Token("string"), + action: ParserAction::Shift(8), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Shift(9), + }, +]; +static ACTION_7: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(4), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(4), + }, + ActionEntry { + terminal: Terminal::Token("newline"), + action: ParserAction::Reduce(4), + }, +]; +static ACTION_8: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(5), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(5), + }, + ActionEntry { + terminal: Terminal::Token("newline"), + action: ParserAction::Reduce(5), + }, +]; +static ACTION_9: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(6), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(6), + }, + ActionEntry { + terminal: Terminal::Token("newline"), + action: ParserAction::Reduce(6), + }, +]; +static ACTION_10: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(7), + }, + ActionEntry { + terminal: Terminal::Token("newline"), + action: ParserAction::Shift(11), + }, +]; +static ACTION_11: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(8), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(8), + }, +]; +static ACTION_12: &[ActionEntry] = &[ + ActionEntry { + terminal: Terminal::Eof, + action: ParserAction::Reduce(9), + }, + ActionEntry { + terminal: Terminal::Token("identifier"), + action: ParserAction::Reduce(9), + }, +]; +static ACTIONS: &[&[ActionEntry]] = &[ + ACTION_0, ACTION_1, ACTION_2, ACTION_3, ACTION_4, ACTION_5, ACTION_6, ACTION_7, ACTION_8, + ACTION_9, ACTION_10, ACTION_11, ACTION_12, +]; + +static GOTO_0: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "document", + state: 1, + }, + GotoEntry { + nonterminal: "$document.0", + state: 2, + }, +]; +static GOTO_1: &[GotoEntry] = &[]; +static GOTO_2: &[GotoEntry] = &[ + GotoEntry { + nonterminal: "assignment", + state: 3, + }, + GotoEntry { + nonterminal: "$assignment.1", + state: 5, + }, +]; +static GOTO_3: &[GotoEntry] = &[]; +static GOTO_4: &[GotoEntry] = &[]; +static GOTO_5: &[GotoEntry] = &[]; +static GOTO_6: &[GotoEntry] = &[GotoEntry { + nonterminal: "$assignment.2", + state: 10, +}]; +static GOTO_7: &[GotoEntry] = &[]; +static GOTO_8: &[GotoEntry] = &[]; +static GOTO_9: &[GotoEntry] = &[]; +static GOTO_10: &[GotoEntry] = &[GotoEntry { + nonterminal: "$assignment.3", + state: 12, +}]; +static GOTO_11: &[GotoEntry] = &[]; +static GOTO_12: &[GotoEntry] = &[]; +static GOTOS: &[&[GotoEntry]] = &[ + GOTO_0, GOTO_1, GOTO_2, GOTO_3, GOTO_4, GOTO_5, GOTO_6, GOTO_7, GOTO_8, GOTO_9, GOTO_10, + GOTO_11, GOTO_12, +]; + +static PRODUCTIONS: &[Production] = &[ + Production { + lhs: "$accept", + rhs: &[Operand { + symbol: Symbol::Nonterminal("document"), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.0", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$document.0", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("$document.0"), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("assignment"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "document", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$document.0"), + actions: &[OperandAction::Alias(0)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$assignment.2", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("number")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$assignment.2", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("string")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$assignment.2", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("identifier")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$assignment.3", + rhs: &[], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "$assignment.3", + rhs: &[Operand { + symbol: Symbol::Terminal(Terminal::Token("newline")), + actions: &[], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, + Production { + lhs: "$assignment.1", + rhs: &[ + Operand { + symbol: Symbol::Terminal(Terminal::Token("identifier")), + actions: &[OperandAction::Field("name")], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Literal("=")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("$assignment.2"), + actions: &[OperandAction::Field("value")], + }, + Operand { + symbol: Symbol::Nonterminal("$assignment.3"), + actions: &[], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }, + Production { + lhs: "assignment", + rhs: &[Operand { + symbol: Symbol::Nonterminal("$assignment.1"), + actions: &[OperandAction::Alias(1)], + }], + reduction: Reduction::Pass, + dynamic_precedence: 0, + }, +]; + +pub static LANGUAGE: LanguageSpec = LanguageSpec { + language: "record-assignments", + version: "v1", + contract: "record-assignments.v1", + grammar_digest: "sha256:9c7e8b0f807fb1bf13198c417ce5b74b90f65d67f054b48188cf5268dba4d1d4", + case_insensitive: false, + root_kind: 0, + kinds: KINDS, + terminals: TERMINALS, + lexical_rules: LEXICAL_RULES, + actions: ACTIONS, + gotos: GOTOS, + productions: PRODUCTIONS, +}; +pub mod contextual_scanner { + // @generated from Scheme contextual Scanner IR; do not edit. + #[allow(unused_imports)] + use gerbil_parser_runtime::scanner::{ + BalancedPair, MarkerPolicy, ScannerAction, ScannerCell, ScannerMatcher, ScannerRule, + ScannerSpec, + }; + pub static SCANNER: ScannerSpec = ScannerSpec { + opcode_contract: "gerbil-parser.contextual-scanner-opcodes.v1", + digest: "sha256:9a7cfe03fd1afef9b234dd6f18c4cc7df302da8f16a3eb6f5b9aae993145b374", + base_grammar_digest: Some( + "sha256:9c7e8b0f807fb1bf13198c417ce5b74b90f65d67f054b48188cf5268dba4d1d4", + ), + initial_mode: "main", + modes: &["main"], + positions: &["name", "value"], + rules: &[ + ScannerRule { + name: "id", + mode: "main", + form: "id", + matcher: ScannerMatcher::Identifier, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "one", + mode: "main", + form: "one", + matcher: ScannerMatcher::Literal("1"), + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "eq", + mode: "main", + form: "eq", + matcher: ScannerMatcher::Literal("="), + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "nl", + mode: "main", + form: "nl", + matcher: ScannerMatcher::NewlineOne, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "ws", + mode: "main", + form: "ws", + matcher: ScannerMatcher::HorizontalWhitespace, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "str", + mode: "main", + form: "str", + matcher: ScannerMatcher::QuotedString(&["\""]), + rank: 0, + action: ScannerAction::Keep, + }, + ], + cells: &[ + ScannerCell { + mode: "main", + position: "name", + form: "id", + terminal: "identifier", + }, + ScannerCell { + mode: "main", + position: "name", + form: "one", + terminal: "number", + }, + ScannerCell { + mode: "main", + position: "name", + form: "eq", + terminal: "punctuation", + }, + ScannerCell { + mode: "main", + position: "name", + form: "nl", + terminal: "newline", + }, + ScannerCell { + mode: "main", + position: "name", + form: "ws", + terminal: "whitespace", + }, + ScannerCell { + mode: "main", + position: "name", + form: "str", + terminal: "string", + }, + ScannerCell { + mode: "main", + position: "value", + form: "id", + terminal: "identifier", + }, + ScannerCell { + mode: "main", + position: "value", + form: "one", + terminal: "number", + }, + ScannerCell { + mode: "main", + position: "value", + form: "eq", + terminal: "punctuation", + }, + ScannerCell { + mode: "main", + position: "value", + form: "nl", + terminal: "newline", + }, + ScannerCell { + mode: "main", + position: "value", + form: "ws", + terminal: "whitespace", + }, + ScannerCell { + mode: "main", + position: "value", + form: "str", + terminal: "string", + }, + ], + }; +} +pub static CONTEXTUAL: gerbil_parser_runtime::ContextualParserSpec = + gerbil_parser_runtime::ContextualParserSpec { + language: &LANGUAGE, + scanner: &contextual_scanner::SCANNER, + parser_digest: "sha256:f6c100c0b9a67933dce8203000d9759545bf049588e28493ece5666660bf5159", + state_positions: &[ + "name", "value", "name", "name", "value", "name", "name", "name", "name", "name", + "name", "name", "name", + ], + }; diff --git a/rust/gerbil-parser-runtime/tests/fixtures/shared_scanner_generated.rs b/rust/gerbil-parser-runtime/tests/fixtures/shared_scanner_generated.rs new file mode 100644 index 00000000..cab21726 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/fixtures/shared_scanner_generated.rs @@ -0,0 +1,613 @@ +// @generated from Scheme contextual Scanner IR; do not edit. +#[allow(unused_imports)] +use gerbil_parser_runtime::scanner::{ + BalancedPair, MarkerPolicy, ScannerAction, ScannerCell, ScannerMatcher, ScannerRule, + ScannerSpec, +}; +pub static SCANNER: ScannerSpec = ScannerSpec { + opcode_contract: "gerbil-parser.contextual-scanner-opcodes.v1", + digest: "sha256:5231420d7457b009e10dddccfce2028dfc27682d158bbcdd7ef0cc5c8a0f85dc", + base_grammar_digest: None, + initial_mode: "command", + modes: &["command", "body"], + positions: &["command", "argument"], + rules: &[ + ScannerRule { + name: "strip", + mode: "command", + form: "open", + matcher: ScannerMatcher::Literal("<<-"), + rank: 30, + action: ScannerAction::ExpectMarker(true), + }, + ScannerRule { + name: "plain", + mode: "command", + form: "open", + matcher: ScannerMatcher::Literal("<<"), + rank: 30, + action: ScannerAction::ExpectMarker(false), + }, + ScannerRule { + name: "word", + mode: "command", + form: "word", + matcher: ScannerMatcher::RegionWord { + stops: &[ + ";;&", "&>>", "<<-", "<<<", "&&", "||", "|&", ";;", ";&", "<<", ">>", "<>", + "<&", ">&", ">|", "&>", ";", "&", "|", "(", ")", "<", ">", + ], + quotes: &[ + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{27}', + escaped: false, + pairs: &[], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{22}', + escaped: true, + pairs: &["${", "$(", "$(("], + }, + gerbil_parser_runtime::scanner::RegionQuote { + delimiter: '\u{60}', + escaped: true, + pairs: &[], + }, + ], + pairs: &[ + gerbil_parser_runtime::scanner::RegionPair { + prefix: "${", + opening: '\u{7b}', + closing: '\u{7d}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "$((", + opening: '\u{28}', + closing: '\u{29}', + depth: 2, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: "<(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + gerbil_parser_runtime::scanner::RegionPair { + prefix: ">(", + opening: '\u{28}', + closing: '\u{29}', + depth: 1, + }, + ], + consume_initial_stop: false, + }, + rank: 0, + action: ScannerAction::EnqueueIfExpecting(MarkerPolicy::ShellQuoteRemoval), + }, + ScannerRule { + name: "space", + mode: "command", + form: "space", + matcher: ScannerMatcher::HorizontalWhitespace, + rank: 0, + action: ScannerAction::Keep, + }, + ScannerRule { + name: "newline", + mode: "command", + form: "newline", + matcher: ScannerMatcher::NewlineOne, + rank: 0, + action: ScannerAction::ActivateNext("body"), + }, + ScannerRule { + name: "marker", + mode: "body", + form: "end", + matcher: ScannerMatcher::MarkerLine, + rank: 10, + action: ScannerAction::FinishMarker { + base: "command", + body: "body", + }, + }, + ScannerRule { + name: "body", + mode: "body", + form: "body", + matcher: ScannerMatcher::BodyLine, + rank: 0, + action: ScannerAction::Keep, + }, + ], + cells: &[ + ScannerCell { + mode: "command", + position: "command", + form: "open", + terminal: "here-open", + }, + ScannerCell { + mode: "command", + position: "command", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "command", + position: "command", + form: "space", + terminal: "space", + }, + ScannerCell { + mode: "command", + position: "command", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "command", + position: "command", + form: "body", + terminal: "body", + }, + ScannerCell { + mode: "command", + position: "command", + form: "end", + terminal: "end", + }, + ScannerCell { + mode: "command", + position: "argument", + form: "open", + terminal: "here-open", + }, + ScannerCell { + mode: "command", + position: "argument", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "command", + position: "argument", + form: "space", + terminal: "space", + }, + ScannerCell { + mode: "command", + position: "argument", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "command", + position: "argument", + form: "body", + terminal: "body", + }, + ScannerCell { + mode: "command", + position: "argument", + form: "end", + terminal: "end", + }, + ScannerCell { + mode: "body", + position: "command", + form: "open", + terminal: "here-open", + }, + ScannerCell { + mode: "body", + position: "command", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "body", + position: "command", + form: "space", + terminal: "space", + }, + ScannerCell { + mode: "body", + position: "command", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "body", + position: "command", + form: "body", + terminal: "body", + }, + ScannerCell { + mode: "body", + position: "command", + form: "end", + terminal: "end", + }, + ScannerCell { + mode: "body", + position: "argument", + form: "open", + terminal: "here-open", + }, + ScannerCell { + mode: "body", + position: "argument", + form: "word", + terminal: "word", + }, + ScannerCell { + mode: "body", + position: "argument", + form: "space", + terminal: "space", + }, + ScannerCell { + mode: "body", + position: "argument", + form: "newline", + terminal: "newline", + }, + ScannerCell { + mode: "body", + position: "argument", + form: "body", + terminal: "body", + }, + ScannerCell { + mode: "body", + position: "argument", + form: "end", + terminal: "end", + }, + ], +}; +pub static TRACES: &[(&str, &[gerbil_parser_runtime::ScannedToken])] = &[ + ( + "cat <<'A' <<-B\r\n$x α\r\nA\r\n\tβ $x\n\tB\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 3, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "here-open", + start: 4, + end: 6, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 6, + end: 9, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 9, + end: 10, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "here-open", + start: 10, + end: 13, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 13, + end: 14, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 14, + end: 16, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "body", + start: 16, + end: 23, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "end", + start: 23, + end: 26, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "body", + start: 26, + end: 33, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "end", + start: 33, + end: 36, + }, + ], + ), + ( + "cat <<\\\\EOF\ntext\n\\EOF\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 3, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "here-open", + start: 4, + end: 6, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 6, + end: 11, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 11, + end: 12, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "body", + start: 12, + end: 17, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "end", + start: 17, + end: 22, + }, + ], + ), + ( + "cat <<\"a\\qb\"\na\\qb\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 3, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "here-open", + start: 4, + end: 6, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 6, + end: 12, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 12, + end: 13, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "end", + start: 13, + end: 18, + }, + ], + ), + ( + "cat <<''\n\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 3, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 3, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "here-open", + start: 4, + end: 6, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 6, + end: 8, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 8, + end: 9, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "end", + start: 9, + end: 10, + }, + ], + ), + ( + "printf %s \"${x:-$(printf '%s' '}')}\"\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 6, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 6, + end: 7, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 7, + end: 9, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 9, + end: 10, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 10, + end: 36, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 36, + end: 37, + }, + ], + ), + ( + "echo α\n\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 5, + end: 7, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 7, + end: 8, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 8, + end: 9, + }, + ], + ), + ( + "echo $((1+(2*3))) <(printf 'α') >(cat)\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 5, + end: 17, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 17, + end: 18, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 18, + end: 32, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 32, + end: 33, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 33, + end: 39, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 39, + end: 40, + }, + ], + ), + ( + "echo `opaque ${x}`\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 5, + end: 18, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 18, + end: 19, + }, + ], + ), + ( + "echo α$(printf 'β')$((2))\n", + &[ + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 0, + end: 4, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "space", + start: 4, + end: 5, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "word", + start: 5, + end: 27, + }, + gerbil_parser_runtime::ScannedToken { + terminal: "newline", + start: 27, + end: 28, + }, + ], + ), +]; diff --git a/rust/gerbil-parser-runtime/tests/unit/artifact_generation.rs b/rust/gerbil-parser-runtime/tests/unit/artifact_generation.rs new file mode 100644 index 00000000..d1c0f7ce --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/artifact_generation.rs @@ -0,0 +1,337 @@ +//! Canonical Scheme GPA1 bytes consumed through the same generated language. +use crate::records_contextual_fixture::generated as language; +use crate::{SyntaxNode, parse, parse_contextual}; +use gerbil_parser_artifact::{ + NativeArtifactView, NativeCatalog, NativeElement, NativeEventKind, NativeNode, +}; +#[path = "../fixtures/artifact_generated.rs"] +mod native; + +fn catalog(contextual: bool) -> &'static NativeCatalog { + static CANONICAL: std::sync::OnceLock = std::sync::OnceLock::new(); + static CONTEXTUAL: std::sync::OnceLock = std::sync::OnceLock::new(); + if contextual { + CONTEXTUAL + .get_or_init(|| NativeCatalog::from_descriptor(native::CONTEXTUAL_DESCRIPTOR).unwrap()) + } else { + CANONICAL.get_or_init(|| NativeCatalog::from_descriptor(native::DESCRIPTOR).unwrap()) + } +} +fn decode<'a>( + payload: &'a [u8], + source: &'a str, +) -> Result, gerbil_parser_artifact::NativeArtifactError> { + NativeArtifactView::decode(payload, source, catalog(false)) +} +fn compare(native: NativeNode<'_, '_>, aot: &SyntaxNode, source: &str) { + let kind = language::LANGUAGE.kinds[aot.kind().0 as usize].name; + assert_eq!(native.kind(), kind); + let range = aot.text_range(); + assert_eq!(native.range().start, usize::from(range.start())); + assert_eq!(native.range().end, usize::from(range.end())); + assert_eq!(native.text(), aot.text()); + assert_eq!( + native.text().as_ptr(), + source[native.range().start..].as_ptr() + ); + let left: Vec<_> = native.children().collect(); + let right: Vec<_> = aot.children_with_tokens().collect(); + assert_eq!(left.len(), right.len()); + for (left, right) in left.into_iter().zip(right) { + match left { + NativeElement::Node(node) => { + assert_eq!(node.parent().unwrap().id(), native.id()); + compare(node, &right.into_node().unwrap(), source); + } + NativeElement::Token(token) => { + let right = right.into_token().unwrap(); + assert_eq!( + token.kind(), + language::LANGUAGE.kinds[right.kind().0 as usize].name + ); + assert_eq!(token.text(), right.text()); + assert_eq!(token.range().start, usize::from(right.text_range().start())); + assert_eq!(token.range().end, usize::from(right.text_range().end())); + assert_eq!( + token.text().as_ptr(), + source[token.range().start..].as_ptr() + ); + } + } + } + for field in native.fields() { + assert!(catalog(false).fields().iter().any(|f| f == field.name())); + assert_eq!( + field.text().as_ptr(), + source[field.range().start..].as_ptr() + ); + } +} +#[test] +fn scheme_native_ast_matches_independent_aot_without_tree_conversion() { + for &(source, payload) in native::CASES { + let source = std::str::from_utf8(source).expect("Scheme source"); + let view = decode(payload, source).expect("canonical GPA1"); + let expected = parse(&language::LANGUAGE, source); + assert_eq!(view.accepted(), expected.is_ok()); + if let Ok(expected) = expected { + let actual = view.root().unwrap(); + compare(actual, &expected.syntax(), source); + assert_eq!(actual.text(), source); + let contextual = + parse_contextual(&language::CONTEXTUAL, source).expect("shared scanner"); + compare(actual, &contextual.syntax(), source); + } else { + assert!(view.root().is_none()); + } + for token in view + .events() + .filter(|event| event.kind == NativeEventKind::Token) + { + let text = view.token_text(token).expect("source slice"); + assert_eq!(text.as_ptr(), source[token.start..].as_ptr()); + } + } +} + +#[test] +fn truncated_extended_and_incompatible_payloads_fail_before_iteration() { + let (source, payload) = native::CASES[1]; + let source = std::str::from_utf8(source).unwrap(); + for length in 0..payload.len() { + assert!(decode(&payload[..length], source).is_err()); + } + let mut extended = payload.to_vec(); + extended.push(0); + assert!(decode(&extended, source).is_err()); + for offset in [0, 4, 8, 12, 16, 48] { + let mut bad = payload.to_vec(); + bad[offset] ^= 255; + assert!(decode(&bad, source).is_err(), "header offset {offset}"); + } + assert_eq!( + decode(payload, "b=1\n").unwrap_err().reason, + "source-digest" + ); + let wrong = String::from_utf8(native::DESCRIPTOR.to_vec()) + .unwrap() + .replace( + catalog(false).grammar_digest(), + "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + ); + let wrong = NativeCatalog::from_descriptor(wrong.as_bytes()).unwrap(); + assert!(NativeArtifactView::decode(payload, source, &wrong).is_err()); +} + +#[test] +fn record_mutations_reject_unknown_tags_symbols_ids_ranges_and_nesting() { + let (source, payload) = native::CASES[1]; + let source = std::str::from_utf8(source).unwrap(); + for (index, event) in decode(payload, source).unwrap().events().enumerate() { + for field in [0, 4, 8, 16, 20] { + let mut bad = payload.to_vec(); + bad[80 + index * 24 + field..80 + index * 24 + field + 4] + .copy_from_slice(&u32::MAX.to_le_bytes()); + assert!( + decode(&bad, source).is_err(), + "event {index} field {field} {:?}", + event.kind + ); + } + } + let mut bad = native::DESCRIPTOR.to_vec(); + bad[0] = b'!'; + assert!(NativeCatalog::from_descriptor(&bad).is_err()); +} + +#[test] +fn borrowed_views_keep_parallel_source_and_payload_lifetimes_independent() { + std::thread::scope(|scope| { + for &(source, payload) in native::CASES { + scope.spawn(move || { + let source = std::str::from_utf8(source).unwrap(); + for _ in 0..16 { + let view = decode(payload, source).unwrap(); + assert_eq!(view.events().len(), (payload.len() - 80) / 24); + if view.accepted() { + assert_eq!(view.root().unwrap().text(), source); + } + } + }); + } + }); +} + +#[test] +fn syntax_status_utf8_boundaries_and_field_identity_are_checked() { + let (source, payload) = native::CASES[1]; + let source = std::str::from_utf8(source).unwrap(); + let mut rejected = payload.to_vec(); + rejected[8] = 1; + assert!(decode(&rejected, source).is_err()); + let view = decode(payload, source).unwrap(); + for (index, event) in view.events().enumerate() { + if event.kind == NativeEventKind::Token && event.start == 0 { + let mut split = payload.to_vec(); + split[80 + index * 24 + 20..80 + index * 24 + 24].copy_from_slice(&1_u32.to_le_bytes()); + assert!(decode(&split, source).is_err()); + } + if event.kind == NativeEventKind::FinishField { + let mut mismatched = payload.to_vec(); + let other = (event.symbol + 1) % u32::try_from(native::FIELD_COUNT).unwrap(); + mismatched[80 + index * 24 + 4..80 + index * 24 + 8] + .copy_from_slice(&other.to_le_bytes()); + assert!(decode(&mismatched, source).is_err()); + } + } +} + +#[test] +fn contextual_native_identity_matches_its_product_and_rejects_canonical_binding() { + for &(source, payload) in native::CONTEXTUAL_CASES { + let source = std::str::from_utf8(source).unwrap(); + let view = NativeArtifactView::decode(payload, source, catalog(true)) + .expect("contextual native GPA1"); + assert_eq!( + decode(payload, source).unwrap_err().reason, + "grammar-digest" + ); + let expected = parse_contextual(&language::CONTEXTUAL, source); + assert_eq!(view.accepted(), expected.is_ok()); + if let Ok(expected) = expected { + compare(view.root().unwrap(), &expected.syntax(), source); + } else { + assert!(view.root().is_none()); + } + } + let (source, payload) = native::CASES[1]; + assert!( + NativeArtifactView::decode(payload, std::str::from_utf8(source).unwrap(), catalog(true)) + .is_err() + ); +} + +#[test] +fn ast_fields_retain_names_values_order_and_declared_ownership() { + let (source, payload) = native::CASES[2]; + let source = std::str::from_utf8(source).unwrap(); + let view = decode(payload, source).unwrap(); + let assignments: Vec<_> = view + .root() + .unwrap() + .children() + .filter_map(|child| match child { + NativeElement::Node(node) => Some(node), + NativeElement::Token(_) => None, + }) + .collect(); + assert_eq!(assignments.len(), 2); + for (assignment, name, value) in [(&assignments[0], "a", "1"), (&assignments[1], "b", "\"β\"")] + { + assert_eq!( + assignment + .field("name") + .map(|field| field.text()) + .collect::>(), + [name] + ); + assert_eq!( + assignment + .field("value") + .map(|field| field.text()) + .collect::>(), + [value] + ); + assert_eq!( + assignment + .fields() + .map(|field| field.name()) + .collect::>(), + ["name", "value"] + ); + } + let invalid = u32::try_from( + catalog(false) + .fields() + .iter() + .position(|field| field == "text") + .unwrap(), + ) + .unwrap(); + let mut bad = payload.to_vec(); + for (index, event) in view.events().enumerate() { + if matches!( + event.kind, + NativeEventKind::StartField | NativeEventKind::FinishField + ) { + bad[80 + index * 24 + 4..80 + index * 24 + 8].copy_from_slice(&invalid.to_le_bytes()); + } + } + assert_eq!(decode(&bad, source).unwrap_err().reason, "field-owner"); +} + +#[test] +fn repeated_zero_width_and_absent_fields_follow_canonical_associations() { + let catalog = NativeCatalog::from_descriptor(native::SCOPES_DESCRIPTOR).unwrap(); + for &(source, payload) in native::SCOPES_CASES { + let source = std::str::from_utf8(source).unwrap(); + let view = NativeArtifactView::decode(payload, source, &catalog).unwrap(); + let root = view.root().unwrap(); + let items: Vec<_> = root.field("item").collect(); + assert_eq!(items.len(), source.len()); + for (offset, field) in items.iter().enumerate() { + assert_eq!(field.text(), "x"); + assert_eq!(field.range().start, offset); + assert_eq!(field.range().end, offset + 1); + assert_eq!(field.children().count(), 1); + } + let marker: Vec<_> = root.field("marker").collect(); + assert_eq!(marker.len(), 1); + assert_eq!(marker[0].text(), ""); + assert_eq!(marker[0].range().start, source.len()); + assert_eq!(marker[0].range().end, source.len()); + assert_eq!(marker[0].children().count(), 1); + assert_eq!(root.field("absent").count(), 0); + assert_eq!(root.fields().count(), items.len() + 1); + } +} + +#[test] +fn accepted_and_rejected_tokens_preserve_scanner_identity_without_an_ast() { + for (contextual, cases) in [(false, native::CASES), (true, native::CONTEXTUAL_CASES)] { + for &(source, payload) in cases { + let source = std::str::from_utf8(source).unwrap(); + let catalog = catalog(contextual); + let view = NativeArtifactView::decode(payload, source, catalog).unwrap(); + let events: Vec<_> = view + .events() + .filter(|e| e.kind == NativeEventKind::Token) + .collect(); + let tokens: Vec<_> = view.tokens().collect(); + assert_eq!(tokens.len(), events.len()); + let mut offset = 0; + for (ordinal, (token, event)) in tokens.iter().zip(events).enumerate() { + assert_eq!(token.id(), u64::try_from(ordinal).unwrap()); + assert_eq!( + token.class(), + catalog.terminals()[event.symbol as usize].name + ); + assert_eq!(token.range().start, offset); + assert_eq!(token.range().end, event.end); + assert_eq!(token.text(), &source[offset..event.end]); + assert_eq!(token.text().as_ptr(), source[offset..].as_ptr()); + assert_eq!(token.parent().is_some(), view.accepted()); + offset = event.end; + } + assert_eq!(offset, source.len()); + } + } + let catalog = NativeCatalog::from_descriptor(native::SCOPES_DESCRIPTOR).unwrap(); + let (source, payload) = native::SCOPES_CASES[1]; + let view = NativeArtifactView::decode(payload, std::str::from_utf8(source).unwrap(), &catalog) + .unwrap(); + for token in view.tokens() { + assert_eq!(token.class(), "item-token"); + assert_eq!(token.kind(), "Identifier"); + } +} diff --git a/rust/gerbil-parser-runtime/tests/unit/bash_scanner.rs b/rust/gerbil-parser-runtime/tests/unit/bash_scanner.rs new file mode 100644 index 00000000..92ea902b --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/bash_scanner.rs @@ -0,0 +1,99 @@ +//! Production Bash profile executes Scheme token/span and rejection controls. +#[path = "../fixtures/generated/bash_scanner.rs"] +mod generated; +use gerbil_parser_runtime::scanner::ContextualScanner; +#[test] +fn production_profile_matches_scheme_control_traces() { + assert!(generated::TRACES.len() >= 40); + assert!(generated::REJECTED.len() >= 4); + for (source, expected) in generated::TRACES { + let scanner = + ContextualScanner::new(&generated::SCANNER, source).expect("admitted profile"); + assert_eq!( + scanner.scan("source").expect("complete scan"), + *expected, + "{source:?}" + ); + } + for source in generated::REJECTED { + assert!( + ContextualScanner::new(&generated::SCANNER, source) + .expect("profile") + .scan("source") + .is_err(), + "{source:?}" + ); + } +} +#[test] +fn production_profile_checkpoints_replay_utf8_and_reject_foreign_owners() { + let source = "cat < = generated::PROFILES + .iter() + .map(|(_, p)| PreparedTextProfile::new(p).unwrap()) + .collect(); + for &(index, source, start, limit, expected) in generated::TRACES { + assert_eq!( + prepared[index].match_prefix(source, start, limit), + expected, + "profile={} source={source:?} span={start}..{limit}", + generated::PROFILES[index].0 + ); + } +} +#[test] +fn binding_names_reject_nullable_profiles_and_invalid_byte_spans() { + static NULLABLE: TextProfile = TextProfile::Run { + class: TextClass::Alphabetic, + minimum: 0, + maximum: None, + }; + static NAME: TextProfile = TextProfile::Run { + class: TextClass::Alphabetic, + minimum: 1, + maximum: None, + }; + assert!(PreparedTextProfile::new(&NULLABLE).is_none()); + let profile = PreparedTextProfile::new(&NAME).unwrap(); + for (start, limit) in [(1, 2), (0, 1), (3, 2), (0, 4)] { + assert_eq!(profile.match_prefix("αx", start, limit), None); + } + assert_eq!(profile.match_prefix("αx", 0, 2), Some(2)); +} diff --git a/rust/gerbil-parser-rowan/tests/unit/classify_first_word_generated.rs b/rust/gerbil-parser-runtime/tests/unit/classify_first_word_generated.rs similarity index 100% rename from rust/gerbil-parser-rowan/tests/unit/classify_first_word_generated.rs rename to rust/gerbil-parser-runtime/tests/unit/classify_first_word_generated.rs diff --git a/rust/gerbil-parser-rowan/tests/unit/classify_or_unknown_generated.rs b/rust/gerbil-parser-runtime/tests/unit/classify_or_unknown_generated.rs similarity index 100% rename from rust/gerbil-parser-rowan/tests/unit/classify_or_unknown_generated.rs rename to rust/gerbil-parser-runtime/tests/unit/classify_or_unknown_generated.rs diff --git a/rust/gerbil-parser-runtime/tests/unit/command_source.rs b/rust/gerbil-parser-runtime/tests/unit/command_source.rs new file mode 100644 index 00000000..43a6e59f --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/command_source.rs @@ -0,0 +1,853 @@ +//! Full Source recognition controls are produced by the independent Scheme engine. +#[path = "../fixtures/generated/command_source.rs"] +mod generated; +use crate::scanner::ContextualScanner; +use crate::{ + CommandForm, CommandInstruction, CommandProgramSpec, CommandTrigger, EventCatalog, + PreparedCommandProgram, PreparedPartProfile, ProjectedNode, ProjectedValue, + build_syntax_events_catalog, +}; +fn fields(root: &ProjectedNode<'_>) -> Vec<&'static str> { + let mut out = Vec::new(); + let mut stack: Vec<_> = root.children().iter().rev().collect(); + while let Some(child) = stack.pop() { + out.push(child.field()); + if let ProjectedValue::Node(node) = child.value() { + stack.extend(node.children().iter().rev()); + } + } + out +} +#[test] +fn source_matches_scheme_events_fields_fifo_links_and_rejections() { + assert_eq!(generated::bash::CONTROLS.len(), 49); + assert!(generated::bash::regions::SCOPES.is_empty()); + assert!(generated::extended::regions::SCOPES.is_empty()); + for (commands, parts, results, regions, scanner, controls) in [ + ( + &generated::bash::COMMAND_PROFILE, + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + &generated::bash::scanner::SCANNER, + generated::bash::CONTROLS, + ), + ( + &generated::extended::COMMAND_PROFILE, + &generated::extended::PART_PROFILE, + &generated::extended::RESULT_PROFILE, + &generated::extended::regions::REGION, + &generated::extended::scanner::SCANNER, + generated::extended::CONTROLS, + ), + ] { + let program = PreparedCommandProgram::new(commands, results).unwrap(); + let parts = PreparedPartProfile::new(parts, results, regions).unwrap(); + for control in controls { + let outcome = ContextualScanner::new(scanner, control.source) + .unwrap() + .scan("source") + .and_then(|tokens| program.parse_scanned(control.source, &tokens, &parts)); + assert_eq!( + outcome.is_ok(), + control.accepted, + "{:?}: {:?}", + control.source, + outcome.as_ref().err() + ); + if let Ok(parse) = outcome { + let events = parse + .events() + .unwrap_or_else(|e| panic!("{:?}: {e:?}", control.source)); + assert_eq!(events, control.events, "events {:?}", control.source); + assert_eq!( + fields(&parse.root), + control.fields, + "fields {:?}", + control.source + ); + assert_eq!( + parse.here_documents, control.links, + "links {:?}", + control.source + ); + let tree = build_syntax_events_catalog( + &EventCatalog { + kinds: results.kinds, + root_kind: parse.root.kind(), + }, + control.source, + &events, + ) + .unwrap(); + assert_eq!(tree.root().text().to_string(), control.source); + } + } + } +} +fn changed_form(mut form: CommandForm) -> &'static CommandProgramSpec { + let mut forms = generated::bash::COMMAND_PROFILE.forms.to_vec(); + let index = forms.iter().position(|f| f.id == form.id).unwrap_or(0); + if index == 0 { + form.id = forms[0].id; + } + forms[index] = form; + Box::leak(Box::new(CommandProgramSpec { + forms: Box::leak(forms.into_boxed_slice()), + ..generated::bash::COMMAND_PROFILE + })) +} +#[test] +fn command_admission_rejects_cycles_nullable_loops_foreign_fields_and_selectors() { + let base = generated::bash::COMMAND_PROFILE.forms[0]; + let mut nested: &'static [CommandInstruction] = &[CommandInstruction::Raw("keyword")]; + for _ in 0..129 { + nested = Box::leak( + vec![CommandInstruction::Node { + field: "command", + kind: "Pipeline", + program: nested, + }] + .into_boxed_slice(), + ); + } + for spec in [ + changed_form(CommandForm { + program: &[CommandInstruction::Call { + field: "body", + form: "unknown-target", + }], + ..base + }), + changed_form(CommandForm { + program: nested, + kind: "Pipeline", + ..base + }), + changed_form(CommandForm { + program: &[CommandInstruction::Many { + trigger: CommandTrigger::Token { + kind: "word", + literals: &[], + }, + program: &[], + }], + ..base + }), + changed_form(CommandForm { + program: &[CommandInstruction::Call { + field: "body", + form: "conditional-branch", + }], + ..base + }), + changed_form(CommandForm { + program: &[CommandInstruction::Raw("not-a-field")], + ..base + }), + changed_form(CommandForm { + trigger: CommandTrigger::Role("separator"), + ..base + }), + changed_form(CommandForm { + trigger: CommandTrigger::Token { + kind: "word", + literals: &["while"], + }, + ..base + }), + changed_form(CommandForm { + program: &[CommandInstruction::Call { + field: "body", + form: "pipeline-prefix", + }], + ..base + }), + changed_form(CommandForm { + id: "pipeline-prefix", + kind: "Pipeline", + trigger: CommandTrigger::Manual(None), + priority: 0, + program: &[CommandInstruction::Raw("keyword")], + }), + ] { + assert!(PreparedCommandProgram::new(spec, &generated::bash::RESULT_PROFILE).is_err()); + } +} +#[test] +fn source_rejects_invalid_tapes_and_foreign_word_owners() { + let program = PreparedCommandProgram::new( + &generated::bash::COMMAND_PROFILE, + &generated::bash::RESULT_PROFILE, + ) + .unwrap(); + let parts = PreparedPartProfile::new( + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + ) + .unwrap(); + let foreign = PreparedPartProfile::new( + &generated::extended::PART_PROFILE, + &generated::extended::RESULT_PROFILE, + &generated::extended::regions::REGION, + ) + .unwrap(); + let source = "echo α"; + let tokens = ContextualScanner::new(&generated::bash::scanner::SCANNER, source) + .unwrap() + .scan("source") + .unwrap(); + assert!(program.parse_scanned(source, &tokens, &foreign).is_err()); + assert!(program.parse_scanned("", &[], &foreign).is_err()); + assert!(program.parse_scanned(source, &tokens[1..], &parts).is_err()); + assert!( + program + .parse_scanned(source, &tokens[..tokens.len() - 1], &parts) + .is_err() + ); +} + +#[test] +fn admission_visits_shared_call_dag_once_without_expanding_execution_paths() { + let names: Vec<&'static str> = (0..32) + .map(|i| Box::leak(format!("dag-{i}").into_boxed_str()) as &'static str) + .collect(); + let mut forms = generated::bash::COMMAND_PROFILE.forms.to_vec(); + for (index, &id) in names.iter().enumerate() { + let program: &'static [CommandInstruction] = if index + 1 == names.len() { + &[CommandInstruction::Raw("keyword")] + } else { + Box::leak( + vec![ + CommandInstruction::Call { + field: "command", + form: names[index + 1], + }, + CommandInstruction::Call { + field: "command", + form: names[index + 1], + }, + ] + .into_boxed_slice(), + ) + }; + forms.push(CommandForm { + id, + kind: "Pipeline", + trigger: CommandTrigger::Manual(None), + priority: 0, + program, + }); + } + let spec = Box::leak(Box::new(CommandProgramSpec { + forms: Box::leak(forms.into_boxed_slice()), + ..generated::bash::COMMAND_PROFILE + })); + PreparedCommandProgram::new(spec, &generated::bash::RESULT_PROFILE).unwrap(); +} + +#[test] +fn one_generated_source_product_executes_both_poo_declarations() { + for (spec, controls) in [ + (&generated::bash::SOURCE, generated::bash::CONTROLS), + (&generated::extended::SOURCE, generated::extended::CONTROLS), + ] { + let engine = crate::PreparedCommandSource::new(spec).unwrap(); + for control in controls { + let parsed = engine.parse(control.source); + assert_eq!(parsed.is_ok(), control.accepted, "{:?}", control.source); + if let Ok(parsed) = parsed { + assert_eq!(parsed.events().unwrap(), control.events); + } + } + } +} + +#[test] +fn source_product_rejects_missing_scanner_positions_and_terminal_publication() { + static NO_SOURCE: crate::scanner::ScannerSpec = crate::scanner::ScannerSpec { + positions: &["other"], + ..generated::bash::scanner::SCANNER + }; + static SOURCE: crate::CommandSourceSpec = crate::CommandSourceSpec { + scanner: &NO_SOURCE, + ..generated::bash::SOURCE + }; + assert!(crate::PreparedCommandSource::new(&SOURCE).is_err()); +} + +fn renamed_id(id: &str) -> &'static str { + Box::leak(format!("private/{id}").into_boxed_str()) +} +fn renamed_program(program: &[CommandInstruction]) -> &'static [CommandInstruction] { + use CommandInstruction::{Balance, Branch, Call, Choose, Many, Node, Optional, Until}; + let rows: Vec<_> = program + .iter() + .map(|step| match *step { + Call { field, form } => Call { + field, + form: renamed_id(form), + }, + Node { + field, + kind, + program, + } => Node { + field, + kind, + program: renamed_program(program), + }, + Optional { trigger, program } => Optional { + trigger, + program: renamed_program(program), + }, + Many { trigger, program } => Many { + trigger, + program: renamed_program(program), + }, + Until { trigger, program } => Until { + trigger, + program: renamed_program(program), + }, + Branch { trigger, yes, no } => Branch { + trigger, + yes: renamed_program(yes), + no: renamed_program(no), + }, + Choose(choices) => Choose(Box::leak( + choices + .iter() + .map(|choice| crate::CommandChoice { + trigger: choice.trigger, + program: renamed_program(choice.program), + }) + .collect::>() + .into_boxed_slice(), + )), + Balance { + open, + close, + depth, + open_field, + close_field, + program, + } => Balance { + open, + close, + depth, + open_field, + close_field, + program: renamed_program(program), + }, + leaf => leaf, + }) + .collect(); + Box::leak(rows.into_boxed_slice()) +} +#[test] +fn private_rule_renaming_retains_scheme_events_and_rejection_controls() { + let original = &generated::bash::COMMAND_PROFILE; + let renamed = Box::leak(Box::new(CommandProgramSpec { + assignment_tail: renamed_id(original.assignment_tail), + pipeline_head: renamed_id(original.pipeline_head), + forms: Box::leak( + original + .forms + .iter() + .map(|form| CommandForm { + id: renamed_id(form.id), + program: renamed_program(form.program), + ..*form + }) + .collect::>() + .into_boxed_slice(), + ), + ..*original + })); + let program = PreparedCommandProgram::new(renamed, &generated::bash::RESULT_PROFILE).unwrap(); + let parts = PreparedPartProfile::new( + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + ) + .unwrap(); + for control in generated::bash::CONTROLS { + let outcome = ContextualScanner::new(&generated::bash::scanner::SCANNER, control.source) + .unwrap() + .scan("source") + .and_then(|tokens| program.parse_scanned(control.source, &tokens, &parts)); + assert_eq!(outcome.is_ok(), control.accepted, "{}", control.source); + if let Ok(parsed) = outcome { + let events = parsed.events().unwrap(); + assert_eq!(events, control.events, "{}", control.source); + assert_eq!(parsed.here_documents, control.links, "{}", control.source); + let tree = build_syntax_events_catalog( + &EventCatalog { + kinds: generated::bash::RESULT_PROFILE.kinds, + root_kind: parsed.root.kind(), + }, + control.source, + &events, + ) + .unwrap(); + assert_eq!(tree.root().text().to_string(), control.source); + } + } +} + +#[test] +fn source_suspends_mixed_compounds_and_releases_deep_results_on_small_stack() { + std::thread::Builder::new() + .stack_size(256 * 1024) + .spawn(|| { + let program = PreparedCommandProgram::new( + &generated::bash::COMMAND_PROFILE, + &generated::bash::RESULT_PROFILE, + ) + .unwrap(); + let parts = PreparedPartProfile::new( + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + ) + .unwrap(); + // Alternation crosses Source List -> Command VM -> Compound repeatedly. + let source = format!("{}echo α; {}", "{ ( ".repeat(512), "); } ; ".repeat(512)); + let tokens = ContextualScanner::new(&generated::bash::scanner::SCANNER, &source) + .unwrap() + .scan("source") + .unwrap(); + let parse = program.parse_scanned(&source, &tokens, &parts).unwrap(); + assert_eq!(parse.root.span(), 0..source.len()); + let mut body_fields = 0; + parse + .walk_events(|event| { + if let crate::CommandEvent::StartField { name: "body", .. } = event { + body_fields += 1; + } + }) + .unwrap(); + assert_eq!(body_fields, 1024); + let events = parse.events().unwrap(); + let tree = build_syntax_events_catalog( + &EventCatalog { + kinds: generated::bash::RESULT_PROFILE.kinds, + root_kind: parse.root.kind(), + }, + &source, + &events, + ) + .unwrap(); + assert_eq!(tree.root().text().to_string(), source); + assert_eq!( + fields(&parse.root).iter().filter(|f| **f == "body").count(), + 1024 + ); + drop(tree); + drop(parse); + // Exercise cleanup with a completed deep child retained by an unfinished parent. + let rejected = format!("{source} |"); + let tokens = ContextualScanner::new(&generated::bash::scanner::SCANNER, &rejected) + .unwrap() + .scan("source") + .unwrap(); + assert!(program.parse_scanned(&rejected, &tokens, &parts).is_err()); + // Failure state must not contaminate the immutable plan's next request. + let tokens = ContextualScanner::new(&generated::bash::scanner::SCANNER, "echo ok") + .unwrap() + .scan("source") + .unwrap(); + assert!(program.parse_scanned("echo ok", &tokens, &parts).is_ok()); + }) + .unwrap() + .join() + .unwrap(); +} + +#[test] +fn source_frame_budget_unwinds_and_keeps_next_parse_independent() { + let program = PreparedCommandProgram::new( + &generated::bash::COMMAND_PROFILE, + &generated::bash::RESULT_PROFILE, + ) + .unwrap(); + let parts = PreparedPartProfile::new( + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + ) + .unwrap(); + let source = format!("{}echo ok; {}", "{ ".repeat(4096), "}; ".repeat(4096)); + let tokens = ContextualScanner::new(&generated::bash::scanner::SCANNER, &source) + .unwrap() + .scan("source") + .unwrap(); + let error = program + .parse_scanned(&source, &tokens, &parts) + .err() + .unwrap(); + assert_eq!(error.reason_kind, "command-source-resource-limit"); + assert!(source.is_char_boundary(error.byte_offset)); + let tokens = ContextualScanner::new(&generated::bash::scanner::SCANNER, "echo ok") + .unwrap() + .scan("source") + .unwrap(); + assert!(program.parse_scanned("echo ok", &tokens, &parts).is_ok()); +} + +#[test] +fn source_budget_also_bounds_a_vm_call_chain_without_source_suspensions() { + let names: Vec<&'static str> = (0..6000) + .map(|i| Box::leak(format!("frame-{i}").into_boxed_str()) as &'static str) + .collect(); + let mut forms = generated::bash::COMMAND_PROFILE.forms.to_vec(); + let brace = forms.iter_mut().find(|f| f.id == "brace-group").unwrap(); + brace.program = Box::leak( + vec![CommandInstruction::Call { + field: "body", + form: names[0], + }] + .into_boxed_slice(), + ); + for (index, &id) in names.iter().enumerate() { + let program: &'static [CommandInstruction] = if index + 1 == names.len() { + &[CommandInstruction::Raw("keyword")] + } else { + Box::leak( + vec![CommandInstruction::Call { + field: "command", + form: names[index + 1], + }] + .into_boxed_slice(), + ) + }; + forms.push(CommandForm { + id, + kind: "Pipeline", + trigger: CommandTrigger::Manual(None), + priority: 0, + program, + }); + } + let spec = Box::leak(Box::new(CommandProgramSpec { + forms: Box::leak(forms.into_boxed_slice()), + ..generated::bash::COMMAND_PROFILE + })); + let program = PreparedCommandProgram::new(spec, &generated::bash::RESULT_PROFILE).unwrap(); + let parts = PreparedPartProfile::new( + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + ) + .unwrap(); + let tokens = ContextualScanner::new(&generated::bash::scanner::SCANNER, "{") + .unwrap() + .scan("source") + .unwrap(); + let error = program.parse_scanned("{", &tokens, &parts).err().unwrap(); + assert_eq!(error.reason_kind, "command-source-resource-limit"); + let tokens = ContextualScanner::new(&generated::bash::scanner::SCANNER, "echo ok") + .unwrap() + .scan("source") + .unwrap(); + assert!(program.parse_scanned("echo ok", &tokens, &parts).is_ok()); +} +// Scheme emits this through the same codec used by the language-independent C ABI. +const NATIVE_CASES: &[u8] = include_bytes!("../fixtures/generated/command_source.bin"); +fn native_bytes<'a>(input: &mut &'a [u8]) -> &'a [u8] { + let (length, rest) = input.split_at(4); + let length = u32::from_le_bytes(length.try_into().unwrap()) as usize; + let (value, rest) = rest.split_at(length); + *input = rest; + value +} +fn canonical_event( + event: crate::CommandEvent, + results: &crate::ResultProfileSpec, +) -> ( + gerbil_parser_artifact::NativeEventKind, + &'static str, + u64, + usize, + usize, +) { + use crate::CommandEvent; + use gerbil_parser_artifact::NativeEventKind as Kind; + match event { + CommandEvent::StartNode { id, kind, offset } => ( + Kind::StartNode, + results.kinds[kind as usize].name, + id, + offset, + 0, + ), + CommandEvent::FinishNode { id, kind, offset } => ( + Kind::FinishNode, + results.kinds[kind as usize].name, + id, + 0, + offset, + ), + CommandEvent::StartField { name, offset } => (Kind::StartField, name, 0, offset, 0), + CommandEvent::FinishField { name, offset } => (Kind::FinishField, name, 0, 0, offset), + CommandEvent::Token { + id, + kind, + start, + end, + } => ( + Kind::Token, + results.kinds[kind as usize].name, + id, + start, + end, + ), + } +} +#[test] +fn source_canonical_publication_matches_scheme_native_ids_fields_ranges_and_trivia() { + use gerbil_parser_artifact::{NativeArtifactView, NativeCatalog, NativeEventKind as Kind}; + let mut input = NATIVE_CASES; + let mut count = 0; + for (name, spec, controls) in [ + ("bash", &generated::bash::SOURCE, generated::bash::CONTROLS), + ( + "extended", + &generated::extended::SOURCE, + generated::extended::CONTROLS, + ), + ] { + let engine = crate::PreparedCommandSource::new(spec).unwrap(); + for control in controls { + assert_eq!(native_bytes(&mut input), name.as_bytes()); + let catalog = NativeCatalog::from_descriptor(native_bytes(&mut input)).unwrap(); + assert_eq!(native_bytes(&mut input), control.source.as_bytes()); + let payload = native_bytes(&mut input); + let accepted = input[0] == 1; + input = &input[1..]; + assert_eq!(accepted, control.accepted); + let view = NativeArtifactView::decode(payload, control.source, &catalog).unwrap(); + assert_eq!(view.accepted(), accepted); + let parsed = engine.parse(control.source); + assert_eq!(parsed.is_ok(), accepted); + if let Ok(parsed) = parsed { + // Compare every canonical record, independently of CST projection. + let expected: Vec<_> = view + .events() + .map(|event| { + let name = match event.kind { + Kind::StartNode | Kind::FinishNode => { + &catalog.kinds()[event.symbol as usize].name + } + Kind::StartField | Kind::FinishField => { + &catalog.fields()[event.symbol as usize] + } + Kind::Token => &catalog.terminals()[event.symbol as usize].name, + }; + (event.kind, name.as_str(), event.id, event.start, event.end) + }) + .collect(); + let mut actual = Vec::new(); + parsed + .walk_events(|event| actual.push(canonical_event(event, spec.results))) + .unwrap(); + assert_eq!(actual, expected, "{name} {:?}", control.source); + assert_eq!(view.root().unwrap().text(), control.source); + // A second publication starts its identifiers again, retaining no state. + let mut replay = Vec::new(); + parsed.walk_events(|event| replay.push(event)).unwrap(); + let mut again = Vec::new(); + parsed.walk_events(|event| again.push(event)).unwrap(); + assert_eq!(again, replay); + } else { + assert!(view.root().is_none()); + let tokens: Vec<_> = view.tokens().collect(); + assert_eq!(tokens.len(), usize::from(!control.source.is_empty())); + if let Some(token) = tokens.first() { + assert_eq!(token.kind(), "unparsed-source"); + assert_eq!(token.text(), control.source); + } + } + for token in view.tokens() { + assert_eq!( + token.text().as_ptr(), + control.source[token.range().start..].as_ptr() + ); + } + // Payload/source identities must be admitted together, including rejects. + let changed = format!("{}x", control.source); + assert_eq!( + NativeArtifactView::decode(payload, &changed, &catalog) + .unwrap_err() + .reason, + "source-digest" + ); + drop(view); + count += 1; + } + } + assert_eq!(input, b""); + assert_eq!(count, 54); +} + +#[test] +fn canonical_source_publication_rejects_a_replaced_root_before_emitting() { + let engine = crate::PreparedCommandSource::new(&generated::bash::SOURCE).unwrap(); + let source = String::from("echo α"); + let equal_source = source.clone(); + assert_ne!(source.as_ptr(), equal_source.as_ptr()); + let mut parsed = engine.parse(&source).unwrap(); + assert_eq!(parsed.source().as_ptr(), source.as_ptr()); + let foreign = engine.parse(&equal_source).unwrap(); + parsed.root = foreign.root; + let mut count = 0; + let error = parsed.walk_events(|_| count += 1).unwrap_err(); + assert_eq!(count, 0); + assert_eq!(error.message, "foreign command Source root"); + assert!(parsed.events().is_err()); + // Equal bytes do not authenticate the original source allocation. + assert!(engine.parse(&source).unwrap().events().is_ok()); +} + +#[test] +fn source_edit_sessions_match_fresh_events_and_publish_owned_source() { + let engine = crate::PreparedCommandSource::new(&generated::bash::SOURCE).unwrap(); + let mut session = engine + .scan_session("echo α\necho tail\n".into(), None) + .unwrap(); + assert_eq!(session.reused_token_count(), 0); + assert_eq!(session.scanned_token_count(), session.tokens().len()); + for source in [ + "echo ββ\necho tail\n", + "echo ββ\necho tail\n", + "echo γ\necho tail\n", + "cat < 0, "{source:?}"); + } + drop(actual); + session = next; + } +} + +#[test] +fn source_sessions_follow_all_scheme_controls_for_both_poo_products() { + for (spec, controls) in [ + (&generated::bash::SOURCE, generated::bash::CONTROLS), + (&generated::extended::SOURCE, generated::extended::CONTROLS), + ] { + let engine = crate::PreparedCommandSource::new(spec).unwrap(); + let mut previous = None; + for control in controls { + match engine.scan_session(control.source.into(), previous.as_ref()) { + Ok(session) => { + { + let parsed = engine.parse_session(&session); + assert_eq!(parsed.is_ok(), control.accepted, "{:?}", control.source); + if let Ok(parsed) = parsed { + assert_eq!(parsed.events().unwrap(), control.events); + } + } + previous = Some(session); + } + Err(_) => assert!(!control.accepted, "{:?}", control.source), + } + } + } +} + +#[test] +fn source_session_rejection_preserves_previous_history_and_checks_plan_identity() { + use crate::scanner::{ContextualScanSession, ScannerSpec}; + static OTHER: ScannerSpec = ScannerSpec { + ..generated::bash::scanner::SCANNER + }; + let engine = crate::PreparedCommandSource::new(&generated::bash::SOURCE).unwrap(); + let old = engine + .scan_session("cat < 0); + assert_eq!( + engine.parse_session(&next).unwrap().events().unwrap(), + engine.parse(next.source()).unwrap().events().unwrap() + ); + let foreign = + ContextualScanSession::new(&OTHER, old.source().into(), "source", Some(&old)).unwrap(); + assert_eq!(foreign.reused_token_count(), 0); + assert!(engine.parse_session(&foreign).is_err()); + // A successful lexical scan can still fail recognition; the parser reports it. + let invalid = engine.scan_session("echo |".into(), Some(&old)).unwrap(); + assert!(engine.parse_session(&invalid).is_err()); + assert_eq!( + engine.parse_session(&old).unwrap().events().unwrap(), + engine.parse(old.source()).unwrap().events().unwrap() + ); +} + +#[test] +fn source_sessions_match_scheme_edit_work_counts() { + let engine = crate::PreparedCommandSource::new(&generated::bash::SOURCE).unwrap(); + let old = engine + .scan_session("cat < = ["α=1\n", "a = 1\r\nb=\"β\"\n", "name=value", ""] + .into_iter() + .map(|source| { + scope.spawn(move || { + for _ in 0..8 { + assert!(parse_contextual(&generated::CONTEXTUAL, "a=").is_err()); + let expected = parse(&generated::LANGUAGE, source).expect("canonical"); + let actual = + parse_contextual(&generated::CONTEXTUAL, source).expect("contextual"); + assert_eq!(actual.syntax().to_string(), source); + assert_eq!( + actual.receipt().source_digest, + expected.receipt().source_digest + ); + } + }) + }) + .collect(); + for worker in workers { + worker.join().expect("source-local receipt"); + } + }); +} diff --git a/rust/gerbil-parser-rowan/tests/unit/event_fold_aot.rs b/rust/gerbil-parser-runtime/tests/unit/event_fold_aot.rs similarity index 89% rename from rust/gerbil-parser-rowan/tests/unit/event_fold_aot.rs rename to rust/gerbil-parser-runtime/tests/unit/event_fold_aot.rs index 24fc0ccf..63a76cec 100644 --- a/rust/gerbil-parser-rowan/tests/unit/event_fold_aot.rs +++ b/rust/gerbil-parser-runtime/tests/unit/event_fold_aot.rs @@ -3,19 +3,19 @@ use super::event_strategy_aot_tests::grammar; mod generated { - use gerbil_parser_rowan::TreeEvent; + use gerbil_parser_runtime::TreeEvent; include!("event_fold_generated.rs"); } mod outline { - use gerbil_parser_rowan::TreeEvent; + use gerbil_parser_runtime::TreeEvent; include!("outline_fold_generated.rs"); } -use gerbil_parser_rowan::{TreeEvent, parse_generated_events}; +use gerbil_parser_runtime::{TreeEvent, parse_generated_events}; #[test] -fn scheme_event_fold_builds_lossless_multiline_rowan_tree() { +fn scheme_event_fold_builds_lossless_multiline_runtime_tree() { let source = "a\nb\n* α\r\nc"; let events = generated::parse_fold_lines(source); assert_eq!( @@ -57,7 +57,7 @@ fn scheme_event_fold_builds_lossless_multiline_rowan_tree() { source, &events, ) - .expect("Scheme event fold satisfies the Rowan event contract"); + .expect("Scheme event fold satisfies the syntax event contract"); assert_eq!(parsed.syntax().to_string(), source); assert_eq!(parsed.syntax().children().count(), 3); assert_eq!( @@ -76,7 +76,7 @@ fn scheme_event_fold_handles_empty_source() { } #[test] -fn scheme_outline_fold_builds_nested_rowan_sections() { +fn scheme_outline_fold_builds_nested_runtime_sections() { let source = "* Parent\n** Child\nbody\n* Peer\n"; let events = outline::parse_outline_lines(source); assert_eq!( @@ -122,7 +122,7 @@ fn scheme_outline_fold_builds_nested_rowan_sections() { ); let parsed = parse_generated_events(&grammar::LANGUAGE, outline::PARSER_DIGEST, source, &events) - .expect("Scheme-authored nesting satisfies the Rowan event contract"); + .expect("Scheme-authored nesting satisfies the syntax event contract"); assert_eq!(parsed.syntax().to_string(), source); assert_eq!(parsed.syntax().children().count(), 2); assert_eq!(parsed.syntax().first_child().unwrap().children().count(), 2); diff --git a/rust/gerbil-parser-rowan/tests/unit/event_fold_generated.rs b/rust/gerbil-parser-runtime/tests/unit/event_fold_generated.rs similarity index 100% rename from rust/gerbil-parser-rowan/tests/unit/event_fold_generated.rs rename to rust/gerbil-parser-runtime/tests/unit/event_fold_generated.rs diff --git a/rust/gerbil-parser-rowan/tests/unit/event_strategy_aot.rs b/rust/gerbil-parser-runtime/tests/unit/event_strategy_aot.rs similarity index 88% rename from rust/gerbil-parser-rowan/tests/unit/event_strategy_aot.rs rename to rust/gerbil-parser-runtime/tests/unit/event_strategy_aot.rs index 3902d1d3..4e7180e8 100644 --- a/rust/gerbil-parser-rowan/tests/unit/event_strategy_aot.rs +++ b/rust/gerbil-parser-runtime/tests/unit/event_strategy_aot.rs @@ -5,10 +5,10 @@ pub(super) mod grammar; #[path = "event_strategy_generated.rs"] mod strategy; -use gerbil_parser_rowan::{TreeEvent, parse_generated_events}; +use gerbil_parser_runtime::{TreeEvent, parse_generated_events}; #[test] -fn scheme_aot_event_algorithm_builds_lossless_rowan_tree() { +fn scheme_aot_event_algorithm_builds_lossless_runtime_tree() { let source = "* α\r\nbody\n"; let events = strategy::parse_event_lines(source); assert_eq!( @@ -34,7 +34,7 @@ fn scheme_aot_event_algorithm_builds_lossless_rowan_tree() { ); let parsed = parse_generated_events(&grammar::LANGUAGE, strategy::PARSER_DIGEST, source, &events) - .expect("Scheme AOT events satisfy Rowan event contract"); + .expect("Scheme AOT events satisfy syntax event contract"); assert_eq!(parsed.syntax().to_string(), source); assert_eq!( parsed.receipt().parser_digest, diff --git a/rust/gerbil-parser-rowan/tests/unit/event_strategy_generated.rs b/rust/gerbil-parser-runtime/tests/unit/event_strategy_generated.rs similarity index 97% rename from rust/gerbil-parser-rowan/tests/unit/event_strategy_generated.rs rename to rust/gerbil-parser-runtime/tests/unit/event_strategy_generated.rs index 5efd77f3..679c7c1a 100644 --- a/rust/gerbil-parser-rowan/tests/unit/event_strategy_generated.rs +++ b/rust/gerbil-parser-runtime/tests/unit/event_strategy_generated.rs @@ -1,5 +1,5 @@ // @generated by gerbil-parser/src/compiler/event-strategy-aot.ss -use gerbil_parser_rowan::TreeEvent; +use gerbil_parser_runtime::TreeEvent; pub const PARSER_DIGEST: &str = "sha256:88bbe27e4b43bf5f5c31b1133325f5eadedbfda0fc9c8defeabdf501f010ed89"; diff --git a/rust/gerbil-parser-rowan/tests/unit/event_strategy_grammar.rs b/rust/gerbil-parser-runtime/tests/unit/event_strategy_grammar.rs similarity index 98% rename from rust/gerbil-parser-rowan/tests/unit/event_strategy_grammar.rs rename to rust/gerbil-parser-runtime/tests/unit/event_strategy_grammar.rs index 1f5cabe5..f3d8fc1f 100644 --- a/rust/gerbil-parser-rowan/tests/unit/event_strategy_grammar.rs +++ b/rust/gerbil-parser-runtime/tests/unit/event_strategy_grammar.rs @@ -1,7 +1,7 @@ -// @generated by gerbil-parser/src/compiler/rust-rowan.ss +// @generated by gerbil-parser/src/compiler/rust-runtime.ss // Source authority: canonical Grammar IR v1; do not edit. -use gerbil_parser_rowan::{ +use gerbil_parser_runtime::{ ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, }; diff --git a/rust/gerbil-parser-rowan/tests/unit/event_tree.rs b/rust/gerbil-parser-runtime/tests/unit/event_tree.rs similarity index 69% rename from rust/gerbil-parser-rowan/tests/unit/event_tree.rs rename to rust/gerbil-parser-runtime/tests/unit/event_tree.rs index 5b4c3465..38a42651 100644 --- a/rust/gerbil-parser-rowan/tests/unit/event_tree.rs +++ b/rust/gerbil-parser-runtime/tests/unit/event_tree.rs @@ -1,6 +1,6 @@ -use super::event_tree::build_rowan_events; +use super::event_tree::{build_syntax_events, build_syntax_events_catalog}; use super::generated_events::parse_generated_events; -use super::model::{KindCategory, KindSpec, LanguageSpec, TreeEvent}; +use super::model::{EventCatalog, KindCategory, KindSpec, LanguageSpec, TreeEvent}; static KINDS: &[KindSpec] = &[ KindSpec { @@ -35,6 +35,11 @@ static LANGUAGE: LanguageSpec = LanguageSpec { productions: &[], }; +static EXTERNAL_CATALOG: EventCatalog = EventCatalog { + root_kind: 0, + kinds: KINDS, +}; + #[test] fn unbounded_outline_events_build_one_lossless_tree() { let source = "* Parent\n** 子\r\nbody\n"; @@ -63,8 +68,8 @@ fn unbounded_outline_events_build_one_lossless_tree() { TreeEvent::FinishNode, TreeEvent::FinishNode, ]; - let green = build_rowan_events(&LANGUAGE, source, &events).expect("valid nested events"); - let root = super::model::SyntaxNode::new_root(green); + let green = build_syntax_events(&LANGUAGE, source, &events).expect("valid nested events"); + let root = green.root(); assert_eq!(root.to_string(), source); assert_eq!(root.descendants().count(), 4); assert_eq!(root.children().count(), 1); @@ -83,7 +88,7 @@ fn malformed_events_fail_closed() { }, TreeEvent::FinishNode, ]; - assert!(build_rowan_events(&LANGUAGE, source, &events).is_ok()); + assert!(build_syntax_events(&LANGUAGE, source, &events).is_ok()); events[1] = TreeEvent::Token { kind: 3, @@ -91,7 +96,7 @@ fn malformed_events_fail_closed() { end: 1, }; assert_eq!( - build_rowan_events(&LANGUAGE, source, &events) + build_syntax_events(&LANGUAGE, source, &events) .expect_err("split UTF-8 codepoint") .reason_kind, "event-range" @@ -103,7 +108,7 @@ fn malformed_events_fail_closed() { end: source.len(), }; assert_eq!( - build_rowan_events(&LANGUAGE, source, &events) + build_syntax_events(&LANGUAGE, source, &events) .expect_err("node identity used as token") .reason_kind, "event-kind" @@ -111,13 +116,48 @@ fn malformed_events_fail_closed() { events[1] = TreeEvent::FinishNode; assert_eq!( - build_rowan_events(&LANGUAGE, source, &events) + build_syntax_events(&LANGUAGE, source, &events) .expect_err("double root finish") .reason_kind, "event-nesting" ); } +#[test] +fn external_events_need_only_a_kind_catalog_and_source_coverage() { + let source = "é\n"; + let events = [ + TreeEvent::StartNode(0), + TreeEvent::Token { + kind: 3, + start: 0, + end: source.len(), + }, + TreeEvent::FinishNode, + ]; + let tree = build_syntax_events_catalog(&EXTERNAL_CATALOG, source, &events) + .expect("external producer does not need LR tables"); + assert_eq!(tree.root().to_string(), source); + + let invalid = EventCatalog { + root_kind: 3, + kinds: KINDS, + }; + assert_eq!( + build_syntax_events_catalog(&invalid, source, &events) + .expect_err("token kind cannot be a root") + .reason_kind, + "invalid-event-catalog" + ); + let missing = [TreeEvent::StartNode(0), TreeEvent::FinishNode]; + assert_eq!( + build_syntax_events_catalog(&EXTERNAL_CATALOG, source, &missing) + .expect_err("external events must cover all source bytes") + .reason_kind, + "event-range" + ); +} + #[test] fn generated_scheme_events_publish_parser_identity_without_rust_recognition() { const PARSER_DIGEST: &str = diff --git a/rust/gerbil-parser-runtime/tests/unit/graph_index.rs b/rust/gerbil-parser-runtime/tests/unit/graph_index.rs new file mode 100644 index 00000000..a60f373c --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/graph_index.rs @@ -0,0 +1,162 @@ +use super::{GraphIndex, GraphIndexError, GraphRecord, GraphRelation}; +use gerbil_parser_artifact::syntax::{TextRange, TextSize}; + +fn record( + id: usize, + parent_id: Option, + children: &[usize], + kind: &'static str, +) -> GraphRecord { + GraphRecord { + id, + parent_id, + child_ids: children.to_vec(), + syntax_kind: 0, + category: "element", + kind, + range: TextRange::empty(TextSize::from(0)), + fields: Vec::new(), + } +} + +fn fixture() -> Vec { + vec![ + record(0, None, &[1, 5], "root"), + record(1, Some(0), &[2, 4], "section"), + record(2, Some(1), &[3], "section"), + record(3, Some(2), &[], "item"), + record(4, Some(1), &[], "item"), + record(5, Some(0), &[6], "section"), + record(6, Some(5), &[], "item"), + ] +} + +#[test] +fn preorder_index_reuses_subtrees_and_relationships() { + let records = fixture(); + let index = GraphIndex::new(&records).unwrap(); + assert_eq!(index.subtree_end(1), Some(5)); + assert_eq!(index.subtree_end(5), Some(7)); + assert!(index.contains(1, 3)); + assert!(!index.contains(1, 6)); + assert_eq!( + index.select( + &records, + 0, + "item", + GraphRelation::DescendantOf, + &[1, 2], + |_| true + ), + Ok(vec![3, 4]) + ); + assert_eq!( + index.select(&records, 1, "item", GraphRelation::ChildOf, &[1], |_| true), + Ok(vec![4]) + ); + assert_eq!( + index.select(&records, 1, "section", GraphRelation::At, &[1], |_| true), + Ok(vec![1]) + ); + assert_eq!( + index.select( + &records, + 1, + "item", + GraphRelation::Any, + &[], + |record| record.id == 3 + ), + Ok(vec![3]) + ); +} + +#[test] +fn malformed_graphs_and_scopes_fail_closed() { + let mut records = fixture(); + records[3].id = 7; + assert!(matches!( + GraphIndex::new(&records), + Err(GraphIndexError::InvalidRecord) + )); + + let mut records = fixture(); + records[1].child_ids.clear(); + assert!(matches!( + GraphIndex::new(&records), + Err(GraphIndexError::InvalidRecord) + )); + + let records = fixture(); + let index = GraphIndex::new(&records).unwrap(); + assert_eq!( + index.select(&records, 10, "item", GraphRelation::Any, &[], |_| true), + Err(GraphIndexError::InvalidScope) + ); + assert_eq!( + index.select(&records, 0, "item", GraphRelation::At, &[10], |_| true), + Err(GraphIndexError::InvalidTarget) + ); +} + +#[test] +fn many_targets_preserve_preorder_and_merge_nested_descendants() { + let records = fixture(); + let index = GraphIndex::new(&records).unwrap(); + let many_targets = [0, 1, 2, 3, 4, 5, 6, 0, 1]; + assert_eq!( + index.select( + &records, + 0, + "item", + GraphRelation::At, + &many_targets, + |_| true + ), + Ok(vec![3, 4, 6]) + ); + assert_eq!( + index.select( + &records, + 0, + "item", + GraphRelation::ChildOf, + &many_targets, + |_| true + ), + Ok(vec![3, 4, 6]) + ); + assert_eq!( + index.select( + &records, + 0, + "item", + GraphRelation::DescendantOf, + &[1, 2, 5], + |_| true + ), + Ok(vec![3, 4, 6]) + ); +} + +#[test] +fn nearest_matching_ancestor_is_strict_and_linear_in_depth() { + let records = fixture(); + let index = GraphIndex::new(&records).unwrap(); + let mut calls = 0; + let nearest = index + .nearest_ancestors_matching(&records, |record| { + calls += 1; + record.kind == "section" + }) + .unwrap(); + assert_eq!( + nearest, + vec![None, None, Some(1), Some(2), Some(1), None, Some(5)] + ); + assert_eq!(calls, records.len()); + assert_eq!( + index.nearest_ancestors_matching(&records[..1], |_| true), + Err(GraphIndexError::InvalidRecord) + ); +} diff --git a/rust/gerbil-parser-runtime/tests/unit/header_delimiters.rs b/rust/gerbil-parser-runtime/tests/unit/header_delimiters.rs new file mode 100644 index 00000000..73eb7979 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/header_delimiters.rs @@ -0,0 +1,76 @@ +//! Execute two Scheme-generated grammars through the shared Rust scanner rules. +#[path = "../fixtures/generated/hl7_header.rs"] +mod hl7; +#[path = "../fixtures/generated/header_record.rs"] +mod record; +use gerbil_parser_runtime::{LanguageSpec, LexicalExpr, LexicalRule, parse}; + +static INVALID_DATA_RULES: [LexicalRule; 3] = [ + record::LANGUAGE.lexical_rules[0], + record::LANGUAGE.lexical_rules[1], + LexicalRule { + expression: LexicalExpr::HeaderData { + prefix: "記", + count: 0, + stops: ";", + }, + ..record::LANGUAGE.lexical_rules[2] + }, +]; +static INVALID_DATA: LanguageSpec = LanguageSpec { + lexical_rules: &INVALID_DATA_RULES, + ..record::LANGUAGE +}; + +#[test] +fn header_aot_two_scheme_languages_parse_and_reject() { + for source in ["記|§α|", "記*$α*", "記|§a b|"] { + let parsed = parse(&record::LANGUAGE, source).expect("independent header record"); + assert_eq!(parsed.syntax().to_string(), source); + assert_eq!( + parsed.receipt().grammar_digest, + record::LANGUAGE.grammar_digest + ); + } + for source in ["記||α|", "記a§αa", "記|§|", "記|§α;|"] { + assert!(parse(&record::LANGUAGE, source).is_err(), "{source}"); + } + for source in [ + include_str!("../../../../languages/hl7/corpus/adt-a08-patient.hl7"), + "MSH*$%!?*LEGACY*AU*FHIR*AU*202609170900**ADT$A08*1*P*2.5.1\r", + ] { + let parsed = parse(&hl7::LANGUAGE, source).expect("HL7 shared header rules"); + assert_eq!(parsed.syntax().to_string(), source); + assert_eq!( + parsed.receipt().grammar_digest, + hl7::LANGUAGE.grammar_digest + ); + } + for source in ["PID|1\r", "MSH|^^\\&|\r", "MSH1^~\\&|\r"] { + assert!(parse(&hl7::LANGUAGE, source).is_err(), "{source}"); + } +} + +#[test] +fn header_aot_admission_rejects_invalid_closed_recipes() { + static INVALID_RULES: [LexicalRule; 3] = [ + LexicalRule { + expression: LexicalExpr::HeaderDelimiter { + prefix: "記", + count: 2, + index: 2, + }, + ..record::LANGUAGE.lexical_rules[0] + }, + record::LANGUAGE.lexical_rules[1], + record::LANGUAGE.lexical_rules[2], + ]; + static INVALID: LanguageSpec = LanguageSpec { + lexical_rules: &INVALID_RULES, + ..record::LANGUAGE + }; + let failure = parse(&INVALID, "記|§α|").expect_err("invalid recipe must reject at admission"); + assert!(failure.diagnostic.message.contains("header delimiter")); + let failure = parse(&INVALID_DATA, "記|§α|").expect_err("invalid header data must reject"); + assert!(failure.diagnostic.message.contains("header data")); +} diff --git a/rust/gerbil-parser-runtime/tests/unit/lexical.rs b/rust/gerbil-parser-runtime/tests/unit/lexical.rs new file mode 100644 index 00000000..5063cad3 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/lexical.rs @@ -0,0 +1,483 @@ +use super::{lexer::lexical_end, model::LexicalExpr}; + +const NUMBER: LexicalExpr = LexicalExpr::NumberLiteral { + prefixes: &["0x", "0o", "0b"], + separator: "_", + suffixes: &["M", "F", "D"], + leading_period: true, + trailing_period: true, +}; + +#[test] +fn profiled_numbers_match_the_scheme_boundaries() { + assert_eq!(lexical_end(&NUMBER, "0xCA_FE", 0), Some(7)); + assert_eq!(lexical_end(&NUMBER, "0o7_55", 0), Some(6)); + assert_eq!(lexical_end(&NUMBER, "0b10_01", 0), Some(7)); + assert_eq!(lexical_end(&NUMBER, "1_000.5E+2F", 0), Some(11)); + assert_eq!(lexical_end(&NUMBER, ".5M", 0), Some(3)); + assert_eq!(lexical_end(&NUMBER, "0x", 0), Some(1)); +} + +#[test] +fn quoted_strings_preserve_doubled_and_backslash_escapes() { + let expression = LexicalExpr::QuotedString(&["\"", "'", "`"]); + assert_eq!(lexical_end(&expression, "`a``b`", 0), Some(6)); + assert_eq!(lexical_end(&expression, "'Ada''s graph'", 0), Some(14)); + assert_eq!(lexical_end(&expression, r#""a\"b""#, 0), Some(6)); + assert_eq!(lexical_end(&expression, "`unterminated", 0), None); +} + +#[test] +fn escaped_quoted_strings_leave_adjacent_strings_separate() { + let expression = LexicalExpr::EscapedQuotedString(&["\""]); + assert_eq!(lexical_end(&expression, "\"a\"\"b\"", 0), Some(3)); + assert_eq!(lexical_end(&expression, r#""a\"b""#, 0), Some(6)); + assert_eq!(lexical_end(&expression, "\"unterminated", 0), None); +} + +#[test] +fn comments_and_choices_take_the_longest_complete_match() { + let expression = LexicalExpr::Choice(&[ + LexicalExpr::LineComment(&["//", "#"]), + LexicalExpr::BlockComment { + opening: "/*", + closing: "*/", + }, + ]); + assert_eq!(lexical_end(&expression, "// note\nnext", 0), Some(7)); + assert_eq!(lexical_end(&expression, "/* note */next", 0), Some(10)); + assert_eq!(lexical_end(&expression, "/* open", 0), None); +} + +#[test] +fn nested_comments_and_heredocs_close_losslessly() { + let nested = LexicalExpr::NestedBlockComment { + opening: "/*", + closing: "*/", + }; + assert_eq!(lexical_end(&nested, "/* /* */ */", 0), Some(11)); + assert_eq!( + lexical_end(&LexicalExpr::Heredoc, "<1.", "<١>字", "<+>*."] { + let parsed = crate::parse(&structured_lexical_generated::LANGUAGE, source).unwrap(); + assert_eq!(parsed.syntax().text().to_string(), source); + } + for source in ["123", "<+1>x.", "<>."] { + assert!(crate::parse(&structured_lexical_generated::LANGUAGE, source).is_err()); + } +} + +#[path = "../fixtures/generated/module_source.rs"] +mod module_source_generated; + +#[test] +fn scheme_generated_module_framing_preserves_unicode_offsets() { + let rule = module_source_generated::LANGUAGE + .lexical_rules + .iter() + .find(|rule| rule.terminal == "text") + .unwrap(); + for &(source, start, expected) in module_source_generated::MODULE_CASES { + let bytes = |end| source.chars().take(end).map(char::len_utf8).sum(); + assert_eq!( + lexical_end(&rule.expression, source, bytes(start)), + expected.map(bytes), + "{source:?} at {start}" + ); + } +} + +#[test] +fn module_framing_executes_in_a_standalone_lossless_parser() { + for source in [ + "preamble\n---- MODULE A ----\n====\nfooter", + "---- MODULE A ----\n---- MODULE B ----\n====\ninner\n====\nfooter", + "---- MODULE A ----\n(* ==== *)\n\"====\"\n====\nfooter", + "---- MODULE 字 ----\n====\nλ", + ] { + let parsed = crate::parse(&module_source_generated::LANGUAGE, source).unwrap(); + assert_eq!(parsed.syntax().text().to_string(), source); + } + for source in [ + "---- MODULE A ----", + "---- MODULE A ----\n---- MODULE B ----\n====", + ] { + assert!(crate::parse(&module_source_generated::LANGUAGE, source).is_err()); + } +} + +#[test] +fn invalid_module_profiles_reject_at_language_admission() { + static BAD: crate::ModuleTextProfile = crate::ModuleTextProfile { + header_border: "-x", + header_word: "MODULE", + end_border: "====", + block_open: "(*", + block_close: "*)", + line_comment: "\\*", + name_extra: "_", + }; + static RULES: [crate::LexicalRule; 7] = [ + crate::LexicalRule { + expression: LexicalExpr::ModuleText(&BAD), + ..module_source_generated::LANGUAGE.lexical_rules[0] + }, + module_source_generated::LANGUAGE.lexical_rules[1], + module_source_generated::LANGUAGE.lexical_rules[2], + module_source_generated::LANGUAGE.lexical_rules[3], + module_source_generated::LANGUAGE.lexical_rules[4], + module_source_generated::LANGUAGE.lexical_rules[5], + module_source_generated::LANGUAGE.lexical_rules[6], + ]; + static LANGUAGE: crate::LanguageSpec = crate::LanguageSpec { + lexical_rules: &RULES, + ..module_source_generated::LANGUAGE + }; + let error = crate::parse(&LANGUAGE, "preamble").unwrap_err(); + assert!(error.diagnostic.message.contains("module framing profile")); +} diff --git a/rust/gerbil-parser-runtime/tests/unit/nullable_boundary.rs b/rust/gerbil-parser-runtime/tests/unit/nullable_boundary.rs new file mode 100644 index 00000000..1f1e347a --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/nullable_boundary.rs @@ -0,0 +1,68 @@ +use super::{Operand, OperandAction, Production, Reduction, Terminal, Token, Value, reduce}; +use crate::engine::model::{Child, Symbol}; + +#[test] +fn nullable_aliases_use_the_following_operand_boundary() { + let production = Production { + lhs: "source-file", + rhs: &[ + Operand { + symbol: Symbol::Nonterminal("empty-before"), + actions: &[OperandAction::Alias(1)], + }, + Operand { + symbol: Symbol::Terminal(Terminal::Token("word")), + actions: &[], + }, + Operand { + symbol: Symbol::Nonterminal("empty-after"), + actions: &[OperandAction::Alias(2)], + }, + ], + reduction: Reduction::Concat, + dynamic_precedence: 0, + }; + for offset in [0, 17, 3] { + let end = offset + 1; + let tokens = [Token { + terminal: "word", + syntax_kind: 0, + text: "a", + start: offset, + end, + }]; + let values = vec![ + Value::Fragment(vec![]), + Value::Token(0), + Value::Fragment(vec![]), + ]; + let expected = Value::Fragment(vec![ + Child { + field: None, + value: Value::Node { + kind: 1, + start: offset, + end: offset, + children: vec![], + }, + }, + Child { + field: None, + value: Value::Token(0), + }, + Child { + field: None, + value: Value::Node { + kind: 2, + start: end, + end, + children: vec![], + }, + }, + ]); + assert_eq!( + reduce(&production, values.into_iter(), &tokens, end), + expected + ); + } +} diff --git a/rust/gerbil-parser-runtime/tests/unit/ordered_results.rs b/rust/gerbil-parser-runtime/tests/unit/ordered_results.rs new file mode 100644 index 00000000..cc5bb027 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/ordered_results.rs @@ -0,0 +1,149 @@ +//! Scheme catalog-constructor parity for ordered repeated command fields. +#[path = "../fixtures/generated/ordered_results.rs"] +mod generated; +use crate::{ + EventCatalog, KindCategory, PreparedResultProfile, ProjectedValue, ResultChildCapture, + ResultProfileSpec, build_syntax_events_catalog, +}; +#[test] +fn ordered_command_construction_matches_scheme_and_lossless_text() { + let profile = PreparedResultProfile::new(&generated::RESULT_PROFILE).unwrap(); + for (at, source) in generated::SOURCES.iter().enumerate() { + let node = match at { + 0 => generated::replay_0(&profile, source), + 1 => generated::replay_1(&profile, source), + 2 => generated::replay_2(&profile, source), + 3 => generated::replay_3(&profile, source), + _ => unreachable!(), + } + .unwrap(); + let events = node.events(); + assert_eq!(events, generated::EVENTS[at]); + let tree = build_syntax_events_catalog( + &EventCatalog { + root_kind: node.kind(), + kinds: generated::RESULT_PROFILE.kinds, + }, + source, + &events, + ) + .unwrap(); + assert_eq!(tree.to_string(), *source); + if at == 2 { + let ProjectedValue::Node(command) = node.children()[0].value() else { + panic!("command node"); + }; + assert_eq!( + command + .children() + .iter() + .map(crate::ProjectedChild::field) + .collect::>(), + ["argument", "name", "argument"] + ); + } + } +} +#[test] +fn catalog_fields_utf8_and_owner_constraints_apply_before_publication() { + static FOREIGN: ResultProfileSpec = ResultProfileSpec { + ..generated::RESULT_PROFILE + }; + let profile = PreparedResultProfile::new(&generated::RESULT_PROFILE).unwrap(); + let other = PreparedResultProfile::new(&FOREIGN).unwrap(); + let word = profile.bind_node("Word").unwrap(); + let source = String::from("α"); + let equal = String::from("α"); + let token = u16::try_from( + generated::RESULT_PROFILE + .kinds + .iter() + .position(|k| k.category == KindCategory::Token) + .unwrap(), + ) + .unwrap(); + assert!(profile.bind_node("unknown").is_err()); + assert!(word.build(&source, 0..1, vec![]).is_err()); + assert!( + word.build( + &source, + 0..2, + vec![ResultChildCapture { + field: "foreign", + value: ProjectedValue::Token { + kind: token, + span: 0..2 + } + }] + ) + .is_err() + ); + assert!( + word.build( + &source, + 0..2, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: 0, + span: 0..2 + } + }] + ) + .is_err() + ); + assert!( + word.build( + &source, + 0..2, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Token { + kind: token, + span: 0..1 + } + }] + ) + .is_err() + ); + let foreign = other + .bind_node("Word") + .unwrap() + .build(&source, 0..2, vec![]) + .unwrap(); + assert!( + word.build( + &source, + 0..2, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Node(foreign) + }] + ) + .is_err() + ); + let detached = word.build(&equal, 0..2, vec![]).unwrap(); + assert!( + word.build( + &source, + 0..2, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Node(detached) + }] + ) + .is_err() + ); + let beyond = word.build(&source, 0..2, vec![]).unwrap(); + assert!( + word.build( + &source, + 0..0, + vec![ResultChildCapture { + field: "part", + value: ProjectedValue::Node(beyond) + }] + ) + .is_err() + ); +} diff --git a/rust/gerbil-parser-rowan/tests/unit/outline_fold_generated.rs b/rust/gerbil-parser-runtime/tests/unit/outline_fold_generated.rs similarity index 100% rename from rust/gerbil-parser-rowan/tests/unit/outline_fold_generated.rs rename to rust/gerbil-parser-runtime/tests/unit/outline_fold_generated.rs diff --git a/rust/gerbil-parser-rowan/tests/unit/owned_first_word_generated.rs b/rust/gerbil-parser-runtime/tests/unit/owned_first_word_generated.rs similarity index 100% rename from rust/gerbil-parser-rowan/tests/unit/owned_first_word_generated.rs rename to rust/gerbil-parser-runtime/tests/unit/owned_first_word_generated.rs diff --git a/rust/gerbil-parser-rowan/tests/unit/owned_rest_after_first_word_generated.rs b/rust/gerbil-parser-runtime/tests/unit/owned_rest_after_first_word_generated.rs similarity index 100% rename from rust/gerbil-parser-rowan/tests/unit/owned_rest_after_first_word_generated.rs rename to rust/gerbil-parser-runtime/tests/unit/owned_rest_after_first_word_generated.rs diff --git a/rust/gerbil-parser-runtime/tests/unit/precedence_capacity.rs b/rust/gerbil-parser-runtime/tests/unit/precedence_capacity.rs new file mode 100644 index 00000000..de22ddf0 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/precedence_capacity.rs @@ -0,0 +1,46 @@ +use super::{ + GlrContext, LanguageSpec, ParserAction, ParserConfiguration, Token, Value, explore_fork, +}; +use crate::engine::selective_glr_tests::{BASE_LANGUAGE, DISTINCT_PRODUCTIONS}; + +#[test] +fn overflow_is_fatal_even_after_a_branch_completes() { + for (score, first, second) in [(i64::MAX - 1, 1, 2), (i64::MIN + 1, -1, -2)] { + let mut productions = DISTINCT_PRODUCTIONS.to_vec(); + productions[3].dynamic_precedence = first; + productions[4].dynamic_precedence = second; + let language = LanguageSpec { + productions: Box::leak(productions.into_boxed_slice()), + ..BASE_LANGUAGE + }; + let configuration = ParserConfiguration { + states: vec![0, 1], + values: vec![Value::Token(0)], + cursor: 1, + score, + }; + let tokens = [Token { + terminal: "identifier", + syntax_kind: 2, + text: "x", + start: 0, + end: 1, + }]; + for branches in [ + [ParserAction::Reduce(3), ParserAction::Reduce(4)], + [ParserAction::Reduce(4), ParserAction::Reduce(3)], + ] { + let error = explore_fork( + &language, + &tokens, + &[0], + &configuration, + &branches, + 0, + &mut GlrContext::default(), + ) + .expect_err("a capacity failure cannot discard a competing branch"); + assert_eq!(error.reason_kind, "dynamic-precedence-overflow"); + } + } +} diff --git a/rust/gerbil-parser-runtime/tests/unit/prepared_module.rs b/rust/gerbil-parser-runtime/tests/unit/prepared_module.rs new file mode 100644 index 00000000..ee583d4d --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/prepared_module.rs @@ -0,0 +1,51 @@ +use super::{ModuleTextProfile, PreparedModuleSource}; +static UNIT: ModuleTextProfile = ModuleTextProfile { + header_border: "~~~", + header_word: "UNIT", + end_border: "!!!", + block_open: "(*", + block_close: "*)", + line_comment: "\\*", + name_extra: "_", +}; +#[test] +fn numeric_extra_character_is_an_explicit_name_role() { + static NUMERIC: ModuleTextProfile = ModuleTextProfile { + name_extra: "0", + ..UNIT + }; + assert_eq!( + PreparedModuleSource::new("~~~ UNIT 000 ~~~", &NUMERIC).end(0), + None + ); + assert_eq!( + PreparedModuleSource::new("~~~ UNIT 111 ~~~", &NUMERIC).end(0), + Some(16) + ); +} +#[test] +fn long_border_run_has_bounded_records_and_independent_source_ownership() { + let source = format!("{} UNIT 字 ~~~\n!!!\nfooter", "~".repeat(100_000)); + let prepared = PreparedModuleSource::new(&source, &UNIT); + assert_eq!(prepared.headers.len(), 1); + assert_eq!(prepared.events.len(), 2); + assert_eq!(prepared.end(0), None); + assert_eq!(prepared.end(source.len() - 6), Some(source.len())); + std::thread::scope(|scope| { + for input in ["preamble\n~~~ UNIT Demo ~~~", "~~~ UNIT 字 ~~~\n!!!\nλ"] { + scope.spawn(move || { + let own = PreparedModuleSource::new(input, &UNIT); + for _ in 0..100 { + assert_eq!( + own.end(0), + if input.starts_with("preamble") { + Some(9) + } else { + None + } + ); + } + }); + } + }); +} diff --git a/rust/gerbil-parser-runtime/tests/unit/prepared_source.rs b/rust/gerbil-parser-runtime/tests/unit/prepared_source.rs new file mode 100644 index 00000000..90218b98 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/prepared_source.rs @@ -0,0 +1,44 @@ +use super::{LexicalExpr, PreparedLexicalSource, lexical_end_prepared}; +const FIELD: LexicalExpr = LexicalExpr::HeaderDelimiter { + prefix: "記", + count: 2, + index: 0, +}; +const BODY: LexicalExpr = LexicalExpr::Choice(&[ + LexicalExpr::HeaderData { + prefix: "記", + count: 2, + stops: ";", + }, + LexicalExpr::HeaderDelimiter { + prefix: "記", + count: 2, + index: 1, + }, +]); +#[test] +fn prepared_headers_deduplicate_nested_rules_and_cache_invalid_results() { + let good = PreparedLexicalSource::new("記|§α|", [&FIELD, &BODY]); + assert_eq!(good.headers.len(), 1); + assert_eq!(good.header("記", 2), Some("|§")); + assert_eq!(lexical_end_prepared(&FIELD, 3, &good), Some(4)); + assert_eq!(lexical_end_prepared(&BODY, 6, &good), Some(8)); + let bad = PreparedLexicalSource::new("記||α|", [&FIELD, &BODY]); + assert_eq!(bad.headers.len(), 1); + assert_eq!(bad.header("記", 2), None); + assert_eq!(lexical_end_prepared(&FIELD, 3, &bad), None); +} +#[test] +fn concurrent_prepared_sources_keep_borrowed_captures_independent() { + std::thread::scope(|scope| { + for source in ["記|§α|", "記*$α*", "記||α|"] { + scope.spawn(move || { + let prepared = PreparedLexicalSource::new(source, [&FIELD, &BODY]); + for _ in 0..100 { + let end = lexical_end_prepared(&FIELD, 3, &prepared); + assert_eq!(end, (!source.starts_with("記||")).then_some(4)); + } + }); + } + }); +} diff --git a/rust/gerbil-parser-rowan/tests/unit/pure_function_aot.rs b/rust/gerbil-parser-runtime/tests/unit/pure_function_aot.rs similarity index 100% rename from rust/gerbil-parser-rowan/tests/unit/pure_function_aot.rs rename to rust/gerbil-parser-runtime/tests/unit/pure_function_aot.rs diff --git a/rust/gerbil-parser-rowan/tests/unit/pure_function_generated.rs b/rust/gerbil-parser-runtime/tests/unit/pure_function_generated.rs similarity index 100% rename from rust/gerbil-parser-rowan/tests/unit/pure_function_generated.rs rename to rust/gerbil-parser-runtime/tests/unit/pure_function_generated.rs diff --git a/rust/gerbil-parser-runtime/tests/unit/records_contextual_fixture.rs b/rust/gerbil-parser-runtime/tests/unit/records_contextual_fixture.rs new file mode 100644 index 00000000..41e4053a --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/records_contextual_fixture.rs @@ -0,0 +1,3 @@ +//! One shared mount for the Scheme-generated contextual records language. +#[path = "../fixtures/records_contextual_generated.rs"] +pub(crate) mod generated; diff --git a/rust/gerbil-parser-runtime/tests/unit/region_boundaries.rs b/rust/gerbil-parser-runtime/tests/unit/region_boundaries.rs new file mode 100644 index 00000000..59f4d7a2 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/region_boundaries.rs @@ -0,0 +1,112 @@ +//! Scheme-owned recognition controls for the shared region execution engine. +#[path = "../fixtures/generated/region_boundaries.rs"] +mod generated; +use super::{PreparedRegionPlan, PreparedRegionSource, RegionScope, RegionSpec}; + +#[test] +fn regions_match_scheme_production_and_scoped_unicode_controls() { + for (spec, scopes, traces) in [ + ( + &generated::bash::REGION, + generated::bash::SCOPES, + generated::bash::TRACES, + ), + ( + &generated::generic::REGION, + generated::generic::SCOPES, + generated::generic::TRACES, + ), + ] { + for &(source, requests) in traces { + let mut prepared = + PreparedRegionSource::new(spec, source, scopes).expect("declared regions"); + for &(operation, at, expected) in requests { + let result = match operation { + 0 => PreparedRegionSource::new(spec, source, &[]) + .unwrap() + .word_end(at), + 1 => prepared.pair_end(at).map(Some), + code => prepared + .quote_end(at, char::from_u32(code).unwrap()) + .map(Some), + }; + let actual = match result { + Err(_) => -1, + Ok(None) => -2, + Ok(Some(end)) => i64::try_from(end).unwrap(), + }; + assert_eq!(actual, expected, "{source:?} operation={operation} at={at}"); + } + } + } +} +#[test] +fn admission_offsets_and_lazy_source_are_bounded() { + static UNKNOWN: &[RegionScope] = &[RegionScope { + prefix: "unknown", + pairs: &[], + }]; + static DUPLICATE: &[RegionScope] = &[ + RegionScope { + prefix: "${", + pairs: &[], + }, + RegionScope { + prefix: "${", + pairs: &[], + }, + ]; + static INVALID: RegionSpec = RegionSpec { + stops: &[""], + ..generated::bash::REGION + }; + assert!(PreparedRegionSource::new(&generated::bash::REGION, "", UNKNOWN).is_err()); + assert!(PreparedRegionSource::new(&generated::bash::REGION, "", DUPLICATE).is_err()); + assert!(PreparedRegionSource::new(&INVALID, "", &[]).is_err()); + let mut source = PreparedRegionSource::new(&generated::bash::REGION, "α'raw", &[]).unwrap(); + // Unrequested unterminated regions remain inert at preparation. + assert!(source.word_end(1).is_err()); + assert!(source.pair_end(usize::MAX).is_err()); + assert!(source.quote_end(2, '"').is_err()); + assert!(source.quote_end(2, '\'').is_err()); +} +#[test] +fn nested_frames_populate_cache_without_suffix_rescans() { + let text = "${x:-${y:-\"中\"}}"; + let mut source = PreparedRegionSource::new(&generated::bash::REGION, text, &[]).unwrap(); + assert_eq!(source.pair_end(0).unwrap(), text.len()); + let pairs = source.ends.pairs.len(); + let quotes = source.ends.quotes.len(); + assert_eq!((pairs, quotes), (2, 1)); + let visits = source.ends.visits; + for _ in 0..100 { + assert_eq!(source.pair_end(5).unwrap(), text.len() - 1); + assert_eq!(source.quote_end(10, '"').unwrap(), text.len() - 2); + } + assert_eq!( + source.ends.visits, visits, + "cached queries do not revisit source characters" + ); + assert_eq!( + (source.ends.pairs.len(), source.ends.quotes.len()), + (pairs, quotes) + ); +} + +#[test] +fn admitted_plan_reuses_tables_with_independent_source_caches() { + let plan = PreparedRegionPlan::new(&generated::bash::REGION, &[]).unwrap(); + let text = format!("{}中{}", "${".repeat(512), "}".repeat(512)); + let mut deep = plan.source(&text); + let mut unrelated = plan.source("${短}"); + assert_eq!(deep.pair_end(0).unwrap(), text.len()); + assert_eq!(deep.ends.pairs.len(), 512); + assert!(deep.ends.visits <= text.chars().count()); + assert!(unrelated.ends.pairs.is_empty()); + assert_eq!(unrelated.pair_end(0).unwrap(), "${短}".len()); + let visits = deep.ends.visits; + for level in 0..512 { + assert_eq!(deep.pair_end(level * 2).unwrap(), text.len() - level); + } + assert_eq!(deep.ends.visits, visits); +} diff --git a/rust/gerbil-parser-runtime/tests/unit/result_projection.rs b/rust/gerbil-parser-runtime/tests/unit/result_projection.rs new file mode 100644 index 00000000..0e00224a --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/result_projection.rs @@ -0,0 +1,198 @@ +use crate::{ + CaptureKind, CaptureSpec, EventCatalog, KindCategory, KindSpec, PreparedResultProfile, + ProjectedNode, ProjectedValue, ProjectionInstruction, ProjectionOpcode, ResultCapture, + ResultNodeSpec, ResultProfileSpec, ResultProjection, build_syntax_events_catalog, +}; +#[path = "../fixtures/generated/result_projections.rs"] +mod generated; +fn fields(node: &ProjectedNode<'_>, output: &mut Vec<(&'static str, &'static str)>) { + for child in node.children() { + output.push(( + generated::RESULT_PROFILE.kinds[usize::from(node.kind())].name, + child.field(), + )); + if let ProjectedValue::Node(node) = child.value() { + fields(node, output); + } + } +} +#[test] +fn production_scheme_word_captures_have_identical_rust_projection_events_and_fields() { + let profile = PreparedResultProfile::new(&generated::RESULT_PROFILE).unwrap(); + for (index, source) in generated::SOURCES.iter().enumerate() { + let node = generated::replay(&profile, source, index).unwrap(); + let events = node.events(); + assert_eq!(events, generated::EXPECTED[index], "source={source:?}"); + let mut actual_fields = Vec::new(); + fields(&node, &mut actual_fields); + assert_eq!(actual_fields, generated::FIELDS[index], "source={source:?}"); + let tree = build_syntax_events_catalog( + &EventCatalog { + root_kind: node.kind(), + kinds: generated::RESULT_PROFILE.kinds, + }, + source, + &events, + ) + .unwrap(); + assert_eq!(tree.to_string(), *source); + } +} +#[test] +fn result_captures_reject_wrong_types_arity_ranges_and_foreign_owners() { + static FOREIGN: ResultProfileSpec = ResultProfileSpec { + kinds: generated::RESULT_PROFILE.kinds, + nodes: generated::RESULT_PROFILE.nodes, + projections: generated::RESULT_PROFILE.projections, + }; + + let profile = PreparedResultProfile::new(&generated::RESULT_PROFILE).unwrap(); + for captures in [ + vec![], + vec![ResultCapture::Absent], + vec![ResultCapture::Parts(vec![])], + vec![ResultCapture::Span(1..2)], + vec![ResultCapture::Span(0..3)], + ] { + assert!(profile.project("LiteralPart", "α", 0..2, captures).is_err()); + } + assert!(profile.project("missing", "α", 0..2, vec![]).is_err()); + assert!( + profile + .project("Word", "α", 1..2, vec![ResultCapture::Parts(vec![])]) + .is_err() + ); + let independent = { + let temporary = PreparedResultProfile::new(&generated::RESULT_PROFILE).unwrap(); + temporary.bind("LiteralPart").unwrap() + }; + assert!( + independent + .project("α", 0..2, vec![ResultCapture::Span(0..2)]) + .is_ok() + ); + let leaf = profile.bind("LiteralPart").unwrap(); + assert!(profile.bind("missing").is_err()); + let node = leaf + .project("α", 0..2, vec![ResultCapture::Span(0..2)]) + .unwrap(); + assert_eq!(node.span(), 0..2); + let foreign = PreparedResultProfile::new(&FOREIGN).unwrap(); + let node = foreign + .project("LiteralPart", "α", 0..2, vec![ResultCapture::Span(0..2)]) + .unwrap(); + assert!( + profile + .project("Word", "α", 0..2, vec![ResultCapture::Parts(vec![node])]) + .is_err() + ); + let first = String::from("α"); + let other = String::from("α"); + let node = profile + .project("LiteralPart", &first, 0..2, vec![ResultCapture::Span(0..2)]) + .unwrap(); + assert!( + profile + .project("Word", &other, 0..2, vec![ResultCapture::Parts(vec![node])]) + .is_err() + ); + let node = profile + .project("LiteralPart", "αx", 0..2, vec![ResultCapture::Span(0..2)]) + .unwrap(); + assert!( + profile + .project("Word", "αx", 2..3, vec![ResultCapture::Parts(vec![node])]) + .is_err() + ); +} +#[test] +fn projection_admission_checks_unused_declarations_and_preserves_node_token_namespaces() { + static EMPTY: ResultProfileSpec = ResultProfileSpec { + kinds: &[], + nodes: &[], + projections: &[], + }; + static KINDS: &[KindSpec] = &[ + KindSpec { + name: "value", + category: KindCategory::Node, + }, + KindSpec { + name: "value", + category: KindCategory::Token, + }, + ]; + static NODES: &[ResultNodeSpec] = &[ResultNodeSpec { + kind: 0, + fields: &["text"], + }]; + static CAPTURES: &[CaptureSpec] = &[CaptureSpec { + name: "text", + kind: CaptureKind::Span, + }]; + static GOOD: ResultProfileSpec = ResultProfileSpec { + kinds: KINDS, + nodes: NODES, + projections: &[ResultProjection { + id: "leaf", + kind: 0, + captures: CAPTURES, + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::Token { + kind: 1, + required: true, + }, + }], + }], + }; + static BAD: ResultProfileSpec = ResultProfileSpec { + kinds: KINDS, + nodes: NODES, + projections: &[ResultProjection { + id: "unused", + kind: 0, + captures: CAPTURES, + instructions: &[ProjectionInstruction { + field: "missing", + opcode: ProjectionOpcode::Token { + kind: 1, + required: true, + }, + }], + }], + }; + static WRONG: ResultProfileSpec = ResultProfileSpec { + kinds: KINDS, + nodes: NODES, + projections: &[ResultProjection { + id: "unused", + kind: 0, + captures: CAPTURES, + instructions: &[ProjectionInstruction { + field: "text", + opcode: ProjectionOpcode::One { required: false }, + }], + }], + }; + let profile = PreparedResultProfile::new(&GOOD).unwrap(); + let node = profile + .bind("leaf") + .unwrap() + .project("中", 0..3, vec![ResultCapture::Span(0..3)]) + .unwrap(); + assert_eq!(node.children()[0].field(), "text"); + let tree = build_syntax_events_catalog( + &EventCatalog { + root_kind: 0, + kinds: GOOD.kinds, + }, + "中", + &node.events(), + ) + .unwrap(); + assert_eq!(tree.to_string(), "中"); + assert!(PreparedResultProfile::new(&EMPTY).is_err()); + assert!(PreparedResultProfile::new(&BAD).is_err()); + assert!(PreparedResultProfile::new(&WRONG).is_err()); +} diff --git a/rust/gerbil-parser-runtime/tests/unit/scanner_obligations.rs b/rust/gerbil-parser-runtime/tests/unit/scanner_obligations.rs new file mode 100644 index 00000000..f3abd0ff --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/scanner_obligations.rs @@ -0,0 +1,32 @@ +//! Ordered convergence compares queue values without changing either checkpoint. +use super::{Obligation, Obligations}; + +#[test] +fn convergence_compares_fifo_values_across_stack_partitions() { + fn marker(text: &str) -> Obligation { + Obligation { + marker: text.into(), + strip_tabs: false, + quoted: false, + } + } + let mut partitioned = Obligations::default(); + for text in ["drop", "A", "B"] { + partitioned.push_back(marker(text)); + } + partitioned.pop_front(); + partitioned.push_back(marker("C")); + let mut queued = Obligations::default(); + for text in ["A", "B", "C"] { + queued.push_back(marker(text)); + } + assert!(partitioned.equivalent(&queued)); + let mut swapped = Obligations::default(); + for text in ["A", "C", "B"] { + swapped.push_back(marker(text)); + } + assert!(!partitioned.equivalent(&swapped)); + assert_eq!(partitioned.len(), 3); + assert_eq!(queued.pop_front().unwrap().marker, "A"); + assert_eq!(partitioned.pop_front().unwrap().marker, "A"); +} diff --git a/rust/gerbil-parser-rowan/tests/unit/selective_glr.rs b/rust/gerbil-parser-runtime/tests/unit/selective_glr.rs similarity index 73% rename from rust/gerbil-parser-rowan/tests/unit/selective_glr.rs rename to rust/gerbil-parser-runtime/tests/unit/selective_glr.rs index 4c4a6299..9cf65f02 100644 --- a/rust/gerbil-parser-rowan/tests/unit/selective_glr.rs +++ b/rust/gerbil-parser-runtime/tests/unit/selective_glr.rs @@ -95,7 +95,7 @@ static B_DISTINCT_RHS: &[Operand] = &[Operand { macro_rules! productions { ($name:ident, $b_rhs:ident, $a_score:expr, $b_score:expr) => { - static $name: &[Production] = &[ + pub(super) static $name: &[Production] = &[ Production { lhs: "$accept", rhs: AUGMENTED_RHS, @@ -134,7 +134,7 @@ productions!(EQUIVALENT_PRODUCTIONS, B_EQUIVALENT_RHS, 0, 0); productions!(DYNAMIC_PRODUCTIONS, B_DISTINCT_RHS, 1, 2); productions!(DISTINCT_PRODUCTIONS, B_DISTINCT_RHS, 0, 0); -const BASE_LANGUAGE: LanguageSpec = LanguageSpec { +pub(super) const BASE_LANGUAGE: LanguageSpec = LanguageSpec { language: "selective-glr-test", version: "v1", contract: "selective-glr-test.v1", @@ -183,6 +183,79 @@ fn dynamic_precedence_selects_the_highest_complete_branch() { assert_eq!(receipt.dynamic_score, 2); } +#[test] +fn accumulated_scores_cross_i32_bounds_without_changing_branch_order() { + for (a, b, root_a, root_b, expected, kind) in [ + ( + i32::MAX, + i32::MAX, + 0, + i32::MAX, + 2 * i64::from(i32::MAX), + "RootB", + ), + ( + i32::MIN, + i32::MIN, + i32::MIN, + 0, + i64::from(i32::MIN), + "RootB", + ), + (i32::MAX, i32::MIN, i32::MIN, i32::MAX, -1, "RootA"), + ] { + let mut productions = DISTINCT_PRODUCTIONS.to_vec(); + productions[1].dynamic_precedence = root_a; + productions[2].dynamic_precedence = root_b; + productions[3].dynamic_precedence = a; + productions[4].dynamic_precedence = b; + let language = Box::leak(Box::new(LanguageSpec { + root_kind: u16::from(kind == "RootB"), + productions: Box::leak(productions.into_boxed_slice()), + ..BASE_LANGUAGE + })); + let result = parse(language, "x"); + if a == i32::MAX && b == i32::MIN { + let error = result.expect_err("exact cancellation retains distinct equal-score roots"); + assert_eq!(error.diagnostic.reason_kind, "selective-glr-ambiguity"); + assert_eq!(error.selective_glr.unwrap().dynamic_score, expected); + } else { + let parsed = result.expect("exact accumulated precedence"); + assert_eq!(parsed.kind_name(parsed.syntax().kind()), Some(kind)); + assert_eq!(parsed.selective_glr_receipt().dynamic_score, expected); + } + } +} + +#[test] +fn score_overflow_rejects_before_mutating_a_reduction() { + use super::model::Value; + use super::parser::{ParserConfiguration, apply_reduce}; + for (score, production) in [(i64::MAX, 3), (i64::MIN, 4)] { + let mut productions = DISTINCT_PRODUCTIONS.to_vec(); + productions[3].dynamic_precedence = 1; + productions[4].dynamic_precedence = -1; + let language = LanguageSpec { + productions: Box::leak(productions.into_boxed_slice()), + ..BASE_LANGUAGE + }; + let mut configuration = ParserConfiguration { + states: vec![0, 1], + values: vec![Value::Token(0)], + cursor: 1, + score, + }; + let before = configuration.clone(); + let error = apply_reduce(&language, &[], None, &mut configuration, production) + .expect_err("score capacity must be checked before reduction"); + assert_eq!(error.reason_kind, "dynamic-precedence-overflow"); + assert_eq!(configuration.states, before.states); + assert_eq!(configuration.values, before.values); + assert_eq!(configuration.cursor, before.cursor); + assert_eq!(configuration.score, before.score); + } +} + #[test] fn equal_score_distinct_forks_fail_closed() { let error = parse(&DISTINCT_LANGUAGE, "x").expect_err("ambiguous fork"); @@ -194,7 +267,7 @@ fn equal_score_distinct_forks_fail_closed() { } #[test] -fn downstream_scanner_uses_the_same_parser_and_rowan_pipeline() { +fn downstream_scanner_uses_the_same_parser_and_runtime_pipeline() { let scanned = [ScannedToken { terminal: "identifier", start: 0, diff --git a/rust/gerbil-parser-runtime/tests/unit/shared_scanner.rs b/rust/gerbil-parser-runtime/tests/unit/shared_scanner.rs new file mode 100644 index 00000000..77601eae --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/shared_scanner.rs @@ -0,0 +1,336 @@ +//! Scheme-generated scanner IR executes identical UTF-8 token traces in Rust. +#[path = "../fixtures/shared_scanner_generated.rs"] +mod generated; +use gerbil_parser_runtime::scanner::ContextualScanner; + +#[test] +fn prefix_replay_reaches_new_source_state_without_relocating_checkpoints() { + use gerbil_parser_runtime::scanner::{ScannerCheckpoint, ScannerSpec}; + static OTHER: ScannerSpec = ScannerSpec { + ..generated::SCANNER + }; + fn advance<'a, 'b>( + scanner: &'a ContextualScanner<'b>, + count: usize, + ) -> ScannerCheckpoint<'a, 'b> { + let mut state = scanner.initial_state(); + for _ in 0..count { + state = scanner.step(&state, "command").unwrap().1; + } + state + } + let old = ContextualScanner::new(&generated::SCANNER, "猫 < {after}" + ); + } + let foreign = ContextualScanner::new(&OTHER, "猫 < = (0..8) + .map(|_| { + scope.spawn(|| { + for _ in 0..8 { + for (source, expected) in generated::TRACES { + let scanner = + ContextualScanner::new(&generated::SCANNER, source).expect("spec"); + assert_eq!(scanner.scan("command").expect("complete scan"), *expected); + } + } + }) + }) + .collect(); + for worker in workers { + worker.join().expect("independent parser thread"); + } + }); +} + +#[test] +fn declared_regions_handle_deep_nesting_and_reject_partial_suffixes() { + let source = format!("echo {}α{}\n", "$(".repeat(5000), ")".repeat(5000)); + let tokens = ContextualScanner::new(&generated::SCANNER, &source) + .expect("region spec") + .scan("command") + .expect("complete nested word"); + assert_eq!(tokens[2].start, 5); + assert_eq!(tokens[2].end, source.len() - 1); + for source in ["echo $((1)", "echo <(cat", "echo `opaque", "echo \"${x}"] { + assert!( + ContextualScanner::new(&generated::SCANNER, source) + .expect("region spec") + .scan("command") + .is_err() + ); + } +} + +#[test] +fn region_declarations_control_initial_stops_and_scalar_depth_admission() { + use gerbil_parser_runtime::scanner::{ + RegionPair, ScannerAction, ScannerMatcher, ScannerRule, ScannerSpec, + }; + static INITIAL: ScannerSpec = ScannerSpec { + rules: &[ScannerRule { + name: "initial", + mode: "command", + form: "word", + rank: 0, + matcher: ScannerMatcher::RegionWord { + stops: &[";"], + quotes: &[], + pairs: &[], + consume_initial_stop: true, + }, + action: ScannerAction::Keep, + }], + ..generated::SCANNER + }; + static BAD_DEPTH: ScannerSpec = ScannerSpec { + rules: &[ScannerRule { + name: "invalid-depth", + mode: "command", + form: "word", + rank: 0, + matcher: ScannerMatcher::RegionWord { + stops: &[";"], + quotes: &[], + pairs: &[RegionPair { + prefix: "α(", + opening: '(', + closing: ')', + depth: 3, + }], + consume_initial_stop: false, + }, + action: ScannerAction::Keep, + }], + ..generated::SCANNER + }; + let tokens = ContextualScanner::new(&INITIAL, ";α") + .expect("consume initial stop") + .scan("command") + .expect("one word"); + assert_eq!(tokens.len(), 1); + assert_eq!((tokens[0].start, tokens[0].end), (0, 3)); + assert!(ContextualScanner::new(&BAD_DEPTH, "α(x)))").is_err()); +} + +#[test] +fn independently_reached_edited_suffix_requires_full_context() { + use gerbil_parser_runtime::scanner::ScannerCheckpoint; + fn advance<'a, 'b>( + scanner: &'a ContextualScanner<'b>, + count: usize, + ) -> ScannerCheckpoint<'a, 'b> { + let mut state = scanner.initial_state(); + for _ in 0..count { + state = scanner.step(&state, "command").expect("step").1; + } + state + } + static OTHER: gerbil_parser_runtime::scanner::ScannerSpec = + gerbil_parser_runtime::scanner::ScannerSpec { + ..generated::SCANNER + }; + let old = ContextualScanner::new(&generated::SCANNER, "cat <>(), + ["* First\n", "* Second\n"] + ); +} + +#[test] +fn graph_projection_reads_nested_node_text_without_reparsing_language_syntax() { + let source = "* Parent\n"; + let root = parse_structural_lines(&LANGUAGE, &HEADING_FIELDS_STRUCTURE, source) + .unwrap() + .syntax(); + let records = project_syntax_graph(&LANGUAGE, &GRAPH_WITH_NODE_TEXT, &root).unwrap(); + let headline = records + .iter() + .find(|record| record.kind == "headline") + .expect("projected section"); + assert_eq!(headline.field("source-line"), Some(source)); + assert_eq!( + headline.field_range("source-line"), + Some(TextRange::new( + 0.into(), + u32::try_from(source.len()).unwrap().into(), + )) + ); +} + +#[test] +fn graph_projection_preserves_declared_empty_fields_without_zero_length_tokens() { + let empty = parse_structural_lines(&LANGUAGE, &STRUCTURE, "") + .unwrap() + .syntax(); + let empty_records = project_syntax_graph(&LANGUAGE, &GRAPH_WITH_EMPTY_VALUE, &empty).unwrap(); + assert_eq!(empty_records[0].field("value"), Some("")); + assert_eq!( + empty_records[0].field_range("value"), + Some(TextRange::empty(0.into())) + ); + + let source = "* Parent\n"; + let populated = parse_structural_lines(&LANGUAGE, &HEADING_FIELDS_STRUCTURE, source) + .unwrap() + .syntax(); + let populated_records = + project_syntax_graph(&LANGUAGE, &GRAPH_WITH_EMPTY_VALUE, &populated).unwrap(); + assert_eq!(populated_records[0].field("value"), Some("Parent")); + assert_eq!(populated_records[0].values("value").count(), 1); +} + +#[test] +fn graph_projection_preserves_repeated_fields_without_merging_neighbors() { + let source = "* Task\nSCHEDULED: <2026-09-24 Thu> DEADLINE: <2026-09-25 Fri>\n"; + let root = parse_structural_lines(&LANGUAGE, &KEY_LINE_STRUCTURE, source) + .unwrap() + .syntax(); + let records = project_syntax_graph(&LANGUAGE, &GRAPH, &root).unwrap(); + let planning = records + .iter() + .find(|record| record.kind == "planning") + .expect("planning graph record"); + assert_eq!( + planning.values("key").collect::>(), + ["SCHEDULED", "DEADLINE"] + ); + assert_eq!( + planning.values("value").collect::>(), + ["<2026-09-24 Thu>", "<2026-09-25 Fri>"] + ); + let value_ranges = planning + .fields + .iter() + .filter(|field| field.name == "value") + .map(|field| field.range) + .collect::>(); + let first = u32::try_from(source.find("<2026-09-24 Thu>").unwrap()).unwrap(); + let second = u32::try_from(source.find("<2026-09-25 Fri>").unwrap()).unwrap(); + assert_eq!( + value_ranges, + [ + TextRange::new(first.into(), (first + 16).into()), + TextRange::new(second.into(), (second + 16).into()), + ] + ); +} diff --git a/rust/gerbil-parser-rowan/tests/unit/structural_key_line_cases.rs b/rust/gerbil-parser-runtime/tests/unit/structural_key_line_cases.rs similarity index 97% rename from rust/gerbil-parser-rowan/tests/unit/structural_key_line_cases.rs rename to rust/gerbil-parser-runtime/tests/unit/structural_key_line_cases.rs index be2b808f..cfb4e8bb 100644 --- a/rust/gerbil-parser-rowan/tests/unit/structural_key_line_cases.rs +++ b/rust/gerbil-parser-runtime/tests/unit/structural_key_line_cases.rs @@ -26,7 +26,7 @@ fn dynamic_keywords_and_heading_adjacent_planning_are_typed_and_lossless() { ); let keys: Vec<_> = planning[0] .children_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 43) .map(|token| token.text().to_string()) .collect(); diff --git a/rust/gerbil-parser-rowan/tests/unit/structural_lines.rs b/rust/gerbil-parser-runtime/tests/unit/structural_lines.rs similarity index 98% rename from rust/gerbil-parser-rowan/tests/unit/structural_lines.rs rename to rust/gerbil-parser-runtime/tests/unit/structural_lines.rs index ac552ef7..7d99472d 100644 --- a/rust/gerbil-parser-rowan/tests/unit/structural_lines.rs +++ b/rust/gerbil-parser-runtime/tests/unit/structural_lines.rs @@ -493,7 +493,7 @@ fn heading_fields_preserve_utf8_title_and_trailing_trivia() { assert_eq!(root.to_string(), source); let title = root .descendants_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .find(|token| token.kind().0 == 23) .expect("typed title token"); assert_eq!(title.text(), "中文 title"); @@ -525,7 +525,7 @@ fn inline_links_preserve_text_and_expose_target_and_description() { assert_eq!(links.len(), 2); let target: Vec<_> = links[0] .children_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 20) .map(|token| token.text().to_owned()) .collect(); @@ -556,7 +556,7 @@ fn block_header_argument_is_typed_without_losing_trivia() { assert_eq!(root.to_string(), source); let arguments: Vec<_> = root .descendants_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 17) .map(|token| token.text().to_owned()) .collect(); @@ -836,13 +836,13 @@ fn key_value_block_emits_typed_lines_with_exact_spans() { assert_eq!(properties[1].to_string(), ":ID: café\n"); let keys: Vec<_> = root .descendants_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 14) .map(|token| token.text().to_string()) .collect(); let values: Vec<_> = root .descendants_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 15) .map(|token| token.text().to_string()) .collect(); diff --git a/rust/gerbil-parser-rowan/tests/unit/structural_list_cases.rs b/rust/gerbil-parser-runtime/tests/unit/structural_list_cases.rs similarity index 94% rename from rust/gerbil-parser-rowan/tests/unit/structural_list_cases.rs rename to rust/gerbil-parser-runtime/tests/unit/structural_list_cases.rs index 69b1187d..c248e953 100644 --- a/rust/gerbil-parser-rowan/tests/unit/structural_list_cases.rs +++ b/rust/gerbil-parser-runtime/tests/unit/structural_list_cases.rs @@ -31,7 +31,7 @@ fn list_strategy_nests_items_and_preserves_continuations_and_ordered_markers() { ); let bullets: Vec<_> = root .descendants_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 36) .map(|token| token.text().to_string()) .collect(); @@ -53,7 +53,7 @@ fn list_indentation_uses_visual_tab_columns_and_rejects_multi_letter_counters() assert_eq!(lists[1].children().count(), 2); let bullets: Vec<_> = root .descendants_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 36) .map(|token| token.text().to_string()) .collect(); diff --git a/rust/gerbil-parser-rowan/tests/unit/structural_named_block_cases.rs b/rust/gerbil-parser-runtime/tests/unit/structural_named_block_cases.rs similarity index 96% rename from rust/gerbil-parser-rowan/tests/unit/structural_named_block_cases.rs rename to rust/gerbil-parser-runtime/tests/unit/structural_named_block_cases.rs index cfc13e91..2e161f40 100644 --- a/rust/gerbil-parser-rowan/tests/unit/structural_named_block_cases.rs +++ b/rust/gerbil-parser-runtime/tests/unit/structural_named_block_cases.rs @@ -47,7 +47,7 @@ fn named_delimited_block_projects_name_and_recovers_at_heading() { assert_eq!(root.to_string(), source); let names: Vec<_> = root .descendants_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 17) .map(|token| token.text().to_owned()) .collect(); @@ -110,7 +110,7 @@ fn required_name_block_rejects_empty_or_malformed_headers() { assert_eq!(root.to_string(), source); let names: Vec<_> = root .descendants_with_tokens() - .filter_map(rowan::NodeOrToken::into_token) + .filter_map(gerbil_parser_artifact::syntax::SyntaxElement::into_token) .filter(|token| token.kind().0 == 17) .map(|token| token.text().to_owned()) .collect(); diff --git a/rust/gerbil-parser-runtime/tests/unit/word_parts.rs b/rust/gerbil-parser-runtime/tests/unit/word_parts.rs new file mode 100644 index 00000000..b47ac9c9 --- /dev/null +++ b/rust/gerbil-parser-runtime/tests/unit/word_parts.rs @@ -0,0 +1,275 @@ +//! Production and inherited POO recipes are recognized independently in Rust. +#[allow(dead_code)] +#[path = "../fixtures/generated/word_parts.rs"] +mod generated; +use crate::{ + EventCatalog, PartGuard, PartOpcode, PartProfileSpec, PartRule, PreparedPartProfile, + PreparedResultProfile, ProjectedNode, ProjectedValue, ResultChildCapture, ResultProfileSpec, + TreeEvent, build_syntax_events_catalog, +}; +fn fields( + root: &ProjectedNode<'_>, + profile: &ResultProfileSpec, +) -> Vec<(&'static str, &'static str)> { + let mut output = Vec::new(); + let mut stack: Vec<_> = root + .children() + .iter() + .rev() + .map(|child| (root, child)) + .collect(); + while let Some((parent, child)) = stack.pop() { + output.push(( + profile.kinds[usize::from(parent.kind())].name, + child.field(), + )); + if let ProjectedValue::Node(node) = child.value() { + stack.extend(node.children().iter().rev().map(|child| (node, child))); + } + } + output +} +#[test] +fn production_and_unicode_inheritance_match_scheme_events_fields_and_outcomes() { + for (parts, results, regions, controls) in [ + ( + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + generated::bash::CONTROLS, + ), + ( + &generated::generic::PART_PROFILE, + &generated::generic::RESULT_PROFILE, + &generated::generic::regions::REGION, + generated::generic::CONTROLS, + ), + ] { + let engine = PreparedPartProfile::new(parts, results, regions).unwrap(); + for control in controls { + let span = control.start..control.end; + let result = match control.mode { + "word" => engine.word(control.source, span).map(Some), + "here" => engine.here_content(control.source, span).map(Some), + "assignment" => engine.assignment(control.source, span), + _ => panic!("unknown control"), + }; + let status = match &result { + Err(_) => 0, + Ok(Some(_)) => 1, + Ok(None) => 2, + }; + assert_eq!( + status, + control.status, + "{} {:?}: {:?}", + control.mode, + control.source, + result.as_ref().err() + ); + if let Ok(Some(node)) = result { + assert_eq!(node.span(), control.start..control.end); + let events = node.events(); + assert_eq!( + events, control.events, + "{} {:?}", + control.mode, control.source + ); + assert_eq!( + fields(&node, results), + control.fields, + "{} {:?}", + control.mode, + control.source + ); + let relative: Vec<_> = events + .into_iter() + .map(|event| match event { + TreeEvent::Token { kind, start, end } => TreeEvent::Token { + kind, + start: start - control.start, + end: end - control.start, + }, + event => event, + }) + .collect(); + let text = &control.source[control.start..control.end]; + let tree = build_syntax_events_catalog( + &EventCatalog { + root_kind: node.kind(), + kinds: results.kinds, + }, + text, + &relative, + ) + .unwrap(); + assert_eq!(tree.to_string(), text); + } + } + } +} +#[test] +fn admission_rejects_rule_region_binding_and_projection_conflicts() { + static UNKNOWN: PartProfileSpec = PartProfileSpec { + assignment: Some(usize::MAX), + ..generated::bash::PART_PROFILE + }; + static RULES: &[PartRule] = &[PartRule { + contexts: &["word"], + prefix: "${", + guard: PartGuard::Any, + opcode: PartOpcode::Parameter { binding: 0 }, + projection: "Word", + opening: 2, + closing: 1, + }]; + static SIGNATURE: PartProfileSpec = PartProfileSpec { + rules: RULES, + ..generated::bash::PART_PROFILE + }; + static BAD_OPENING: PartProfileSpec = PartProfileSpec { + rules: &[PartRule { + opening: 1, + ..RULES[0] + }], + ..generated::bash::PART_PROFILE + }; + static VALID_RULE: PartRule = PartRule { + projection: "ParameterExpansion", + opcode: PartOpcode::Parameter { binding: 1 }, + ..RULES[0] + }; + static DUPLICATE: PartProfileSpec = PartProfileSpec { + rules: &[VALID_RULE, VALID_RULE], + ..generated::bash::PART_PROFILE + }; + static NULLABLE: PartProfileSpec = PartProfileSpec { + bindings: &[crate::BindingSpec { + name: &crate::TextProfile::Literal(""), + prefixes: &[], + operators: &[], + subscript: None, + }], + assignment: None, + ..generated::bash::PART_PROFILE + }; + static BAD_SCOPE: PartProfileSpec = PartProfileSpec { + bindings: &[ + generated::bash::PART_PROFILE.bindings[0], + crate::BindingSpec { + subscript: Some(crate::SubscriptSpec { + scopes: &[], + ..generated::bash::PART_PROFILE.bindings[1].subscript.unwrap() + }), + ..generated::bash::PART_PROFILE.bindings[1] + }, + generated::bash::PART_PROFILE.bindings[2], + ], + ..generated::bash::PART_PROFILE + }; + static WRONG_PAIR: PartProfileSpec = PartProfileSpec { + rules: &[PartRule { + opcode: PartOpcode::Pair, + closing: 2, + ..VALID_RULE + }], + ..generated::bash::PART_PROFILE + }; + for spec in [ + &UNKNOWN, + &SIGNATURE, + &BAD_OPENING, + &DUPLICATE, + &NULLABLE, + &BAD_SCOPE, + &WRONG_PAIR, + ] { + assert!( + PreparedPartProfile::new( + spec, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION + ) + .is_err() + ); + } +} +#[test] +fn bounded_source_and_deep_composition_preserve_full_source_ownership() { + let engine = PreparedPartProfile::new( + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + ) + .unwrap(); + assert!(engine.word("α", 1..2).is_err()); + assert!(engine.assignment("α", 0..1).is_err()); + assert!(engine.word("\"α\"tail", 0..3).is_err()); + let text = format!("{}中{}", "${x:-".repeat(512), "}".repeat(512)); + let node = engine.word(&text, 0..text.len()).unwrap(); + let events = node.events(); + let tree = build_syntax_events_catalog( + &EventCatalog { + root_kind: node.kind(), + kinds: generated::bash::RESULT_PROFILE.kinds, + }, + &text, + &events, + ) + .unwrap(); + assert_eq!(tree.to_string(), text); +} + +#[test] +fn recognized_word_nodes_move_into_ordered_command_fields_with_one_catalog() { + let results = PreparedResultProfile::new(&generated::bash::RESULT_PROFILE).unwrap(); + let words = crate::PreparedPartProfile::new( + &generated::bash::PART_PROFILE, + &generated::bash::RESULT_PROFILE, + &generated::bash::regions::REGION, + ) + .unwrap(); + let source = "α${x:-中}"; + let first = words.word(source, 0..2).unwrap(); + let second = words.word(source, 2..source.len()).unwrap(); + let command = results + .bind_node("SimpleCommand") + .unwrap() + .build( + source, + 0..source.len(), + vec![ + ResultChildCapture { + field: "name", + value: ProjectedValue::Node(first), + }, + ResultChildCapture { + field: "argument", + value: ProjectedValue::Node(second), + }, + ], + ) + .unwrap(); + let root = results + .bind_node("BashFile") + .unwrap() + .build( + source, + 0..source.len(), + vec![ResultChildCapture { + field: "command", + value: ProjectedValue::Node(command), + }], + ) + .unwrap(); + let tree = build_syntax_events_catalog( + &EventCatalog { + root_kind: root.kind(), + kinds: generated::bash::RESULT_PROFILE.kinds, + }, + source, + &root.events(), + ) + .unwrap(); + assert_eq!(tree.to_string(), source); +} diff --git a/scripts/generate-reductions.ss b/scripts/generate-reductions.ss new file mode 100644 index 00000000..dc4f2a23 --- /dev/null +++ b/scripts/generate-reductions.ss @@ -0,0 +1,18 @@ +;;; Build entry for any module exporting one admitted language descriptor. +(import :gerbil/expander + (only-in :std/misc/ports read-all-as-string) + (only-in :gerbil-parser/language-build-support make-fused-reduction-strategy emit-build-strategy) + (only-in :gerbil-parser/src/language/module-input language-module-descriptor)) +(def (main . args) + (unless (and (= (length args) 3) (member (car args) '("module" "check"))) + (error "expected module|check grammar-module output-path" args)) + (let* ((strategy (make-fused-reduction-strategy (language-module-descriptor (cadr args)))) + (emit (lambda (port) (emit-build-strategy strategy port)))) + (if (equal? (car args) "module") + (call-with-output-file (caddr args) emit) + (let ((expected (call-with-output-string emit)) (repeated (call-with-output-string emit)) + (actual (call-with-input-file (caddr args) read-all-as-string))) + (unless (and (equal? expected repeated) (equal? expected actual)) + (error "generated fused reduction product is stale or nondeterministic" (caddr args))) + (displayln "GENERATED-REDUCTIONS-OK " (cadr args)))))) +(export main) diff --git a/scripts/generate-source-parser.ss b/scripts/generate-source-parser.ss new file mode 100644 index 00000000..e14562fc --- /dev/null +++ b/scripts/generate-source-parser.ss @@ -0,0 +1,18 @@ +;;; Build entry for any module exporting one admitted language descriptor. +(import :gerbil/expander + (only-in :std/misc/ports read-all-as-string) + (only-in :gerbil-parser/language-build-support make-recursive-source-strategy emit-build-strategy) + (only-in :gerbil-parser/src/language/module-input language-module-descriptor)) +(def (main . args) + (unless (and (= (length args) 3) (member (car args) '("module" "check"))) + (error "expected module|check grammar-module output-path" args)) + (let* ((strategy (make-recursive-source-strategy (language-module-descriptor (cadr args)))) + (emit (lambda (port) (emit-build-strategy strategy port)))) + (if (equal? (car args) "module") + (call-with-output-file (caddr args) emit) + (let ((expected (call-with-output-string emit)) (repeated (call-with-output-string emit)) + (actual (call-with-input-file (caddr args) read-all-as-string))) + (unless (and (equal? expected repeated) (equal? expected actual)) + (error "generated recursive source product is stale or nondeterministic" (caddr args))) + (displayln "GENERATED-HCL-RECURSIVE-OK " (cadr args)))))) +(export main) diff --git a/scripts/preparation-cache.ss b/scripts/preparation-cache.ss new file mode 100644 index 00000000..0dcd8f0d --- /dev/null +++ b/scripts/preparation-cache.ss @@ -0,0 +1,264 @@ +;;; Content admission for the complete compiled conformance preparation owner. +(import (only-in :std/crypto/digest Digest::sha256 digest-update! digest-final!) + (only-in :std/os/flock open-output-file/lock) + (only-in :std/os/device device-close)) + +(def (preparation-files root) + ;; Preserve sorted depth-first identity with one accumulator. Report actual + ;; discovered files, including the post-link admission walk. + (let ((files '()) (count 0)) + (def (walk path) + (when (file-exists? path) + (if (eq? (file-type path) 'directory) + (for-each (lambda (name) + (unless (string-prefix? "." name) + (walk (path-expand name path)))) + (list-sort stringstring index)) store)) +(def (copy-preparation-product from to) + (create-directory* (path-directory to)) + (when (file-exists? to) (delete-file to)) + ;; Gambit's copy-file creates a new mode; preserve executable/read permissions + ;; explicitly because they are part of the admitted product identity. + (call-with-input-file from + (lambda (input) + (call-with-output-file (list path: to permissions: (file-mode from)) + (lambda (output) + (let (buffer (make-u8vector 65536)) + (let loop () + (let (count (read-subu8vector buffer 0 (u8vector-length buffer) input)) + (unless (zero? count) + (write-subu8vector buffer 0 count output) + (loop)))))))))) +(def (restore-preparation-products! record store admit?) + (let ((entries (caddr record)) (total (length (caddr record))) (index -1)) + (and (every (lambda (entry) + (set! index (+ index 1)) + (let (stored (preparation-store-file store index)) + (and (admit? (car entry)) (file-exists? stored) + (equal? (preparation-file-digest stored) (cadr entry)) + (= (file-info-mode (file-info stored)) (caddr entry))))) entries) + (begin + (set! index -1) + (for-each (lambda (entry) + (set! index (+ index 1)) + (copy-preparation-product (preparation-store-file store index) (car entry)) + (when (or (zero? (modulo (+ index 1) 64)) + (= (+ index 1) total)) + (displayln "PREPARATION-RESTORED files=" (+ index 1) + " last=" (car entry)) (force-output))) entries) + (displayln "CONFORMANCE-CACHE-RESTORED") (force-output) + #t)))) +(def (save-preparation-products! record store) + (create-directory* store) + (let ((receipt (path-expand "receipt.sexp" store)) + (total (length (caddr record))) (index -1)) + (when (file-exists? receipt) (delete-file receipt)) + (for-each (lambda (entry) + (set! index (+ index 1)) + (copy-preparation-product (car entry) (preparation-store-file store index)) + (when (or (zero? (modulo (+ index 1) 64)) + (= (+ index 1) total)) + (displayln "PREPARATION-STORED files=" (+ index 1) + " last=" (car entry)) (force-output))) + (caddr record)) + (call-with-output-file receipt (lambda (port) (write record port))) + (displayln "PREPARATION-STORED-RECEIPT files=" (+ index 1)) (force-output))) + +(def (call-with-preparation-cache receipt inputs outputs prepare + (store #f) (admit? (lambda (_) #f))) + ;; The caller owns all writes. Reject simultaneous owners rather than admitting + ;; a receipt while another compiler can mutate the same products. + (let (lock (string-append receipt ".lock")) + (create-directory* (path-directory receipt)) + (let (owner (open-output-file/lock lock 0)) + (try + (let* ((identity (preparation-snapshot (inputs))) + (local (preparation-record receipt)) + (saved (or local (and store (preparation-record (path-expand "receipt.sexp" store))))) + (_ (when (and (not local) saved store (equal? identity (cadr saved))) + (unless (restore-preparation-products! saved store admit?) (set! saved #f)))) + (products (outputs))) + (if (and saved + (equal? identity (cadr saved)) + (pair? products) + (equal? (preparation-snapshot products) (caddr saved))) + (begin + (unless local + (call-with-output-file receipt (lambda (port) (write saved port)))) + (displayln "CONFORMANCE-CACHE-HIT") (force-output)) + (begin + ;; Failed rebuilds must not leave a usable old certificate. + (when (file-exists? receipt) (delete-file receipt)) + (displayln "CONFORMANCE-CACHE-MISS") (force-output) + (prepare) + (unless (equal? identity (preparation-snapshot (inputs))) + (error "conformance preparation inputs changed during build")) + (let (products (outputs)) + (unless (pair? products) (error "conformance preparation produced no artifacts")) + (let (record (list 'conformance-preparation-v1 identity + (preparation-snapshot products))) + (call-with-output-file receipt (lambda (port) (write record port))) + (displayln "PREPARATION-RECEIPT-READY files=" (length (caddr record))) + (force-output) + (when store (save-preparation-products! record store))))))) + (finally (device-close owner)))))) + +(def (prepare-conformance!) + (let* ((library (path-expand "lib" (getenv "GERBIL_PATH"))) + (binary (path-expand "bin/gerbil-parser-conformance" (getenv "GERBIL_PATH"))) + (home (getenv "GERBIL_BUILD_PREFIX" (gerbil-home))) + (owned (conformance-owned-modules)) + (helpers (filter (lambda (name) (not (conformance-source-only? name))) owned)) + (owned-product? (conformance-module-product-matcher library owned)) + (helper-product? (conformance-module-product-matcher library helpers))) + (def (inputs) + ;; Conservative closure: all project sources and dependency products, + ;; including macro interfaces, plus the actual SDK and compiler identity. + (append + (apply append (map preparation-files '("src" "languages" "t" "include"))) + (filter (lambda (path) (string-suffix? ".ss" path)) + (preparation-files "scripts")) + (filter (lambda (path) (string-suffix? ".ss" path)) + (list-sort stringstring (cadr form))) + (string-suffix? "-test" (symbol->string (cadr form))))) + (cdr imports))) + (imported (map (lambda (row) (let (name (symbol->string (cadr row))) (substring name 15 (string-length name)))) rows)) + (names (apply append (map cddr rows))) + (used (filter (lambda (name) (member name names)) (test-plan-symbols execution)))) + (unless (and (equal? (list-sort stringstring a) (symbol->string b))) names) + (list-sort (lambda (a b) (stringstring a) (symbol->string b))) used))) + (error "conformance test ownership or Suite execution is incomplete" owners imported names used)) + (map (lambda (owner) (string-append owner ".ss")) owners))) +(def (conformance-covered-test-files) + (let ((catalog (test-plan-definition "build-language-conformance.ss" 'conformance-modules)) + (imports (call-with-input-file "t/conformance-main.ss" + (lambda (port) + (let loop () + (let (form (read port)) + (cond ((eof-object? form) (error "missing conformance imports")) + ((and (pair? form) (eq? (car form) 'import)) form) + (else (loop)))))))) + (execution (test-plan-definition "t/conformance-main.ss" 'suites))) + (unless (and (pair? catalog) (eq? (car catalog) 'quote)) + (error "expected literal conformance module catalog")) + (let* ((covered (validate-conformance-test-plan (cadr catalog) imports execution)) + (source-rows (conformance-source-suites))) + (unless (and (test-plan-unique? (map car source-rows)) + (andmap (lambda (row) + (and (member (string-append (car row) ".ss") covered) + (pair? (cdr row)) + (test-plan-unique? (append (conformance-imported-test-names (car row)) (cdr row))) + (andmap (lambda (name) (and (symbol? name) (string-suffix? "-test" (symbol->string name)))) (cdr row)))) source-rows)) + (error "invalid or overlapping source Suite declarations" source-rows)) + covered))) + +(def (conformance-imported-test-names file) + (call-with-input-file "t/conformance-main.ss" + (lambda (port) + (let loop () + (let (form (read port)) + (cond ((eof-object? form) (error "missing conformance imports" file)) + ((and (pair? form) (eq? (car form) 'import)) + (let (row (find (lambda (row) + (and (pair? row) (eq? (car row) 'only-in) + (equal? (symbol->string (cadr row)) + (string-append ":gerbil-parser/" file)))) (cdr form))) + (unless row (error "missing conformance test import" file)) + (cddr row))) + (else (loop)))))))) +(def (validate-test-export-coverage! expected actual) + (let (less? (lambda (a b) (stringstring a) (symbol->string b)))) + (unless (equal? (list-sort less? expected) (list-sort less? actual)) + (error "conformance imports omit or repeat exported test Suites" expected actual)))) + +(def (validate-test-suite-ownership! compiled source actual) + (unless (and (test-plan-unique? (append compiled source)) + (test-plan-unique? actual)) + (error "overlapping or repeated test Suite ownership" compiled source actual)) + (validate-test-export-coverage! (append compiled source) actual) + source) +(def (source-test-suite-selected? name compiled) + (and (string-suffix? "-test" (symbol->string name)) (not (memq name compiled)))) + +(def (conformance-source-suites) + (let (form (test-plan-definition "t/conformance-main.ss" 'source-suites)) + (unless (and (pair? form) (eq? (car form) 'quote) (list? (cadr form))) + (error "expected literal source Suite declarations")) + (cadr form))) +(def (conformance-source-suite-names module) + (let (row (assoc module (conformance-source-suites))) + (if row (cdr row) '()))) diff --git a/scripts/tests/compiler-process-test.ss b/scripts/tests/compiler-process-test.ss new file mode 100644 index 00000000..1de00a8f --- /dev/null +++ b/scripts/tests/compiler-process-test.ss @@ -0,0 +1,28 @@ +;;; Exercise the real native compiler process owner, including concurrent jobs. +(displayln "COMPILER-PROCESS-START") (force-output) +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(call-with-compiled-interface-trace + (lambda () (load "build-language-conformance-link.ss"))) +(import (only-in :std/misc/process process-error?)) +(let* ((root (path-expand ".data/compiler-process-controls")) + (probe (path-expand "compiler-pipe" root))) + (create-directory* root) + (run-compiler [(getenv "GERBIL_GCC" "gcc") + "scripts/tests/fixtures/compiler-pipe.c" "-o" probe]) + (def (capture command) + (call-with-output-u8vector + (lambda (output) + (parameterize ((current-output-port output)) (run-compiler command))))) + (unless (equal? (capture [probe "--burst"]) (make-u8vector 262144 120)) + (error "compiler output was lost under pipe pressure")) + (unless (equal? (capture [probe "--fragment"]) + (string->utf8 "fragmentdiagnostic")) + (error "compiler fragment or stderr bytes were changed")) + ;; Use the same workgroup and process owner as conformance preparation. + (run-static-jobs (make-list 8 probe) (lambda (path) (run-compiler [path]))) + (unless (with-catch process-error? + (lambda () (run-compiler [probe "--failure"]) #f)) + (error "compiler process failure was not propagated"))) +(displayln "COMPILER-PROCESS-OK") (force-output) +(exit 0) diff --git a/scripts/tests/conformance-runner-test.ss b/scripts/tests/conformance-runner-test.ss new file mode 100644 index 00000000..490bdf81 --- /dev/null +++ b/scripts/tests/conformance-runner-test.ss @@ -0,0 +1,34 @@ +#!/usr/bin/env gxi +(import :std/test + (only-in :std/test/base TestConfig current-test-config VERBOSITY-CASE + test-suite! test-result-ok?)) +;; Exercise the production functions themselves without expanding language packs. +(call-with-input-file "t/conformance-main.ss" + (lambda (port) + (let loop () + (let (form (read port)) + (unless (eof-object? form) + (when (and (pair? form) (eq? (car form) 'def) + (pair? (cadr form)) + (memq (caadr form) '(validate-conformance-suites! run-suite-row!))) + (eval form)) + (loop)))))) +(def (main mode . _) + (let* ((owner (current-thread)) + (good (test-suite "control" (test-case "caller ownership" (check (current-thread) => owner))))) + (cond + ((equal? mode "success") + (validate-conformance-suites! (list (list "left" good))) + (run-suite-row! (list "left" good)) + (run-suite-row! (list "right" (test-suite "second" (test-case "same caller" (check (current-thread) => owner))))) + (displayln "CONFORMANCE-CONTROL-OK success")) + ((equal? mode "duplicate") + (validate-conformance-suites! (list (list "left" good) (list "right" good))) + (error "duplicate Suite was accepted")) + ((equal? mode "failure") + (run-suite-row! (list "failure" (test-suite "failure" (test-case "negative control" (check #f => #t))))) + (error "failed Suite was accepted")) + ((equal? mode "timeout") + (run-suite-row! (list "timeout" (test-suite "timeout" (test-case "blocked" (thread-sleep! 2)))) 0.1) + (error "timed-out Suite was accepted")) + (else (error "unknown control mode" mode))))) diff --git a/scripts/tests/fixtures/compiler-pipe.c b/scripts/tests/fixtures/compiler-pipe.c new file mode 100644 index 00000000..fc39c0c0 --- /dev/null +++ b/scripts/tests/fixtures/compiler-pipe.c @@ -0,0 +1,26 @@ +/* External compilers require blocking stdout/stderr, including under fan-out. */ +#include +#include +#include +#include +int main(int argc, char **argv) { + if ((fcntl(STDOUT_FILENO, F_GETFL) & O_NONBLOCK) || + (fcntl(STDERR_FILENO, F_GETFL) & O_NONBLOCK)) { + fprintf(stderr, "compiler inherited a nonblocking output descriptor\n"); + return 42; + } + if (argc == 2 && !strcmp(argv[1], "--burst")) { + char bytes[4096]; + memset(bytes, 'x', sizeof bytes); + for (int i = 0; i < 64; ++i) + if (fwrite(bytes, 1, sizeof bytes, stdout) != sizeof bytes) return 43; + return fflush(stdout) ? 44 : 0; + } + if (argc == 2 && !strcmp(argv[1], "--fragment")) { + fputs("fragment", stdout); fflush(stdout); fputs("diagnostic", stderr); return 0; + } + if (argc == 2 && !strcmp(argv[1], "--failure")) return 17; + puts("COMPILER-PIPE-OK"); + + return 0; +} diff --git a/scripts/tests/history-commit-test.ss b/scripts/tests/history-commit-test.ss new file mode 100644 index 00000000..44cd423f --- /dev/null +++ b/scripts/tests/history-commit-test.ss @@ -0,0 +1,89 @@ +;;; Actual same-owner callback re-entry, not a simulated parser lifecycle. +(displayln "HISTORY-COMMIT-START") (force-output) +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(preload-module "gerbil-parser/src/ffi/language-handles") +(preload-module "gerbil-parser/languages/bash/parser") +(preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") +(import (only-in :gerbil-parser/src/ffi/language-handles + register-language! release-language! language-handle-parse + language-handle-parse/publish language-handle-scan-counts language-handle-count + language-handle-descriptor) + (only-in :gerbil-parser/languages/bash/parser bash-source-language) + (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(def (require! value message) (unless value (error message))) +(def (fails? thunk) (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def (counts handle) + (let-values (((scanned reused) (language-handle-scan-counts handle))) (list scanned reused))) +(def initial "cat <utf8 initial)) + (let (publish (lambda (_) + (set! inner-payload (language-handle-parse handle (string->utf8 nested))) + (set! committed (counts handle)) + (if raises? (error "outer callback failed after inner commit") #t))) + (if raises? + (require! (fails? (lambda () (language-handle-parse/publish handle (string->utf8 outer) publish))) + "outer failure swallowed") + (require! (equal? (language-handle-parse/publish handle (string->utf8 outer) publish) + (language-handle-parse fresh (string->utf8 outer))) "outer payload changed"))) + (require! (and (= (car committed) 1) (> (cadr committed) 0)) "inner history not measured") + (require! (equal? committed (counts handle)) "outer completion overwrote newer history") + (require! (equal? inner-payload (language-handle-parse fresh (string->utf8 nested))) "inner payload differs from fresh") + (displayln "HISTORY-NESTED-COMMIT-OK raises=" raises?) (force-output) + (finally (release-language! handle) (release-language! fresh))))) '(#f #t)) + +(let (handle (register-language! bash-source-language)) + (try + (language-handle-parse handle (string->utf8 initial)) + (let (before (counts handle)) + (language-handle-parse/publish handle (string->utf8 outer) + (lambda (_) + (require! (fails? (lambda () (language-handle-parse handle #u8(255)))) "invalid nested source accepted") + #t)) + (require! (equal? before (counts handle)) "superseded outer committed after failed newer attempt")) + (displayln "HISTORY-FAILED-SUPERSESSION-OK") (force-output) + (finally (release-language! handle)))) + +(let (handle (register-language! bash-source-language)) + (try + (language-handle-parse handle (string->utf8 initial)) + (let (before (counts handle)) + (language-handle-parse/publish handle (string->utf8 outer) + (lambda (_) + (language-handle-parse/publish handle (string->utf8 nested) (lambda (_) #f)) + #t)) + (require! (equal? before (counts handle)) "rejected newer publication allowed stale outer commit")) + (displayln "HISTORY-REJECTED-SUPERSESSION-OK") (force-output) + (finally (release-language! handle)))) + +(let ((handle (register-language! bash-source-language)) + (other (register-language! bash-source-language))) + (try + (language-handle-parse handle (string->utf8 initial)) + (language-handle-parse/publish handle (string->utf8 outer) + (lambda (_) + (language-handle-descriptor handle) + (language-handle-parse other (string->utf8 nested)) + #t)) + (language-handle-parse other (string->utf8 outer)) + (require! (equal? (counts handle) (counts other)) "unrelated handle or metadata query superseded request") + (displayln "HISTORY-HANDLE-ISOLATION-OK") (force-output) + (finally (release-language! handle) (release-language! other)))) + +(let (handle (register-language! bash-source-language)) + (language-handle-parse/publish handle (string->utf8 initial) + (lambda (_) (release-language! handle) #t)) + (require! (fails? (lambda () (counts handle))) "released handle retained readable history") + (displayln "HISTORY-RELEASE-OK") (force-output)) +(require! (= baseline (language-handle-count)) "history controls leaked handles") +(displayln "HISTORY-COMMIT-CONTROLS-OK") (force-output) +(test-child-process-exit! 0) diff --git a/scripts/tests/preparation-test.ss b/scripts/tests/preparation-test.ss new file mode 100644 index 00000000..524b11d3 --- /dev/null +++ b/scripts/tests/preparation-test.ss @@ -0,0 +1,138 @@ +(load "scripts/preparation-cache.ss") +(import (only-in :std/io/tempfile make-temporary-file-name)) +(def root (make-temporary-file-name "parser-preparation-controls")) +(create-directory* root) +(def input (path-expand "input.ss" root)) +(def output (path-expand "output.o" root)) +(def receipt (path-expand "receipt.sexp" root)) +(def count 0) +;; Static phase products are outputs too, not new dependency inputs. +(def library (path-expand "lib" root)) +(for-each + (lambda (suffix) + (unless (conformance-module-product? + (path-expand (string-append "static/gerbil-parser__t__fixture" suffix) library) + library '("t/fixture")) + (error "owned static phase product admitted as input" suffix))) + '(".scm" "~0.scm" "~1.scm" "~0.c")) +(when (conformance-module-product? + (path-expand "static/gerbil-parser__t__fixture-other~0.scm" library) + library '("t/fixture")) + (error "foreign static module excluded from inputs")) +(displayln "CACHE-CONTROL-OK static-phase-ownership") +(def (put path text) + (when (file-exists? path) (delete-file path)) + (call-with-output-file path (lambda (port) (display text port)))) +;; Inventory identity includes ordering and hidden-entry exclusion, not just a +;; set of paths. Exercise the production walk independently of cache receipts. +(def discovery (path-expand "discovery" root)) +(create-directory* (path-expand "b" discovery)) +(create-directory* (path-expand ".hidden-directory" discovery)) +(for-each (lambda (name) (put (path-expand name discovery) name)) + '("z.ss" "a.ss" "b/c.ss" ".hidden.ss" ".hidden-directory/x.ss")) +(unless (equal? (preparation-files discovery) + (map (lambda (name) (path-expand name discovery)) '("a.ss" "b/c.ss" "z.ss"))) + (error "preparation discovery changed sorted identity or hidden-entry admission")) +(unless (and (null? (preparation-files (path-expand "missing" discovery))) + (equal? (preparation-files (path-expand "a.ss" discovery)) (list (path-expand "a.ss" discovery)))) + (error "preparation discovery changed missing or single-file admission")) +(displayln "CACHE-CONTROL-OK discovery-identity") +(def (prepare) + (set! count (+ count 1)) + (put output (number->string count))) +(def (run) + (call-with-preparation-cache receipt (lambda () (list input)) + (lambda () (if (file-exists? output) (list output) '())) prepare)) +(def (check-count expected) + (unless (= count expected) (error "wrong preparation count" expected count))) +(put input "a") +(when (file-exists? receipt) (delete-file receipt)) +(run) (check-count 1) +(run) (check-count 1) (displayln "CACHE-CONTROL-OK unchanged") +(put input "b") (run) (check-count 2) (displayln "CACHE-CONTROL-OK input-content") +(put output "corrupt") (run) (check-count 3) (displayln "CACHE-CONTROL-OK output-content") +(delete-file output) (run) (check-count 4) (displayln "CACHE-CONTROL-OK missing-output") +(put receipt "broken (") (run) (check-count 5) (displayln "CACHE-CONTROL-OK corrupt-receipt") +(put input "c") +(with-catch + (lambda (_) (void)) + (lambda () (call-with-preparation-cache receipt (lambda () (list input)) + (lambda () (list output)) (lambda () (error "expected preparation failure"))))) +(when (file-exists? receipt) (error "failed preparation published a receipt")) +(run) (check-count 6) (displayln "CACHE-CONTROL-OK failure-retry") +(def store (path-expand "store" root)) +(def (stored-run) + (call-with-preparation-cache receipt (lambda () (list input)) + (lambda () (if (file-exists? output) (list output) '())) prepare store + (lambda (path) (equal? path output)))) +;; Publish a fresh content record into the independently restorable store. +(delete-file receipt) (stored-run) (check-count 7) +(delete-file receipt) (delete-file output) +(stored-run) (check-count 7) (displayln "CACHE-CONTROL-OK store-restore") +(delete-file receipt) (delete-file output) +(put (preparation-store-file store 0) "corrupt") +(stored-run) (check-count 8) (displayln "CACHE-CONTROL-OK corrupt-store") +(delete-file receipt) (delete-file output) +(put input "changed-after-store") +(stored-run) (check-count 9) (displayln "CACHE-CONTROL-OK store-input-change") +;; Mutating an input during preparation can never publish a usable record. +(delete-file receipt) +(with-catch + (lambda (_) (void)) + (lambda () (call-with-preparation-cache receipt (lambda () (list input)) + (lambda () (list output)) (lambda () (put input "mutation"))))) +(when (file-exists? receipt) (error "input mutation published a receipt")) +(displayln "CACHE-CONTROL-OK input-mutation") +;; Compiler memo publication must not mutate the admitted source closure. +(def library (path-expand "lib" root)) +(def interface (path-expand "dependency.ssi" library)) +(create-directory* library) +(put interface "immutable-interface") +(def generated + (map (lambda (name) + (path-expand (string-append "gerbil-parser/" name "/digest.scm") library)) + '("compiled-language-artifacts" "compiled-language-parser-cache" + "compiled-language-declaration-cache" "compiled-language-program-cache"))) +(set! generated (cons (path-expand "static/foreign-module.o.options.sexp" library) generated)) +(def (memo-inputs) + (filter (lambda (path) (not (or (conformance-generated-cache? path library) + (conformance-generated-object? path library)))) + (preparation-files library))) +(def (memo-prepare) + (prepare) + (for-each (lambda (path) + (create-directory* (path-directory path)) + (put path "derived")) generated)) +(def (memo-run) + (call-with-preparation-cache receipt memo-inputs + (lambda () (list output)) memo-prepare)) +(memo-run) (check-count 10) +(unless (equal? (memo-inputs) (list interface)) + (error "compiler memo products leaked into inputs" (memo-inputs))) +(memo-run) (check-count 10) +(put (car generated) "new-derived-content") +(memo-run) (check-count 10) +(displayln "CACHE-CONTROL-OK generated-memo-products") +(put interface "changed-interface") +(memo-run) (check-count 11) +(displayln "CACHE-CONTROL-OK dependency-interface") +(for-each + (lambda (path) + (when (conformance-generated-cache? path library) + (error "generated-cache exclusion admitted a sibling" path))) + (list interface + (path-expand "gerbil-parser/compiled-language-artifacts-extra/digest.scm" library) + (path-expand "other/compiled-language-artifacts/digest.scm" library) + (path-expand "gerbil-parser/compiled-language-artifacts" library))) +(displayln "CACHE-CONTROL-OK generated-cache-boundaries") +(for-each + (lambda (path) + (when (conformance-generated-object? path library) + (error "native object exclusion admitted an input" path))) + (list interface (path-expand "dependency.o.options.sexp" library) + (path-expand "static/dependency.scm" library) + (path-expand "static/dependency.ssi" library) + (path-expand "static-extra/foreign-module.o.options.sexp" library))) +(displayln "CACHE-CONTROL-OK generated-object-boundaries") +(delete-file-or-directory root #t) +(displayln "OK") diff --git a/scripts/tests/process-qualification-test.ss b/scripts/tests/process-qualification-test.ss new file mode 100644 index 00000000..a9434b26 --- /dev/null +++ b/scripts/tests/process-qualification-test.ss @@ -0,0 +1,32 @@ +;;; Exercise production process contracts with real native children. +(load "test-processes.ss") +(def make-case + (eval '(lambda (name command expected marker budget idle) + (process-case name command expected marker budget idle)))) +(def root (path-expand ".data/process-qualification-controls")) +(create-directory* root) +(def (probe name code expected marker budget idle) + (qualify-process! + (make-case name ["gxi" "-e" code] expected marker budget idle) root)) +(def (reject! name thunk) + (unless (with-catch (lambda (_) #t) (lambda () (thunk) #f)) + (error "negative process control was accepted" name)) + (displayln "PROCESS-CONTROL-OK " name) (force-output)) +(unless (= 17 (probe "exit-status" "(exit 17)" 17 #f 2 1)) + (error "normal exit status changed")) +(displayln "PROCESS-CONTROL-OK exit-status") +(probe "fragments" "(display \"fragment\") (force-output) (display \"diagnostic\" (current-error-port)) (force-output (current-error-port)) (exit 0)" + 0 "fragmentdiagnostic" 2 1) +(displayln "PROCESS-CONTROL-OK partial-output") +(reject! "wrong-exit" (lambda () (probe "wrong-exit" "(exit 17)" 0 #f 2 1))) +(reject! "missing-marker" (lambda () (probe "missing-marker" "(exit 0)" 0 "MISSING" 2 1))) +(reject! "idle-timeout" (lambda () (probe "idle" "(thread-sleep! 2) (exit 0)" 0 #f 2 0.1))) +(reject! "total-timeout" + (lambda () (probe "total" "(let loop () (display \".\") (force-output) (thread-sleep! 0.02) (loop))" + 0 #f 0.15 0.1))) +;; EOF is not process completion: the total fence also covers process-status. +(reject! "closed-output-timeout" + (lambda () (probe "closed-output" "(close-output-port (current-output-port)) (close-output-port (current-error-port)) (thread-sleep! 2) (exit 0)" + 0 #f 0.15 #f))) +(displayln "PROCESS-QUALIFICATION-CONTROLS-OK") (force-output) +(exit 0) diff --git a/scripts/tests/test-plan-test.ss b/scripts/tests/test-plan-test.ss new file mode 100644 index 00000000..ce9a6bd7 --- /dev/null +++ b/scripts/tests/test-plan-test.ss @@ -0,0 +1,24 @@ +#!/usr/bin/env gxi +(load "scripts/test-plan.ss") +(def (control name thunk) + (unless (thunk) (error "test plan control failed" name)) + (displayln "TEST-PLAN-CONTROL-OK " name) (force-output)) +(def (rejected? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def (main . _) + (let ((catalog '("t/a-test" "t/helper")) + (imports '(import (only-in :gerbil-parser/t/a-test a-test))) + (execution '(list (cons "a" (list a-test))))) + (control 'exact-owner (lambda () (equal? (validate-conformance-test-plan catalog imports execution) '("t/a-test.ss")))) + (control 'missing-owner (lambda () (rejected? (lambda () (validate-conformance-test-plan '("t/a-test" "t/b-test") imports execution))))) + (control 'missing-suite (lambda () (rejected? (lambda () (validate-conformance-test-plan catalog imports '(list)))))) + (control 'repeated-suite (lambda () (rejected? (lambda () (validate-conformance-test-plan catalog imports '(list a-test a-test)))))) + (control 'repeated-owner (lambda () (rejected? (lambda () (validate-conformance-test-plan '("t/a-test" "t/a-test") imports execution))))) + (control 'unimported-export (lambda () (rejected? (lambda () (validate-test-export-coverage! '(a-test) '(a-test b-test)))))) + (control 'exact-exports (lambda () (validate-test-export-coverage! '(b-test a-test) '(a-test b-test)) #t)) + (control 'split-ownership (lambda () (equal? (validate-test-suite-ownership! '(a-test) '(b-test) '(b-test a-test)) '(b-test)))) + (control 'overlapping-ownership (lambda () (rejected? (lambda () (validate-test-suite-ownership! '(a-test) '(a-test) '(a-test)))))) + (control 'unowned-export (lambda () (rejected? (lambda () (validate-test-suite-ownership! '(a-test) '() '(a-test b-test)))))) + (control 'source-selection (lambda () (equal? (filter (lambda (name) (source-test-suite-selected? name '(a-test))) '(a-test b-test helper)) '(b-test)))) + (control 'repository-plan (lambda () (> (length (conformance-covered-test-files)) 0)))) + (displayln "OK")) diff --git a/src/build-support/rust-rowan-aot.ss b/src/build-support/rust-runtime-aot.ss similarity index 52% rename from src/build-support/rust-rowan-aot.ss rename to src/build-support/rust-runtime-aot.ss index 7834d26f..949c796e 100644 --- a/src/build-support/rust-rowan-aot.ss +++ b/src/build-support/rust-runtime-aot.ss @@ -1,15 +1,15 @@ ;;; -*- Gerbil -*- -;;; Declarative ASP Building API owner for the optional Rowan generator. +;;; Declarative ASP Building API owner for the optional Rust generator. (import (only-in :asp-gerbil-scheme/building-api asp-gerbil-scheme-library-package-prototype asp-gerbil-scheme-native-pkg-config-options asp-gerbil-scheme-package-spec!)) -(export rust-rowan-aot-runtime-modules - rust-rowan-aot-package - rust-rowan-aot-build-spec) +(export rust-runtime-aot-runtime-modules + rust-runtime-aot-package + rust-runtime-aot-build-spec) -(def rust-rowan-aot-runtime-modules +(def rust-runtime-aot-runtime-modules '("src/compiler/funcs" "src/compiler/lr" "src/runtime/token" @@ -17,25 +17,29 @@ "src/runtime/reduce" "src/runtime/funcs" "src/runtime/observability" + "src/runtime/lr-completion" "src/runtime/lr-parser" + "src/runtime/module-source" + "src/runtime/lexical-source" "src/runtime/scan" "src/compiler/machine" "src/language/descriptor" "src/runtime/identity" "src/runtime/language-artifact" "src/language/grammar" - "rust-rowan-grammar-support" - "src/compiler/rust-rowan" - "src/ffi/rust-rowan-aot-v1")) + "rust-runtime-grammar-support" + "src/compiler/rust-runtime" + "src/language/module-input" + "src/ffi/rust-runtime-aot")) (asp-gerbil-scheme-package-spec! - (rust-rowan-aot-package @ asp-gerbil-scheme-library-package-prototype) - (spec rust-rowan-aot-build-spec) + (rust-runtime-aot-package @ asp-gerbil-scheme-library-package-prototype) + (spec rust-runtime-aot-build-spec) (modules - (append rust-rowan-aot-runtime-modules - '("src/ffi/rust-rowan-aot-main.ss"))) + (append rust-runtime-aot-runtime-modules + '("src/ffi/rust-runtime-aot-main.ss"))) (role 'build-support) - (product-entry-modules '("src/ffi/rust-rowan-aot-main.ss")) + (product-entry-modules '("src/ffi/rust-runtime-aot-main.ss")) (native-capabilities '(tls)) (pkg-config-libs '("openssl")) (nix-deps '("openssl")) @@ -45,6 +49,6 @@ (native-spec (append (map (lambda (module) `(gxc: ,module)) - rust-rowan-aot-runtime-modules) - '((exe: "src/ffi/rust-rowan-aot-main" - bin: "gerbil-parser-rowan-aot"))))) + rust-runtime-aot-runtime-modules) + '((exe: "src/ffi/rust-runtime-aot-main" + bin: "gerbil-parser-runtime-aot"))))) diff --git a/src/cli.ss b/src/cli.ss index d8303f81..bdd2f7b9 100644 --- a/src/cli.ss +++ b/src/cli.ss @@ -3,8 +3,8 @@ (import (only-in ./compiler/parser-ir parser-ir-canonical) (only-in ./runtime/artifact parse-artifact-success?) - (only-in ../languages/arithmetic/v1/parser - arithmetic-parser-ir parse-arithmetic-v1)) + (only-in ../languages/arithmetic/parser + arithmetic-parser-ir parse-arithmetic)) (export gparse-build gparse-inspect gparse-check @@ -28,7 +28,7 @@ ;; : (-> String Fixnum) (def (gparse-check source) - (let (artifact (parse-arithmetic-v1 source)) + (let (artifact (parse-arithmetic source)) (write-line artifact) (if (parse-artifact-success? artifact) 0 1))) diff --git a/src/compiler/arithmetic-drive.ss b/src/compiler/arithmetic-drive.ss new file mode 100644 index 00000000..70a27a3b --- /dev/null +++ b/src/compiler/arithmetic-drive.ss @@ -0,0 +1,2 @@ +;;; Generated from arithmetic Parser IR; regenerate with generate-fused-lr.ss. +(begin (import (only-in :gerbil-parser/src/runtime/lr-parser lr-runtime-lexical-mode-catalog) (only-in :gerbil-parser/src/runtime/recognition make-recognition-child make-recognition-fragment recognition-child-field recognition-child-value) (only-in :gerbil-parser/src/runtime/reduce recognition-children-field recognition-children-alias) (only-in :gerbil-parser/src/runtime/funcs recognition-sequence->list recognition-sequence-for-action recognition-sequence-append recognition-sequence-start) (only-in :gerbil-parser/src/runtime/token token-kind token-lexeme token-start token-end)) (export direct-drive direct-grammar-digest) (def direct-grammar-digest "sha256:fed71c7da68ff163c284a2f0e9d3b5e192649ea124862eb20d06961d17712fe0") (def (direct-drive runtime next-input after-shift) (let ((modes (lr-runtime-lexical-mode-catalog runtime)) (input-end-offset 0)) (def (fallback) (values 'fallback #f)) (let loop ((states '(0)) (semantic-values '()) (rest '()) (actions 0) (shifts 0)) (case (car states) ((0) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 4))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (if (pair? rest) (let ((next-states (cons 9 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (if (pair? rest) (let ((next-states (cons 11 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (if (pair? rest) (let ((next-states (cons 12 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (if (pair? rest) (let ((next-states (cons 15 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (if (pair? rest) (let ((next-states (cons 16 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) (else (fallback))))) ((1) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 7))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((null? rest) (let (children (and (pair? semantic-values) (recognition-sequence->list (car semantic-values)))) (if (and (pair? children) (null? (cdr children)) (not (recognition-child-field (car children)))) (values 'accepted (list (recognition-child-value (car children)) rest)) (fallback)))) (else (fallback))))) ((2) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 6))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (if (pair? rest) (let ((next-states (cons 17 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (if (pair? rest) (let ((next-states (cons 18 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (if (pair? rest) (let ((next-states (cons 20 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (if (pair? rest) (let ((next-states (cons 21 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'SourceFile (recognition-sequence-for-action (recognition-children-field 'expression (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 1) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((3) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((4) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((5) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((6) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((7) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((8) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 2) ((9) 23) ((13) 24) ((19) 25) ((22) 26) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((9) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 4))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (if (pair? rest) (let ((next-states (cons 9 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (if (pair? rest) (let ((next-states (cons 11 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (if (pair? rest) (let ((next-states (cons 12 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (if (pair? rest) (let ((next-states (cons 15 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (if (pair? rest) (let ((next-states (cons 16 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) (else (fallback))))) ((10) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'GroupedExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 6) ((9) 6) ((13) 6) ((19) 6) ((22) 6) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'GroupedExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 6) ((9) 6) ((13) 6) ((19) 6) ((22) 6) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'GroupedExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 6) ((9) 6) ((13) 6) ((19) 6) ((22) 6) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'GroupedExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 6) ((9) 6) ((13) 6) ((19) 6) ((22) 6) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'GroupedExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 6) ((9) 6) ((13) 6) ((19) 6) ((22) 6) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'GroupedExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 6) ((9) 6) ((13) 6) ((19) 6) ((22) 6) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((11) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 5))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((12) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 5))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((0) 13) ((9) 13) ((13) 13) ((19) 13) ((22) 13) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((13) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 4))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (if (pair? rest) (let ((next-states (cons 9 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (if (pair? rest) (let ((next-states (cons 11 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (if (pair? rest) (let ((next-states (cons 12 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (if (pair? rest) (let ((next-states (cons 15 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (if (pair? rest) (let ((next-states (cons 16 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) (else (fallback))))) ((14) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'PrefixExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 5) ((9) 5) ((13) 5) ((19) 5) ((22) 5) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'PrefixExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 5) ((9) 5) ((13) 5) ((19) 5) ((22) 5) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'PrefixExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 5) ((9) 5) ((13) 5) ((19) 5) ((22) 5) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'PrefixExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 5) ((9) 5) ((13) 5) ((19) 5) ((22) 5) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'PrefixExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 5) ((9) 5) ((13) 5) ((19) 5) ((22) 5) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'PrefixExpression (recognition-sequence-for-action v0) offset)) (target (case (car remaining-states) ((0) 5) ((9) 5) ((13) 5) ((19) 5) ((22) 5) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((15) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NameExpression (recognition-sequence-for-action (recognition-children-field 'name (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 7) ((9) 7) ((13) 7) ((19) 7) ((22) 7) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NameExpression (recognition-sequence-for-action (recognition-children-field 'name (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 7) ((9) 7) ((13) 7) ((19) 7) ((22) 7) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NameExpression (recognition-sequence-for-action (recognition-children-field 'name (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 7) ((9) 7) ((13) 7) ((19) 7) ((22) 7) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NameExpression (recognition-sequence-for-action (recognition-children-field 'name (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 7) ((9) 7) ((13) 7) ((19) 7) ((22) 7) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NameExpression (recognition-sequence-for-action (recognition-children-field 'name (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 7) ((9) 7) ((13) 7) ((19) 7) ((22) 7) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NameExpression (recognition-sequence-for-action (recognition-children-field 'name (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 7) ((9) 7) ((13) 7) ((19) 7) ((22) 7) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((16) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NumberExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 8) ((9) 8) ((13) 8) ((19) 8) ((22) 8) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NumberExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 8) ((9) 8) ((13) 8) ((19) 8) ((22) 8) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NumberExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 8) ((9) 8) ((13) 8) ((19) 8) ((22) 8) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NumberExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 8) ((9) 8) ((13) 8) ((19) 8) ((22) 8) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NumberExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 8) ((9) 8) ((13) 8) ((19) 8) ((22) 8) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NumberExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (target (case (car remaining-states) ((0) 8) ((9) 8) ((13) 8) ((19) 8) ((22) 8) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((17) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 5))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((18) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 5))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 19) ((23) 19) ((24) 19) ((25) 19) ((26) 19) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((19) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 4))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (if (pair? rest) (let ((next-states (cons 9 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (if (pair? rest) (let ((next-states (cons 11 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (if (pair? rest) (let ((next-states (cons 12 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (if (pair? rest) (let ((next-states (cons 15 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (if (pair? rest) (let ((next-states (cons 16 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) (else (fallback))))) ((20) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 5))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((21) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 5))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (let* ((remaining-states (cdr states)) (remaining-values (cdr semantic-values)) (v0 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (target (case (car remaining-states) ((2) 22) ((23) 22) ((24) 22) ((25) 22) ((26) 22) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((22) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 4))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "(")) (if (pair? rest) (let ((next-states (cons 9 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (if (pair? rest) (let ((next-states (cons 11 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (if (pair? rest) (let ((next-states (cons 12 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'identifier)) (if (pair? rest) (let ((next-states (cons 15 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (eq? (token-kind (car rest)) 'number)) (if (pair? rest) (let ((next-states (cons 16 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) (else (fallback))))) ((23) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 3))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (if (pair? rest) (let ((next-states (cons 17 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (if (pair? rest) (let ((next-states (cons 18 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (if (pair? rest) (let ((next-states (cons 20 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (if (pair? rest) (let ((next-states (cons 21 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (if (pair? rest) (let ((next-states (cons 27 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) (else (fallback))))) ((24) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 1))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr (cdr states))) (remaining-values (cdr (cdr semantic-values))) (v0 (car (cdr semantic-values))) (v1 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'operator (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operand (recognition-sequence-for-action v1) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 14) ((9) 14) ((13) 14) ((19) 14) ((22) 14) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr (cdr states))) (remaining-values (cdr (cdr semantic-values))) (v0 (car (cdr semantic-values))) (v1 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'operator (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operand (recognition-sequence-for-action v1) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 14) ((9) 14) ((13) 14) ((19) 14) ((22) 14) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr (cdr states))) (remaining-values (cdr (cdr semantic-values))) (v0 (car (cdr semantic-values))) (v1 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'operator (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operand (recognition-sequence-for-action v1) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 14) ((9) 14) ((13) 14) ((19) 14) ((22) 14) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr (cdr states))) (remaining-values (cdr (cdr semantic-values))) (v0 (car (cdr semantic-values))) (v1 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'operator (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operand (recognition-sequence-for-action v1) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 14) ((9) 14) ((13) 14) ((19) 14) ((22) 14) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr (cdr states))) (remaining-values (cdr (cdr semantic-values))) (v0 (car (cdr semantic-values))) (v1 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'operator (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operand (recognition-sequence-for-action v1) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 14) ((9) 14) ((13) 14) ((19) 14) ((22) 14) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr (cdr states))) (remaining-values (cdr (cdr semantic-values))) (v0 (car (cdr semantic-values))) (v1 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'operator (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operand (recognition-sequence-for-action v1) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 14) ((9) 14) ((13) 14) ((19) 14) ((22) 14) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((25) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 2))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 3) ((9) 3) ((13) 3) ((19) 3) ((22) 3) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 3) ((9) 3) ((13) 3) ((19) 3) ((22) 3) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (if (pair? rest) (let ((next-states (cons 20 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (if (pair? rest) (let ((next-states (cons 21 states)) (next-values (cons (list (make-recognition-child #f (car rest))) semantic-values)) (next-actions (fx+ actions 1)) (next-shifts (fx+ shifts 1))) (after-shift (car rest) next-states next-values next-actions next-shifts) (loop next-states next-values (cdr rest) next-actions next-shifts)) (fallback))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 3) ((9) 3) ((13) 3) ((19) 3) ((22) 3) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 3) ((9) 3) ((13) 3) ((19) 3) ((22) 3) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((26) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 1))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 4) ((9) 4) ((13) 4) ((19) 4) ((22) 4) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 4) ((9) 4) ((13) 4) ((19) 4) ((22) 4) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 4) ((9) 4) ((13) 4) ((19) 4) ((22) 4) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 4) ((9) 4) ((13) 4) ((19) 4) ((22) 4) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 4) ((9) 4) ((13) 4) ((19) 4) ((22) 4) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (target (case (car remaining-states) ((0) 4) ((9) 4) ((13) 4) ((19) 4) ((22) 4) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) ((27) (let (rest (if (null? rest) (let (input-token (next-input (vector-ref modes 0))) (if input-token (begin (set! input-end-offset (max input-end-offset (token-end input-token))) (list input-token)) '())) rest)) (cond ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "+")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'expression (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2)) (target (case (car remaining-states) ((0) 10) ((9) 10) ((13) 10) ((19) 10) ((22) 10) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "-")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'expression (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2)) (target (case (car remaining-states) ((0) 10) ((9) 10) ((13) 10) ((19) 10) ((22) 10) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "*")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'expression (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2)) (target (case (car remaining-states) ((0) 10) ((9) 10) ((13) 10) ((19) 10) ((22) 10) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) "/")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'expression (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2)) (target (case (car remaining-states) ((0) 10) ((9) 10) ((13) 10) ((19) 10) ((22) 10) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((and (pair? rest) (string? (token-lexeme (car rest))) (string=? (token-lexeme (car rest)) ")")) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'expression (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2)) (target (case (car remaining-states) ((0) 10) ((9) 10) ((13) 10) ((19) 10) ((22) 10) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) ((null? rest) (let* ((remaining-states (cdr (cdr (cdr states)))) (remaining-values (cdr (cdr (cdr semantic-values)))) (v0 (car (cdr (cdr semantic-values)))) (v1 (car (cdr semantic-values))) (v2 (car semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'expression (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2)) (target (case (car remaining-states) ((0) 10) ((9) 10) ((13) 10) ((19) 10) ((22) 10) (else #f)))) (if target (loop (cons target remaining-states) (cons value remaining-values) rest (fx+ actions 1) shifts) (fallback)))) (else (fallback))))) (else (fallback))))))) diff --git a/src/compiler/bound-ir.ss b/src/compiler/bound-ir.ss index bb3c680a..103dcec3 100644 --- a/src/compiler/bound-ir.ss +++ b/src/compiler/bound-ir.ss @@ -4,7 +4,8 @@ (import (only-in ../grammar/algebra grammar-expression-references grammar-expression-terminals) - (only-in ../runtime/identity sha256-text)) + (only-in :std/list/list-builder with-list-builder) + (only-in ../runtime/identity sha256-bytes)) (export +bound-grammar-ir-schema+ bind-grammar-ir bound-grammar-ir-ref @@ -42,6 +43,21 @@ (def (canonical value) (call-with-output-string (lambda (port) (write value port)))) +;;; Preserve the canonical writer while emitting digest input directly in UTF-8. +;;; Implicit port encoding is not part of the canonical identity contract. +(def (canonical-digest/raw value) + (sha256-bytes + (call-with-output-u8vector '(char-encoding: UTF-8 eol-encoding: lf) + (lambda (port) (write value port))))) + +;;; Most declaration rows have no references. Retain this immutable datum's +;;; digest once, while returning an independent mutable string to every caller. +;;; Nonempty declarations keep the exact canonical byte writer and hash path. +(def +empty-reference-digest+ (canonical-digest/raw '())) +(def (canonical-digest value) + (if (null? value) (string-copy +empty-reference-digest+) + (canonical-digest/raw value))) + ;;; Binding identity is structured package data. Native grammar names are not ;;; rewritten into a second slash-delimited namespace. ;; : (-> Symbol Symbol Datum Alist) @@ -57,10 +73,28 @@ (cons 'location origin) (cons 'generated? #f))) -;; : (-> Alist Symbol Symbol String Alist) -(def (declaration-source source-map namespace name origin) - (let* ((section (assq namespace source-map)) - (row (and section (assoc name (cdr section))))) +;;; One invocation owns the lookup index. Retain association entries so a +;;; present false source remains distinct from absence; first occurrences win. +(def (index-declaration-sources source-map) + (let (sections (make-table test: eq?)) + (for-each + (lambda (namespace) + (let (section (assq namespace source-map)) + (when section + (let (names (make-table test: equal?)) + (for-each + (lambda (row) + (unless (table-ref names (car row) #f) + (table-set! names (car row) row))) + (cdr section)) + (table-set! sections namespace names))))) + '(syntax-kind terminal lexical-rule rule field)) + sections)) + +;; : (-> SourceIndex Symbol Datum String Alist) +(def (declaration-source source-index namespace name origin) + (let* ((section (table-ref source-index namespace #f)) + (row (and section (table-ref section name #f)))) (if row (cdr row) (default-source origin)))) ;; : (-> Symbol Symbol Datum Alist) @@ -88,19 +122,19 @@ (list (reference-binding-id owner 'syntax-kind (cadr row)))) (else '()))) -;; : (-> Symbol Symbol List String List Alist Alist) -(def (bind-row owner namespace row origin lineage source-map) +;; : (-> Symbol Symbol List String List SourceIndex Alist) +(def (bind-row owner namespace row origin lineage source-index) (let* ((name (car row)) (scope (and (eq? namespace 'field) (cadr row))) (identity-name (if scope (list scope name) name)) (identity (binding-id owner namespace identity-name)) - (source (declaration-source source-map namespace identity-name origin)) + (source (declaration-source source-index namespace identity-name origin)) (references (row-references owner namespace row)) (declaration (list owner namespace row source lineage references))) (list (cons 'bindingId identity) - (cons 'declarationId (sha256-text (canonical declaration))) + (cons 'declarationId (canonical-digest declaration)) (cons 'namespace namespace) (cons 'name name) (cons 'scope scope) @@ -109,29 +143,26 @@ (cons 'source source) (cons 'expansionLineage lineage) (cons 'references references) - (cons 'referenceDigest (sha256-text (canonical references)))))) + (cons 'referenceDigest (canonical-digest references))))) ;;; Keep a section-local name index while preserving declaration order. A ;;; duplicate declaration fails before any partially bound section escapes. -;; : (forall (r b) (-> Symbol Symbol [r] String List Alist [b])) +;; : (forall (r b) (-> Symbol Symbol [r] String List SourceIndex [b])) ;; : (-> GrammarOwner GrammarNamespace DeclarationRows OriginModule ExpansionLineage BoundDeclarations) -(def (bind-section owner namespace rows origin lineage source-map) - (let ((seen (make-table test: eq?)) - (bound '())) - (for-each - (lambda (row) - (let (name (and (pair? row) (car row))) - (unless (symbol? name) - (error "bound grammar declaration requires a symbolic identity" - namespace row)) - (when (table-ref seen name #f) - (error "duplicate bound grammar declaration" namespace name)) - (table-set! seen name #t) - (set! bound - (cons (bind-row owner namespace row origin lineage source-map) - bound)))) - rows) - (reverse bound))) +(def (bind-section owner namespace rows origin lineage source-index) + (let (seen (make-table test: eq?)) + (with-list-builder (emit) + (for-each + (lambda (row) + (let (name (and (pair? row) (car row))) + (unless (symbol? name) + (error "bound grammar declaration requires a symbolic identity" + namespace row)) + (when (table-ref seen name #f) + (error "duplicate bound grammar declaration" namespace name)) + (table-set! seen name #t) + (emit (bind-row owner namespace row origin lineage source-index)))) + rows)))) ;; : (forall (a) (-> [(Pair Symbol a)] Symbol (Maybe a))) ;; : (-> Alist Symbol Datum) @@ -155,43 +186,42 @@ (terminals (grammar-ir-value grammar 'terminals)) (lexical-rules (grammar-ir-value grammar 'lexical-rules)) (rules (grammar-ir-value grammar 'rules)) - (fields - (apply append - (map (lambda (row) - (map (lambda (field) (list field (car row))) - (caddr row))) - syntax-kinds))) + (source-index (index-declaration-sources source-map)) (sections (list (cons 'syntax-kind - (bind-section owner 'syntax-kind syntax-kinds origin lineage source-map)) + (bind-section owner 'syntax-kind syntax-kinds origin lineage source-index)) (cons 'terminal - (bind-section owner 'terminal terminals origin lineage source-map)) + (bind-section owner 'terminal terminals origin lineage source-index)) (cons 'lexical-rule - (bind-section owner 'lexical-rule lexical-rules origin lineage source-map)) + (bind-section owner 'lexical-rule lexical-rules origin lineage source-index)) (cons 'rule - (bind-section owner 'rule rules origin lineage source-map)) + (bind-section owner 'rule rules origin lineage source-index)) ;; Field identity is qualified by its node owner, so the same field ;; spelling in two syntax kinds is not a namespace collision. (cons 'field - (map (lambda (row) - (bind-row owner 'field - row - origin lineage source-map)) - fields))))) - (let* ((bindings (apply append (map cdr sections))) - (binding-ids (make-table test: equal?)) + (with-list-builder (emit) + (for-each + (lambda (row) + (for-each + (lambda (field) + (emit (bind-row owner 'field (list field (car row)) + origin lineage source-index))) + (caddr row))) + syntax-kinds)))))) + (let* ((binding-ids (make-table test: equal?)) + (binding-count 0) (references - (apply append - (map (lambda (binding) - (bound-grammar-ir-ref binding 'references)) - bindings)))) - (for-each - (lambda (binding) - (table-set! binding-ids - (bound-grammar-ir-ref binding 'bindingId) - #t)) - bindings) + (with-list-builder (emit) + (for-each + (lambda (section) + (for-each + (lambda (binding) + (set! binding-count (+ binding-count 1)) + (table-set! binding-ids (bound-grammar-ir-ref binding 'bindingId) #t) + (for-each emit (bound-grammar-ir-ref binding 'references))) + (cdr section))) + sections)))) ;; Admission closes the sidecar over the exact declaration set. Missing ;; targets are never retained as advisory or deferred references. (for-each @@ -205,11 +235,11 @@ (cons 'originModule origin) (cons 'expansionLineage lineage) (cons 'sections sections) - (cons 'bindingCount (length bindings)) + (cons 'bindingCount binding-count) (cons 'referenceCount (length references)) - (cons 'referenceDigest (sha256-text (canonical references))) - (cons 'grammarDigest (sha256-text (canonical grammar))) - (cons 'identityDigest (sha256-text (canonical sections))))))) + (cons 'referenceDigest (canonical-digest references)) + (cons 'grammarDigest (canonical-digest grammar)) + (cons 'identityDigest (canonical-digest sections)))))) ;; : (forall (a) (-> [(Pair Symbol a)] String)) ;; : (-> BoundGrammarIR CanonicalBoundGrammarText) diff --git a/src/compiler/build-strategy.ss b/src/compiler/build-strategy.ss new file mode 100644 index 00000000..56aa0d2a --- /dev/null +++ b/src/compiler/build-strategy.ss @@ -0,0 +1,64 @@ +;;; -*- Gerbil -*- +;;; Common POO strategy binding/admission; providers belong to engine extensions. +(import (only-in :clan/poo/object .o .cc .ref .slot? object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ../language/descriptor language-grammar? language-grammar-ir language-grammar-machine) + (only-in ./machine parser-machine-grammar-digest parser-machine-ir)) +(export BuildStrategy. BuildStrategyContract make-bound-build-strategy emit-build-strategy + build-strategy-common-shape? declare-build-strategy-provider) + +;;; Opaque engine registration keeps a recipe override from substituting an +;;; unchecked writer. A provider owns one output format, admission and emitter. +(defstruct build-strategy-provider (name format admit emit)) +(def (declare-build-strategy-provider name format admit emit) + (unless (and (symbol? name) (symbol? format) (procedure? admit) (procedure? emit)) + (error "invalid engine build strategy provider" name format)) + (make-build-strategy-provider name format admit emit)) + +(def (build-strategy-common-shape? candidate) + (and (object? candidate) + (andmap (lambda (slot) (.slot? candidate slot)) '(descriptor digest metadata provider kind format)) + (language-grammar? (.ref candidate 'descriptor)) + (build-strategy-provider? (.ref candidate 'provider)) + (eq? (.ref candidate 'kind) (build-strategy-provider-name (.ref candidate 'provider))) + (eq? (.ref candidate 'format) (build-strategy-provider-format (.ref candidate 'provider))) + (equal? (.ref candidate 'digest) + (parser-machine-grammar-digest (language-grammar-machine (.ref candidate 'descriptor)))) + (eq? (language-grammar-ir (.ref candidate 'descriptor)) + (parser-machine-ir (language-grammar-machine (.ref candidate 'descriptor)))) + (object? (.ref candidate 'metadata)))) + +(def (build-strategy-shape? candidate) + (with-catch (lambda (_) #f) + (lambda () + (and (build-strategy-common-shape? candidate) + ((build-strategy-provider-admit (.ref candidate 'provider)) candidate))))) + +(define-type (BuildStrategyContract @ PooFlowContract.) + identity: 'gerbil-parser/build-strategy + .classify: (lambda (candidate context) + (let (accepted? (build-strategy-shape? candidate)) + (poo-flow-classification-evidence + 'gerbil-parser/build-strategy candidate accepted? + (if accepted? '() '((expected gerbil-parser/build-strategy))) context)))) + +(def BuildStrategy. + (.o (:: self) + descriptor: #f provider: #f + (digest (parser-machine-grammar-digest (language-grammar-machine (.ref self 'descriptor)))) + metadata: (.o) + (kind (build-strategy-provider-name (.ref self 'provider))) + (format (build-strategy-provider-format (.ref self 'provider))) + (.emit (lambda (port) (emit-build-strategy self port))))) + +(def (make-bound-build-strategy descriptor prototype) + (let (candidate (.cc prototype 'descriptor descriptor)) + (validate BuildStrategyContract candidate) + candidate)) + +;;; The common dispatcher has no strategy-kind or language-name branches. +;;; Writers must materialize their output before touching the caller's port. +(def (emit-build-strategy strategy port) + (validate BuildStrategyContract strategy) + ((build-strategy-provider-emit (.ref strategy 'provider)) strategy port)) diff --git a/src/compiler/concise-language.ss b/src/compiler/concise-language.ss new file mode 100644 index 00000000..b2dddbf4 --- /dev/null +++ b/src/compiler/concise-language.ss @@ -0,0 +1,496 @@ +;;; -*- Gerbil -*- +;;; Concise language elaboration: collect authored syntax, validate rules, derive +;;; catalogs, and submit a declaration to the existing canonical compiler. +;;; Runtime identifiers are supplied by the public macro's hygienic context. + +(import (only-in :gerbil/core/expander + syntax-case syntax with-syntax datum->syntax syntax->datum + stx-map stx-list? syntax->list identifier? raise-syntax-error) + (only-in :gerbil/expander core-expand1) + (only-in :std/list/list delete-duplicates/hash) + (only-in ../grammar/algebra grammar-expression-header?) + (only-in ./language-artifact project-language-catalog + make-language-declaration expand-language-declaration-syntax)) +(export expand-lexical-language-syntax lower-source-rule-overlays lower-source-rule-precedences) + +(def (expand-concise-language-syntax stx expand-lexical-rows compile-parser bind-grammar-ir + assembly-binding descriptor-binding + author-entry begin-entry define-entry (author-syntax stx)) + (def (syntax-failure message (offending stx)) + (raise-syntax-error #f message offending author-syntax)) + (def (require condition message) + (unless condition (syntax-failure message))) + (def (unique values) + (delete-duplicates/hash values)) + ;; These associations live only during this declaration's expansion. Pair + ;; identity distinguishes equal expressions at different author occurrences. + (def expression-sources (make-table test: eq?)) + (def expression-operands (make-table test: eq?)) + (def rule-sources (make-table test: eq?)) + (def rule-declarations (make-table test: eq?)) + (def node-kind-sources (make-table test: eq?)) + (def macro-lineage '()) + (def grammar-constructors + '(node seq choice optional repeat repeat1 field + prec empty literal layout-start layout-next layout-end + token reference)) + (def (expand-surface-syntax expression (author-call #f)) + (syntax-case expression () + ((head argument ...) + (let (head-value (syntax->datum #'head)) + (if (memq head-value grammar-constructors) + (let (datum + (cons head-value + (stx-map (lambda (child) + (expand-surface-syntax child author-call)) + #'(argument ...)))) + (table-set! expression-sources datum (or author-call expression)) + (table-set! expression-operands datum + (stx-map (lambda (child) (or author-call child)) + #'(head argument ...))) + datum) + (let (expanded (core-expand1 expression)) + (when (eq? expanded expression) + (raise-syntax-error #f "unknown concise GrammarExpr constructor" + (or author-call #'head) stx)) + (set! macro-lineage + (cons head-value macro-lineage)) + ;; Arbitrary transformers do not promise a parameter-origin map. + ;; Keep the outer author invocation as conservative blame context. + (expand-surface-syntax expanded (or author-call expression)))))) + (atom (syntax->datum #'atom)))) + (syntax-case stx + (identity root lex rules extras keywords recoveries + conflicts case-insensitive) + ((_ prefix + (identity language-value version-value contract-value) + (root root-name) + (lex lexical-row ...) + (rules rule-row ...) + (extras extra-name ...) + (keywords keyword-row ...) + (recoveries recovery-row ...) + (conflicts conflict-policy) + (case-insensitive case-insensitive-value) + option ...) + (identifier? #'prefix) + (let* ((options (syntax->datum #'(option ...))) + (prefix-name (syntax->datum #'prefix)) + (root-value (syntax->datum #'root-name)) + (lexical-rows (expand-lexical-rows #'(lexical-row ...) stx)) + (rule-rows + (stx-map + (lambda (row) + (syntax-case row () + ((name expression ...) + (let (result + (cons (syntax->datum #'name) + (stx-map expand-surface-syntax + #'(expression ...)))) + (table-set! rule-declarations result row) + (table-set! rule-sources result + (stx-map (lambda (value) value) #'(expression ...))) + result)) + (_ (syntax-failure + "invalid concise rule declaration")))) + #'(rule-row ...))) + (token-names + (map (lambda (row) + (require (and (list? row) + (= (length row) 3) + (symbol? (car row)) + (symbol? (cadr row))) + "invalid concise lexical declaration") + (car row)) + lexical-rows)) + (token-kinds (unique (map cadr lexical-rows))) + (rule-names + (map (lambda (row) + (require (and (list? row) + (pair? (cdr row)) + (symbol? (car row))) + "invalid concise rule declaration") + (car row)) + rule-rows)) + (node-rows '())) + (require (memq root-value rule-names) + "concise language root must name a declared rule") + (require (every (lambda (row) + (and (pair? row) (memq (car row) '(node-fields catalog)))) + options) + "unknown concise language option") + (require (= (length options) (length (unique (map car options)))) + "duplicate concise language option") + (def (published-catalog inferred-kinds inferred-terminals) + (with-catch (lambda (condition) (syntax-failure (error-message condition))) + (lambda () + (project-language-catalog inferred-kinds inferred-terminals + (assq 'catalog options))))) + (def (record-node! kind fields) + (require (symbol? kind) + "concise node kind must be an identifier") + (let (current (assq kind node-rows)) + (if current + ;; Alternatives of the same public node contribute to one schema. + (set-cdr! current (unique (append (cdr current) fields))) + (set! node-rows + (append node-rows (list (cons kind fields))))))) + ;; Field inference consumes only fully validated, normalized rules. + ;; Transparent references retain recursive field propagation; named + ;; nodes stop it, and visited names bound traversal of recursive helpers. + (def (surface-fields expression rules (seen '())) + (def (rule-fields name) + (if (memq name seen) '() + (let (row (assq name rules)) + (if row + (surface-fields (cadr row) rules (cons name seen)) + '())))) + (case (car expression) + ((alias) '()) + ((reference) (rule-fields (cadr expression))) + ((field) + (cons (cadr expression) + (surface-fields (caddr expression) rules seen))) + ((prec) + (surface-fields (cadddr expression) rules seen)) + ((optional repeat repeat1) + (surface-fields (cadr expression) rules seen)) + ((seq choice) + (apply append + (map (lambda (child) (surface-fields child rules seen)) + (cdr expression)))) + (else '()))) + (def (collect-node-fields! expression rules) + (case (car expression) + ((alias) + ;; Match the prior normalizer's postorder node registration. + (collect-node-fields! (caddr expression) rules) + (record-node! (cadr expression) + (unique (surface-fields (caddr expression) rules)))) + ((field) (collect-node-fields! (caddr expression) rules)) + ((prec) (collect-node-fields! (cadddr expression) rules)) + ((optional repeat repeat1) + (collect-node-fields! (cadr expression) rules)) + ((seq choice) + (for-each (lambda (child) (collect-node-fields! child rules)) + (cdr expression))) + (else (void)))) + (def (surface-body expressions sources) + (require (pair? expressions) + "concise rule or node requires an expression") + (if (null? (cdr expressions)) + (surface-expression (car expressions) (car sources)) + (cons 'seq (map surface-expression expressions sources)))) + (def (surface-expression expression source) + (def blame (table-ref expression-sources expression source)) + (def operands (table-ref expression-operands expression #f)) + (def (operand-source index) + (if operands (list-ref operands index) blame)) + (def (syntax-failure message (offending blame)) + (raise-syntax-error #f message offending stx)) + (def (require condition message (offending blame)) + (unless condition (syntax-failure message offending))) + (cond + ((string? expression) + (require (grammar-expression-header? (list 'literal expression)) + "invalid canonical GrammarExpr metadata") + (list 'literal expression)) + ((symbol? expression) + (cond + ((and (memq expression token-names) (memq expression rule-names)) + (syntax-failure + "ambiguous concise identifier; use token or reference")) + ((memq expression token-names) (list 'token expression)) + ((memq expression rule-names) (list 'reference expression)) + (else + (syntax-failure + "unresolved concise grammar identifier")))) + ((not (and (pair? expression) (stx-list? expression))) + (syntax-failure "invalid concise grammar expression")) + (else + (case (car expression) + ((node alias) + (require (and (pair? (cdr expression)) + (symbol? (cadr expression)) + (pair? (cddr expression))) + "invalid concise node expression") + (let ((kind (cadr expression)) + (body (surface-body (cddr expression) (cddr operands)))) + ;; Keep the first authored occurrence for cross-rule kind checks. + (unless (table-ref node-kind-sources kind #f) + (table-set! node-kind-sources kind (operand-source 1))) + (list 'alias kind body))) + ((seq sequence choice) + (require (pair? (cdr expression)) + "concise sequence or choice cannot be empty") + (cons (if (eq? (car expression) 'sequence) + 'seq + (car expression)) + (map surface-expression (cdr expression) (cdr operands)))) + ((optional repeat repeat1) + (require (= (length expression) 2) + "invalid concise unary grammar expression") + (list (car expression) + (surface-expression (cadr expression) (operand-source 1)))) + ((field) + (require (and (= (length expression) 3) + (symbol? (cadr expression))) + "invalid concise field expression") + (list 'field (cadr expression) + (surface-expression (caddr expression) (operand-source 2)))) + ((prec precedence) + (require (= (length expression) 4) + "invalid concise precedence expression") + (require (grammar-expression-header? + (cons 'precedence (cdr expression))) + "invalid canonical GrammarExpr metadata") + (list 'prec (cadr expression) (caddr expression) + (surface-expression (cadddr expression) (operand-source 3)))) + ((empty) + (require (null? (cdr expression)) + "invalid concise empty expression") + expression) + ((layout-end) + (require (every (lambda (boundary) + (and (string? boundary) + (positive? (string-length boundary)))) + (cdr expression)) + "invalid concise layout closing boundary") + expression) + ((literal layout-start layout-next) + (require (and (= (length expression) 2) + (string? (cadr expression))) + "invalid concise literal expression") + (require (grammar-expression-header? expression) + "invalid canonical GrammarExpr metadata") + expression) + ((token) + (require (= (length expression) 2) + "explicit token names an unknown lexical declaration") + (require (memq (cadr expression) token-names) + "explicit token names an unknown lexical declaration" (operand-source 1)) + expression) + ((reference) + (require (= (length expression) 2) + "explicit reference names an unknown rule") + (require (memq (cadr expression) rule-names) + "explicit reference names an unknown rule" (operand-source 1)) + expression) + (else + (syntax-failure + "unknown concise GrammarExpr constructor")))))) + (let* ((normalized-rules + (map (lambda (row) + (list (car row) (surface-body (cdr row) (table-ref rule-sources row)))) + rule-rows)) + (root-rule (assq root-value normalized-rules)) + (root-expression (and root-rule (cadr root-rule))) + (root-kind + (and (pair? root-expression) + (eq? (car root-expression) 'alias) + (cadr root-expression)))) + (unless root-kind + (let* ((row (assq root-value rule-rows)) + (sources (table-ref rule-sources row)) + (source (if (null? (cdr sources)) + (table-ref expression-sources (cadr row) (car sources)) + (table-ref rule-declarations row)))) + (syntax-failure + "concise language root rule must construct one node" source))) + (for-each + (lambda (row) (collect-node-fields! (cadr row) normalized-rules)) + normalized-rules) + ;; Optional fields retain a published ABI even when an alternative + ;; does not emit them. Ordinary packs infer their complete catalog. + (let (declared (assq 'node-fields options)) + (when declared + (for-each + (lambda (row) + (require (and (pair? row) (assq (car row) node-rows) + (list? (cdr row)) (every symbol? (cdr row))) + "node-fields requires a declared node and field identifiers") + (let (node (assq (car row) node-rows)) + (set-cdr! node (unique (append (cdr row) (cdr node)))))) + (cdr declared)))) + (for-each + (lambda (kind) + (when (assq kind node-rows) + (syntax-failure + "concise syntax kind is both a node and a token" + (table-ref node-kind-sources kind)))) + token-kinds) + (let* ((root-node (assq root-kind node-rows)) + (ordered-nodes + (cons root-node + (filter (lambda (row) + (not (eq? (car row) root-kind))) + node-rows))) + (syntax-kinds + (append + (map (lambda (row) + (list (car row) 'node (cdr row))) + ordered-nodes) + (map (lambda (kind) + (list kind 'token '(text))) + token-kinds))) + (terminals + (map (lambda (row) (list (car row) (cadr row))) + lexical-rows)) + (lexical-rules + (map (lambda (row) (list (car row) (caddr row))) + lexical-rows)) + (catalog-values + (call-with-values + (lambda () (published-catalog syntax-kinds terminals)) list)) + (syntax-kinds (car catalog-values)) + (terminals (cadr catalog-values)) + (declaration + (make-language-declaration + author-syntax #'prefix + (stx-map (lambda (value) value) + #'(language-value version-value contract-value)) + (map (lambda (row) + (cons (car row) (datum->syntax #'prefix (cdr row)))) + `((syntax-kinds ,@syntax-kinds) + (terminals ,@terminals) + (lexical-rules ,@lexical-rules) + (rules ,@normalized-rules) + (extras ,@(syntax->datum #'(extra-name ...))) + (keywords ,@(syntax->datum #'(keyword-row ...))) + (parser-entrypoints (,root-value parse pure)) + (recoveries ,@(syntax->datum #'(recovery-row ...))) + (flow (source lexical) (lexical parser) (parser cst)))) + (syntax->datum #'conflict-policy) + (syntax->datum #'case-insensitive-value) + `(deflanguage ,@(reverse (unique macro-lineage))) '()))) + (expand-language-declaration-syntax + declaration compile-parser bind-grammar-ir + assembly-binding descriptor-binding begin-entry define-entry))))) + ((_ prefix identity-row root-row lex-row rules-row clause ...) + (and (identifier? #'prefix) + (eq? (car (syntax->datum #'identity-row)) 'identity) + (eq? (car (syntax->datum #'root-row)) 'root) + (eq? (car (syntax->datum #'lex-row)) 'lex) + (eq? (car (syntax->datum #'rules-row)) 'rules)) + (let* ((rows (stx-map (lambda (row) row) #'(clause ...))) + (names (map (lambda (row) + (let (datum (syntax->datum row)) + (and (pair? datum) (car datum)))) rows))) + (require (every (lambda (name) + (memq name '(extras keywords recoveries conflicts + case-insensitive node-fields catalog))) names) + "unknown concise language option") + (require (= (length names) (length (unique names))) + "duplicate concise language option") + (let ((sections (map cons names rows))) + (def (section name default) + (let (row (assq name sections)) + (if row (cdr row) (datum->syntax #'prefix default)))) + (with-syntax ((@author-entry author-entry) + (extras-row (section 'extras '(extras))) + (keywords-row (section 'keywords '(keywords))) + (recoveries-row (section 'recoveries '(recoveries))) + (conflicts-row (section 'conflicts '(conflicts reject))) + (case-row (section 'case-insensitive '(case-insensitive #f))) + ((option ...) + (filter (lambda (row) + (memq (car (syntax->datum row)) '(node-fields catalog))) + rows))) + (expand-concise-language-syntax + #'(@author-entry prefix identity-row root-row lex-row rules-row + extras-row keywords-row recoveries-row conflicts-row case-row option ...) + expand-lexical-rows compile-parser bind-grammar-ir + assembly-binding descriptor-binding author-entry + begin-entry define-entry author-syntax))))) + (_ (raise-syntax-error #f "invalid concise language declaration" stx)))) + +;;; The public lexical vocabulary owns syntax policy; rule elaboration and +;;; publication remain in the common compiler. No release identity or VM flow +;;; is accepted from the author declaration. +(def (expand-lexical-language-syntax stx expand-lexical-rows compile-parser bind-grammar-ir + assembly-binding descriptor-binding + author-entry begin-entry define-entry) + (syntax-case stx () + ((_ prefix (syntax-tag (vocabulary (root-tag root-name) (lex-tag lexical-row ...) clause ...)) + (rules-tag rule-row ...)) + (equal? (map syntax->datum (list #'syntax-tag #'vocabulary #'root-tag #'lex-tag #'rules-tag)) + '(syntax lexical root lex rules)) + (let* ((rows (stx-map values #'(clause ...))) + (names (map (lambda (row) + (let (datum (syntax->datum row)) + (and (pair? datum) (car datum)))) rows))) + (unless (and (every (lambda (name) + (memq name '(extras keywords recoveries conflicts case-insensitive + node-fields catalog))) names) + (= (length names) (length (delete-duplicates/hash names)))) + (raise-syntax-error #f + "lexical syntax requires distinct syntax policies; identity, flow and backends belong to other owners" stx)) + (expand-concise-language-syntax + #'(deflanguage prefix (identity #f #f #f) (root root-name) + (lex lexical-row ...) (rules rule-row ...) clause ...) + expand-lexical-rows compile-parser bind-grammar-ir + assembly-binding descriptor-binding author-entry + begin-entry define-entry stx))) + (_ (raise-syntax-error #f "lexical syntax requires a root and lexical declarations" stx)))) + +;;; User AST constructors become engine-only aliases after source admission. +;;; The original source witness and replacement are lowered symmetrically. +(def (lower-source-rule-overlays rows owner namespace version commit) + (def (lower expression) + (syntax-case expression (node seq prec alias sequence precedence) + ((node kind body ...) + (and (identifier? #'kind) (pair? (syntax->list #'(body ...)))) + (let (children (stx-map lower #'(body ...))) + (with-syntax ((value (if (null? (cdr children)) (car children) + (datum->syntax namespace (cons 'sequence children))))) + #'(alias kind value)))) + ((seq body ...) + (with-syntax (((child ...) (stx-map lower #'(body ...)))) #'(sequence child ...))) + ((prec association level body) + (with-syntax ((child (lower #'body))) #'(precedence association level child))) + ((alias argument ...) + (raise-syntax-error #f "alias is an engine constructor; author rules use node" expression owner)) + ((sequence argument ...) + (raise-syntax-error #f "sequence is an engine constructor; author rules use seq" expression owner)) + ((precedence argument ...) + (raise-syntax-error #f "precedence is an engine constructor; author rules use prec" expression owner)) + ((head argument ...) + (with-syntax (((child ...) (stx-map lower #'(argument ...)))) #'(head child ...))) + (_ expression))) + + (stx-map + (lambda (row) + (syntax-case row () + ((name provenance original replacement) + (identifier? #'name) + (with-syntax ((before (lower #'original)) (after (lower #'replacement)) + (receipt (source-rule-receipt #'provenance owner namespace version commit))) + #'(name receipt before after))) + (_ (raise-syntax-error #f "source rule overlay requires name, provenance and two construction witnesses" row owner)))) + rows)) + +(def (source-rule-receipt provenance owner namespace version commit) + (let* ((origin (syntax->datum provenance)) + (qualified? (and (list? origin) (= (length origin) 2) + (memq (car origin) '(normalization disambiguation)) + (symbol? (cadr origin)))) + (label (if qualified? (cadr origin) origin))) + (unless (symbol? label) + (raise-syntax-error #f "source provenance requires an identifier or a typed normalization/disambiguation identifier" provenance owner)) + (datum->syntax namespace + (append (list '(schema . "gerbil-parser.iso-bnf-rule-overlay.v1") + (cons 'namespace (syntax->datum namespace)) (cons 'name label)) + (if qualified? (list (cons 'kind (car origin))) '()) + (list (cons 'sourceVersion (syntax->datum version)) + (cons 'upstreamCommit (syntax->datum commit))))))) + +(def (lower-source-rule-precedences rows owner namespace version commit) + (stx-map + (lambda (row) + (syntax-case row () + ((name provenance association level) + (identifier? #'name) + (with-syntax ((receipt (source-rule-receipt #'provenance owner namespace version commit))) + #'(name receipt association level))) + (_ (raise-syntax-error #f "source precedence requires rule, provenance, association and level" row owner)))) + rows)) diff --git a/src/compiler/contextual-dispatch.ss b/src/compiler/contextual-dispatch.ss new file mode 100644 index 00000000..a3948151 --- /dev/null +++ b/src/compiler/contextual-dispatch.ss @@ -0,0 +1,153 @@ +;;; -*- Gerbil -*- +;;; Compiler-side resolution of contextual declaration methods. The output is +;;; closed data; parsing never consults POO objects or method procedures. + +(import (only-in ../modules/parser/contextual-objects + contextual-role? contextual-role-ref + contextual-method-name contextual-method-mode + contextual-method-position contextual-method-form + contextual-method-result) + (only-in ../runtime/identity sha256-text)) +(export compile-contextual-dispatch contextual-dispatch-ref) + +(def +contextual-dispatch-schema+ "gerbil-parser.contextual-dispatch.v1") +(defstruct contextual-candidate (origin name mode position form result) + transparent: #t) + +(def (contextual-dispatch-ref ir key) + (let (row (assq key ir)) + (and row (cdr row)))) + +(def (catalog? values) + (and (pair? values) + (list? values) + (andmap symbol? values) + (not (memq 'any values)) + (let loop ((rest values) (seen '())) + (or (null? rest) + (and (not (memq (car rest) seen)) + (loop (cdr rest) (cons (car rest) seen))))))) + +(def (role-candidate role method) + (make-contextual-candidate + (contextual-role-ref role 'name) + (contextual-method-name method) + (contextual-method-mode method) + (contextual-method-position method) + (contextual-method-form method) + (contextual-method-result method))) + +(def (candidate-row candidate) + (list (contextual-candidate-origin candidate) + (contextual-candidate-name candidate) + (contextual-candidate-mode candidate) + (contextual-candidate-position candidate) + (contextual-candidate-form candidate) + (contextual-candidate-result candidate))) + +(def (candidate-origin candidate) + (list (contextual-candidate-origin candidate) + (contextual-candidate-name candidate))) + +(def (known-selector? selector catalog) + (or (eq? selector 'any) (memq selector catalog))) + +(def (matches? selector actual) + (or (eq? selector 'any) (eq? selector actual))) + +(def (method-matches? candidate mode position form) + (and (matches? (contextual-candidate-mode candidate) mode) + (matches? (contextual-candidate-position candidate) position) + (matches? (contextual-candidate-form candidate) form))) + +(def (selector-at-least-as-specific? left right) + (or (eq? left right) (eq? right 'any))) + +(def (dominates? left right) + (and + (selector-at-least-as-specific? + (contextual-candidate-mode left) (contextual-candidate-mode right)) + (selector-at-least-as-specific? + (contextual-candidate-position left) (contextual-candidate-position right)) + (selector-at-least-as-specific? + (contextual-candidate-form left) (contextual-candidate-form right)) + (or (not (eq? (contextual-candidate-mode left) + (contextual-candidate-mode right))) + (not (eq? (contextual-candidate-position left) + (contextual-candidate-position right))) + (not (eq? (contextual-candidate-form left) + (contextual-candidate-form right)))))) + +(def (maximal-methods matching) + (filter + (lambda (row) + (not (any (lambda (other) (dominates? other row)) matching))) + matching)) + +(def (resolve-cell rows mode position form) + (let* ((matching + (filter (lambda (row) (method-matches? row mode position form)) rows)) + (maximal (maximal-methods matching))) + (if (null? maximal) + #f + (let (result (contextual-candidate-result (car maximal))) + (unless (andmap (lambda (candidate) + (eq? (contextual-candidate-result candidate) result)) + maximal) + (error "ambiguous contextual dispatch" + (list mode position form) (map candidate-row maximal))) + (list result (map candidate-origin maximal)))))) + +;;; All resolution happens here. A method tie with the same result retains +;;; both origins in the receipt; a tie with different results is an error. +(def (compile-contextual-dispatch roles modes positions forms) + (unless (and (list? roles) (andmap contextual-role? roles) + (catalog? modes) (catalog? positions) (catalog? forms)) + (error "invalid contextual dispatch declaration")) + (let (candidates + (apply append + (map (lambda (role) + (map (lambda (method) (role-candidate role method)) + (contextual-role-ref role 'methods))) + roles))) + (for-each + (lambda (candidate) + (unless (and (symbol? (contextual-candidate-name candidate)) + (symbol? (contextual-candidate-mode candidate)) + (symbol? (contextual-candidate-position candidate)) + (symbol? (contextual-candidate-form candidate)) + (symbol? (contextual-candidate-result candidate)) + (known-selector? (contextual-candidate-mode candidate) modes) + (known-selector? (contextual-candidate-position candidate) + positions) + (known-selector? (contextual-candidate-form candidate) forms)) + (error "unknown contextual dispatch selector" + (candidate-row candidate)))) + candidates) + (let* ((cells + (apply append + (map + (lambda (mode) + (apply append + (map + (lambda (position) + (map + (lambda (form) + (list mode position form + (resolve-cell candidates mode position + form))) + forms)) + positions))) + modes))) + (body + (list (cons 'schema +contextual-dispatch-schema+) + (cons 'modes modes) + (cons 'positions positions) + (cons 'forms forms) + (cons 'methods (map candidate-row candidates)) + (cons 'cells cells))) + (digest + (sha256-text + (call-with-output-string + (lambda (port) (write body port)))))) + (append body (list (cons 'digest digest)))))) diff --git a/src/compiler/contextual-parser-ir.ss b/src/compiler/contextual-parser-ir.ss new file mode 100644 index 00000000..314fd913 --- /dev/null +++ b/src/compiler/contextual-parser-ir.ss @@ -0,0 +1,183 @@ +;;; -*- Gerbil -*- +;;; Contextual declarations lower to one closed parser product before runtime. + +(import (only-in ./contextual-dispatch + compile-contextual-dispatch contextual-dispatch-ref) + (only-in ./contextual-scanner-ir compile-contextual-scanner) + (only-in ./parser-ir parser-ir-ref) + (only-in ../modules/parser/contextual-objects + make-contextual-method make-contextual-role + make-contextual-scan-rule) + (only-in ../runtime/contextual-ir make-contextual-parser-ir) + (only-in ../runtime/identity sha256-text)) +(export compile-contextual-parser + compile-contextual-parser/declaration) + +(def (canonical value) + (call-with-output-string (lambda (port) (write value port)))) + +(def (unique-symbols? values) + (and (list? values) + (andmap symbol? values) + (let loop ((rest values) (seen '())) + (or (null? rest) + (and (not (memq (car rest) seen)) + (loop (cdr rest) (cons (car rest) seen))))))) + +(def (subset? left right) + (andmap (lambda (value) (member value right)) left)) + +(def (lookahead? value known) + (match value + (['token name] + (and (symbol? name) (member (list 'terminal 'token name) known))) + (['literal text] + (and (string? text) (member (list 'terminal 'literal text) known))) + (['eof] (member '(terminal eof) known)) + (else #f))) + +(def (unique-lookaheads? values known) + (and (list? values) + (andmap (lambda (value) (lookahead? value known)) values) + (let loop ((rest values) (seen '())) + (or (null? rest) + (and (not (member (car rest) seen)) + (loop (cdr rest) (cons (car rest) seen))))))) + +(def (valid-position-clause? clause positions terminals) + (match clause + ([name required forbidden] + (and (memq name positions) + (unique-lookaheads? required terminals) + (unique-lookaheads? forbidden terminals) + (not (any (lambda (term) (member term forbidden)) required)))) + (else #f))) + +(def (clause-matches? clause expected) + (and (subset? (cadr clause) expected) + (not (any (lambda (term) (member term expected)) + (caddr clause))))) + +(def (clause-dominates? left right) + (and (subset? (cadr right) (cadr left)) + (subset? (caddr right) (caddr left)) + (or (> (length (cadr left)) (length (cadr right))) + (> (length (caddr left)) (length (caddr right)))))) + +(def (state-position state expected clauses) + (let* ((matching (filter (lambda (clause) + (clause-matches? clause expected)) + clauses)) + (maximal + (filter + (lambda (clause) + (not (any (lambda (other) + (clause-dominates? other clause)) + matching))) + matching))) + (unless (and (pair? maximal) + (andmap (lambda (clause) + (eq? (car clause) (caar maximal))) + maximal)) + (error "unresolved contextual LR position" state expected maximal)) + (cons state (caar maximal)))) + +(def (compile-position-table parser-ir positions clauses) + (let* ((actions (cdr (assq 'actions + (parser-ir-ref parser-ir 'lr-spec)))) + (terminals + (and (vector? actions) + (apply append (map (lambda (row) (map car row)) + (vector->list actions)))))) + (unless (and (vector? actions) + (pair? clauses) (list? clauses) + (andmap (lambda (clause) + (valid-position-clause? + clause positions terminals)) + clauses)) + (error "invalid contextual LR position declaration" clauses)) + (let loop ((index 0) (result '())) + (if (= index (vector-length actions)) + (reverse result) + (let (expected + (map (lambda (row) (cdr (car row))) + (vector-ref actions index))) + (loop (+ index 1) + (cons (state-position index expected clauses) + result))))))) + +(def (validate-results dispatch parser-ir) + (let (terminals (map car (parser-ir-ref parser-ir 'terminals))) + (for-each + (lambda (cell) + (when (list-ref cell 3) + (unless (memq (car (list-ref cell 3)) terminals) + (error "contextual result is not a grammar terminal" cell)))) + (contextual-dispatch-ref dispatch 'cells)))) + +;;; Clauses have the closed shape (position required-lookaheads +;;; forbidden-lookaheads), with (token name), (literal text), and (eof) rows. +;;; A more specific clause must add a requirement or +;;; exclusion; incomparable matches with different positions fail compilation. +(def (compile-contextual-parser parser-ir grammar-digest roles + modes positions forms scan-rules + initial-mode position-clauses) + (unless (and (equal? (parser-ir-ref parser-ir 'schema) + "gerbil-parser.parser-ir.v1") + (string? grammar-digest) + (unique-symbols? modes) + (unique-symbols? positions) + (unique-symbols? forms)) + (error "invalid contextual parser declaration")) + (let* ((dispatch + (compile-contextual-dispatch roles modes positions forms)) + (_checked (validate-results dispatch parser-ir)) + (scanner + (compile-contextual-scanner scan-rules dispatch initial-mode + grammar-digest)) + (state-positions + (compile-position-table parser-ir positions position-clauses)) + (construction (list (cons 'syntax-kinds (parser-ir-ref parser-ir 'syntax-kinds)) + (cons 'terminals (parser-ir-ref parser-ir 'terminals)) + (cons 'projections (parser-ir-ref parser-ir 'rules))))) + (make-contextual-parser-ir + (list (cons 'dialect 'lr) (cons 'program parser-ir)) scanner + (list (cons 'state-positions state-positions)) construction + (parser-ir-ref parser-ir 'root-kind) + (list (cons 'base-grammar-digest grammar-digest) + (cons 'parser-ir-digest (sha256-text (canonical parser-ir))))))) + +;;; Datum projection used only by canonical compiler expansion. The language +;;; file declares rows; POO objects exist solely while compiling the product. +(def (compile-contextual-parser/declaration + parser-ir grammar-digest role-rows modes positions forms scan-rows + initial-mode position-clauses) + (let ((roles + (map + (lambda (row) + (match row + ([name methods] + (make-contextual-role + name + (map + (lambda (method) + (match method + ([method-name mode position form result] + (make-contextual-method + method-name mode position form result)) + (else (error "invalid contextual method row" method)))) + methods))) + (else (error "invalid contextual role row" row)))) + role-rows)) + (rules + (map + (lambda (row) + (match row + ([name mode form matcher rank action] + (make-contextual-scan-rule + name mode form matcher rank action)) + (else (error "invalid contextual scan row" row)))) + scan-rows))) + (compile-contextual-parser + parser-ir grammar-digest roles modes positions forms rules + initial-mode position-clauses))) diff --git a/src/compiler/contextual-program.ss b/src/compiler/contextual-program.ss new file mode 100644 index 00000000..68be68f4 --- /dev/null +++ b/src/compiler/contextual-program.ss @@ -0,0 +1,46 @@ +;;; Source declarations admit once to closed IR and its owned executable lowering. +(import (only-in :clan/poo/object .ref) + (only-in ../runtime/contextual-ir make-contextual-parser-ir contextual-ir-copy) + (only-in ../language/command-profile compile-command-profile admit-command-plan command-plan-recipe command-plan-kind) + (only-in ../language/part-profile compile-part-profile part-plan-recipe admit-part-plan) + (only-in ../language/scanner-profile compile-scanner-profile) + (only-in ../language/result-profile compile-result-profile result-plan-recipe result-plan-catalog) + (only-in ../runtime/region-scanner region-plan-specification prepare-region-plan)) +(export compile-command-contextual-program compile-line-contextual-program + contextual-program? contextual-program-ir contextual-program-regions + contextual-program-scanner contextual-program-results contextual-program-parts contextual-program-commands) +(defstruct contextual-program (owned-ir regions owned-scanner results parts commands)) +(def (contextual-program-scanner program) + (contextual-ir-copy (contextual-program-owned-scanner program))) +(def (contextual-program-ir program) (contextual-ir-copy (contextual-program-owned-ir program))) +(def (construction results) + (append (result-plan-catalog results) (list (cons 'projections (caddr (result-plan-recipe results)))))) +(def (compile-command-contextual-program candidate) + (let* ((results (compile-result-profile (.ref candidate 'results))) + (regions (prepare-region-plan (region-plan-specification (.ref candidate 'regions)))) + (scanner-profile (.ref candidate 'scanner)) + (scanner (compile-scanner-profile scanner-profile)) + (commands (admit-command-plan (compile-command-profile (.ref candidate 'commands)) results scanner-profile)) + (parts (admit-part-plan (compile-part-profile (.ref candidate 'parts)) results regions)) + (command-data (command-plan-recipe commands)) + (root (command-plan-kind commands 'BashFile)) + (ir (make-contextual-parser-ir + (list (cons 'dialect 'command) + (cons 'program (list (cons 'commands command-data) (cons 'parts (part-plan-recipe parts))))) + scanner + (list (cons 'regions (region-plan-specification regions)) + (cons 'entries (list-ref command-data 4))) + (construction results) root))) + (make-contextual-program ir regions scanner results parts commands))) +(def (compile-line-contextual-program candidate) + (let* ((results (compile-result-profile (.ref candidate 'results))) + (root (.ref candidate 'root-kind)) (token (.ref candidate 'token-kind)) + (prefix (string-copy (.ref candidate 'required-prefix))) + (_ (unless (assq token (cdr (assq 'terminals (result-plan-catalog results)))) + (error "line recognition token is outside construction vocabulary" token))) + (ir (make-contextual-parser-ir + (list (cons 'dialect 'lines) + (cons 'program (list (cons 'token-kind token) (cons 'required-prefix prefix)))) + '((operation . newline-inclusive)) '((entries . (lines))) + (construction results) root))) + (make-contextual-program ir #f #f results #f #f))) diff --git a/src/compiler/contextual-scanner-ir.ss b/src/compiler/contextual-scanner-ir.ss new file mode 100644 index 00000000..10f50c1e --- /dev/null +++ b/src/compiler/contextual-scanner-ir.ss @@ -0,0 +1,221 @@ +;;; -*- Gerbil -*- +;;; Closed contextual scanner rules and a digest bound to compiler dispatch. + +(import (only-in ../modules/parser/contextual-objects + contextual-scan-rule? contextual-scan-rule-name + contextual-scan-rule-mode contextual-scan-rule-form + contextual-scan-rule-matcher contextual-scan-rule-rank + contextual-scan-rule-action) + (only-in ./contextual-dispatch contextual-dispatch-ref) + (only-in ../runtime/contextual-scanner + +contextual-scanner-opcode-contract+) + (only-in ../runtime/identity sha256-text) + (only-in ../runtime/region-scanner valid-region-specification?)) +(export compile-contextual-scanner contextual-scanner-ir-ref) + +(def +contextual-scanner-ir-schema+ "gerbil-parser.contextual-scanner-ir.v1") + +(def (contextual-scanner-ir-ref ir key) + (let (row (assq key ir)) (and row (cdr row)))) + +(def (nonempty-strings? values) + (and (pair? values) (list? values) + (andmap (lambda (value) + (and (string? value) (> (string-length value) 0))) + values))) + +(def (unique-by? values project) + (let loop ((rest values) (seen '())) + (or (null? rest) + (let (key (project (car rest))) + (and (not (member key seen)) + (loop (cdr rest) (cons key seen))))))) + +(def (matcher? expression) + (match expression + (['literal value] + (and (string? value) (> (string-length value) 0))) + (['literals values] (nonempty-strings? values)) + (['unless-prefix prefixes exceptions child] + (and (nonempty-strings? prefixes) (list? exceptions) + (or (null? exceptions) (nonempty-strings? exceptions)) (pair? child) (not (eq? (car child) 'unless-prefix)) (matcher? child))) + (['line-prefix prefix separator] + (and (string? prefix) (positive? (string-length prefix)) + (string? separator) (= (string-length separator) 1) + (not (memv (string-ref separator 0) (string->list prefix))))) + ((or ['marker-line-at separator] ['body-line-at separator]) + (and (string? separator) (= (string-length separator) 1))) + (['horizontal-whitespace+] #t) + (['newline] #t) + (['newline-one] #t) + (['identifier] #t) + (['marker-line] #t) + (['body-line] #t) + (['quoted-string delimiters] (nonempty-strings? delimiters)) + (['region-word spec] (valid-region-specification? spec)) + (['balanced-word stops quotes pairs] + (and (nonempty-strings? stops) + (nonempty-strings? quotes) + (andmap (lambda (quote) (= (string-length quote) 1)) quotes) + (unique-by? quotes (lambda (quote) quote)) + (list? pairs) + (andmap + (lambda (pair) + (match pair + ([prefix opening closing] + (and (string? prefix) (> (string-length prefix) 0) + (char? opening) (char? closing) + (char=? (string-ref prefix + (- (string-length prefix) 1)) + opening))) + (else #f))) + pairs)) + (unique-by? pairs car)) + (else #f))) + +(def (action-matcher-compatible? action matcher) + (let (opcode (car (if (eq? (car matcher) 'unless-prefix) (cadddr matcher) matcher))) + (match action + ('keep (not (memq opcode '(marker-line marker-line-at)))) + ((or ['expect-marker _] ['expect-marker-in _ _]) + (memq opcode '(literal literals))) + ((or ['enqueue-if-expecting _] ['enqueue-marker-in _ _]) + (memq opcode '(balanced-word region-word identifier quoted-string))) + (['activate-next _] (memq opcode '(newline-one literal))) + (['finish-marker _ _] (memq opcode '(marker-line marker-line-at))) + (else #f)))) + +(def (action? expression modes) + (match expression + ('keep #t) + (['expect-marker strip-tabs?] (boolean? strip-tabs?)) + (['expect-marker-in strip-tabs? mode] (and (boolean? strip-tabs?) (memq mode modes))) + (['enqueue-marker-in policy mode] (and (memq policy '(raw shell-quote-removal)) (memq mode modes))) + (['enqueue-if-expecting policy] + (memq policy '(raw shell-quote-removal))) + (['activate-next body-mode] (memq body-mode modes)) + (['finish-marker base-mode body-mode] + (and (memq base-mode modes) (memq body-mode modes))) + (else #f))) + +(def (scan-rule-row rule) + (list (contextual-scan-rule-name rule) + (contextual-scan-rule-mode rule) + (contextual-scan-rule-form rule) + (contextual-scan-rule-matcher rule) + (contextual-scan-rule-rank rule) + (contextual-scan-rule-action rule))) + +(def (canonical value) + (call-with-output-string (lambda (port) (write value port)))) + +(def (valid-dispatch-digest? dispatch) + (and (list? dispatch) + (string? (contextual-dispatch-ref dispatch 'digest)) + (equal? + (sha256-text + (canonical + (filter (lambda (row) (not (eq? (car row) 'digest))) dispatch))) + (contextual-dispatch-ref dispatch 'digest)))) + +(def (same-candidate? left right) + (and (eq? (contextual-scan-rule-mode left) + (contextual-scan-rule-mode right)) + (equal? (contextual-scan-rule-matcher left) + (contextual-scan-rule-matcher right)) + (= (contextual-scan-rule-rank left) + (contextual-scan-rule-rank right)))) + +(def (validate-deterministic-rules rules) + (let loop ((remaining rules)) + (when (pair? remaining) + (for-each + (lambda (other) + (when (same-candidate? (car remaining) other) + (error "duplicate contextual scan candidate" + (contextual-scan-rule-name (car remaining)) + (contextual-scan-rule-name other)))) + (cdr remaining)) + (loop (cdr remaining))))) + +(def (index-dispatch-cells cells) + (let ((index (make-table test: equal?)) + (missing (gensym 'missing-cell))) + (for-each + (lambda (cell) + (let (key (take cell 3)) + (unless (eq? (table-ref index key missing) missing) + (error "duplicate contextual dispatch cell" key)) + (table-set! index key (list-ref cell 3)))) + cells) + index)) + +(def (cell-result index mode position form) + (table-ref index (list mode position form) #f)) + +(def (validate-rule rule modes forms) + (unless (and (contextual-scan-rule? rule) + (symbol? (contextual-scan-rule-name rule)) + (memq (contextual-scan-rule-mode rule) modes) + (memq (contextual-scan-rule-form rule) forms) + (matcher? (contextual-scan-rule-matcher rule)) + (integer? (contextual-scan-rule-rank rule)) + (action? (contextual-scan-rule-action rule) modes) + (action-matcher-compatible? + (contextual-scan-rule-action rule) + (contextual-scan-rule-matcher rule))) + (error "invalid contextual scan rule" rule))) + +;;; The dispatch table resolves token kinds for every scanner rule and parser +;;; position. No POO method or language callback enters the scanner runtime. +(def (compile-contextual-scanner rules dispatch initial-mode + (grammar-digest #f)) + (let ((modes (contextual-dispatch-ref dispatch 'modes)) + (positions (contextual-dispatch-ref dispatch 'positions)) + (forms (contextual-dispatch-ref dispatch 'forms))) + (unless (and (list? rules) (pair? rules) + (memq initial-mode modes) + (equal? (contextual-dispatch-ref dispatch 'schema) + "gerbil-parser.contextual-dispatch.v1") + (valid-dispatch-digest? dispatch) + (or (not grammar-digest) (string? grammar-digest))) + (error "invalid contextual scanner declaration")) + (def cell-index + (index-dispatch-cells (contextual-dispatch-ref dispatch 'cells))) + (for-each (lambda (rule) (validate-rule rule modes forms)) rules) + (unless (unique-by? rules contextual-scan-rule-name) + (error "duplicate contextual scan rule identity")) + (validate-deterministic-rules rules) + (for-each + (lambda (mode) + (unless (any (lambda (rule) + (eq? (contextual-scan-rule-mode rule) mode)) + rules) + (error "contextual scanner mode has no rules" mode))) + modes) + (for-each + (lambda (rule) + (for-each + (lambda (position) + (unless (cell-result cell-index + (contextual-scan-rule-mode rule) + position + (contextual-scan-rule-form rule)) + (error "scan rule has no contextual result" + (contextual-scan-rule-name rule) position))) + positions)) + rules) + (let* ((body + (list (cons 'schema +contextual-scanner-ir-schema+) + (cons 'opcode-contract +contextual-scanner-opcode-contract+) + (cons 'initial-mode initial-mode) + (cons 'dispatch-digest + (contextual-dispatch-ref dispatch 'digest)) + (cons 'grammar-digest grammar-digest) + (cons 'modes modes) + (cons 'positions positions) + (cons 'rules (map scan-rule-row rules)) + (cons 'cells (contextual-dispatch-ref dispatch 'cells)))) + (digest + (sha256-text (canonical body)))) + (append body (list (cons 'digest digest)))))) diff --git a/src/compiler/event-fold-analysis.ss b/src/compiler/event-fold-analysis.ss new file mode 100644 index 00000000..e5542469 --- /dev/null +++ b/src/compiler/event-fold-analysis.ss @@ -0,0 +1,249 @@ +;;; Build-time dependence and effect analysis over already admitted Fold IR. +;;; Unknown forms are opaque, never evidence of purity. No request cache. +(import (only-in :std/list/list delete-duplicates/hash duplicates append-map)) +(export fold-expression-analysis fold-statement-analysis fold-zero-support-preserved? + fold-shared-scans fold-rewrite-scans fold-prefix-guard fold-eliminate-inert-branches + fold-share-statement-scans) + +;; An entry support is an invariant, not necessarily an untouched cell. +;; Whole ordered regions may reset that support to zero, but may not change it +;; transiently: a subsequent consumer could observe the intermediate value. +;; Frames, publication and unknown handoffs remain explicit owner boundaries. +(def (fold-zero-support-preserved? forms support) + (or (null? support) + (every + (lambda (form) + (case (car form) + ((if) + (and (null? (vector-ref (fold-expression-analysis (cadr form)) 2)) + (fold-zero-support-preserved? (caddr form) support) + (fold-zero-support-preserved? (cadddr form) support))) + ((set-uint set-bool) + (or (not (memq (cadr form) support)) + (and (eq? (car form) 'set-uint) (equal? (caddr form) '(uint 0))))) + (else #f))) forms))) + +;;; Drop discarded observations only after admitting the absence of effects. +;;; Queries, errors and unknown predicates retain their original execution. +(def (fold-eliminate-inert-branches forms (cursor #f)) + (def (cursor-offset? value) + (and cursor + (or (equal? value (list 'line-index cursor)) + (and (pair? value) (eq? (car value) 'line-step) + (cursor-offset? (cadr value)))))) + (def (ordered? from until) + (or (equal? from until) + (and (pair? until) (eq? (car until) 'line-step) + (ordered? from (cadr until))))) + (def (pure-observation? predicate) + (or (null? (vector-ref (fold-expression-analysis predicate) 2)) + (and (pair? predicate) + (case (car predicate) + ((and or not) (every pure-observation? (cdr predicate))) + ((line-bytes-any-in? line-bytes-all-in? line-bytes-in-set?) + ;; The emitted loop validates its frame before entering. Each + ;; step clamps to end; ordered cursor-derived ranges are total. + (and (cursor-offset? (cadr predicate)) + (cursor-offset? (caddr predicate)) + (ordered? (cadr predicate) (caddr predicate)))) + (else #f))))) + (append-map + (lambda (form) + (case (car form) + ((if) + (let ((yes (fold-eliminate-inert-branches (caddr form) cursor)) + (no (fold-eliminate-inert-branches (cadddr form) cursor))) + (if (and (null? yes) (null? no) + (pure-observation? (cadr form))) + '() (list (list 'if (cadr form) yes no))))) + ((join-once) + (list (list 'join-once (cadr form) + (fold-eliminate-inert-branches (caddr form) cursor) + (fold-eliminate-inert-branches (cadddr form) cursor)))) + ((for-line-bytes) + (list (append (take form 4) + (list (fold-eliminate-inert-branches (list-ref form 4) (cadr form)))))) + ((with-source-bounds) + (list (append (take form 3) + (list (fold-eliminate-inert-branches (cadddr form)))))) + (else (list form)))) forms)) + +;; Summary fields: state reads, state writes, effects. Source bounds/cursors +;; remain lexical dependencies; admitted reuse may cross statements, never frames. +(def (summary reads writes effects) (vector reads writes effects)) +(def (unique items) (delete-duplicates/hash items from-end?: #t)) +(def (merge summaries) + (summary (unique (append-map (cut vector-ref <> 0) summaries)) + (unique (append-map (cut vector-ref <> 1) summaries)) + (unique (append-map (cut vector-ref <> 2) summaries)))) +(def (fold-expression-analysis value) + (if (not (pair? value)) (summary '() '() '()) + (let* ((op (car value)) + (children (merge (map fold-expression-analysis (cdr value))))) + (case op + ((state state-offset stack-top stack-nonempty?) + (summary (list (cadr value)) '() '())) + ((uint bool %line-prefix-hit offset line-index line-prefix-end line-content-end line-trim-end + line-step line-marker-end line-marker-level line-indent-column + uint-add uint-multiply uint-positive? uint-equal? + uint-greater? uint-not-equal? offset-less? not and or + line-starts-with line-starts-with-ascii-ci line-prefix-boundary-ascii-ci + line-marker-ascii-ci line-blank? line-has-word-after-prefix? + line-has-key-after-prefix?) children) + ((uint-divide) + ;; A necessary-prefix rejection cannot suppress division by zero. + (let (divisor (caddr value)) + (if (and (pair? divisor) (eq? (car divisor) 'uint) + (number? (cadr divisor)) (> (cadr divisor) 0)) children + (merge (list children (summary '() '() '(division-error))))))) + ;; Bounded scans are total only when starting at admitted frame bounds. + ((line-skip-horizontal line-scan-word line-scan-key line-scan-until + line-scan-nonspace-until line-trim-end-from line-physical-end) + (if (safe-offset? (cadr value)) children + (merge (list children (summary '() '() '(range-error)))))) + ((line-byte-equal?) + (if (safe-offset? (cadr value)) children + (merge (list children (summary '() '() '(range-error)))))) + ((line-bytes-any-in? line-bytes-all-in? line-bytes-in-set?) + ;; Member tables are admitted constants, not expression programs. + (merge (list (fold-expression-analysis (cadr value)) + (fold-expression-analysis (caddr value)) + (summary '() '() '(range-error))))) + ((future-line-marker-before-boundary? future-heading-title? + future-named-line-marker-before-boundary?) + (merge (list children (summary '() '() '(query-cache))))) + (else (merge (list children (summary '() '() '(opaque))))))))) + +(def (safe-offset? value) + (or (memq value '(start end)) + (and (pair? value) + (case (car value) + ((line-index line-prefix-end line-content-end line-trim-end) #t) + ((line-step) (safe-offset? (cadr value))) + (else #f))))) + +(def (fold-statement-analysis forms) + (merge + (map + (lambda (form) + (case (car form) + ((set-bool set-uint) + (merge (list (fold-expression-analysis (caddr form)) + (summary '() (list (cadr form)) '())))) + ((if join-once) + (merge (list (fold-expression-analysis (cadr form)) + (fold-statement-analysis (caddr form)) + (fold-statement-analysis (cadddr form))))) + ((for-line-bytes) + (merge (list (fold-expression-analysis (caddr form)) + (fold-expression-analysis (cadddr form)) + (fold-statement-analysis (list-ref form 4)) + (summary '() '() '(iteration))))) + ((with-source-bounds) + (merge (list (fold-expression-analysis (cadr form)) + (fold-expression-analysis (caddr form)) + (fold-statement-analysis (cadddr form)) + (summary '() '() '(bounds range-error))))) + ((token start-node finish-node) + (merge (list (merge (map fold-expression-analysis (cdr form))) + (summary '() '() '(events))))) + ((call-source-helper) (summary '() '() '(helper events range-error))) + ((scan-list-marker) (summary '() (list-tail form 4) '(allocation))) + ((push-frame pop-frame open-level close-frame close-through + close-frames-while close-all close-all-frames) + (merge (list (merge (map fold-expression-analysis (cddr form))) + (summary (list (cadr form)) (list (cadr form)) '(stack events))))) + (else (summary '() '() '(opaque))))) forms))) + +(def (shared-scan? value) + (or (equal? value '(line-content-end)) (equal? value '(line-trim-end)) + (equal? value '(line-skip-horizontal start)) + (and (pair? value) (memq (car value) '(line-scan-until line-scan-word line-scan-key)) + (eq? (cadr value) 'start) + (null? (vector-ref (fold-expression-analysis value) 0)) + (null? (vector-ref (fold-expression-analysis value) 2))))) +(def (scan-occurrences value) + (cond ((shared-scan? value) (list value)) + ((not (pair? value)) '()) + (else (append-map scan-occurrences (cdr value))))) +(def (guaranteed-scans value) + (cond ((shared-scan? value) (list value)) + ((not (pair? value)) '()) + ((memq (car value) '(and or)) + (if (null? (cdr value)) '() (guaranteed-scans (cadr value)))) + ;; Only these operators evaluate all children unconditionally. + ((memq (car value) '(offset-less? offset uint-add uint-multiply uint-divide + uint-positive? uint-equal? uint-not-equal? uint-greater? + line-step not)) + (append-map guaranteed-scans (cdr value))) + (else '()))) +(def (fold-shared-scans expression) + (let ((repeated (duplicates (scan-occurrences expression))) + (facts (fold-expression-analysis expression))) + ;; Hoisting must not cross an error/cache effect, even with a guaranteed + ;; scan. Its original first evaluation dominates every reused occurrence. + (if (not (null? (vector-ref facts 2))) '() + (filter (lambda (scan) (assoc scan repeated)) + (unique (guaranteed-scans expression)))))) +(def (fold-rewrite-scans expression replacements) + (cond ((assoc expression replacements) => (lambda (entry) `(line-index ,(cdr entry)))) + ((not (pair? expression)) expression) + (else (cons (car expression) + (map (cut fold-rewrite-scans <> replacements) (cdr expression)))))) + +;; Straight-line source facts are lexical values, not request memoization. +;; Only a total source scan already guaranteed by the first statement may +;; dominate later statements. Never cross a loop, frame or opaque handoff. +(def (fold-share-statement-scans forms (delimiter-only? #f)) + (def (barrier? form) + (any (cut memq <> '(opaque helper bounds iteration)) + (vector-ref (fold-statement-analysis (list form)) 2))) + (def (leading form) + (case (car form) + ((set-bool set-uint) (guaranteed-scans (caddr form))) + ((if) (guaranteed-scans (cadr form))) + ((token) (append (guaranteed-scans (caddr form)) + (guaranteed-scans (cadddr form)))) + (else '()))) + (def (occurs? scan value) + (or (equal? scan value) + (and (pair? value) (or (occurs? scan (car value)) + (occurs? scan (cdr value)))))) + (let loop ((rest forms)) + (cond ((null? rest) '()) + ((barrier? (car rest)) (cons (car rest) (loop (cdr rest)))) + (else + (let split ((tail rest) (region '())) + (if (or (null? tail) (barrier? (car tail))) + (let* ((region (reverse region)) + (scans (filter (lambda (scan) + (and (or (not delimiter-only?) + (memq (car scan) '(line-scan-until line-scan-word line-scan-key))) + (every (cut eq? <> 'events) + (vector-ref (fold-statement-analysis (list (car rest))) 2)) + (occurs? scan (cdr region)))) + (unique (leading (car rest)))))) + (if (null? scans) (cons (car rest) (loop (cdr rest))) + (cons (list '%shared-scans scans region) (loop tail)))) + (split (cdr tail) (cons (car tail) region)))))))) + +(def (fold-prefix-guard expression) + (and (null? (vector-ref (fold-expression-analysis expression) 2)) + (let guard ((value expression)) + (and (pair? value) + (case (car value) + ((and) (any guard (cdr value))) + ((line-byte-equal?) + (and (safe-offset? (cadr value)) + (vector (cadr value) (list (caddr value))))) + ((line-starts-with line-starts-with-ascii-ci + line-prefix-boundary-ascii-ci line-marker-ascii-ci) + (let (bytes (string->utf8 (cadr value))) + (and (> (u8vector-length bytes) 0) + (let* ((byte (u8vector-ref bytes 0)) + (ci? (not (eq? (car value) 'line-starts-with))) + (members (cond ((and ci? (<= 65 byte 90)) (list byte (+ byte 32))) + ((and ci? (<= 97 byte 122)) (list byte (- byte 32))) + (else (list byte))))) + (vector 'start members))))) + (else #f)))))) diff --git a/src/compiler/event-fold-aot.ss b/src/compiler/event-fold-aot.ss index c82e58da..11287620 100644 --- a/src/compiler/event-fold-aot.ss +++ b/src/compiler/event-fold-aot.ss @@ -1,944 +1,17 @@ ;;; -*- Gerbil -*- -;;; A closed Scheme event fold: execute the algorithm here and serialize its -;;; typed transitions for gerbil-scheme-rust-ir. No Rust source templates. - -(import (only-in :std/encoding/json json->string) - (only-in ../language/descriptor language-grammar-machine) - (only-in ../runtime/identity sha256-text) - (only-in ./machine parser-machine-grammar-digest) - (only-in ./event-list-marker.ss - event-line-indent-column scan-event-list-marker) - (only-in ./event-strategy-aot source-line-events kind-index line-starts-with?)) +;;; Public event-fold AOT surface; execution and IR lowering are separate modules. +(import (only-in ./event-fold-runtime.ss + run-event-fold prepare-event-fold-program run-event-fold-program) + (only-in ./event-fold-ir.ss event-fold-ir-json)) (export define-event-fold-parser run-event-fold event-fold-ir-json) -(def (fold-state states name) - (let (entry (assq name states)) - (unless entry (error "undeclared event fold state" name)) - (cdr entry))) - -(def (rust-state-name name) - (list->string - (map (lambda (character) - (if (char=? character #\-) #\_ character)) - (string->list (symbol->string name))))) - -(def (validate-state-names states) - (let loop ((rest states) (seen [])) - (unless (null? rest) - (let (name (rust-state-name (caar rest))) - (when (member name seen) - (error "duplicate event fold state" name)) - (loop (cdr rest) (cons name seen)))))) - -(def (fold-state-of-type states name predicate) - (let (value (fold-state states name)) - (unless (predicate value) (error "event fold state has wrong type" name)) - value)) - -(def (fold-unsigned? value) - (and (exact-integer? value) (>= value 0))) - -(def (fold-update-state states name value) - (map (lambda (entry) - (if (eq? (car entry) name) (cons name value) entry)) - states)) - -(def (fold-marker-byte value) - (unless (and (string? value) (= (string-length value) 1) - (< (char->integer (string-ref value 0)) 128)) - (error "event fold marker must be one ASCII byte" value)) - (char->integer (string-ref value 0))) - -(def (ascii-lower-byte value) - (if (and (<= 65 value) (<= value 90)) (+ value 32) value)) - -(def (line-starts-with-ascii-ci? line prefix) - (let ((actual (string->utf8 line)) (expected (string->utf8 prefix))) - (and (>= (u8vector-length actual) (u8vector-length expected)) - (let loop ((index 0)) - (or (= index (u8vector-length expected)) - (and (= (ascii-lower-byte (u8vector-ref actual index)) - (ascii-lower-byte (u8vector-ref expected index))) - (loop (+ index 1)))))))) - -(def (fold-line-blank? line) - (let (bytes (string->utf8 line)) - (let loop ((index 0)) - (or (= index (u8vector-length bytes)) - (and (memv (u8vector-ref bytes index) '(9 10 13 32)) - (loop (+ index 1))))))) - -(def (fold-line-prefix-boundary? line prefix marker?) - (and (line-starts-with-ascii-ci? line prefix) - (let* ((bytes (string->utf8 line)) - (start (u8vector-length (string->utf8 prefix))) - (end (u8vector-length bytes))) - (if marker? - (let loop ((cursor start)) - (or (= cursor end) - (and (memv (u8vector-ref bytes cursor) '(9 10 13 32)) - (loop (+ cursor 1))))) - (or (= start end) - (memv (u8vector-ref bytes start) '(9 10 13 32))))))) - -(def (fold-future-line-end bytes start) - (let (size (u8vector-length bytes)) - (let loop ((cursor start)) - (if (or (= cursor size) (memv (u8vector-ref bytes cursor) '(10 13))) - (if (= cursor size) cursor - (if (and (= (u8vector-ref bytes cursor) 13) - (< (+ cursor 1) size) - (= (u8vector-ref bytes (+ cursor 1)) 10)) - (+ cursor 2) (+ cursor 1))) - (loop (+ cursor 1)))))) - -(def (fold-future-marker-line? bytes start end marker indent?) - (let* ((prefix (string->utf8 marker)) - (begin (if indent? - (let skip ((cursor start)) - (if (and (< cursor end) - (memv (u8vector-ref bytes cursor) '(9 32))) - (skip (+ cursor 1)) cursor)) - start)) - (prefix-end (+ begin (u8vector-length prefix)))) - (and (<= prefix-end end) - (let match ((cursor begin) (index 0)) - (or (= index (u8vector-length prefix)) - (and (= (ascii-lower-byte (u8vector-ref bytes cursor)) - (ascii-lower-byte (u8vector-ref prefix index))) - (match (+ cursor 1) (+ index 1))))) - (let tail ((cursor prefix-end)) - (or (= cursor end) - (and (memv (u8vector-ref bytes cursor) '(9 10 13 32)) - (tail (+ cursor 1)))))))) - -(def (fold-future-heading? bytes start end marker separator) - (let run ((cursor start)) - (and (< cursor end) - (if (= (u8vector-ref bytes cursor) marker) - (run (+ cursor 1)) - (and (> cursor start) - (= (u8vector-ref bytes cursor) separator)))))) - -(def (fold-future-key-value-line? bytes start end marker) - (let skip ((cursor start)) - (if (and (< cursor end) (memv (u8vector-ref bytes cursor) '(9 32))) - (skip (+ cursor 1)) - (and (< cursor end) (= (u8vector-ref bytes cursor) marker) - (let key ((cursor (+ cursor 1)) (key-start (+ cursor 1))) - (and (< cursor end) - (let (byte (u8vector-ref bytes cursor)) - (cond - ((= byte marker) - (and (> cursor key-start) - (or (= (+ cursor 1) end) - (memv (u8vector-ref bytes (+ cursor 1)) - '(9 10 13 32))))) - ((memv byte '(9 10 13 32)) #f) - (else (key (+ cursor 1) key-start)))))))))) - -(def (fold-future-marker-before-boundary? bytes from target stop - heading-marker heading-separator - indent? stop-at-heading? body-key-marker) - (let search ((cursor from)) - (if (>= cursor (u8vector-length bytes)) #f - (let (end (fold-future-line-end bytes cursor)) - (cond - ((and stop-at-heading? - (fold-future-heading? bytes cursor end - heading-marker heading-separator)) #f) - ((and stop (fold-future-marker-line? bytes cursor end stop indent?)) #f) - ((fold-future-marker-line? bytes cursor end target indent?) #t) - ((and body-key-marker - (not (fold-future-key-value-line? bytes cursor end - body-key-marker))) #f) - (else (search end))))))) - -(def (fold-ascii-prefix? value) - (and (string? value) - (every (lambda (character) (< (char->integer character) 128)) - (string->list value)))) - -(def (fold-key-byte? byte) - (or (and (<= 48 byte) (<= byte 57)) - (and (<= 65 byte) (<= byte 90)) - (and (<= 97 byte) (<= byte 122)) - (memv byte '(45 95)))) - -(def (fold-line-has-key-after-prefix? line prefix) - (and (line-starts-with-ascii-ci? line prefix) - (let* ((bytes (string->utf8 line)) - (begin (u8vector-length (string->utf8 prefix))) - (end (u8vector-length bytes))) - (let loop ((cursor begin)) - (if (and (< cursor end) (fold-key-byte? (u8vector-ref bytes cursor))) - (loop (+ cursor 1)) - (and (> cursor begin) (< cursor end) - (= (u8vector-ref bytes cursor) 58))))))) - -(def (fold-offset expression line start end states indices) - (cond - ((eq? expression 'start) start) - ((eq? expression 'end) end) - ((not (pair? expression)) - (error "invalid event fold source offset" expression)) - (else - (case (car expression) - ((line-prefix-end) - (min end (+ start (u8vector-length (string->utf8 (cadr expression)))))) - ((line-marker-end) - (+ start (fold-uint (cons 'line-marker-level (cdr expression)) - line start end states indices))) - ((line-skip-horizontal line-scan-word line-scan-key) - (let* ((bytes (string->utf8 line)) - (cursor (fold-offset (cadr expression) line start end states indices))) - (let loop ((offset cursor)) - (if (and (< offset end) - ((if (eq? (car expression) 'line-skip-horizontal) - (lambda (byte) (memv byte '(9 32))) - (if (eq? (car expression) 'line-scan-key) - fold-key-byte? - (lambda (byte) (not (memv byte '(9 10 13 32)))))) - (u8vector-ref bytes (- offset start)))) - (loop (+ offset 1)) offset)))) - ((line-scan-nonspace-until line-scan-until) - (unless (= (length expression) 3) - (error "invalid event fold bounded delimiter scan" expression)) - (let* ((bytes (string->utf8 line)) - (cursor (fold-offset (cadr expression) line start end states indices)) - (delimiter (fold-marker-byte (caddr expression)))) - (let loop ((offset cursor)) - (if (and (< offset end) - (not (= (u8vector-ref bytes (- offset start)) delimiter)) - (or (eq? (car expression) 'line-scan-until) - (not (memv (u8vector-ref bytes (- offset start)) - '(9 10 13 32))))) - (loop (+ offset 1)) offset)))) - ((line-step) - (min end (+ 1 (fold-offset (cadr expression) line start end states indices)))) - ((line-trim-end) - (let (bytes (string->utf8 line)) - (let loop ((cursor end)) - (if (and (> cursor start) - (memv (u8vector-ref bytes (- cursor start 1)) - '(9 10 11 12 13 32))) - (loop (- cursor 1)) cursor)))) - ((line-trim-end-from) - (let ((bytes (string->utf8 line)) - (floor (fold-offset (cadr expression) line start end states indices))) - (let loop ((cursor end)) - (if (and (> cursor floor) - (memv (u8vector-ref bytes (- cursor start 1)) - '(9 10 11 12 13 32))) - (loop (- cursor 1)) cursor)))) - ((line-content-end) - (let (bytes (string->utf8 line)) - (let loop ((cursor end)) - (if (and (> cursor start) - (memv (u8vector-ref bytes (- cursor start 1)) '(10 13))) - (loop (- cursor 1)) cursor)))) - ((line-index) - (let (entry (assq (cadr expression) indices)) - (unless entry (error "unbound event fold line index" expression)) - (cdr entry))) - ((state-offset) - (fold-state-of-type states (cadr expression) fold-unsigned?)) - (else (error "unsupported event fold source offset" expression)))))) - -(def (fold-line-has-word-after-prefix? line prefix) - (and (line-starts-with-ascii-ci? line prefix) - (let* ((end (u8vector-length (string->utf8 line))) - (cursor (fold-offset - (list 'line-skip-horizontal - (list 'line-prefix-end prefix)) line 0 end '() '()))) - (and (< cursor end) - (not (memv (u8vector-ref (string->utf8 line) cursor) - '(9 10 13 32))))))) - -(def (fold-offset-ir expression states indices) - (cond - ((memq expression '(start end)) (symbol->string expression)) - ((not (pair? expression)) - (error "invalid event fold source offset" expression)) - (else - (case (car expression) - ((line-prefix-end) - (unless (and (= (length expression) 2) - (fold-ascii-prefix? (cadr expression))) - (error "invalid event fold line prefix offset" expression)) - (hash ("kind" "line_prefix_end") ("value" (cadr expression)))) - ((line-marker-end) - (unless (= (length expression) 3) - (error "invalid event fold line marker offset" expression)) - (hash ("kind" "line_marker_end") - ("marker" (fold-marker-byte (cadr expression))) - ("separator" (fold-marker-byte (caddr expression))))) - ((line-skip-horizontal line-scan-word line-scan-key line-step) - (unless (= (length expression) 2) - (error "invalid event fold scanner offset" expression)) - (hash ("kind" (rust-state-name (car expression))) - ("from" (fold-offset-ir (cadr expression) states indices)))) - ((line-scan-nonspace-until line-scan-until) - (unless (= (length expression) 3) - (error "invalid event fold bounded delimiter scan" expression)) - (hash ("kind" (rust-state-name (car expression))) - ("from" (fold-offset-ir (cadr expression) states indices)) - ("delimiter" (fold-marker-byte (caddr expression))))) - ((line-trim-end) - (unless (= (length expression) 1) - (error "invalid event fold line trim offset" expression)) - (hash ("kind" "line_trim_end"))) - ((line-trim-end-from) - (unless (= (length expression) 2) - (error "invalid event fold bounded line trim offset" expression)) - (hash ("kind" "line_trim_end_from") - ("from" (fold-offset-ir (cadr expression) states indices)))) - ((line-content-end) - (unless (= (length expression) 1) - (error "invalid event fold line content offset" expression)) - (hash ("kind" "line_content_end"))) - ((line-index) - (unless (and (= (length expression) 2) - (memq (cadr expression) indices)) - (error "unbound event fold line index" expression)) - (hash ("kind" "line_index") - ("name" (rust-state-name (cadr expression))))) - ((state-offset) - (unless (= (length expression) 2) - (error "invalid event fold state offset" expression)) - (fold-state-of-type states (cadr expression) fold-unsigned?) - (hash ("kind" "state_offset") - ("name" (rust-state-name (cadr expression))))) - (else (error "unsupported event fold source offset" expression)))))) - -;; Final transitions have no current source line. A token can still refer to -;; offsets retained in typed state by an earlier line transition. -(def (fold-final-token-offset? expression) - (and (pair? expression) - (eq? (car expression) 'state-offset) - (= (length expression) 2))) - -(def (fold-uint expression line start end states indices) - (case (car expression) - ((uint) - (unless (and (= (length expression) 2) - (exact-integer? (cadr expression)) (>= (cadr expression) 0)) - (error "invalid event fold unsigned value" expression)) - (cadr expression)) - ((state) (fold-state-of-type states (cadr expression) fold-unsigned?)) - ((offset) (fold-offset (cadr expression) line start end states indices)) - ((line-indent-column) - (event-line-indent-column line (cadr expression))) - ((stack-top) - (let (stack (fold-state-of-type states (cadr expression) list?)) - (if (null? stack) 0 (car stack)))) - ((uint-add uint-multiply uint-divide) - (let ((left (fold-uint (cadr expression) line start end states indices)) - (right (fold-uint (caddr expression) line start end states indices))) - (case (car expression) - ((uint-add) (+ left right)) - ((uint-multiply) (* left right)) - (else (quotient left (max 1 right)))))) - ((line-marker-level) - (let* ((marker (integer->char (fold-marker-byte (cadr expression)))) - (separator (integer->char (fold-marker-byte (caddr expression)))) - (size (string-length line))) - (let loop ((index 0)) - (if (and (< index size) (char=? (string-ref line index) marker)) - (loop (+ index 1)) - (if (and (> index 0) (< index size) - (char=? (string-ref line index) separator)) - index 0))))) - (else (error "unsupported event fold unsigned expression" expression)))) - -(def (fold-uint-ir expression states indices allow-line?) - (case (car expression) - ((uint) - (unless (and (= (length expression) 2) - (exact-integer? (cadr expression)) (>= (cadr expression) 0)) - (error "invalid event fold unsigned value" expression)) - (hash ("kind" "usize") ("value" (cadr expression)))) - ((state) - (fold-state-of-type states (cadr expression) fold-unsigned?) - (hash ("kind" "state") ("name" (rust-state-name (cadr expression))))) - ((offset) - (unless allow-line? (error "event fold final transition has no line" expression)) - (hash ("kind" "offset") - ("value" (fold-offset-ir (cadr expression) states indices)))) - ((line-indent-column) - (unless (and allow-line? (= (length expression) 2) - (exact-integer? (cadr expression)) (> (cadr expression) 0)) - (error "invalid event fold indentation width" expression)) - (hash ("kind" "line_indent_column") - ("tab_width" (cadr expression)))) - ((stack-top) - (fold-state-of-type states (cadr expression) list?) - (hash ("kind" "stack_top") - ("stack" (rust-state-name (cadr expression))))) - ((uint-add uint-multiply uint-divide) - (unless (= (length expression) 3) - (error "invalid event fold unsigned arithmetic" expression)) - (hash ("kind" (case (car expression) - ((uint-add) "add") - ((uint-multiply) "multiply") - (else "divide"))) - ("left" (fold-uint-ir (cadr expression) states indices allow-line?)) - ("right" (fold-uint-ir (caddr expression) states indices allow-line?)))) - ((line-marker-level) - (unless allow-line? (error "event fold final transition has no line" expression)) - (hash ("kind" "line_marker_level") - ("marker" (fold-marker-byte (cadr expression))) - ("separator" (fold-marker-byte (caddr expression))))) - (else (error "unsupported event fold unsigned expression" expression)))) - -(def (fold-predicate expression source-bytes line start end states indices) - (unless (pair? expression) - (error "event fold predicate must be structured" expression)) - (case (car expression) - ((bool) - (unless (and (= (length expression) 2) (boolean? (cadr expression))) - (error "invalid event fold boolean" expression)) - (cadr expression)) - ((state) (fold-state-of-type states (cadr expression) boolean?)) - ((uint-positive?) - (> (fold-uint (cadr expression) line start end states indices) 0)) - ((uint-equal?) - (unless (= (length expression) 3) - (error "invalid event fold unsigned equality" expression)) - (= (fold-uint (cadr expression) line start end states indices) - (fold-uint (caddr expression) line start end states indices))) - ((uint-not-equal?) - (unless (= (length expression) 3) - (error "invalid event fold unsigned inequality" expression)) - (not (= (fold-uint (cadr expression) line start end states indices) - (fold-uint (caddr expression) line start end states indices)))) - ((uint-greater?) - (unless (= (length expression) 3) - (error "invalid event fold unsigned comparison" expression)) - (> (fold-uint (cadr expression) line start end states indices) - (fold-uint (caddr expression) line start end states indices))) - ((offset-less?) - (unless (= (length expression) 3) - (error "invalid event fold offset comparison" expression)) - (< (fold-offset (cadr expression) line start end states indices) - (fold-offset (caddr expression) line start end states indices))) - ((stack-nonempty?) - (pair? (fold-state-of-type states (cadr expression) list?))) - ((line-starts-with) - (unless (and (= (length expression) 2) (string? (cadr expression))) - (error "invalid event fold prefix" expression)) - (line-starts-with? line (cadr expression))) - ((line-starts-with-ascii-ci) - (unless (and (= (length expression) 2) (string? (cadr expression))) - (error "invalid event fold ASCII prefix" expression)) - (line-starts-with-ascii-ci? line (cadr expression))) - ((line-prefix-boundary-ascii-ci line-marker-ascii-ci) - (unless (and (= (length expression) 2) - (fold-ascii-prefix? (cadr expression))) - (error "invalid event fold ASCII line marker" expression)) - (fold-line-prefix-boundary? - line (cadr expression) (eq? (car expression) 'line-marker-ascii-ci))) - ((line-blank?) - (unless (= (length expression) 1) - (error "invalid event fold blank-line predicate" expression)) - (fold-line-blank? line)) - ((future-line-marker-before-boundary?) - (unless (and (= (length expression) 8) - (fold-ascii-prefix? (cadr expression)) - (> (string-length (cadr expression)) 0) - (fold-ascii-prefix? (caddr expression)) - (boolean? (list-ref expression 5)) - (boolean? (list-ref expression 6)) - (or (equal? (list-ref expression 7) "") - (and (string? (list-ref expression 7)) - (= (string-length (list-ref expression 7)) 1)))) - (error "invalid event fold future marker search" expression)) - (fold-future-marker-before-boundary? - source-bytes end (cadr expression) - (and (> (string-length (caddr expression)) 0) (caddr expression)) - (fold-marker-byte (cadddr expression)) - (fold-marker-byte (list-ref expression 4)) - (list-ref expression 5) (list-ref expression 6) - (and (> (string-length (list-ref expression 7)) 0) - (fold-marker-byte (list-ref expression 7))))) - ((line-has-word-after-prefix?) - (unless (and (= (length expression) 2) - (fold-ascii-prefix? (cadr expression))) - (error "invalid event fold word-prefix predicate" expression)) - (fold-line-has-word-after-prefix? line (cadr expression))) - ((line-has-key-after-prefix?) - (unless (and (= (length expression) 2) - (fold-ascii-prefix? (cadr expression))) - (error "invalid event fold key-prefix predicate" expression)) - (fold-line-has-key-after-prefix? line (cadr expression))) - ((line-byte-equal?) - (unless (= (length expression) 3) - (error "invalid event fold byte predicate" expression)) - (let ((at (fold-offset (cadr expression) line start end states indices)) - (byte (caddr expression)) - (bytes (string->utf8 line))) - (and (< at end) (= (u8vector-ref bytes (- at start)) byte)))) - ((line-bytes-all-in? line-bytes-any-in?) - (unless (and (= (length expression) 4) - (list? (cadddr expression)) - (every (lambda (byte) (and (exact-integer? byte) - (<= 0 byte 255))) - (cadddr expression))) - (error "invalid event fold byte-set predicate" expression)) - (let ((from (fold-offset (cadr expression) line start end states indices)) - (until (fold-offset (caddr expression) line start end states indices)) - (bytes (string->utf8 line))) - (unless (and (<= start from) (<= from until) (<= until end)) - (error "event fold byte range outside source line" expression)) - (let loop ((cursor from)) - (if (eq? (car expression) 'line-bytes-all-in?) - (or (= cursor until) - (and (memv (u8vector-ref bytes (- cursor start)) - (cadddr expression)) - (loop (+ cursor 1)))) - (and (< cursor until) - (or (and (memv (u8vector-ref bytes (- cursor start)) - (cadddr expression)) #t) - (loop (+ cursor 1)))))))) - ((not) (not (fold-predicate (cadr expression) source-bytes line start end states indices))) - ((and or) - (unless (>= (length expression) 3) - (error "event fold boolean combination needs two predicates" expression)) - (let (evaluate (lambda (part) - (fold-predicate part source-bytes line start end states indices))) - (if (eq? (car expression) 'and) - (every evaluate (cdr expression)) - (if (ormap evaluate (cdr expression)) #t #f)))) - (else (error "unsupported event fold predicate" expression)))) - -(def (fold-frame-finishes count) - (unless (and (exact-integer? count) (> count 0) (<= count 255)) - (error "event fold frame close arity must be positive" count)) - (let loop ((remaining count) (events [])) - (if (= remaining 0) events - (loop (- remaining 1) (cons '(finish) events))))) - -(def (fold-statements statements source-bytes line start end states indices) - (let loop ((rest statements) (state states) (events [])) - (if (null? rest) (cons state events) - (let* ((form (car rest)) - (step - (case (car form) - ((set-bool) - (let (value (fold-predicate (caddr form) source-bytes line start end state indices)) - (fold-state-of-type state (cadr form) boolean?) - (cons (fold-update-state state (cadr form) value) []))) - ((set-uint) - (let (value (fold-uint (caddr form) line start end state indices)) - (fold-state-of-type state (cadr form) fold-unsigned?) - (cons (fold-update-state state (cadr form) value) []))) - ((close-through) - (let* ((stack (fold-state-of-type state (cadr form) list?)) - (level (fold-uint (caddr form) line start end state indices))) - (let close ((rest stack) (events [])) - (if (and (pair? rest) (>= (car rest) level)) - (close (cdr rest) (cons '(finish) events)) - (cons (fold-update-state state (cadr form) rest) events))))) - ((open-level) - (let* ((stack (fold-state-of-type state (cadr form) list?)) - (level (fold-uint (caddr form) line start end state indices))) - (cons (fold-update-state state (cadr form) (cons level stack)) - (list (list 'start (cadddr form)))))) - ((close-all) - (let (stack (fold-state-of-type state (cadr form) list?)) - (cons (fold-update-state state (cadr form) []) - (map (lambda (_) '(finish)) stack)))) - ((start-node) (cons state (list (list 'start (cadr form))))) - ((token) - (let ((from (fold-offset (caddr form) line start end state indices)) - (to (fold-offset (cadddr form) line start end state indices))) - (cons state (if (= from to) [] - (list (list 'token (cadr form) from to)))))) - ((finish-node) (cons state (list '(finish)))) - ((if) - (fold-statements - (if (fold-predicate (cadr form) source-bytes line start end state indices) - (caddr form) (cadddr form)) - source-bytes line start end state indices)) - ((for-line-bytes) - (let ((from (fold-offset (caddr form) line start end state indices)) - (until (fold-offset (cadddr form) line start end state indices))) - (unless (and (<= start from) (<= from until) (<= until end)) - (error "event fold iteration outside source line" form)) - (let iterate ((cursor from) (current state) (reversed [])) - (if (= cursor until) (cons current (reverse reversed)) - (let (step (fold-statements (list-ref form 4) source-bytes line start end - current - (cons (cons (cadr form) cursor) - indices))) - (iterate (+ cursor 1) (car step) - (foldl cons reversed (cdr step)))))))) - ((scan-list-marker) - (unless (= (length form) 10) - (error "invalid event fold list marker statement" form)) - (let* ((marker (scan-event-list-marker - line start (cadr form) (caddr form) - (cadddr form))) - (present (list-ref form 4)) - (column (list-ref form 5)) - (ordered (list-ref form 6)) - (bullet-start (list-ref form 7)) - (bullet-end (list-ref form 8)) - (content-start (list-ref form 9))) - (fold-state-of-type state present boolean?) - (fold-state-of-type state column fold-unsigned?) - (fold-state-of-type state ordered boolean?) - (for-each (lambda (name) - (fold-state-of-type state name fold-unsigned?)) - (list bullet-start bullet-end content-start)) - (let (updated (fold-update-state state present (if marker #t #f))) - (if marker - (cons (fold-update-state - (fold-update-state - (fold-update-state - (fold-update-state - (fold-update-state updated column (vector-ref marker 0)) - ordered (vector-ref marker 1)) - bullet-start (vector-ref marker 2)) - bullet-end (vector-ref marker 3)) - content-start (vector-ref marker 4)) []) - (cons updated []))))) - ((push-frame) - (let ((stack (fold-state-of-type state (cadr form) list?)) - (value (fold-uint (caddr form) line start end state indices))) - (cons (fold-update-state state (cadr form) (cons value stack)) []))) - ((close-frames-while) - (let (closes (fold-frame-finishes (list-ref form 3))) - (let close ((current state) (reversed [])) - (let (stack (fold-state-of-type current (cadr form) list?)) - (if (and (pair? stack) - (fold-predicate (caddr form) source-bytes line start end current indices)) - (close (fold-update-state current (cadr form) (cdr stack)) - (foldl cons reversed closes)) - (cons current (reverse reversed))))))) - ((close-all-frames) - (let ((stack (fold-state-of-type state (cadr form) list?)) - (closes (fold-frame-finishes (caddr form)))) - (cons (fold-update-state state (cadr form) []) - (apply append (map (lambda (_) closes) stack))))) - (else (error "unsupported event fold statement" form))))) - (loop (cdr rest) (car step) (append events (cdr step))))))) - -(def (run-event-fold source root initial line-forms finish-forms) - (let ((source-bytes (string->utf8 source)) - (states (map (lambda (entry) - (cons (car entry) (if (equal? (cadr entry) '(uint-stack)) - [] (cadr entry)))) initial))) - (let* ((events - (source-line-events - source root - (lambda (line start end) - (let (step (fold-statements line-forms source-bytes line start end states '())) - (set! states (car step)) - (cdr step))))) - (closing (fold-statements finish-forms source-bytes "" 0 0 states '()))) - (append (reverse (cdr (reverse events))) (cdr closing) '((finish)))))) - -(def (fold-predicate-ir expression states indices allow-line?) - (unless (pair? expression) - (error "event fold predicate must be structured" expression)) - (case (car expression) - ((bool) - (unless (and (= (length expression) 2) (boolean? (cadr expression))) - (error "invalid event fold boolean" expression)) - (hash ("kind" "bool") ("value" (cadr expression)))) - ((state) - (fold-state-of-type states (cadr expression) boolean?) - (hash ("kind" "state") ("name" (rust-state-name (cadr expression))))) - ((uint-positive?) - (hash ("kind" "usize_positive") - ("value" (fold-uint-ir (cadr expression) states indices allow-line?)))) - ((uint-equal?) - (unless (= (length expression) 3) - (error "invalid event fold unsigned equality" expression)) - (hash ("kind" "usize_equal") - ("left" (fold-uint-ir (cadr expression) states indices allow-line?)) - ("right" (fold-uint-ir (caddr expression) states indices allow-line?)))) - ((uint-not-equal?) - (unless (= (length expression) 3) - (error "invalid event fold unsigned inequality" expression)) - (hash ("kind" "usize_not_equal") - ("left" (fold-uint-ir (cadr expression) states indices allow-line?)) - ("right" (fold-uint-ir (caddr expression) states indices allow-line?)))) - ((uint-greater?) - (unless (= (length expression) 3) - (error "invalid event fold unsigned comparison" expression)) - (hash ("kind" "usize_greater") - ("left" (fold-uint-ir (cadr expression) states indices allow-line?)) - ("right" (fold-uint-ir (caddr expression) states indices allow-line?)))) - ((offset-less?) - (unless allow-line? - (error "event fold final transition has no source line" expression)) - (unless (= (length expression) 3) - (error "invalid event fold offset comparison" expression)) - (hash ("kind" "offset_less") - ("left" (fold-offset-ir (cadr expression) states indices)) - ("right" (fold-offset-ir (caddr expression) states indices)))) - ((stack-nonempty?) - (unless (= (length expression) 2) - (error "invalid event fold stack predicate" expression)) - (fold-state-of-type states (cadr expression) list?) - (hash ("kind" "stack_nonempty") - ("stack" (rust-state-name (cadr expression))))) - ((line-starts-with) - (unless allow-line? - (error "event fold final transition has no source line" expression)) - (unless (and (= (length expression) 2) (string? (cadr expression))) - (error "invalid event fold prefix" expression)) - (hash ("kind" "line_starts_with") ("value" (cadr expression)))) - ((line-starts-with-ascii-ci) - (unless allow-line? - (error "event fold final transition has no source line" expression)) - (unless (and (= (length expression) 2) (string? (cadr expression)) - (every (lambda (character) (< (char->integer character) 128)) - (string->list (cadr expression)))) - (error "invalid event fold ASCII prefix" expression)) - (hash ("kind" "line_starts_with_ascii_case_insensitive") - ("value" (cadr expression)))) - ((line-prefix-boundary-ascii-ci line-marker-ascii-ci) - (unless allow-line? - (error "event fold final transition has no source line" expression)) - (unless (and (= (length expression) 2) - (fold-ascii-prefix? (cadr expression))) - (error "invalid event fold ASCII line marker" expression)) - (hash ("kind" (if (eq? (car expression) 'line-marker-ascii-ci) - "line_marker_ascii_case_insensitive" - "line_prefix_boundary_ascii_case_insensitive")) - ("value" (cadr expression)))) - ((line-blank?) - (unless allow-line? - (error "event fold final transition has no source line" expression)) - (unless (= (length expression) 1) - (error "invalid event fold blank-line predicate" expression)) - (hash ("kind" "line_blank"))) - ((future-line-marker-before-boundary?) - (unless (and allow-line? (= (length expression) 8) - (fold-ascii-prefix? (cadr expression)) - (> (string-length (cadr expression)) 0) - (fold-ascii-prefix? (caddr expression)) - (boolean? (list-ref expression 5)) - (boolean? (list-ref expression 6)) - (or (equal? (list-ref expression 7) "") - (and (string? (list-ref expression 7)) - (= (string-length (list-ref expression 7)) 1)))) - (error "invalid event fold future marker search" expression)) - (hash ("kind" "future_line_marker_before_boundary") - ("target" (cadr expression)) - ("stop" (caddr expression)) - ("heading_marker" (fold-marker-byte (cadddr expression))) - ("heading_separator" (fold-marker-byte (list-ref expression 4))) - ("indent" (list-ref expression 5)) - ("stop_at_heading" (list-ref expression 6)) - ("body_key_marker" - (if (equal? (list-ref expression 7) "") 0 - (fold-marker-byte (list-ref expression 7)))))) - ((line-has-word-after-prefix?) - (unless allow-line? - (error "event fold final transition has no source line" expression)) - (unless (and (= (length expression) 2) - (fold-ascii-prefix? (cadr expression))) - (error "invalid event fold word-prefix predicate" expression)) - (hash ("kind" "line_has_word_after_prefix") - ("value" (cadr expression)))) - ((line-has-key-after-prefix?) - (unless allow-line? - (error "event fold final transition has no source line" expression)) - (unless (and (= (length expression) 2) - (fold-ascii-prefix? (cadr expression))) - (error "invalid event fold key-prefix predicate" expression)) - (hash ("kind" "line_has_key_after_prefix") - ("value" (cadr expression)))) - ((line-byte-equal?) - (unless (and allow-line? (= (length expression) 3) - (exact-integer? (caddr expression)) - (<= 0 (caddr expression) 255)) - (error "invalid event fold byte predicate" expression)) - (hash ("kind" "line_byte_equal") - ("at" (fold-offset-ir (cadr expression) states indices)) - ("value" (caddr expression)))) - ((line-bytes-all-in? line-bytes-any-in?) - (unless (and allow-line? (= (length expression) 4) - (list? (cadddr expression)) - (every (lambda (byte) (and (exact-integer? byte) - (<= 0 byte 255))) - (cadddr expression))) - (error "invalid event fold byte-set predicate" expression)) - (hash ("kind" (if (eq? (car expression) 'line-bytes-all-in?) - "line_bytes_all_in" "line_bytes_any_in")) - ("from" (fold-offset-ir (cadr expression) states indices)) - ("until" (fold-offset-ir (caddr expression) states indices)) - ("values" (list->vector (cadddr expression))))) - ((not) - (hash ("kind" "not") - ("value" (fold-predicate-ir (cadr expression) states indices allow-line?)))) - ((and or) - (unless (>= (length expression) 3) - (error "event fold boolean combination needs two predicates" expression)) - (let ((kind (symbol->string (car expression))) - (parts (map (lambda (part) - (fold-predicate-ir part states indices allow-line?)) - (cdr expression)))) - (foldl (lambda (left right) - (hash ("kind" kind) ("left" left) ("right" right))) - (car parts) (cdr parts)))) - (else (error "unsupported event fold predicate" expression)))) - -(def (fold-statements-ir grammar statements states indices allow-line?) - (list->vector - (map - (lambda (form) - (case (car form) - ((set-bool) - (fold-state-of-type states (cadr form) boolean?) - (hash ("kind" "set_bool") ("name" (rust-state-name (cadr form))) - ("value" (fold-predicate-ir (caddr form) states indices allow-line?)))) - ((set-uint) - (fold-state-of-type states (cadr form) fold-unsigned?) - (hash ("kind" "set_usize") ("name" (rust-state-name (cadr form))) - ("value" (fold-uint-ir (caddr form) states indices allow-line?)))) - ((close-through) - (fold-state-of-type states (cadr form) list?) - (hash ("kind" "close_through_level") - ("stack" (rust-state-name (cadr form))) - ("level" (fold-uint-ir (caddr form) states indices allow-line?)))) - ((open-level) - (fold-state-of-type states (cadr form) list?) - (hash ("kind" "open_level") - ("stack" (rust-state-name (cadr form))) - ("level" (fold-uint-ir (caddr form) states indices allow-line?)) - ("syntax_kind" (kind-index grammar (cadddr form) 'node)))) - ((close-all) - (fold-state-of-type states (cadr form) list?) - (hash ("kind" "close_all_levels") - ("stack" (rust-state-name (cadr form))))) - ((start-node) - (hash ("kind" "start_node") - ("syntax_kind" (kind-index grammar (cadr form) 'node)))) - ((token) - (unless (= (length form) 4) - (error "event fold token requires source offsets" form)) - (unless (or allow-line? - (and (fold-final-token-offset? (caddr form)) - (fold-final-token-offset? (cadddr form)))) - (error "event fold final transition has no source offsets" form)) - (hash ("kind" "token") - ("syntax_kind" (kind-index grammar (cadr form) 'token)) - ("start" (fold-offset-ir (caddr form) states indices)) - ("end" (fold-offset-ir (cadddr form) states indices)))) - ((finish-node) (hash ("kind" "finish_node"))) - ((if) - (hash ("kind" "if") - ("condition" (fold-predicate-ir (cadr form) states indices allow-line?)) - ("consequent" (fold-statements-ir grammar (caddr form) states indices allow-line?)) - ("alternate" (fold-statements-ir grammar (cadddr form) states indices allow-line?)))) - ((for-line-bytes) - (unless (and allow-line? (= (length form) 5) - (symbol? (cadr form)) - (not (assq (cadr form) states)) - (not (memq (cadr form) indices))) - (error "invalid event fold source-line iteration" form)) - (hash ("kind" "for_line_bytes") - ("index" (rust-state-name (cadr form))) - ("from" (fold-offset-ir (caddr form) states indices)) - ("until" (fold-offset-ir (cadddr form) states indices)) - ("body" (fold-statements-ir grammar (list-ref form 4) states - (cons (cadr form) indices) allow-line?)))) - ((scan-list-marker) - (unless (and allow-line? (= (length form) 10) - (string? (cadr form)) - (boolean? (caddr form)) - (exact-integer? (cadddr form)) - (> (cadddr form) 0)) - (error "invalid event fold list marker statement" form)) - (fold-state-of-type states (list-ref form 4) boolean?) - (fold-state-of-type states (list-ref form 5) fold-unsigned?) - (fold-state-of-type states (list-ref form 6) boolean?) - (for-each (lambda (name) - (fold-state-of-type states name fold-unsigned?)) - (list (list-ref form 7) (list-ref form 8) (list-ref form 9))) - (hash ("kind" "scan_list_marker") - ("marker" - (hash ("unordered" (cadr form)) - ("ordered" (caddr form)) - ("tab_width" (cadddr form)) - ("present" (rust-state-name (list-ref form 4))) - ("column" (rust-state-name (list-ref form 5))) - ("ordered_slot" (rust-state-name (list-ref form 6))) - ("bullet_start" (rust-state-name (list-ref form 7))) - ("bullet_end" (rust-state-name (list-ref form 8))) - ("content_start" (rust-state-name (list-ref form 9))))))) - ((push-frame) - (fold-state-of-type states (cadr form) list?) - (hash ("kind" "push_frame") - ("stack" (rust-state-name (cadr form))) - ("value" (fold-uint-ir (caddr form) states indices allow-line?)))) - ((close-frames-while) - (fold-state-of-type states (cadr form) list?) - (fold-frame-finishes (list-ref form 3)) - (hash ("kind" "close_frames_while") - ("stack" (rust-state-name (cadr form))) - ("condition" (fold-predicate-ir (caddr form) states indices allow-line?)) - ("finish_count" (list-ref form 3)))) - ((close-all-frames) - (fold-state-of-type states (cadr form) list?) - (fold-frame-finishes (caddr form)) - (hash ("kind" "close_all_frames") - ("stack" (rust-state-name (cadr form))) - ("finish_count" (caddr form)))) - (else (error "unsupported event fold statement" form)))) - statements))) - -(def (event-fold-ir-json name grammar root initial line-forms finish-forms) - (let* ((states (map (lambda (entry) - (unless (and (= (length entry) 2) (symbol? (car entry)) - (or (boolean? (cadr entry)) - (fold-unsigned? (cadr entry)) - (equal? (cadr entry) '(uint-stack)))) - (error "event fold requires typed state" entry)) - (cons (car entry) (if (equal? (cadr entry) '(uint-stack)) - [] (cadr entry)))) initial)) - (initial-ir - (list->vector - (map (lambda (declaration) - (let ((name (rust-state-name (car declaration))) - (value (cadr declaration))) - (cond - ((boolean? value) - (hash ("kind" "let_bool") ("name" name) ("value" value))) - ((number? value) - (hash ("kind" "let_usize") ("name" name) ("value" value))) - (else (hash ("kind" "let_usize_stack") ("name" name)))))) - initial))) - (digest - (sha256-text - (call-with-output-string - (lambda (port) - (write (list 'event-fold.v1 - (parser-machine-grammar-digest - (language-grammar-machine grammar)) - name root initial line-forms finish-forms) port)))))) - (validate-state-names states) - (json->string - (hash ("schema" "gerbil-scheme-rust.event-function-ir.v1") - ("name" (symbol->string name)) - ("root_kind" (kind-index grammar root 'node)) - ("parser_digest" digest) - ("initial" initial-ir) - ("line" (fold-statements-ir grammar line-forms states '() #t)) - ("finish" (fold-statements-ir grammar finish-forms states '() #f))) - sort-keys: #t))) - (defrules define-event-fold-parser () ((_ scheme-name rust-name grammar root source initial (line-form ...) (finish-form ...)) (begin - (def (scheme-name source) - (run-event-fold source 'root 'initial - '(line-form ...) '(finish-form ...))) + (def scheme-name + (let (program (prepare-event-fold-program + 'root 'initial '(line-form ...) '(finish-form ...))) + (lambda (source) (run-event-fold-program program source)))) (def rust-name (event-fold-ir-json 'rust-name grammar 'root 'initial '(line-form ...) '(finish-form ...)))))) diff --git a/src/compiler/event-fold-byte-dispatch.ss b/src/compiler/event-fold-byte-dispatch.ss new file mode 100644 index 00000000..9910a2d1 --- /dev/null +++ b/src/compiler/event-fold-byte-dispatch.ss @@ -0,0 +1,209 @@ +;;; Prepare ordered byte decision regions before procedure expansion. +;;; Only a validated loop byte and total guard prefixes may select a region. +(import (only-in :std/list/list delete-duplicates/hash) + (only-in :std/list/list-builder with-list-builder) + (only-in ./event-fold-analysis fold-expression-analysis)) +(export fold-byte-dispatch fold-byte-prefilter fold-necessary-bytes fold-byte-regions fold-line-negative-region) + +;; One shared negative classification across ordered line consumers. Retain +;; each original transition once, guarded by the frame-local classification. +;; No successful suffix/state match follows from a positive classification. +;; Frames, loops and helper handoffs remain opaque boundaries. +(def (fold-line-negative-region forms) + (def prefixes '()) + (def (necessary predicate) + (cond + ((and (pair? predicate) + (memq (car predicate) '(line-starts-with line-starts-with-ascii-ci + line-prefix-boundary-ascii-ci line-marker-ascii-ci + line-has-word-after-prefix? line-has-key-after-prefix?))) + (let (bytes (u8vector->list (string->utf8 (cadr predicate)))) + (and (pair? bytes) (list (cons 'start bytes))))) + ((and (pair? predicate) (eq? (car predicate) 'line-byte-equal?) + (member (cadr predicate) '(start (line-skip-horizontal start)))) + (list (cons (if (eq? (cadr predicate) 'start) 'start 'horizontal) + (list (caddr predicate))))) + ((and (pair? predicate) (eq? (car predicate) 'line-bytes-any-in?) + (member (cadr predicate) '(start (line-skip-horizontal start))) + (equal? (caddr predicate) (list 'line-step (cadr predicate)))) + (and (pair? (list-ref predicate 3)) + (map (lambda (byte) + (cons (if (eq? (cadr predicate) 'start) 'start 'horizontal) + (list byte))) (list-ref predicate 3)))) + ((and (pair? predicate) (eq? (car predicate) 'and)) + (let loop ((rest (conjunction-terms predicate))) + (and (pair? rest) + (or (necessary (car rest)) + (and (null? (vector-ref (fold-expression-analysis (car rest)) 2)) + (loop (cdr rest))))))) + ((and (pair? predicate) (eq? (car predicate) 'or)) + (let (alternatives (map necessary (cdr predicate))) + (and (every list? alternatives) (foldr append '() alternatives)))) + (else #f))) + (def (walk body) + (with-list-builder (emit) + (for-each + (lambda (form) + (case (car form) + ((if) + (let (needed (necessary (cadr form))) + (if needed + (begin + (set! prefixes (append needed prefixes)) + (emit (list 'if (list 'and '(%line-prefix-hit) (cadr form)) + (walk (caddr form)) (walk (cadddr form))))) + (emit (list 'if (cadr form) (walk (caddr form)) (walk (cadddr form))))))) + ((join-once) + (emit (list 'join-once (cadr form) (walk (caddr form)) (walk (cadddr form))))) + (else (emit form)))) body))) + (let ((residual (walk forms))) + (and (pair? prefixes) (not (equal? forms residual)) + (vector (delete-duplicates/hash prefixes from-end?: #t) residual)))) + +;; One classification for the entire validated byte-loop body, not one guard +;; per independent state machine. Descend only through decisions and lexical +;; joins: source frames/helpers/nested iterations stay in their owners. +;; A mismatch bypasses total guard reads, never queries/errors/effects. +(def (fold-byte-regions forms cursor) + (def (cofactor byte) + (def (walk body) + (with-list-builder (emit) + (for-each + (lambda (form) + (case (car form) + ((if) + (let (members (fold-necessary-bytes (cadr form) cursor)) + (if (and members (not (memv byte members))) + (for-each emit (walk (cadddr form))) + (emit (list 'if (cadr form) (walk (caddr form)) (walk (cadddr form))))))) + ((join-once) + (emit (list 'join-once (cadr form) (walk (caddr form)) (walk (cadddr form))))) + (else (emit form)))) body))) + (walk forms)) + (let ((fallback (cofactor -1)) (by-body (make-hash-table)) (groups '())) + (for-each + (lambda (byte) + (let (body (cofactor byte)) + (unless (equal? body fallback) + (let (group (or (hash-get by-body body) + (let (created (vector '() body)) + (hash-put! by-body body created) + (set! groups (cons created groups)) created))) + (vector-set! group 0 (cons byte (vector-ref group 0))))))) + (iota 256)) + (and (pair? groups) + (vector (map (lambda (group) + (cons (reverse (vector-ref group 0)) (vector-ref group 1))) + (reverse groups)) fallback)))) + +;; Literal source patterns lower to nested conjunctions. Flatten only `and`; +;; the original short-circuit order and all effect barriers remain intact. +(def (conjunction-terms predicate) + ;; The standard builder emits each leaf once, preserving short-circuit order + ;; without copying flattened child lists at every enclosing conjunction. + (with-list-builder (emit) + (def (visit term) + (if (and (pair? term) (eq? (car term) 'and)) + (for-each visit (cdr term)) + (emit term))) + (visit predicate))) + +;; A byte condition is necessary only before any observable leaf. For an +;; ordered choice every alternative must supply such a condition. Keep each +;; original alternative intact: a byte match is not a full pattern match. +(def (fold-necessary-bytes predicate cursor) + (cond + ((byte-leaf? predicate cursor) (list (caddr predicate))) + ((and (pair? predicate) (eq? (car predicate) 'line-bytes-any-in?) + (equal? (cadr predicate) (list 'line-index cursor)) + (equal? (caddr predicate) (list 'line-step (cadr predicate)))) + (list-ref predicate 3)) + ((and (pair? predicate) (eq? (car predicate) 'and)) + (let loop ((rest (conjunction-terms predicate))) + (and (pair? rest) + (or (fold-necessary-bytes (car rest) cursor) + (and (null? (vector-ref (fold-expression-analysis (car rest)) 2)) + (loop (cdr rest))))))) + ((and (pair? predicate) (eq? (car predicate) 'or)) + (let (alternatives (map (cut fold-necessary-bytes <> cursor) (cdr predicate))) + (and (every list? alternatives) + (delete-duplicates/hash (foldr append '() alternatives) from-end?: #t)))) + (else #f))) + +(def (necessary-test members cursor) + `(line-bytes-any-in? (line-index ,cursor) (line-step (line-index ,cursor)) ,members)) + +;; A validated loop byte can reject an entire state-dependent decision before +;; reading its slots. Only total prefixes can move behind the byte guard; +;; opaque operations, queries and range errors remain in their original order. +(def (fold-byte-prefilter predicate cursor) + (def (byte-test? value) + (and (pair? value) + (or (byte-leaf? value cursor) + (and (eq? (car value) 'line-bytes-any-in?) + (equal? (cadr value) (list 'line-index cursor)) + (equal? (caddr value) (list 'line-step (cadr value))))))) + (cond + ((and (pair? predicate) (eq? (car predicate) 'or)) + (let (members (fold-necessary-bytes predicate cursor)) + (and members `(and ,(necessary-test members cursor) ,predicate)))) + (else + (and (pair? predicate) (eq? (car predicate) 'and) + (let loop ((rest (conjunction-terms predicate)) (before '())) + (cond ((null? rest) #f) + ((byte-test? (car rest)) + (and (pair? before) + (cons 'and (cons (car rest) + (append (reverse before) (cdr rest)))))) + ((and (pair? (car rest)) (eq? (caar rest) 'or) + (fold-necessary-bytes (car rest) cursor)) + => (lambda (members) + (and (pair? before) + (cons 'and (cons (necessary-test members cursor) + (append (reverse before) rest)))))) + ((null? (vector-ref (fold-expression-analysis (car rest)) 2)) + (loop (cdr rest) (cons (car rest) before))) + (else #f))))))) +(def (byte-leaf? predicate cursor) + (and (pair? predicate) (eq? (car predicate) 'line-byte-equal?) + (equal? (cadr predicate) (list 'line-index cursor)))) +(def (guard predicate cursor) + (cond ((byte-leaf? predicate cursor) (cons (caddr predicate) '(bool #t))) + ((and (pair? predicate) (eq? (car predicate) 'and)) + (let loop ((rest (conjunction-terms predicate)) (before '())) + (cond ((null? rest) #f) + ((byte-leaf? (car rest) cursor) + (cons (caddr (car rest)) + (let ((terms (append (reverse before) (cdr rest)))) + (if (null? terms) '(bool #t) (cons 'and terms))))) + ;; Never bypass errors, queries, allocation or opaque leaves. + ((null? (vector-ref (fold-expression-analysis (car rest)) 2)) + (loop (cdr rest) (cons (car rest) before))) + (else #f)))) + (else #f))) +(def (fold-byte-dispatch form cursor) + (def (finish entries fallback) + (let ((bytes (delete-duplicates/hash (map car entries) from-end?: #t))) + ;; A singleton keeps its original branch, not an extra dispatch layer. + (and (> (length bytes) 1) + (vector + (map (lambda (byte) + (cons byte + (let choose ((rest entries)) + (if (null? rest) fallback + (let ((entry (car rest))) + (cond ((not (= byte (car entry))) (choose (cdr rest))) + ((equal? (cadr entry) '(bool #t)) (caddr entry)) + (else (list (list 'if (cadr entry) (caddr entry) + (choose (cdr rest))))))))))) bytes) + fallback)))) + (let collect ((form form) (entries '())) + (let ((fact (and (pair? form) (eq? (car form) 'if) + (guard (cadr form) cursor)))) + (if fact + (let ((fallback (cadddr form)) + (next (cons (list (car fact) (cdr fact) (caddr form)) entries))) + (if (and (= (length fallback) 1) (eq? (caar fallback) 'if)) + (collect (car fallback) next) + (finish (reverse next) fallback))) + (finish (reverse entries) (list form)))))) diff --git a/src/compiler/event-fold-byte-set.ss b/src/compiler/event-fold-byte-set.ss new file mode 100644 index 00000000..9bef8747 --- /dev/null +++ b/src/compiler/event-fold-byte-set.ss @@ -0,0 +1,29 @@ +;;; -*- Gerbil -*- +;;; Request-local preparation of immutable grammar byte sets. +(export current-fold-byte-set-cache fold-byte-set-for fold-marker-byte) + +;; Shared scalar admission must not import the interpreter/runtime closure. +(def (fold-marker-byte value) + (unless (and (string? value) (= (string-length value) 1) + (< (char->integer (string-ref value 0)) 128)) + (error "event fold marker must be one ASCII byte" value)) + (char->integer (string-ref value 0))) + +(def current-fold-byte-set-cache (make-parameter #f)) + +(def (fold-byte-set-for values expression) + (let* ((cache (current-fold-byte-set-cache)) + (prepared (and cache (table-ref cache values #f)))) + (or prepared + (begin + (unless (list? values) + (error "invalid event fold byte-set predicate" expression)) + (let (table (make-u8vector 256 0)) + (for-each + (lambda (byte) + (unless (and (exact-integer? byte) (<= 0 byte 255)) + (error "invalid event fold byte-set predicate" expression)) + (u8vector-set! table byte 1)) + values) + (when cache (table-set! cache values table)) + table))))) diff --git a/src/compiler/event-fold-call-plan.ss b/src/compiler/event-fold-call-plan.ss new file mode 100644 index 00000000..fd990912 --- /dev/null +++ b/src/compiler/event-fold-call-plan.ss @@ -0,0 +1,113 @@ +;;; Fuse unique generated call edges, never duplicate a shared consumer. +;;; This runs once at compilation, before module partitioning. No value cache. +(import (only-in ./event-fold-scheme-context fold-scheme-fresh-name)) +(export fold-fuse-single-use-calls) + +(def (private? form) + (and (pair? form) (eq? (car form) 'def) (pair? (cadr form)) + (string-prefix? "%event-fold-" (symbol->string (caadr form))) + ;; Helper entrypoints retain their frame/ownership boundary. + (string-contains (symbol->string (caadr form)) "-expression-"))) + +(def (size code) + (if (or (not (pair? code)) (eq? (car code) 'quote)) 1 + (+ 1 (size (car code)) (size (cdr code))))) + +;; Alpha-convert binders before moving a body into its caller. Initializers +;; retain let/let* scope, named-loop recursion stays local, quotes stay opaque. +(def (fresh-body code env) + (def (atom value) + (let (entry (and (symbol? value) (assq value env))) + (if entry (cdr entry) value))) + (cond + ((not (pair? code)) (atom code)) + ((eq? (car code) 'quote) code) + ((memq (car code) '(let let*)) + (let* ((named? (symbol? (cadr code))) + (raw-bindings (if named? (caddr code) (cadr code))) + (bindings (if (and (pair? raw-bindings) (symbol? (car raw-bindings))) + (list raw-bindings) raw-bindings)) + (loop-name (and named? (fold-scheme-fresh-name 'fused-loop))) + (body-env (if named? (cons (cons (cadr code) loop-name) env) env)) + (out '())) + (for-each + (lambda (binding) + (let* ((name (fold-scheme-fresh-name 'fused-local)) + (value (fresh-body (cadr binding) + (if (eq? (car code) 'let*) body-env env)))) + (set! out (cons (list name value) out)) + (set! body-env (cons (cons (car binding) name) body-env)))) bindings) + (append (list (car code)) (if named? (list loop-name) '()) + (list (reverse out)) + (map (cut fresh-body <> body-env) (if named? (cdddr code) (cddr code)))))) + ((eq? (car code) 'lambda) + (let* ((names (map (lambda (_) (fold-scheme-fresh-name 'fused-local)) (cadr code))) + (body-env (append (map cons (cadr code) names) env))) + `(lambda ,names ,@(map (cut fresh-body <> body-env) (cddr code))))) + (else (cons (fresh-body (car code) env) (fresh-body (cdr code) env))))) + +(def (fold-fuse-single-use-calls forms (budget 256)) + (let ((directory (make-hash-table-eq)) (uses (make-hash-table-eq))) + (for-each (lambda (form) + (when (private? form) (hash-put! directory (caadr form) form))) forms) + (def (count code) + (cond + ((symbol? code) + (when (hash-get directory code) + (hash-put! uses code (+ 1 (or (hash-get uses code) 0))))) + ((and (pair? code) (not (eq? (car code) 'quote))) + (count (car code)) (count (cdr code))))) + ;; Signatures bind names; export/value references count and prevent removal. + (for-each (lambda (form) + (count (if (and (pair? form) (eq? (car form) 'def)) (cddr form) form))) forms) + (let ((active (make-hash-table-eq))) + (def (walk code) + (cond + ((or (not (pair? code)) (eq? (car code) 'quote)) code) + ((and (symbol? (car code)) (hash-get directory (car code))) + (let* ((name (car code)) (definition (hash-get directory name)) + (parameters (cdadr definition)) (arguments (cdr code))) + (if (and (= (or (hash-get uses name) 0) 1) + (not (hash-get active name)) + (= (length parameters) (length arguments)) + ;; Generated closure arguments are scalar references; + ;; do not change effectful argument evaluation order. + (every (lambda (arg) (not (pair? arg))) arguments) + (<= (size (cddr definition)) budget)) + (begin + (hash-put! active name #t) + (let* ((names (map (lambda (_) (fold-scheme-fresh-name 'fused-arg)) parameters)) + (body (map (cut fresh-body <> (map cons parameters names)) (cddr definition))) + (expanded (map walk body))) + (hash-put! active name #f) + ;; A chain is bounded by its expanded size, not each leaf. + (if (<= (size expanded) budget) + (begin + `(let ,(map list names arguments) ,@expanded)) + code))) + code))) + (else (cons (walk (car code)) (walk (cdr code)))))) + ;; Only fuse top-level ordered definitions and public bodies. Unknown + ;; recursive cycles are still rejected by the module graph validator. + (let (rewritten + (map (lambda (form) + (if (and (pair? form) (eq? (car form) 'def)) + (append (take form 2) (map walk (cddr form))) form)) forms)) + ;; Failed parent admission may retain an original call after child + ;; expansion. Remove only unreachable definitions from the final graph, + ;; never from speculative expansion side effects. + (let ((final (make-hash-table-eq)) (reachable (make-hash-table-eq))) + (for-each (lambda (form) + (when (private? form) (hash-put! final (caadr form) form))) rewritten) + (def (visit code) + (cond + ((symbol? code) + (when (and (hash-get final code) (not (hash-get reachable code))) + (hash-put! reachable code #t) + (visit (cddr (hash-get final code))))) + ((and (pair? code) (not (eq? (car code) 'quote))) + (visit (car code)) (visit (cdr code))))) + (for-each (lambda (form) + (unless (private? form) (visit form))) rewritten) + (filter (lambda (form) + (or (not (private? form)) (hash-get reachable (caadr form)))) rewritten)))))) diff --git a/src/compiler/event-fold-consumer-state.ss b/src/compiler/event-fold-consumer-state.ss new file mode 100644 index 00000000..bbb1f35e --- /dev/null +++ b/src/compiler/event-fold-consumer-state.ss @@ -0,0 +1,45 @@ +;;; Consumer-local selector facts follow control flow and declared support. +;;; No runtime cache or all-machine admission guard is generated. +(import (only-in ./event-fold-analysis fold-expression-analysis fold-statement-analysis)) +(export fold-consumer-selector-body) + +(def (fold-consumer-selector-body forms) + (def (pure? predicate) + (null? (vector-ref (fold-expression-analysis predicate) 2))) + (def (known predicate facts) + (let (entry (assoc predicate facts)) (and entry (list 'bool (cdr entry))))) + (def (select predicate truth facts) + (let (facts (cons (cons predicate truth) facts)) + (case (car predicate) + ((not) (select (cadr predicate) (not truth) facts)) + ((and or) + (if (eq? truth (eq? (car predicate) 'and)) + (foldl (lambda (part selected) (select part truth selected)) facts (cdr predicate)) facts)) + (else facts)))) + (def (invalidate forms facts) + (let (effect (fold-statement-analysis forms)) + (if (any (cut memq <> '(opaque helper bounds)) (vector-ref effect 2)) '() + (filter (lambda (fact) + (not (any (cut memq <> (vector-ref effect 1)) + (vector-ref (fold-expression-analysis (car fact)) 0)))) facts)))) + (def (common left right) (filter (cut member <> right) left)) + (def (walk forms facts) + (let loop ((rest forms) (facts facts) (out '())) + (if (null? rest) (vector (reverse out) facts) + (let (form (car rest)) + (if (eq? (car form) 'if) + (let* ((predicate (cadr form)) (pure (pure? predicate)) + (value (and pure (known predicate facts)))) + (if value + (let (chosen (walk (if (cadr value) (caddr form) (cadddr form)) facts)) + (loop (cdr rest) (vector-ref chosen 1) + (append (reverse (vector-ref chosen 0)) out))) + (let* ((basis (if pure facts '())) + (yes (walk (caddr form) (if pure (select predicate #t basis) basis))) + (no (walk (cadddr form) (if pure (select predicate #f basis) basis)))) + (loop (cdr rest) (common (vector-ref yes 1) (vector-ref no 1)) + (cons (list 'if predicate (vector-ref yes 0) (vector-ref no 0)) out))))) + ;; Loops, lexical joins and helper/frame handoffs remain opaque + ;; here. No selector truth is carried into another iteration. + (loop (cdr rest) (invalidate (list form) facts) (cons form out))))))) + (vector-ref (walk forms '()) 0)) diff --git a/src/compiler/event-fold-future.ss b/src/compiler/event-fold-future.ss new file mode 100644 index 00000000..c8a59b61 --- /dev/null +++ b/src/compiler/event-fold-future.ss @@ -0,0 +1,337 @@ +;;; -*- Gerbil -*- +;;; Source-backed future-line queries for Scheme event folds. +(import (only-in ./funcs.ss ascii-lower-byte + compiler-position-index-add! + compiler-position-index-freeze! + compiler-position-vector-next + compiler-u8vector-prefix?)) +(export ascii-lower-byte current-future-scan-cache + fold-future-cache-for fold-future-index-for + fold-future-named-index fold-future-named-index-result + fold-future-heading-title? fold-future-heading-spec? + fold-future-marker-before-boundary? fold-source-slices-equal?) + +;; Cache suffix answers for one parse request; dynamic parameters keep nested +;; and concurrent parses independent. +(def current-future-scan-cache (make-parameter #f)) + +(def (fold-future-line-end bytes start) + (let (size (u8vector-length bytes)) + (let loop ((cursor start)) + (if (or (= cursor size) (memv (u8vector-ref bytes cursor) '(10 13))) + (if (= cursor size) cursor + (if (and (= (u8vector-ref bytes cursor) 13) + (< (+ cursor 1) size) + (= (u8vector-ref bytes (+ cursor 1)) 10)) + (+ cursor 2) (+ cursor 1))) + (loop (+ cursor 1)))))) + +(def (fold-future-marker-line? bytes start end prefix indent?) + (let* ((begin (if indent? + (let skip ((cursor start)) + (if (and (< cursor end) + (memv (u8vector-ref bytes cursor) '(9 32))) + (skip (+ cursor 1)) cursor)) + start)) + (prefix-end (+ begin (u8vector-length prefix)))) + (and (<= prefix-end end) + (let match ((cursor begin) (index 0)) + (or (= index (u8vector-length prefix)) + (and (= (ascii-lower-byte (u8vector-ref bytes cursor)) + (ascii-lower-byte (u8vector-ref prefix index))) + (match (+ cursor 1) (+ index 1))))) + (let tail ((cursor prefix-end)) + (or (= cursor end) + (and (memv (u8vector-ref bytes cursor) '(9 10 13 32)) + (tail (+ cursor 1)))))))) + +(def (fold-source-slices-equal? bytes left-from left-until + right-from right-until ascii-ci?) + (let (size (u8vector-length bytes)) + (and (<= 0 left-from left-until size) + (<= 0 right-from right-until size) + (= (- left-until left-from) (- right-until right-from)) + (let loop ((left left-from) (right right-from)) + (or (= left left-until) + (and (let ((a (u8vector-ref bytes left)) + (b (u8vector-ref bytes right))) + (= (if ascii-ci? (ascii-lower-byte a) a) + (if ascii-ci? (ascii-lower-byte b) b))) + (loop (+ left 1) (+ right 1)))))))) + +(def (fold-future-cache-for query) + (let* ((request-cache + (or (current-future-scan-cache) + (let (created (make-hash-table)) + (current-future-scan-cache created) + created))) + (cached (hash-get request-cache query))) + (or cached + (let (created (make-hash-table)) + (hash-put! request-cache query created) + created)))) + +(def (fold-future-index-for query build) + (let* ((request-cache + (or (current-future-scan-cache) + (let (created (make-hash-table)) + (current-future-scan-cache created) + created))) + (cached (hash-get request-cache query))) + (or cached + (let (index (build)) + (hash-put! request-cache query index) + index)))) + +(def (fold-normalize-marker bytes from until indent? ascii-ci?) + (let* ((start (if indent? + (let skip ((cursor from)) + (if (and (< cursor until) + (memv (u8vector-ref bytes cursor) '(9 32))) + (skip (+ cursor 1)) cursor)) + from)) + (end (let trim ((cursor until)) + (if (and (> cursor start) + (memv (u8vector-ref bytes (- cursor 1)) + '(9 10 13 32))) + (trim (- cursor 1)) cursor))) + (key (subu8vector bytes start end))) + (when ascii-ci? + (let loop ((index 0)) + (when (< index (u8vector-length key)) + (u8vector-set! key index + (ascii-lower-byte (u8vector-ref key index))) + (loop (+ index 1))))) + key)) + +(def (fold-named-marker-key bytes from until prefix suffix ascii-ci?) + (and (exact-integer? from) (exact-integer? until) + (<= 0 from) (< from until) (<= until (u8vector-length bytes)) + (let (key + (u8vector-append (string->utf8 prefix) + (subu8vector bytes from until) + (string->utf8 suffix))) + (fold-normalize-marker key 0 (u8vector-length key) #f ascii-ci?)))) + +;; The map keys are complete normalized marker lines. This preserves the +;; existing prefix/name/suffix and whitespace rules while indexing candidate +;; names in one source scan, including documents with a different name per opener. +(def (fold-future-named-index bytes target-prefix parent-prefix + stop heading-marker heading-separator + indent? stop-at-heading? ascii-ci? + parent? stop-ascii-ci?) + (let ((targets (make-hash-table)) + (parents (if (and parent? (not (eq? ascii-ci? stop-ascii-ci?))) + (make-hash-table) + #f)) + (target-keys '()) + (parent-keys '()) + (fixed-stops '()) + (headings '()) + (target-prefix-bytes (string->utf8 target-prefix)) + (stop-bytes (and stop (string->utf8 stop))) + (parent-prefix-bytes (and parent? (string->utf8 parent-prefix)))) + (let scan ((cursor 0)) + (when (< cursor (u8vector-length bytes)) + (let* ((end (fold-future-line-end bytes cursor)) + (begin (if indent? + (let skip ((at cursor)) + (if (and (< at end) + (memv (u8vector-ref bytes at) '(9 32))) + (skip (+ at 1)) at)) + cursor)) + (target-candidate? + (compiler-u8vector-prefix? + bytes begin end target-prefix-bytes ascii-ci?)) + (parent-candidate? + (and parent? + (compiler-u8vector-prefix? + bytes begin end parent-prefix-bytes stop-ascii-ci?)))) + (when (or target-candidate? (and (not parents) parent-candidate?)) + (set! target-keys + (compiler-position-index-add! + targets target-keys + (fold-normalize-marker bytes cursor end indent? ascii-ci?) + cursor))) + (when (and parents parent-candidate?) + (set! parent-keys + (compiler-position-index-add! + parents parent-keys + (fold-normalize-marker bytes cursor end indent? + stop-ascii-ci?) + cursor))) + (when (and stop-bytes + (fold-future-marker-line? bytes cursor end stop-bytes indent?)) + (set! fixed-stops (cons cursor fixed-stops))) + (when (and stop-at-heading? + (fold-future-heading? bytes cursor end + heading-marker heading-separator)) + (set! headings (cons cursor headings))) + (scan end)))) + (compiler-position-index-freeze! targets target-keys) + (when parents + (compiler-position-index-freeze! parents parent-keys)) + (vector targets (or parents targets) (list->vector fixed-stops) + (list->vector headings)))) + +;; Position vectors are descending because the source scan conses positions. +(def (fold-earlier-position left right) + (cond + ((not left) right) + ((not right) left) + (else (min left right)))) + +(def (fold-future-named-index-result index bytes from + name-from name-until prefix suffix + ascii-ci? stop-name-from stop-name-until + stop-prefix stop-suffix stop-ascii-ci?) + (let* ((target-key + (fold-named-marker-key bytes name-from name-until + prefix suffix ascii-ci?)) + (parent-key + (and stop-name-from stop-name-until + (fold-named-marker-key bytes stop-name-from stop-name-until + stop-prefix stop-suffix stop-ascii-ci?))) + (target (and target-key + (compiler-position-vector-next + (hash-get (vector-ref index 0) target-key) from))) + (parent (and parent-key + (compiler-position-vector-next + (hash-get (vector-ref index 1) parent-key) + from))) + (boundary + (fold-earlier-position + parent + (fold-earlier-position + (compiler-position-vector-next (vector-ref index 2) from) + (compiler-position-vector-next (vector-ref index 3) from))))) + (and target (or (not boundary) (< target boundary))))) + +;; Each searched line receives the suffix result, so later queries start from +;; a known answer instead of scanning the same suffix again. +(def (fold-future-line-search bytes from cache classify) + (def (finish result visited) + (for-each (lambda (position) + (hash-put! cache position (if result 1 2))) + visited) + result) + (let search ((cursor from) (visited '())) + (cond + ((>= cursor (u8vector-length bytes)) (finish #f visited)) + ((hash-get cache cursor) => + (lambda (answer) (finish (= answer 1) visited))) + (else + (let* ((end (fold-future-line-end bytes cursor)) + (decision (classify cursor end)) + (visited (cons cursor visited))) + (case decision + ((match) (finish #t visited)) + ((stop) (finish #f visited)) + (else (search end visited)))))))) + +(def (fold-future-heading? bytes start end marker separator) + (let run ((cursor start)) + (and (< cursor end) + (if (= (u8vector-ref bytes cursor) marker) + (run (+ cursor 1)) + (and (> cursor start) + (= (u8vector-ref bytes cursor) separator)))))) + +(def (fold-heading-title-line? bytes start end marker separator + min-level title-bytes) + (let (level + (let count ((cursor start)) + (if (and (< cursor end) + (= (u8vector-ref bytes cursor) marker)) + (count (+ cursor 1)) (- cursor start)))) + (and (>= level min-level) + (< (+ start level) end) + (= (u8vector-ref bytes (+ start level)) separator) + (let* ((begin + (let skip ((cursor (+ start level 1))) + (if (and (< cursor end) + (memv (u8vector-ref bytes cursor) '(9 32))) + (skip (+ cursor 1)) cursor))) + (until + (let trim ((cursor end)) + (if (and (> cursor begin) + (memv (u8vector-ref bytes (- cursor 1)) + '(9 10 13 32))) + (trim (- cursor 1)) cursor)))) + (and (= (- until begin) (u8vector-length title-bytes)) + (let compare ((cursor begin) (index 0)) + (or (= index (u8vector-length title-bytes)) + (and (= (u8vector-ref bytes cursor) + (u8vector-ref title-bytes index)) + (compare (+ cursor 1) (+ index 1)))))))))) + +(def (fold-future-heading-title? bytes from marker separator min-level title + cache) + (let (title-bytes (string->utf8 title)) + (fold-future-line-search + bytes from cache + (lambda (start end) + (if (fold-heading-title-line? bytes start end marker separator + min-level title-bytes) + 'match 'continue))))) + +(def (fold-future-heading-spec? expression) + (and (= (length expression) 5) + (string? (cadr expression)) + (string? (caddr expression)) + (not (equal? (cadr expression) (caddr expression))) + (or (pair? (list-ref expression 3)) + (and (exact-integer? (list-ref expression 3)) + (> (list-ref expression 3) 0))) + (string? (list-ref expression 4)) + (> (string-length (list-ref expression 4)) 0) + (not (ormap (lambda (character) + (memv character '(#\newline #\return))) + (string->list (list-ref expression 4)))))) + +(def (fold-future-key-value-line? bytes start end marker) + (let skip ((cursor start)) + (if (and (< cursor end) (memv (u8vector-ref bytes cursor) '(9 32))) + (skip (+ cursor 1)) + (and (< cursor end) (= (u8vector-ref bytes cursor) marker) + (let key ((cursor (+ cursor 1)) (key-start (+ cursor 1))) + (and (< cursor end) + (let (byte (u8vector-ref bytes cursor)) + (cond + ((= byte marker) + (if (or (= (+ cursor 1) end) + (memv (u8vector-ref bytes (+ cursor 1)) + '(9 10 13 32))) + (> cursor key-start) + (key (+ cursor 1) key-start))) + ((memv byte '(9 10 13 32)) #f) + (else (key (+ cursor 1) key-start)))))))))) + +(def (fold-future-marker-before-boundary? bytes from target stop + heading-marker heading-separator + indent? stop-at-heading? body-key-marker + cache) + ;; Encode query markers once, not once per searched source line. Cached + ;; answers and terminal positions allocate no marker buffers at all. + (cond + ((>= from (u8vector-length bytes)) #f) + ((hash-get cache from) => (lambda (answer) (= answer 1))) + (else + ;; Generated products pass private constant byte literals; dynamic callers + ;; retain the checked text route. No request buffer or cache is shared. + (let ((target-bytes (if (u8vector? target) target (string->utf8 target))) + (stop-bytes (and stop (if (u8vector? stop) stop (string->utf8 stop))))) + (fold-future-line-search + bytes from cache + (lambda (cursor end) + (cond + ((and stop-at-heading? + (fold-future-heading? bytes cursor end + heading-marker heading-separator)) 'stop) + ((and stop-bytes + (fold-future-marker-line? bytes cursor end stop-bytes indent?)) 'stop) + ((fold-future-marker-line? bytes cursor end target-bytes indent?) 'match) + ((and body-key-marker + (not (fold-future-key-value-line? bytes cursor end + body-key-marker))) 'stop) + (else 'continue)))))))) diff --git a/src/compiler/event-fold-ir.ss b/src/compiler/event-fold-ir.ss new file mode 100644 index 00000000..066b1077 --- /dev/null +++ b/src/compiler/event-fold-ir.ss @@ -0,0 +1,504 @@ +;;; -*- Gerbil -*- +;;; Serialize the Scheme event fold as typed, source-owned AOT IR. +(import (only-in :std/encoding/json json->string) + (only-in ../language/descriptor language-grammar-machine) + (only-in ../runtime/identity sha256-text) + (only-in ./machine parser-machine-grammar-digest) + (only-in ./event-strategy-aot kind-index) + (only-in ./event-fold-future.ss fold-future-heading-spec?) + (only-in ./event-fold-runtime.ss + fold-ascii-prefix? fold-frame-finishes + fold-initial-states fold-marker-byte + fold-offset-ir fold-state-of-type fold-static-name-set fold-uint-ir + fold-join-marks-handled? + fold-unsigned? rust-state-name validate-state-names)) +(export event-fold-ir event-fold-ir-json) +(def (fold-predicate-ir expression states indices allow-line?) + (unless (pair? expression) + (error "event fold predicate must be structured" expression)) + (case (car expression) + ((bool) + (unless (and (= (length expression) 2) (boolean? (cadr expression))) + (error "invalid event fold boolean" expression)) + (hash ("kind" "bool") ("value" (cadr expression)))) + ((state) + (fold-state-of-type states (cadr expression) boolean?) + (hash ("kind" "state") ("name" (rust-state-name (cadr expression))))) + ((uint-positive?) + (hash ("kind" "usize_positive") + ("value" (fold-uint-ir (cadr expression) states indices allow-line?)))) + ((uint-equal?) + (unless (= (length expression) 3) + (error "invalid event fold unsigned equality" expression)) + (hash ("kind" "usize_equal") + ("left" (fold-uint-ir (cadr expression) states indices allow-line?)) + ("right" (fold-uint-ir (caddr expression) states indices allow-line?)))) + ((uint-not-equal?) + (unless (= (length expression) 3) + (error "invalid event fold unsigned inequality" expression)) + (hash ("kind" "usize_not_equal") + ("left" (fold-uint-ir (cadr expression) states indices allow-line?)) + ("right" (fold-uint-ir (caddr expression) states indices allow-line?)))) + ((uint-greater?) + (unless (= (length expression) 3) + (error "invalid event fold unsigned comparison" expression)) + (hash ("kind" "usize_greater") + ("left" (fold-uint-ir (cadr expression) states indices allow-line?)) + ("right" (fold-uint-ir (caddr expression) states indices allow-line?)))) + ((offset-less?) + (unless allow-line? + (error "event fold final transition has no source line" expression)) + (unless (= (length expression) 3) + (error "invalid event fold offset comparison" expression)) + (hash ("kind" "offset_less") + ("left" (fold-offset-ir (cadr expression) states indices)) + ("right" (fold-offset-ir (caddr expression) states indices)))) + ((stack-nonempty?) + (unless (= (length expression) 2) + (error "invalid event fold stack predicate" expression)) + (fold-state-of-type states (cadr expression) list?) + (hash ("kind" "stack_nonempty") + ("stack" (rust-state-name (cadr expression))))) + ((line-starts-with) + (unless allow-line? + (error "event fold final transition has no source line" expression)) + (unless (and (= (length expression) 2) (string? (cadr expression))) + (error "invalid event fold prefix" expression)) + (hash ("kind" "line_starts_with") ("value" (cadr expression)))) + ((line-starts-with-ascii-ci) + (unless allow-line? + (error "event fold final transition has no source line" expression)) + (unless (and (= (length expression) 2) (string? (cadr expression)) + (every (lambda (character) (< (char->integer character) 128)) + (string->list (cadr expression)))) + (error "invalid event fold ASCII prefix" expression)) + (hash ("kind" "line_starts_with_ascii_case_insensitive") + ("value" (cadr expression)))) + ((line-prefix-boundary-ascii-ci line-marker-ascii-ci) + (unless allow-line? + (error "event fold final transition has no source line" expression)) + (unless (and (= (length expression) 2) + (fold-ascii-prefix? (cadr expression))) + (error "invalid event fold ASCII line marker" expression)) + (hash ("kind" (if (eq? (car expression) 'line-marker-ascii-ci) + "line_marker_ascii_case_insensitive" + "line_prefix_boundary_ascii_case_insensitive")) + ("value" (cadr expression)))) + ((line-blank?) + (unless allow-line? + (error "event fold final transition has no source line" expression)) + (unless (= (length expression) 1) + (error "invalid event fold blank-line predicate" expression)) + (hash ("kind" "line_blank"))) + ((future-line-marker-before-boundary?) + (unless (and allow-line? (= (length expression) 8) + (fold-ascii-prefix? (cadr expression)) + (> (string-length (cadr expression)) 0) + (fold-ascii-prefix? (caddr expression)) + (boolean? (list-ref expression 5)) + (boolean? (list-ref expression 6)) + (or (equal? (list-ref expression 7) "") + (and (string? (list-ref expression 7)) + (= (string-length (list-ref expression 7)) 1)))) + (error "invalid event fold future marker search" expression)) + (hash ("kind" "future_line_marker_before_boundary") + ("target" (cadr expression)) + ("stop" (caddr expression)) + ("heading_marker" (fold-marker-byte (cadddr expression))) + ("heading_separator" (fold-marker-byte (list-ref expression 4))) + ("indent" (list-ref expression 5)) + ("stop_at_heading" (list-ref expression 6)) + ("body_key_marker" + (if (equal? (list-ref expression 7) "") 0 + (fold-marker-byte (list-ref expression 7)))))) + ((future-heading-title?) + (unless (and allow-line? (fold-future-heading-spec? expression)) + (error "invalid event fold future heading" expression)) + (hash ("kind" "future_heading_title") + ("heading_marker" (fold-marker-byte (cadr expression))) + ("heading_separator" (fold-marker-byte (caddr expression))) + ("min_level" + (fold-uint-ir + (let (level (list-ref expression 3)) + (if (pair? level) level `(uint ,level))) + states indices allow-line?)) + ("title" (list-ref expression 4)))) + ((future-named-line-marker-before-boundary?) + (unless (and allow-line? (memv (length expression) '(11 16)) + (fold-ascii-prefix? (list-ref expression 3)) + (> (string-length (list-ref expression 3)) 0) + (fold-ascii-prefix? (list-ref expression 4)) + (fold-ascii-prefix? (list-ref expression 5)) + (boolean? (list-ref expression 8)) + (boolean? (list-ref expression 9)) + (boolean? (list-ref expression 10)) + (or (= (length expression) 11) + (and (fold-ascii-prefix? (list-ref expression 13)) + (> (string-length (list-ref expression 13)) 0) + (fold-ascii-prefix? (list-ref expression 14)) + (boolean? (list-ref expression 15)) + (equal? (list-ref expression 5) "")))) + (error "invalid event fold future named marker search" expression)) + (let (payload + (hash ("kind" "future_named_line_marker_before_boundary") + ("name_from" (fold-offset-ir (cadr expression) states indices)) + ("name_until" (fold-offset-ir (caddr expression) states indices)) + ("target_prefix" (list-ref expression 3)) + ("target_suffix" (list-ref expression 4)) + ("stop" (list-ref expression 5)) + ("heading_marker" (fold-marker-byte (list-ref expression 6))) + ("heading_separator" (fold-marker-byte (list-ref expression 7))) + ("indent" (list-ref expression 8)) + ("stop_at_heading" (list-ref expression 9)) + ("ascii_case_insensitive" (list-ref expression 10)))) + (when (= (length expression) 16) + (hash-put! payload "stop_name_from" + (fold-offset-ir (list-ref expression 11) states indices)) + (hash-put! payload "stop_name_until" + (fold-offset-ir (list-ref expression 12) states indices)) + (hash-put! payload "stop_prefix" (list-ref expression 13)) + (hash-put! payload "stop_suffix" (list-ref expression 14)) + (hash-put! payload "stop_ascii_case_insensitive" + (list-ref expression 15))) + payload)) + ((line-has-word-after-prefix?) + (unless allow-line? + (error "event fold final transition has no source line" expression)) + (unless (and (= (length expression) 2) + (fold-ascii-prefix? (cadr expression))) + (error "invalid event fold word-prefix predicate" expression)) + (hash ("kind" "line_has_word_after_prefix") + ("value" (cadr expression)))) + ((line-has-key-after-prefix?) + (unless allow-line? + (error "event fold final transition has no source line" expression)) + (unless (and (= (length expression) 2) + (fold-ascii-prefix? (cadr expression))) + (error "invalid event fold key-prefix predicate" expression)) + (hash ("kind" "line_has_key_after_prefix") + ("value" (cadr expression)))) + ((line-byte-equal?) + (unless (and allow-line? (= (length expression) 3) + (exact-integer? (caddr expression)) + (<= 0 (caddr expression) 255)) + (error "invalid event fold byte predicate" expression)) + (hash ("kind" "line_byte_equal") + ("at" (fold-offset-ir (cadr expression) states indices)) + ("value" (caddr expression)))) + ((line-bytes-all-in? line-bytes-any-in?) + (unless (and allow-line? (= (length expression) 4) + (list? (cadddr expression)) + (every (lambda (byte) (and (exact-integer? byte) + (<= 0 byte 255))) + (cadddr expression))) + (error "invalid event fold byte-set predicate" expression)) + (hash ("kind" (if (eq? (car expression) 'line-bytes-all-in?) + "line_bytes_all_in" "line_bytes_any_in")) + ("from" (fold-offset-ir (cadr expression) states indices)) + ("until" (fold-offset-ir (caddr expression) states indices)) + ("values" (list->vector (cadddr expression))))) + ((line-bytes-in-set?) + (unless (and allow-line? (= (length expression) 4)) + (error "invalid event fold name-set predicate" expression)) + (hash ("kind" "line_bytes_in_set") + ("from" (fold-offset-ir (cadr expression) states indices)) + ("until" (fold-offset-ir (caddr expression) states indices)) + ("values" (list->vector + (fold-static-name-set (cadddr expression)))))) + ((source-slices-equal? source-slices-equal-ascii-ci?) + (unless (and allow-line? (= (length expression) 5)) + (error "invalid event fold source-slice comparison" expression)) + (hash ("kind" "source_slices_equal") + ("left_from" (fold-offset-ir (cadr expression) states indices)) + ("left_until" (fold-offset-ir (caddr expression) states indices)) + ("right_from" (fold-offset-ir (cadddr expression) states indices)) + ("right_until" + (fold-offset-ir (list-ref expression 4) states indices)) + ("ascii_case_insensitive" + (eq? (car expression) 'source-slices-equal-ascii-ci?)))) + ((not) + (hash ("kind" "not") + ("value" (fold-predicate-ir (cadr expression) states indices allow-line?)))) + ((and or) + (unless (>= (length expression) 3) + (error "event fold boolean combination needs two predicates" expression)) + (let ((kind (symbol->string (car expression))) + (parts (map (lambda (part) + (fold-predicate-ir part states indices allow-line?)) + (cdr expression)))) + (foldl (lambda (left right) + (hash ("kind" kind) ("left" left) ("right" right))) + (car parts) (cdr parts)))) + (else (error "unsupported event fold predicate" expression)))) + +(def (fold-final-token-offset? expression) + (and (pair? expression) + (eq? (car expression) (quote state-offset)) + (= (length expression) 2))) + +(def (fold-statements-ir grammar statements states indices allow-line? (helpers '())) + (list->vector + (map + (lambda (form) + (case (car form) + ((set-bool) + (fold-state-of-type states (cadr form) boolean?) + (hash ("kind" "set_bool") ("name" (rust-state-name (cadr form))) + ("value" (fold-predicate-ir (caddr form) states indices allow-line?)))) + ((set-uint) + (fold-state-of-type states (cadr form) fold-unsigned?) + (hash ("kind" "set_usize") ("name" (rust-state-name (cadr form))) + ("value" (fold-uint-ir (caddr form) states indices allow-line?)))) + ((close-through) + (fold-state-of-type states (cadr form) list?) + (hash ("kind" "close_through_level") + ("stack" (rust-state-name (cadr form))) + ("level" (fold-uint-ir (caddr form) states indices allow-line?)))) + ((open-level) + (fold-state-of-type states (cadr form) list?) + (hash ("kind" "open_level") + ("stack" (rust-state-name (cadr form))) + ("level" (fold-uint-ir (caddr form) states indices allow-line?)) + ("syntax_kind" (kind-index grammar (cadddr form) 'node)))) + ((close-all) + (fold-state-of-type states (cadr form) list?) + (hash ("kind" "close_all_levels") + ("stack" (rust-state-name (cadr form))))) + ((start-node) + (hash ("kind" "start_node") + ("syntax_kind" (kind-index grammar (cadr form) 'node)))) + ((token) + (unless (= (length form) 4) + (error "event fold token requires source offsets" form)) + (unless (or allow-line? + (and (fold-final-token-offset? (caddr form)) + (fold-final-token-offset? (cadddr form)))) + (error "event fold final token requires saved source offsets" form)) + (hash ("kind" "token") + ("syntax_kind" (kind-index grammar (cadr form) 'token)) + ("start" (fold-offset-ir (caddr form) states indices)) + ("end" (fold-offset-ir (cadddr form) states indices)))) + ((finish-node) (hash ("kind" "finish_node"))) + ((if) + (hash ("kind" "if") + ("condition" (fold-predicate-ir (cadr form) states indices allow-line?)) + ("consequent" (fold-statements-ir grammar (caddr form) states indices + allow-line? helpers)) + ("alternate" (fold-statements-ir grammar (cadddr form) states indices + allow-line? helpers)))) + ((join-once) + (unless (and (= (length form) 4) (symbol? (cadr form)) + (not (assq (cadr form) states)) + (not (memq (cadr form) indices)) + (pair? (caddr form)) (pair? (cadddr form)) + (fold-join-marks-handled? (caddr form) (cadr form))) + (error "invalid event fold join" form)) + (let (local-states (cons (cons (cadr form) #f) states)) + (hash ("kind" "join_once") + ("handled" (rust-state-name (cadr form))) + ("branches" (fold-statements-ir grammar (caddr form) + local-states indices + allow-line? helpers)) + ("continuation" (fold-statements-ir grammar (cadddr form) + local-states indices + allow-line? helpers))))) + ((for-line-bytes) + (unless (and allow-line? (= (length form) 5) + (symbol? (cadr form)) + (not (assq (cadr form) states)) + (not (memq (cadr form) indices))) + (error "invalid event fold source-line iteration" form)) + (hash ("kind" "for_line_bytes") + ("index" (rust-state-name (cadr form))) + ("from" (fold-offset-ir (caddr form) states indices)) + ("until" (fold-offset-ir (cadddr form) states indices)) + ("body" (fold-statements-ir grammar (list-ref form 4) states + (cons (cadr form) indices) + allow-line? helpers)))) + ((with-source-bounds) + (unless (and (= (length form) 4) + (or allow-line? + (and (pair? (cadr form)) + (eq? (caadr form) 'state-offset) + (pair? (caddr form)) + (eq? (caaddr form) 'state-offset)))) + (error "event fold final bounds require saved source offsets" form)) + (hash ("kind" "with_source_bounds") + ("from" (fold-offset-ir (cadr form) states indices)) + ("until" (fold-offset-ir (caddr form) states indices)) + ("body" (fold-statements-ir grammar (cadddr form) + states indices #t helpers)))) + ((call-source-helper) + (unless (and (memv (length form) '(4 5)) + (symbol? (cadr form)) + (assq (cadr form) helpers) + (or allow-line? + (and (pair? (caddr form)) + (eq? (caaddr form) 'state-offset) + (pair? (cadddr form)) + (eq? (car (cadddr form)) 'state-offset)))) + (error "unknown or invalid event helper call" form)) + (let* ((helper (assq (cadr form) helpers)) + (parameters (if (= (length helper) 4) (cadddr helper) '())) + (arguments (if (= (length form) 5) (list-ref form 4) '()))) + (unless (and (list? arguments) + (= (length parameters) (length arguments))) + (error "event helper argument arity" form)) + (hash ("kind" "call_source_helper") + ("name" (rust-state-name (cadr form))) + ("from" (fold-offset-ir (caddr form) states indices)) + ("until" (fold-offset-ir (cadddr form) states indices)) + ("arguments" + (list->vector + (map (lambda (argument) + (fold-uint-ir argument states indices allow-line?)) + arguments)))))) + ((scan-list-marker) + (unless (and allow-line? (= (length form) 10) + (string? (cadr form)) + (boolean? (caddr form)) + (exact-integer? (cadddr form)) + (> (cadddr form) 0)) + (error "invalid event fold list marker statement" form)) + (fold-state-of-type states (list-ref form 4) boolean?) + (fold-state-of-type states (list-ref form 5) fold-unsigned?) + (fold-state-of-type states (list-ref form 6) boolean?) + (for-each (lambda (name) + (fold-state-of-type states name fold-unsigned?)) + (list (list-ref form 7) (list-ref form 8) (list-ref form 9))) + (hash ("kind" "scan_list_marker") + ("marker" + (hash ("unordered" (cadr form)) + ("ordered" (caddr form)) + ("tab_width" (cadddr form)) + ("present" (rust-state-name (list-ref form 4))) + ("column" (rust-state-name (list-ref form 5))) + ("ordered_slot" (rust-state-name (list-ref form 6))) + ("bullet_start" (rust-state-name (list-ref form 7))) + ("bullet_end" (rust-state-name (list-ref form 8))) + ("content_start" (rust-state-name (list-ref form 9))))))) + ((push-frame) + (fold-state-of-type states (cadr form) list?) + (hash ("kind" "push_frame") + ("stack" (rust-state-name (cadr form))) + ("value" (fold-uint-ir (caddr form) states indices allow-line?)))) + ((pop-frame) + (unless (= (length form) 2) + (error "invalid event fold frame pop" form)) + (fold-state-of-type states (cadr form) list?) + (hash ("kind" "pop_frame") + ("stack" (rust-state-name (cadr form))))) + ((close-frame) + (unless (= (length form) 3) + (error "invalid event fold frame close" form)) + (fold-state-of-type states (cadr form) list?) + (fold-frame-finishes (caddr form)) + (hash ("kind" "close_frame") + ("stack" (rust-state-name (cadr form))) + ("finish_count" (caddr form)))) + ((close-frames-while) + (fold-state-of-type states (cadr form) list?) + (fold-frame-finishes (list-ref form 3)) + (hash ("kind" "close_frames_while") + ("stack" (rust-state-name (cadr form))) + ("condition" (fold-predicate-ir (caddr form) states indices allow-line?)) + ("finish_count" (list-ref form 3)))) + ((close-all-frames) + (fold-state-of-type states (cadr form) list?) + (fold-frame-finishes (caddr form)) + (hash ("kind" "close_all_frames") + ("stack" (rust-state-name (cadr form))) + ("finish_count" (caddr form)))) + (else (error "unsupported event fold statement" form)))) + statements))) + +(def (fold-initial-ir initial) + (list->vector + (map (lambda (declaration) + (let ((name (rust-state-name (car declaration))) + (value (cadr declaration))) + (cond + ((boolean? value) + (hash ("kind" "let_bool") ("name" name) ("value" value))) + ((number? value) + (hash ("kind" "let_usize") ("name" name) ("value" value))) + (else (hash ("kind" "let_usize_stack") ("name" name)))))) + initial))) + +(def (event-fold-ir name grammar root initial line-forms finish-forms + (helpers '()) (parameters '())) + (let* ((states (fold-initial-states initial)) + (helper-ir + (list->vector + (map (lambda (helper) + (unless (and (memv (length helper) '(3 4)) + (symbol? (car helper))) + (error "invalid event helper declaration" helper)) + (let ((local-states (fold-initial-states (cadr helper))) + (parameters (if (= (length helper) 4) + (cadddr helper) '()))) + (validate-state-names local-states) + (unless (and (list? parameters) + (every symbol? parameters)) + (error "invalid event helper parameters" helper)) + (validate-state-names + (map (lambda (parameter) (cons parameter 0)) parameters)) + (for-each + (lambda (parameter) + (fold-state-of-type local-states parameter fold-unsigned?)) + parameters) + (hash ("name" (rust-state-name (car helper))) + ("initial" (fold-initial-ir (cadr helper))) + ("parameters" + (list->vector (map rust-state-name parameters))) + ("body" (fold-statements-ir grammar (caddr helper) + local-states '() #t helpers))))) + helpers))) + (digest + (sha256-text + (call-with-output-string + (lambda (port) + (write (append + (list 'event-fold.v1 + (parser-machine-grammar-digest + (language-grammar-machine grammar)) + name root initial line-forms finish-forms) + (if (null? helpers) '() (list helpers)) + (if (null? parameters) '() (list parameters))) port))))) + (payload + (hash ("schema" "gerbil-scheme-rust.event-function-ir.v1") + ("name" (symbol->string name)) + ("root_kind" (kind-index grammar root 'node)) + ("parser_digest" digest) + ("initial" (fold-initial-ir initial)) + ("line" (fold-statements-ir grammar line-forms states '() #t helpers)) + ("finish" (fold-statements-ir grammar finish-forms states '() #f helpers))))) + (validate-state-names states) + (validate-state-names + (map (lambda (helper) (cons (car helper) #f)) helpers)) + (when (pair? parameters) + (hash-put! payload "parameters" + (list->vector + (map (lambda (parameter) + (unless (and (= (length parameter) 3) + (symbol? (car parameter)) + (symbol? (cadr parameter)) + (fold-unsigned? (caddr parameter))) + (error "invalid event fold parameter" parameter)) + (unless (= (fold-state-of-type states (cadr parameter) + fold-unsigned?) + (caddr parameter)) + (error "event fold parameter default differs from state" + parameter)) + (hash ("name" (rust-state-name (car parameter))) + ("state" (rust-state-name (cadr parameter))) + ("default" (caddr parameter)))) + parameters)))) + (when (pair? helpers) (hash-put! payload "helpers" helper-ir)) + payload)) + +(def (event-fold-ir-json name grammar root initial line-forms finish-forms + (helpers '()) (parameters '())) + (json->string + (event-fold-ir name grammar root initial line-forms finish-forms helpers parameters) + sort-keys: #t)) diff --git a/src/compiler/event-fold-ordinary-run.ss b/src/compiler/event-fold-ordinary-run.ss new file mode 100644 index 00000000..d53642d2 --- /dev/null +++ b/src/compiler/event-fold-ordinary-run.ss @@ -0,0 +1,162 @@ +;;; Build-time admission of overwrite-only common-byte runs. No value cache. +(import (only-in :std/list/list delete-duplicates/hash) + (only-in ./event-fold-zero-state fold-zero-state-plan fold-zero-state-support) + (only-in ./event-fold-byte-dispatch fold-byte-regions) + (only-in ./event-fold-analysis fold-expression-analysis fold-statement-analysis fold-eliminate-inert-branches)) +(export fold-ordinary-run-plan fold-consumer-run-plan) + +;; A reset/increment transition is summarized in the scanner, not executed +;; against owner slots at every byte. Only an exact current-byte class and a +;; single self-dependent unsigned update are admitted. The residual body must +;; not observe or write the summarized cell; guards must not depend on it. +(def (counter-transition form cursor) + (and (eq? (car form) 'if) + (pair? (cadr form)) (eq? (caadr form) 'line-bytes-any-in?) + (equal? (cadadr form) `(line-index ,cursor)) + (equal? (caddr (cadr form)) `(line-step (line-index ,cursor))) + (= (length (caddr form)) 1) (= (length (cadddr form)) 1) + (let* ((yes (caaddr form)) (no (car (cadddr form))) + (name (and (eq? (car yes) 'set-uint) (cadr yes)))) + (and name + (equal? yes `(set-uint ,name (uint-add (state ,name) (uint 1)))) + (equal? no `(set-uint ,name (uint 0))) + (cons name (list-ref (cadr form) 3)))))) + +(def (extract-counters forms cursor) + (let ((summaries '())) + ;; Only unconditional updates can be moved out of their statement owner. + ;; Do not descend through a conditional: its execution predicate matters. + (let (body + (foldr (lambda (form rest) + (let (counter (counter-transition form cursor)) + (if counter + (begin (set! summaries (cons counter summaries)) rest) + (cons form rest)))) '() forms)) + (vector body summaries)))) + +(def (fold-consumer-run-plan forms cursor (guards '())) + ;; Derive a common interval from the complete consumer body, without an + ;; all-idle premise. Consumer selectors may stay active: their support must + ;; be disjoint from every write in the interval. Only last-byte overwrites + ;; and isolated exact counters are admitted; publication retains source order. + (let* ((cofactor (and (pair? guards) (fold-zero-state-plan forms #t guards))) + (initial (extract-counters (if cofactor (vector-ref cofactor 1) forms) cursor)) + (regions (fold-byte-regions (vector-ref initial 0) cursor))) + (and regions + (let* ((common (fold-eliminate-inert-branches (vector-ref regions 1) cursor)) + (body common) + (summaries (vector-ref initial 1)) + (writes (vector-ref (fold-statement-analysis body) 1)) + (counts (map car summaries))) + (def (total? value) + (null? (fold-eliminate-inert-branches + (list (list 'if value '() '())) cursor))) + (def (admitted? forms) + (every + (lambda (form) + (case (car form) + ((if) + (let (selector (fold-expression-analysis (cadr form))) + (and (null? (vector-ref selector 2)) + (not (any (cut memq <> (append writes counts)) + (vector-ref selector 0))) + ;; Cursor-dependent selectors can change mid-interval. + (not (contains-cursor? (cadr form))) + (admitted? (caddr form)) (admitted? (cadddr form))))) + ((set-bool set-uint) + (and (null? (vector-ref (fold-expression-analysis (caddr form)) 0)) + (total? (caddr form)))) + (else #f))) forms)) + (def (contains-cursor? value) + (and (pair? value) + (or (memq (car value) '(line-index line-step line-byte-equal? + line-bytes-any-in? line-bytes-all-in? line-bytes-in-set?)) + (any contains-cursor? (cdr value))))) + (and (= (length counts) (length (delete-duplicates/hash counts))) + (not (any (cut memq <> (append writes counts)) guards)) + (not (any (cut memq <> writes) counts)) + (not (any (cut memq <> counts) + (vector-ref (fold-statement-analysis body) 0))) + (admitted? body) + (vector guards (foldr append '() (map car (vector-ref regions 0))) + '(bool #t) body summaries)))))) + +(def (fold-ordinary-run-plan forms cursor) + (let* ((idle (fold-idle-run-plan forms cursor)) + (selected + (or (fold-consumer-run-plan forms cursor) + (and idle + ;; Remove only a consumer premise with a complete transfer witness. + ;; The remaining consumers may be active. No runtime mode cache or + ;; subset enumeration is introduced: one bounded pass over supports. + (foldl + (lambda (name plan) + (let* ((remaining (filter (lambda (cell) (not (eq? cell name))) + (vector-ref plan 0))) + (witness (fold-consumer-run-plan forms cursor remaining))) + (or witness plan))) idle (vector-ref idle 0)))))) + (if (and idle selected + (equal? (vector-ref idle 2) (vector-ref selected 2)) + (< (length (vector-ref idle 1)) (length (vector-ref selected 1))) + (every (cut memv <> (vector-ref selected 1)) (vector-ref idle 1))) + (let* ((support (filter (lambda (name) (not (memq name (vector-ref selected 0)))) + (vector-ref idle 0))) + (writes (append (vector-ref (fold-statement-analysis (vector-ref selected 3)) 1) + (map car (vector-ref selected 4))))) + (if (and (pair? support) (not (any (cut memq <> writes) support))) + ;; Select a precompiled membership table once per interval. Both + ;; transfers are admitted, and selector support is unchanged while + ;; scanning. This is not a mode cache or a second source traversal. + (vector (vector-ref selected 0) (vector-ref selected 1) + (vector-ref selected 2) (vector-ref selected 3) + (vector-ref selected 4) (vector support (vector-ref idle 1))) + selected)) selected))) + +(def (fold-idle-run-plan forms cursor) + (let* ((idle (fold-zero-state-support forms #t)) + (extracted (extract-counters (if idle (vector-ref idle 1) forms) cursor)) + (summaries (vector-ref extracted 1)) + (regions (fold-byte-regions (vector-ref extracted 0) cursor))) + (and regions + (let* ((guard-cells (if idle (vector-ref idle 0) '())) + ;; Region rejection removes unreachable effect barriers. Apply + ;; the same explicit entry facts again, not a cached state. + (refined (fold-zero-state-plan (vector-ref regions 1) #t guard-cells)) + (common (fold-eliminate-inert-branches + (if refined (vector-ref refined 1) (vector-ref regions 1)) cursor)) + ;; A skipped-prefix guard becomes permanently true after the + ;; cursor reaches its stable bound; never cross a bound write. + (prefix (and (= (length common) 1) + (eq? (caar common) 'if) + (null? (caddar common)) + (pair? (cadar common)) + (eq? (caadar common) 'offset-less?) + (equal? (cadadr (car common)) (list 'line-index cursor)) + (pair? (caddr (cadar common))) + (eq? (car (caddr (cadar common))) 'state-offset) + (cadr (caddr (cadar common))))) + (writes (if prefix (cadddr (car common)) common)) + (targets (map (lambda (form) (and (memq (car form) '(set-bool set-uint)) + (cadr form))) writes))) + (and (let ((names (map car summaries))) + (and (= (length names) (length (delete-duplicates/hash names))) + (not (any (cut memq <> guard-cells) names)) + (not (any (cut memq <> targets) names)) + (not (any (cut memq <> names) + (vector-ref (fold-statement-analysis common) 0))))) + (every (lambda (form) + (and (memq (car form) '(set-bool set-uint)) + (null? (vector-ref (fold-expression-analysis (caddr form)) 0)) + ;; Reuse the total-observation admission. Bounds + ;; derive from the validated current cursor only. + (null? (fold-eliminate-inert-branches + (list (list 'if (caddr form) '() '())) cursor)))) + writes) + (not (any (cut memq <> targets) guard-cells)) + (or (not prefix) (not (memq prefix targets))) + (vector guard-cells + (foldr append '() (map car (vector-ref regions 0))) + (if prefix + `(not (offset-less? (line-index ,cursor) (state-offset ,prefix))) + '(bool #t)) + writes summaries)))))) diff --git a/src/compiler/event-fold-prefix-family.ss b/src/compiler/event-fold-prefix-family.ss new file mode 100644 index 00000000..d54f2d3d --- /dev/null +++ b/src/compiler/event-fold-prefix-family.ss @@ -0,0 +1,58 @@ +;;; -*- Gerbil -*- +;;; Stage homogeneous pure prefix alternatives into shared byte decisions. +;;; This is a private lowering of existing admitted IR, not a new author DSL. +(export fold-scheme-prefix-family fold-scheme-negative-prefix-test) + +;; A conservative shared recognition trie. ASCII folding widens case-sensitive +;; triggers; a false result therefore rejects all original trigger predicates. +;; True retains the untouched original program, including suffixes and effects. +(def (fold-scheme-negative-prefix-test paths (cursor 'start)) + (if (null? paths) #f + (prefix-tree-code (prefix-tree (map (lambda (path) (map lower-byte path)) paths)) cursor #t))) + +(def (prefix-alternative? opcode form) + (and (list? form) (= (length form) 2) (eq? (car form) opcode) + (string? (cadr form)) (positive? (string-length (cadr form))))) + +(def (lower-byte byte) + (if (<= 65 byte 90) (+ byte 32) byte)) + +;; Build-local lists are pure. Shorter terminals absorb longer alternatives: +;; all members are total prefix predicates, with no state reads or effects. +(def (prefix-tree paths) + (if (any null? paths) + #t + (let (groups + (foldl (lambda (path groups) + (let (byte (car path)) + (cons (cons byte (cons (cdr path) (cond ((assv byte groups) => cdr) (else '())))) + (filter (lambda (entry) (not (= byte (car entry)))) groups)))) + '() paths)) + (map (lambda (entry) (cons (car entry) (prefix-tree (cdr entry)))) + (list-sort (lambda (left right) (< (car left) (car right))) groups))))) + +(def (prefix-tree-code tree cursor ascii-ci?) + (if (eq? tree #t) + #t + `(and (< ,cursor end) + (let (byte (u8vector-ref source-bytes ,cursor)) + (case ,(if ascii-ci? '(ascii-lower-byte byte) 'byte) + ,@(map (lambda (entry) + `((,(car entry)) + (let (next (+ ,cursor 1)) + ,(prefix-tree-code (cdr entry) 'next ascii-ci?)))) tree) + (else #f)))))) + +(def (fold-scheme-prefix-family alternatives) + (and (<= 4 (length alternatives) 32) + (let* ((opcode (and (pair? (car alternatives)) (caar alternatives))) + (ascii-ci? (eq? opcode 'line-starts-with-ascii-ci))) + (and (memq opcode '(line-starts-with line-starts-with-ascii-ci)) + (every (cut prefix-alternative? opcode <>) alternatives) + ;; Keep native expansion bounded. Other forms retain ordinary + ;; Boolean lowering; this is not a second runtime implementation. + (let (paths (map (lambda (form) + (let (bytes (u8vector->list (string->utf8 (cadr form)))) + (if ascii-ci? (map lower-byte bytes) bytes))) alternatives)) + (and (<= (foldl (lambda (path size) (+ size (length path))) 0 paths) 512) + (prefix-tree-code (prefix-tree paths) 'start ascii-ci?))))))) diff --git a/src/compiler/event-fold-program.ss b/src/compiler/event-fold-program.ss new file mode 100644 index 00000000..cdaae70a --- /dev/null +++ b/src/compiler/event-fold-program.ss @@ -0,0 +1,69 @@ +;;; -*- Gerbil -*- +;;; Engine-owned static strategy data, never request buffers or mutable slots. +(import (only-in ./event-fold-state-frame.ss prepare-fold-state-layout)) +(export prepare-event-fold-program fold-initial-states snapshot-fold-data event-fold-program? + event-fold-program-root event-fold-program-layout + event-fold-program-line-forms event-fold-program-finish-forms + event-fold-program-helpers + event-fold-helper-layout event-fold-helper-forms event-fold-helper-parameters + fold-helper-ref) + +(defstruct event-fold-program (root layout line-forms finish-forms helpers) final: #t) +(defstruct event-fold-helper (layout forms parameters) final: #t) + +;; Preserve DAG sharing while taking ownership of grammar data. Reject cycles +;; and executable/mutable foreign values before constructing a reusable plan. +(def (snapshot-fold-data value) + (let ((copies (make-table test: eq?)) (active (make-table test: eq?))) + (def (copy value) + (cond + ((pair? value) + (when (table-ref active value #f) (error "cyclic event fold program")) + (or (table-ref copies value #f) + (begin + (table-set! active value #t) + (let (result (cons (copy (car value)) (copy (cdr value)))) + (table-set! active value) + (table-set! copies value result) + result)))) + ((string? value) + (or (table-ref copies value #f) + (let (result (string-copy value)) + (table-set! copies value result) result))) + ((or (null? value) (symbol? value) (number? value) + (boolean? value) (char? value)) value) + (else (error "event fold program requires closed data")))) + (copy value))) + +(def (fold-initial-states initial) + (map (lambda (entry) + (unless (and (list? entry) (= (length entry) 2) (symbol? (car entry)) + (or (boolean? (cadr entry)) + (and (exact-integer? (cadr entry)) (>= (cadr entry) 0)) + (equal? (cadr entry) '(uint-stack)))) + (error "event fold requires typed state" entry)) + (cons (car entry) (if (equal? (cadr entry) '(uint-stack)) '() (cadr entry)))) + initial)) + +(def (initial-layout initial) + (prepare-fold-state-layout (fold-initial-states initial))) + +(def (prepare-event-fold-program root initial line-forms finish-forms (helpers '())) + (let* ((owned (snapshot-fold-data (list root initial line-forms finish-forms helpers))) + (directory (make-table test: eq?))) + (for-each + (lambda (helper) + (unless (and (list? helper) (memv (length helper) '(3 4)) + (symbol? (car helper))) + (error "invalid event fold helper" helper)) + ;; Match the existing first-declaration lookup policy. + (unless (table-ref directory (car helper) #f) + (table-set! directory (car helper) + (make-event-fold-helper (initial-layout (cadr helper)) (caddr helper) + (if (= (length helper) 4) (cadddr helper) '()))))) + (list-ref owned 4)) + (make-event-fold-program (car owned) (initial-layout (cadr owned)) + (caddr owned) (cadddr owned) directory))) + +(def (fold-helper-ref helpers name) + (table-ref helpers name #f)) diff --git a/src/compiler/event-fold-runtime.ss b/src/compiler/event-fold-runtime.ss new file mode 100644 index 00000000..e9f6a4c6 --- /dev/null +++ b/src/compiler/event-fold-runtime.ss @@ -0,0 +1,764 @@ +;;; -*- Gerbil -*- +;;; Execute closed event folds over source bytes; shared typed admission for AOT. +(import (only-in :std/string/utf8 utf8->string string-utf8-length) + (only-in ../runtime/event-fold-lines + current-fold-line-view fold-line-bytes fold-key-byte? fold-ascii-prefix? + line-starts-with-ascii-ci? fold-line-prefix-boundary? fold-line-blank? + fold-line-has-key-after-prefix? fold-line-has-word-after-prefix?) + (only-in ./event-fold-byte-set.ss + current-fold-byte-set-cache fold-byte-set-for fold-marker-byte) + (only-in ./event-fold-state-frame.ss + instantiate-fold-state-frame fold-state-frame? + fold-frame-ref fold-frame-update! fold-frame-bound? + fold-frame-bind! fold-frame-unbind!) + (only-in ./event-fold-program.ss + prepare-event-fold-program fold-initial-states event-fold-program? + event-fold-program-root event-fold-program-layout + event-fold-program-line-forms event-fold-program-finish-forms + event-fold-program-helpers fold-helper-ref + event-fold-helper-layout event-fold-helper-forms + event-fold-helper-parameters) + (only-in ./event-list-marker.ss + event-line-indent-column scan-event-list-marker) + (only-in ./event-fold-future.ss + ascii-lower-byte current-future-scan-cache + fold-future-cache-for fold-future-index-for + fold-future-named-index fold-future-named-index-result + fold-future-heading-title? fold-future-heading-spec? + fold-future-marker-before-boundary? + fold-source-slices-equal?) + (only-in ../runtime/source-lines fold-source-lines line-starts-with?)) +(export prepare-event-fold-program run-event-fold-program + run-event-fold fold-ascii-prefix? fold-frame-finishes + fold-initial-states fold-marker-byte + fold-offset-ir fold-state-of-type fold-static-name-set fold-uint-ir + fold-unsigned? fold-join-marks-handled? rust-state-name validate-state-names + current-fold-line-view fold-line-bytes fold-key-byte? + line-starts-with-ascii-ci? fold-line-prefix-boundary? fold-line-blank? + fold-line-has-word-after-prefix? fold-line-has-key-after-prefix?) + +(def (fold-join-marks-handled? forms handled) + (any (lambda (form) + (and (pair? form) + (case (car form) + ((set-bool) + (and (= (length form) 3) (eq? (cadr form) handled) + (equal? (caddr form) '(bool #t)))) + ((if) + (and (= (length form) 4) + (or (fold-join-marks-handled? (caddr form) handled) + (fold-join-marks-handled? (cadddr form) handled)))) + ((for-line-bytes) + (and (= (length form) 5) + (fold-join-marks-handled? (list-ref form 4) handled))) + ((with-source-bounds) + (and (= (length form) 4) + (fold-join-marks-handled? (cadddr form) handled))) + (else #f)))) + forms)) +(def (fold-state states name) + (if (fold-state-frame? states) + (fold-frame-ref states name) + (let (entry (assq name states)) + (unless entry (error "undeclared event fold state" name)) + (cdr entry)))) + +(def (rust-state-name name) + (list->string + (map (lambda (character) + (if (char=? character #\-) #\_ character)) + (string->list (symbol->string name))))) + +(def (validate-state-names states) + (let loop ((rest states) (seen [])) + (unless (null? rest) + (let (name (rust-state-name (caar rest))) + (when (member name seen) + (error "duplicate event fold state" name)) + (loop (cdr rest) (cons name seen)))))) + +(def (fold-state-of-type states name predicate) + (let (value (fold-state states name)) + (unless (predicate value) (error "event fold state has wrong type" name)) + value)) + +(def (fold-static-name-set values) + (unless (and (pair? values) + (every (lambda (value) + (and (string? value) + (> (string-length value) 0) + (every (lambda (character) + (< (char->integer character) 128)) + (string->list value)))) + values) + (let loop ((rest values)) + (or (null? (cdr rest)) + (and (string= value 0))) + +(def (fold-offset expression line start end states indices) + (cond + ((eq? expression 'start) start) + ((eq? expression 'end) end) + ((not (pair? expression)) + (error "invalid event fold source offset" expression)) + (else + (case (car expression) + ((line-prefix-end) + (min end (+ start (string-utf8-length (cadr expression))))) + ((line-marker-end) + (+ start (fold-uint (cons 'line-marker-level (cdr expression)) + line start end states indices))) + ((line-skip-horizontal line-scan-word line-scan-key) + (let* ((bytes (fold-line-bytes line)) + (cursor (fold-offset (cadr expression) line start end states indices))) + (let loop ((offset cursor)) + (if (and (< offset end) + ((if (eq? (car expression) 'line-skip-horizontal) + (lambda (byte) (memv byte '(9 32))) + (if (eq? (car expression) 'line-scan-key) + fold-key-byte? + (lambda (byte) (not (memv byte '(9 10 13 32)))))) + (u8vector-ref bytes (- offset start)))) + (loop (+ offset 1)) offset)))) + ((line-scan-nonspace-until line-scan-until) + (unless (= (length expression) 3) + (error "invalid event fold bounded delimiter scan" expression)) + (let* ((bytes (fold-line-bytes line)) + (cursor (fold-offset (cadr expression) line start end states indices)) + (delimiter (fold-marker-byte (caddr expression)))) + (let loop ((offset cursor)) + (if (and (< offset end) + (not (= (u8vector-ref bytes (- offset start)) delimiter)) + (or (eq? (car expression) 'line-scan-until) + (not (memv (u8vector-ref bytes (- offset start)) + '(9 10 13 32))))) + (loop (+ offset 1)) offset)))) + ((line-physical-end) + (unless (= (length expression) 2) + (error "invalid event fold physical-line scan" expression)) + (let* ((bytes (fold-line-bytes line)) + (from (fold-offset (cadr expression) line start end states indices))) + (let loop ((cursor (max start (min from end)))) + (if (and (< cursor end) + (not (memv (u8vector-ref bytes (- cursor start)) '(10 13)))) + (loop (+ cursor 1)) cursor)))) + ((line-step) + (min end (+ 1 (fold-offset (cadr expression) line start end states indices)))) + ((line-trim-end) + (let (bytes (fold-line-bytes line)) + (let loop ((cursor end)) + (if (and (> cursor start) + (memv (u8vector-ref bytes (- cursor start 1)) + '(9 10 11 12 13 32))) + (loop (- cursor 1)) cursor)))) + ((line-trim-end-from) + (let ((bytes (fold-line-bytes line)) + (floor (fold-offset (cadr expression) line start end states indices))) + (let loop ((cursor end)) + (if (and (> cursor floor) + (memv (u8vector-ref bytes (- cursor start 1)) + '(9 10 11 12 13 32))) + (loop (- cursor 1)) cursor)))) + ((line-content-end) + (let (bytes (fold-line-bytes line)) + (let loop ((cursor end)) + (if (and (> cursor start) + (memv (u8vector-ref bytes (- cursor start 1)) '(10 13))) + (loop (- cursor 1)) cursor)))) + ((line-index) + (let (entry (assq (cadr expression) indices)) + (unless entry (error "unbound event fold line index" expression)) + (cdr entry))) + ((state-offset) + (fold-state-of-type states (cadr expression) fold-unsigned?)) + (else (error "unsupported event fold source offset" expression)))))) + + +(def (fold-offset-ir expression states indices) + (cond + ((memq expression '(start end)) (symbol->string expression)) + ((not (pair? expression)) + (error "invalid event fold source offset" expression)) + (else + (case (car expression) + ((line-prefix-end) + (unless (and (= (length expression) 2) + (fold-ascii-prefix? (cadr expression))) + (error "invalid event fold line prefix offset" expression)) + (hash ("kind" "line_prefix_end") ("value" (cadr expression)))) + ((line-marker-end) + (unless (= (length expression) 3) + (error "invalid event fold line marker offset" expression)) + (hash ("kind" "line_marker_end") + ("marker" (fold-marker-byte (cadr expression))) + ("separator" (fold-marker-byte (caddr expression))))) + ((line-skip-horizontal line-scan-word line-scan-key + line-physical-end line-step) + (unless (= (length expression) 2) + (error "invalid event fold scanner offset" expression)) + (hash ("kind" (rust-state-name (car expression))) + ("from" (fold-offset-ir (cadr expression) states indices)))) + ((line-scan-nonspace-until line-scan-until) + (unless (= (length expression) 3) + (error "invalid event fold bounded delimiter scan" expression)) + (hash ("kind" (rust-state-name (car expression))) + ("from" (fold-offset-ir (cadr expression) states indices)) + ("delimiter" (fold-marker-byte (caddr expression))))) + ((line-trim-end) + (unless (= (length expression) 1) + (error "invalid event fold line trim offset" expression)) + (hash ("kind" "line_trim_end"))) + ((line-trim-end-from) + (unless (= (length expression) 2) + (error "invalid event fold bounded line trim offset" expression)) + (hash ("kind" "line_trim_end_from") + ("from" (fold-offset-ir (cadr expression) states indices)))) + ((line-content-end) + (unless (= (length expression) 1) + (error "invalid event fold line content offset" expression)) + (hash ("kind" "line_content_end"))) + ((line-index) + (unless (and (= (length expression) 2) + (memq (cadr expression) indices)) + (error "unbound event fold line index" expression)) + (hash ("kind" "line_index") + ("name" (rust-state-name (cadr expression))))) + ((state-offset) + (unless (= (length expression) 2) + (error "invalid event fold state offset" expression)) + (fold-state-of-type states (cadr expression) fold-unsigned?) + (hash ("kind" "state_offset") + ("name" (rust-state-name (cadr expression))))) + (else (error "unsupported event fold source offset" expression)))))) + +(def (fold-uint expression line start end states indices) + (case (car expression) + ((uint) + (unless (and (= (length expression) 2) + (exact-integer? (cadr expression)) (>= (cadr expression) 0)) + (error "invalid event fold unsigned value" expression)) + (cadr expression)) + ((state) (fold-state-of-type states (cadr expression) fold-unsigned?)) + ((offset) (fold-offset (cadr expression) line start end states indices)) + ((line-indent-column) + (event-line-indent-column line (cadr expression))) + ((stack-top) + (let (stack (fold-state-of-type states (cadr expression) list?)) + (if (null? stack) 0 (car stack)))) + ((uint-add uint-multiply uint-divide) + (let ((left (fold-uint (cadr expression) line start end states indices)) + (right (fold-uint (caddr expression) line start end states indices))) + (case (car expression) + ((uint-add) (+ left right)) + ((uint-multiply) (* left right)) + (else (quotient left (max 1 right)))))) + ((line-marker-level) + (let* ((marker (integer->char (fold-marker-byte (cadr expression)))) + (separator (integer->char (fold-marker-byte (caddr expression)))) + (size (string-length line))) + (let loop ((index 0)) + (if (and (< index size) (char=? (string-ref line index) marker)) + (loop (+ index 1)) + (if (and (> index 0) (< index size) + (char=? (string-ref line index) separator)) + index 0))))) + (else (error "unsupported event fold unsigned expression" expression)))) + +(def (fold-uint-ir expression states indices allow-line?) + (case (car expression) + ((uint) + (unless (and (= (length expression) 2) + (exact-integer? (cadr expression)) (>= (cadr expression) 0)) + (error "invalid event fold unsigned value" expression)) + (hash ("kind" "usize") ("value" (cadr expression)))) + ((state) + (fold-state-of-type states (cadr expression) fold-unsigned?) + (hash ("kind" "state") ("name" (rust-state-name (cadr expression))))) + ((offset) + (unless allow-line? (error "event fold final transition has no line" expression)) + (hash ("kind" "offset") + ("value" (fold-offset-ir (cadr expression) states indices)))) + ((line-indent-column) + (unless (and allow-line? (= (length expression) 2) + (exact-integer? (cadr expression)) (> (cadr expression) 0)) + (error "invalid event fold indentation width" expression)) + (hash ("kind" "line_indent_column") + ("tab_width" (cadr expression)))) + ((stack-top) + (fold-state-of-type states (cadr expression) list?) + (hash ("kind" "stack_top") + ("stack" (rust-state-name (cadr expression))))) + ((uint-add uint-multiply uint-divide) + (unless (= (length expression) 3) + (error "invalid event fold unsigned arithmetic" expression)) + (hash ("kind" (case (car expression) + ((uint-add) "add") + ((uint-multiply) "multiply") + (else "divide"))) + ("left" (fold-uint-ir (cadr expression) states indices allow-line?)) + ("right" (fold-uint-ir (caddr expression) states indices allow-line?)))) + ((line-marker-level) + (unless allow-line? (error "event fold final transition has no line" expression)) + (hash ("kind" "line_marker_level") + ("marker" (fold-marker-byte (cadr expression))) + ("separator" (fold-marker-byte (caddr expression))))) + (else (error "unsupported event fold unsigned expression" expression)))) + +(def (fold-predicate expression source-bytes line start end states indices) + (unless (pair? expression) + (error "event fold predicate must be structured" expression)) + (case (car expression) + ((bool) + (unless (and (= (length expression) 2) (boolean? (cadr expression))) + (error "invalid event fold boolean" expression)) + (cadr expression)) + ((state) (fold-state-of-type states (cadr expression) boolean?)) + ((uint-positive?) + (> (fold-uint (cadr expression) line start end states indices) 0)) + ((uint-equal?) + (unless (= (length expression) 3) + (error "invalid event fold unsigned equality" expression)) + (= (fold-uint (cadr expression) line start end states indices) + (fold-uint (caddr expression) line start end states indices))) + ((uint-not-equal?) + (unless (= (length expression) 3) + (error "invalid event fold unsigned inequality" expression)) + (not (= (fold-uint (cadr expression) line start end states indices) + (fold-uint (caddr expression) line start end states indices)))) + ((uint-greater?) + (unless (= (length expression) 3) + (error "invalid event fold unsigned comparison" expression)) + (> (fold-uint (cadr expression) line start end states indices) + (fold-uint (caddr expression) line start end states indices))) + ((offset-less?) + (unless (= (length expression) 3) + (error "invalid event fold offset comparison" expression)) + (< (fold-offset (cadr expression) line start end states indices) + (fold-offset (caddr expression) line start end states indices))) + ((stack-nonempty?) + (pair? (fold-state-of-type states (cadr expression) list?))) + ((line-starts-with) + (unless (and (= (length expression) 2) (string? (cadr expression))) + (error "invalid event fold prefix" expression)) + (line-starts-with? line (cadr expression))) + ((line-starts-with-ascii-ci) + (unless (and (= (length expression) 2) (string? (cadr expression))) + (error "invalid event fold ASCII prefix" expression)) + (line-starts-with-ascii-ci? line (cadr expression))) + ((line-prefix-boundary-ascii-ci line-marker-ascii-ci) + (unless (and (= (length expression) 2) + (fold-ascii-prefix? (cadr expression))) + (error "invalid event fold ASCII line marker" expression)) + (fold-line-prefix-boundary? + line (cadr expression) (eq? (car expression) 'line-marker-ascii-ci))) + ((line-blank?) + (unless (= (length expression) 1) + (error "invalid event fold blank-line predicate" expression)) + (fold-line-blank? line)) + ((future-line-marker-before-boundary?) + (unless (and (= (length expression) 8) + (fold-ascii-prefix? (cadr expression)) + (> (string-length (cadr expression)) 0) + (fold-ascii-prefix? (caddr expression)) + (boolean? (list-ref expression 5)) + (boolean? (list-ref expression 6)) + (or (equal? (list-ref expression 7) "") + (and (string? (list-ref expression 7)) + (= (string-length (list-ref expression 7)) 1)))) + (error "invalid event fold future marker search" expression)) + (fold-future-marker-before-boundary? + source-bytes end (cadr expression) + (and (> (string-length (caddr expression)) 0) (caddr expression)) + (fold-marker-byte (cadddr expression)) + (fold-marker-byte (list-ref expression 4)) + (list-ref expression 5) (list-ref expression 6) + (and (> (string-length (list-ref expression 7)) 0) + (fold-marker-byte (list-ref expression 7))) + (fold-future-cache-for expression))) + ((future-heading-title?) + (unless (fold-future-heading-spec? expression) + (error "invalid event fold future heading" expression)) + (let (minimum + (max 1 + (let (level (list-ref expression 3)) + (if (pair? level) + (fold-uint level line start end states indices) + level)))) + (fold-future-heading-title? + source-bytes end (fold-marker-byte (cadr expression)) + (fold-marker-byte (caddr expression)) minimum + (list-ref expression 4) + (fold-future-cache-for (list expression minimum))))) + ((future-named-line-marker-before-boundary?) + (unless (and (memv (length expression) '(11 16)) + (fold-ascii-prefix? (list-ref expression 3)) + (> (string-length (list-ref expression 3)) 0) + (fold-ascii-prefix? (list-ref expression 4)) + (fold-ascii-prefix? (list-ref expression 5)) + (boolean? (list-ref expression 8)) + (boolean? (list-ref expression 9)) + (boolean? (list-ref expression 10)) + (or (= (length expression) 11) + (and (fold-ascii-prefix? (list-ref expression 13)) + (> (string-length (list-ref expression 13)) 0) + (fold-ascii-prefix? (list-ref expression 14)) + (boolean? (list-ref expression 15)) + (equal? (list-ref expression 5) "")))) + (error "invalid event fold future named marker search" expression)) + (let* ((name-from + (fold-offset (cadr expression) line start end states indices)) + (name-until + (fold-offset (caddr expression) line start end states indices)) + (parent? (= (length expression) 16)) + (stop-name-from + (and parent? + (fold-offset (list-ref expression 11) + line start end states indices))) + (stop-name-until + (and parent? + (fold-offset (list-ref expression 12) + line start end states indices))) + (index + (fold-future-index-for + (list 'future-named-index expression) + (lambda () + (fold-future-named-index + source-bytes + (list-ref expression 3) + (if parent? (list-ref expression 13) "") + (and (> (string-length (list-ref expression 5)) 0) + (list-ref expression 5)) + (fold-marker-byte (list-ref expression 6)) + (fold-marker-byte (list-ref expression 7)) + (list-ref expression 8) (list-ref expression 9) + (list-ref expression 10) + parent? + (and parent? (list-ref expression 15))))))) + (fold-future-named-index-result + index source-bytes end name-from name-until + (list-ref expression 3) (list-ref expression 4) + (list-ref expression 10) + stop-name-from stop-name-until + (if parent? (list-ref expression 13) "") + (if parent? (list-ref expression 14) "") + (and parent? (list-ref expression 15))))) + ((line-has-word-after-prefix?) + (unless (and (= (length expression) 2) + (fold-ascii-prefix? (cadr expression))) + (error "invalid event fold word-prefix predicate" expression)) + (fold-line-has-word-after-prefix? line (cadr expression))) + ((line-has-key-after-prefix?) + (unless (and (= (length expression) 2) + (fold-ascii-prefix? (cadr expression))) + (error "invalid event fold key-prefix predicate" expression)) + (fold-line-has-key-after-prefix? line (cadr expression))) + ((line-byte-equal?) + (unless (= (length expression) 3) + (error "invalid event fold byte predicate" expression)) + (let ((at (fold-offset (cadr expression) line start end states indices)) + (byte (caddr expression)) + (bytes (fold-line-bytes line))) + (and (< at end) (= (u8vector-ref bytes (- at start)) byte)))) + ((line-bytes-all-in? line-bytes-any-in?) + (unless (= (length expression) 4) + (error "invalid event fold byte-set predicate" expression)) + (let ((members (fold-byte-set-for (cadddr expression) expression)) + (from (fold-offset (cadr expression) line start end states indices)) + (until (fold-offset (caddr expression) line start end states indices)) + (bytes (fold-line-bytes line))) + (unless (and (<= start from) (<= from until) (<= until end)) + (error "event fold byte range outside source line" expression)) + (let loop ((cursor from)) + (if (eq? (car expression) 'line-bytes-all-in?) + (or (= cursor until) + (and (= 1 (u8vector-ref members + (u8vector-ref bytes (- cursor start)))) + (loop (+ cursor 1)))) + (and (< cursor until) + (or (= 1 (u8vector-ref members + (u8vector-ref bytes (- cursor start)))) + (loop (+ cursor 1)))))))) + ((line-bytes-in-set?) + (unless (= (length expression) 4) + (error "invalid event fold name-set predicate" expression)) + (let* ((names (fold-static-name-set (cadddr expression))) + (from (fold-offset (cadr expression) line start end states indices)) + (until (fold-offset (caddr expression) line start end states indices)) + (bytes (fold-line-bytes line))) + (unless (and (<= start from) (<= from until) (<= until end)) + (error "event fold name range outside source line" expression)) + (ormap (lambda (name) + (and (= (- until from) (string-length name)) + (let loop ((index 0)) + (or (= index (string-length name)) + (and (= (u8vector-ref bytes (+ (- from start) index)) + (char->integer (string-ref name index))) + (loop (+ index 1))))))) + names))) + ((source-slices-equal? source-slices-equal-ascii-ci?) + (unless (= (length expression) 5) + (error "invalid event fold source-slice comparison" expression)) + (let* ((left-from (fold-offset (cadr expression) line start end states indices)) + (left-until (fold-offset (caddr expression) line start end states indices)) + (right-from (fold-offset (cadddr expression) line start end states indices)) + (right-until (fold-offset (list-ref expression 4) + line start end states indices)) + (ascii-ci? (eq? (car expression) + 'source-slices-equal-ascii-ci?))) + (fold-source-slices-equal? source-bytes left-from left-until + right-from right-until ascii-ci?))) + ((not) (not (fold-predicate (cadr expression) source-bytes line start end states indices))) + ((and or) + (unless (>= (length expression) 3) + (error "event fold boolean combination needs two predicates" expression)) + ;; Keep short-circuit evaluation without allocating an escaping closure. + (let ((all? (eq? (car expression) 'and))) + (let loop ((rest (cdr expression))) + (if (null? rest) all? + (if (fold-predicate (car rest) source-bytes line start end states indices) + (if all? (loop (cdr rest)) #t) + (if all? #f (loop (cdr rest)))))))) + (else (error "unsupported event fold predicate" expression)))) + +(def (fold-frame-finishes count) + (unless (and (exact-integer? count) (> count 0) (<= count 255)) + (error "event fold frame close arity must be positive" count)) + (let loop ((remaining count) (events [])) + (if (= remaining 0) events + (loop (- remaining 1) (cons '(finish) events))))) + +(def (fold-statements statements source-bytes line start end states indices helpers + (active-helpers '()) (reversed '())) + (let (view (current-fold-line-view)) + (if (and view (eq? line (car view))) + (fold-statements/line-view statements source-bytes line start end + states indices helpers active-helpers reversed) + (parameterize ((current-fold-line-view (cons line (string->utf8 line)))) + (fold-statements/line-view statements source-bytes line start end + states indices helpers active-helpers reversed))))) + +;; Thread one persistent reversed event chain through nested execution. +;; Only the source-line/output boundary restores forward order; no shared +;; mutable queue or continuation-sensitive builder escapes the request. +(def (fold-statements/line-view statements source-bytes line start end states + indices helpers active-helpers reversed) + (let (state states) + (let loop ((rest statements) (reversed reversed)) + (if (null? rest) reversed + (let* ((form (car rest)) + (step + (case (car form) + ((set-bool) + (let (value (fold-predicate (caddr form) source-bytes line start end state indices)) + (fold-state-of-type state (cadr form) boolean?) + (fold-frame-update! state (cadr form) value) + reversed)) + ((set-uint) + (let (value (fold-uint (caddr form) line start end state indices)) + (fold-state-of-type state (cadr form) fold-unsigned?) + (fold-frame-update! state (cadr form) value) + reversed)) + ((close-through) + (let* ((stack (fold-state-of-type state (cadr form) list?)) + (level (fold-uint (caddr form) line start end state indices))) + (let close ((rest stack) (events reversed)) + (if (and (pair? rest) (>= (car rest) level)) + (close (cdr rest) (cons '(finish) events)) + (begin (fold-frame-update! state (cadr form) rest) + events))))) + ((open-level) + (let* ((stack (fold-state-of-type state (cadr form) list?)) + (level (fold-uint (caddr form) line start end state indices))) + (fold-frame-update! state (cadr form) (cons level stack)) + (cons (list 'start (cadddr form)) reversed))) + ((close-all) + (let (stack (fold-state-of-type state (cadr form) list?)) + (fold-frame-update! state (cadr form) []) + (foldl (lambda (_ events) (cons '(finish) events)) + reversed stack))) + ((start-node) (cons (list 'start (cadr form)) reversed)) + ((token) + (let ((from (fold-offset (caddr form) line start end state indices)) + (to (fold-offset (cadddr form) line start end state indices))) + (if (= from to) reversed + (cons (list 'token (cadr form) from to) reversed)))) + ((finish-node) (cons '(finish) reversed)) + ((if) + (fold-statements + (if (fold-predicate (cadr form) source-bytes line start end state indices) + (caddr form) (cadddr form)) + source-bytes line start end state indices helpers active-helpers reversed)) + ((join-once) + (unless (and (= (length form) 4) (symbol? (cadr form)) + (not (fold-frame-bound? state (cadr form))) + (not (memq (cadr form) indices)) + (pair? (caddr form)) (pair? (cadddr form)) + (fold-join-marks-handled? (caddr form) (cadr form))) + (error "invalid event fold join" form)) + (let* ((branch + (fold-statements + (caddr form) source-bytes line start end + (fold-frame-bind! state (cadr form) #f) + indices helpers active-helpers reversed)) + (continuation + (if (fold-state-of-type state (cadr form) boolean?) + branch + (fold-statements + (cadddr form) source-bytes line start end + state indices helpers active-helpers branch)))) + (fold-frame-unbind! state (cadr form)) + continuation)) + ((for-line-bytes) + (let ((from (fold-offset (caddr form) line start end state indices)) + (until (fold-offset (cadddr form) line start end state indices))) + (unless (and (<= start from) (<= from until) (<= until end)) + (error "event fold iteration outside source line" form)) + (let iterate ((cursor from) (reversed reversed)) + (if (= cursor until) reversed + (let (step (fold-statements (list-ref form 4) source-bytes line start end + state + (cons (cons (cadr form) cursor) + indices) helpers active-helpers reversed)) + (iterate (+ cursor 1) + step)))))) + ((with-source-bounds) + (unless (= (length form) 4) + (error "invalid event fold source bounds" form)) + (let ((from (fold-offset (cadr form) line start end state indices)) + (until (fold-offset (caddr form) line start end state indices))) + (unless (and (<= 0 from) (<= from until) + (<= until (u8vector-length source-bytes))) + (error "event fold bounds outside source" form)) + (fold-statements + (cadddr form) source-bytes + (utf8->string (subu8vector source-bytes from until)) + from until state indices helpers active-helpers reversed))) + ((call-source-helper) + (unless (memv (length form) '(4 5)) + (error "invalid event helper call" form)) + (let* ((helper (fold-helper-ref helpers (cadr form))) + (from (fold-offset (caddr form) line start end state indices)) + (until (fold-offset (cadddr form) line start end state indices))) + (unless (and helper (not (memq (cadr form) active-helpers))) + (error "unknown or recursive event helper" (cadr form))) + (unless (and (<= 0 from) (<= from until) + (<= until (u8vector-length source-bytes))) + (error "event helper bounds outside source" form)) + (let* ((parameters (event-fold-helper-parameters helper)) + (arguments (if (= (length form) 5) + (list-ref form 4) '())) + (helper-state (instantiate-fold-state-frame + (event-fold-helper-layout helper)))) + (unless (= (length parameters) (length arguments)) + (error "event helper argument arity" form)) + (for-each + (lambda (parameter argument) + (fold-state-of-type helper-state parameter fold-unsigned?) + (fold-frame-update! + helper-state parameter + (fold-uint argument line start end state indices))) + parameters arguments) + (fold-statements + (event-fold-helper-forms helper) source-bytes + (utf8->string (subu8vector source-bytes from until)) + from until helper-state '() + helpers (cons (cadr form) active-helpers) reversed)))) + ((scan-list-marker) + (unless (= (length form) 10) + (error "invalid event fold list marker statement" form)) + (let* ((marker (scan-event-list-marker + line start (cadr form) (caddr form) + (cadddr form))) + (present (list-ref form 4)) + (column (list-ref form 5)) + (ordered (list-ref form 6)) + (bullet-start (list-ref form 7)) + (bullet-end (list-ref form 8)) + (content-start (list-ref form 9))) + (fold-state-of-type state present boolean?) + (fold-state-of-type state column fold-unsigned?) + (fold-state-of-type state ordered boolean?) + (for-each (lambda (name) + (fold-state-of-type state name fold-unsigned?)) + (list bullet-start bullet-end content-start)) + (fold-frame-update! state present (if marker #t #f)) + (when marker + (fold-frame-update! state column (vector-ref marker 0)) + (fold-frame-update! state ordered (vector-ref marker 1)) + (fold-frame-update! state bullet-start (vector-ref marker 2)) + (fold-frame-update! state bullet-end (vector-ref marker 3)) + (fold-frame-update! state content-start (vector-ref marker 4))) + reversed)) + ((push-frame) + (let ((stack (fold-state-of-type state (cadr form) list?)) + (value (fold-uint (caddr form) line start end state indices))) + (fold-frame-update! state (cadr form) (cons value stack)) + reversed)) + ((pop-frame) + (let (stack (fold-state-of-type state (cadr form) list?)) + (fold-frame-update! state (cadr form) + (if (pair? stack) (cdr stack) [])) + reversed)) + ((close-frame) + (let ((stack (fold-state-of-type state (cadr form) list?)) + (closes (fold-frame-finishes (caddr form)))) + (if (pair? stack) + (begin (fold-frame-update! state (cadr form) (cdr stack)) + (foldl cons reversed closes)) + reversed))) + ((close-frames-while) + (let (closes (fold-frame-finishes (list-ref form 3))) + (let close ((reversed reversed)) + (let (stack (fold-state-of-type state (cadr form) list?)) + (if (and (pair? stack) + (fold-predicate (caddr form) source-bytes line start end state indices)) + (begin (fold-frame-update! state (cadr form) (cdr stack)) + (close (foldl cons reversed closes))) + reversed))))) + ((close-all-frames) + (let ((stack (fold-state-of-type state (cadr form) list?)) + (closes (fold-frame-finishes (caddr form)))) + (fold-frame-update! state (cadr form) []) + (foldl (lambda (_ events) (foldl cons events closes)) + reversed stack))) + (else (error "unsupported event fold statement" form))))) + (loop (cdr rest) step)))))) + +(def (run-event-fold source root initial line-forms finish-forms + (helpers '()) (overrides '())) + (run-event-fold-program + (prepare-event-fold-program root initial line-forms finish-forms helpers) + source overrides)) + +(def (run-event-fold-program program source (overrides '())) + (unless (event-fold-program? program) (error "event fold requires prepared program")) + (parameterize ((current-future-scan-cache #f) + (current-fold-byte-set-cache (make-table test: eq?))) + (let ((source-bytes (string->utf8 source)) + (states (instantiate-fold-state-frame (event-fold-program-layout program))) + (root (event-fold-program-root program)) + (line-forms (event-fold-program-line-forms program)) + (finish-forms (event-fold-program-finish-forms program)) + (helpers (event-fold-program-helpers program))) + (for-each (lambda (override) + (fold-state-of-type states (car override) fold-unsigned?) + (unless (fold-unsigned? (cdr override)) + (error "invalid event fold parameter override" override)) + (fold-frame-update! states (car override) (cdr override))) + overrides) + (let* ((reversed + (fold-source-lines + source (list (list 'start root)) + (lambda (line start end reversed) + (fold-statements line-forms source-bytes line start end + states '() helpers '() reversed)))) + (closing + (fold-statements finish-forms source-bytes "" 0 0 + states '() helpers '() reversed))) + (reverse (cons '(finish) closing)))))) diff --git a/src/compiler/event-fold-scheme-build.ss b/src/compiler/event-fold-scheme-build.ss new file mode 100644 index 00000000..95aecb1c --- /dev/null +++ b/src/compiler/event-fold-scheme-build.ss @@ -0,0 +1,17 @@ +;;; -*- Gerbil -*- +;;; Build-time static products only; requests never import the compiler. +(import (only-in :gerbil/compiler compile-module)) +(export compile-event-fold-scheme-units) + +(def (compile-event-fold-scheme-units directory names) + ;; The engine emits names in dependency order. Retain native SSI/SSXI and + ;; static Scheme inputs, without compiling unused dynamic loading objects. + (for-each + (lambda (name) + (displayln "... compile native fold unit " name) + (force-output) + (compile-module + (path-expand (string-append name ".ss") directory) + [invoke-gsc: #f keep-scm: #t optimize: #t generate-ssxi: #t + output-dir: (path-expand "lib" directory)])) + names)) diff --git a/src/compiler/event-fold-scheme-context.ss b/src/compiler/event-fold-scheme-context.ss new file mode 100644 index 00000000..5d7e1eb6 --- /dev/null +++ b/src/compiler/event-fold-scheme-context.ss @@ -0,0 +1,123 @@ +;;; -*- Gerbil -*- +;;; Build-local interning of code, never memoization of request values. +(export current-fold-scheme-counter current-fold-scheme-prefix + current-fold-scheme-expressions current-fold-scheme-definitions + current-fold-scheme-diagnostics current-fold-scheme-idle-plan fold-scheme-observed + current-fold-scheme-tape-kinds current-fold-scheme-effect-only fold-scheme-event + fold-scheme-fresh-name fold-scheme-share-expression fold-scheme-share-statements) + +(def current-fold-scheme-counter (make-parameter 0)) +(def current-fold-scheme-prefix (make-parameter 'native)) +(def current-fold-scheme-expressions (make-parameter #f)) +(def current-fold-scheme-definitions (make-parameter '())) +(def current-fold-scheme-diagnostics (make-parameter #f)) +(def current-fold-scheme-idle-plan (make-parameter #f)) +(def current-fold-scheme-tape-kinds (make-parameter #f)) +(def current-fold-scheme-effect-only (make-parameter #f)) +(def (fold-scheme-event tag (kind #f) (from 0) (until 0)) + (let (kinds (current-fold-scheme-tape-kinds)) + (if kinds + (case tag + ((finish) '(fold-tape-finish! events)) + (else + (let (entry (assq kind kinds)) + (unless entry (error "missing compiled Fold kind" kind)) + (if (eq? tag 'start) `(fold-tape-start! events ,(cadr entry)) + `(fold-tape-token! events ,(cadr entry) ,from ,until))))) + (case tag + ((finish) '(set! events (cons '(finish) events))) + ((start) `(set! events (cons '(start ,kind) events))) + ((token) `(set! events (cons (list 'token ',kind ,from ,until) events))))))) +(def (fold-scheme-observed name code) + (case (current-fold-scheme-diagnostics) + ((counts) `(begin (fold-cost-count! ',name) ,code)) + ((#f) code) + (else `(call-with-fold-cost ',name (lambda () ,code))))) + +(def (fold-scheme-fresh-name prefix) + (let (ordinal (current-fold-scheme-counter)) + (current-fold-scheme-counter (+ ordinal 1)) + (string->symbol + (string-append "%event-fold-" (symbol->string (current-fold-scheme-prefix)) + "-" (symbol->string prefix) "-" (number->string ordinal))))) + +(def (contains-variable? code name) + (cond ((eq? code name) #t) + ((not (pair? code)) #f) + ((eq? (car code) 'quote) #f) + (else (or (contains-variable? (car code) name) + (contains-variable? (cdr code) name))))) + +(def (rename-variables code names) + (cond + ((symbol? code) (let (entry (assq code names)) (if entry (cdr entry) code))) + ((not (pair? code)) code) + ((eq? (car code) 'quote) code) + (else (cons (rename-variables (car code) names) + (rename-variables (cdr code) names))))) + +(def (substantial-code? code) + ;; A bounded walk avoids traversing large constants or counting entire trees. + (let loop ((pending (list code)) (remaining 24)) + (cond ((<= remaining 0) #t) + ((null? pending) #f) + (else + (let (node (car pending)) + (if (and (pair? node) (not (eq? (car node) 'quote))) + (loop (cons (car node) (cons (cdr node) (cdr pending))) (- remaining 1)) + (loop (cdr pending) (- remaining 1)))))))) + +(def (writes-variable? code name) + (and (pair? code) (not (eq? (car code) 'quote)) + (or (and (eq? (car code) 'set!) (eq? (cadr code) name)) + (writes-variable? (car code) name) + (writes-variable? (cdr code) name)))) + +(def (share-code code bindings indices statements?) + (let (cache (current-fold-scheme-expressions)) + (if (and cache (substantial-code? code)) + (let* ((free + (filter (lambda (name) (contains-variable? code name)) + (map cdr (filter (lambda (entry) (symbol? (cdr entry))) + (append bindings indices))))) + (formals + (map (lambda (index) + (string->symbol (string-append "%fold-free-" (number->string index)))) + (iota (length free)))) + (normalized (rename-variables code (map cons free formals))) + ;; Derive the private calling convention from the lowered body. + ;; Nested callees have already been projected, so unused source, + ;; bounds and owner arguments disappear transitively. This is + ;; build-time closure conversion, not request caching or inlining. + (base (filter (cut contains-variable? normalized <>) + (if statements? '(source-bytes start end slots events) + '(source-bytes start end slots)))) + (key (list statements? formals normalized))) + ;; A mutable join temporary stays in its lexical owner. Base slots + ;; are vector cells shared explicitly, not free-variable mutation. + (if (and statements? (any (lambda (name) (writes-variable? code name)) free)) + code + (let (name (or (table-ref cache key #f) + (let (name (fold-scheme-fresh-name 'expression)) + (table-set! cache key name) + (current-fold-scheme-definitions + (cons `(def (,name ,@base ,@formals) + ,(fold-scheme-observed name normalized)) + (current-fold-scheme-definitions))) + name))) + `(,name ,@base ,@free)))) + code))) + +(def (fold-scheme-share-expression code bindings indices) + (share-code code bindings indices #f)) + +(def (fold-scheme-share-statements statements bindings indices) + ;; A direct tape is a private mutable owner. Internal transitions need no + ;; owner-valued result: publication and the outer source fold retain it. + ;; Keep list-producing Fold's result threading unchanged. + (let* ((effect-only? (and (current-fold-scheme-tape-kinds) + (current-fold-scheme-effect-only))) + (code (if effect-only? `(begin ,@statements) `(begin ,@statements events))) + (shared (share-code code bindings indices #t))) + (if (eq? code shared) statements + (list (if effect-only? shared `(set! events ,shared)))))) diff --git a/src/compiler/event-fold-scheme-expressions.ss b/src/compiler/event-fold-scheme-expressions.ss new file mode 100644 index 00000000..acc54799 --- /dev/null +++ b/src/compiler/event-fold-scheme-expressions.ss @@ -0,0 +1,260 @@ +;;; -*- Gerbil -*- +;;; Build-time lowering only. Generated requests never dispatch expression data. +(import (only-in :std/string/utf8 string-utf8-length) + (only-in ./event-fold-byte-set fold-byte-set-for fold-marker-byte) + (only-in ./event-fold-prefix-family fold-scheme-prefix-family) + (only-in ./event-fold-scheme-context fold-scheme-share-expression)) +(export fold-scheme-offset fold-scheme-uint fold-scheme-predicate fold-scheme-condition) + +;; Bindings map admitted names to generated lvalues, not runtime name tables. +(def (slot bindings name) + (let (entry (assq name bindings)) + (unless entry (error "unbound native fold slot" name)) + (cdr entry))) + +(def (fold-scheme-offset expression bindings indices) + (fold-scheme-share-expression + (fold-scheme-offset/body expression bindings indices) bindings indices)) + +(def (fold-scheme-offset/body expression bindings indices) + (def (offset value) (fold-scheme-offset value bindings indices)) + (def (uint value) (fold-scheme-uint value bindings indices)) + (cond + ((memq expression '(start end)) expression) + (else + (case (car expression) + ((line-prefix-end) + `(min end (+ start ,(string-utf8-length (cadr expression))))) + ((line-marker-end) + `(+ start ,(uint (cons 'line-marker-level (cdr expression))))) + ((line-index) (slot indices (cadr expression))) + ((state-offset) (slot bindings (cadr expression))) + ((line-step) `(min end (+ 1 ,(offset (cadr expression))))) + ((line-skip-horizontal line-scan-word line-scan-key + line-scan-nonspace-until line-scan-until line-physical-end) + (let (fact (and (equal? expression '(line-skip-horizontal start)) + (assq '%frame-horizontal indices))) + (if fact (cdr fact) + (let* ((kind (car expression)) + (from (offset (cadr expression))) + (test + (case kind + ((line-skip-horizontal) '(memv byte '(9 32))) + ((line-scan-word) '(not (memv byte '(9 10 13 32)))) + ((line-scan-key) '(fold-key-byte? byte)) + ((line-physical-end) '(not (memv byte '(10 13)))) + (else + `(and (not (= byte ,(fold-marker-byte (caddr expression)))) + ,@(if (eq? kind 'line-scan-until) '() + '((not (memv byte '(9 10 13 32)))))))))) + `(let (bytes source-bytes) + (let loop ((cursor ,(if (eq? kind 'line-physical-end) + `(max start (min ,from end)) from))) + (if (and (< cursor end) + (let (byte (u8vector-ref bytes cursor)) ,test)) + (loop (+ cursor 1)) cursor))))))) + ((line-trim-end line-trim-end-from line-content-end) + (let (fact (and (eq? (car expression) 'line-content-end) + (assq '%shared-content-end indices))) + (if fact (cdr fact) + `(let (bytes source-bytes) + (let ((floor ,(if (eq? (car expression) 'line-trim-end-from) + (offset (cadr expression)) 'start))) + (let loop ((cursor end)) + (if (and (> cursor floor) + (memv (u8vector-ref bytes (- cursor 1)) + ',(if (eq? (car expression) 'line-content-end) + '(10 13) '(9 10 11 12 13 32)))) + (loop (- cursor 1)) cursor))))))) + (else (error "unadmitted native fold offset" expression)))))) + +(def (fold-scheme-uint expression bindings indices) + (fold-scheme-share-expression + (fold-scheme-uint/body expression bindings indices) bindings indices)) + +(def (fold-scheme-uint/body expression bindings indices) + (def (uint value) (fold-scheme-uint value bindings indices)) + (case (car expression) + ((uint) (cadr expression)) + ((state) (slot bindings (cadr expression))) + ((offset) (fold-scheme-offset (cadr expression) bindings indices)) + ((line-indent-column) `(event-byte-indent-column source-bytes start end ,(cadr expression))) + ((stack-top) `(let (stack ,(slot bindings (cadr expression))) + (if (null? stack) 0 (car stack)))) + ((uint-add uint-multiply uint-divide) + (let ((left (uint (cadr expression))) (right (uint (caddr expression)))) + (case (car expression) + ((uint-add) `(+ ,left ,right)) + ((uint-multiply) `(* ,left ,right)) + (else `(quotient ,left (max 1 ,right)))))) + ((line-marker-level) + `(fold-byte-marker-level source-bytes start end + ,(fold-marker-byte (cadr expression)) + ,(fold-marker-byte (caddr expression)))) + (else (error "unadmitted native fold unsigned expression" expression)))) + +(def (fold-scheme-predicate expression bindings indices) + (fold-scheme-share-expression + (fold-scheme-predicate/body expression bindings indices) bindings indices)) + +(def current-fold-condition (make-parameter #f)) +(def (fold-scheme-condition expression bindings indices) + (parameterize ((current-fold-condition #t)) + (fold-scheme-predicate expression bindings indices))) + +(def (bounded-boolean-code operator operands bindings indices) + ;; Preserve left-to-right short circuiting, but do not feed a monolithic + ;; branch chain to the native compiler. Shared groups remain ordinary + ;; native calls over this request's explicit slots and source bounds. + (if (<= (length operands) 8) + `(,operator ,@operands) + (let group ((rest operands) (part '()) (count 0) (groups '())) + (cond + ((or (null? rest) (= count 8)) + (let (groups + (cons (fold-scheme-share-expression `(,operator ,@(reverse part)) + bindings indices) + groups)) + (if (null? rest) + (bounded-boolean-code operator (reverse groups) bindings indices) + (group rest '() 0 groups)))) + (else (group (cdr rest) (cons (car rest) part) (+ count 1) groups)))))) + +(def (fold-scheme-predicate/body expression bindings indices) + (def (predicate value) (fold-scheme-predicate value bindings indices)) + (def (uint value) (fold-scheme-uint value bindings indices)) + (def (offset value) (fold-scheme-offset value bindings indices)) + (case (car expression) + ((bool) (cadr expression)) + ((%line-prefix-hit) (cdr (assq '%line-prefix-hit indices))) + ((state) (slot bindings (cadr expression))) + ((uint-positive?) `(> ,(uint (cadr expression)) 0)) + ((uint-equal? uint-greater? uint-not-equal?) + (let (comparison `(,(if (eq? (car expression) 'uint-greater?) '> '=) + ,(uint (cadr expression)) ,(uint (caddr expression)))) + (if (eq? (car expression) 'uint-not-equal?) `(not ,comparison) comparison))) + ((offset-less?) `(< ,(offset (cadr expression)) ,(offset (caddr expression)))) + ((stack-nonempty?) `(pair? ,(slot bindings (cadr expression)))) + ((not) `(not ,(predicate (cadr expression)))) + ;; The interpreter normalizes successful Boolean combinations to #t. + ((and or) + (let (code (or (and (eq? (car expression) 'or) + (fold-scheme-prefix-family (cdr expression))) + (bounded-boolean-code (car expression) + (map predicate (cdr expression)) bindings indices))) + ;; Branch conditions observe truthiness only. Boolean state publication + ;; still normalizes, and ordered short-circuit/query evaluation stays. + (if (current-fold-condition) code `(if ,code #t #f)))) + ((line-starts-with) + `(fold-byte-prefix? source-bytes start end ',(string->utf8 (cadr expression)) #f)) + ((line-starts-with-ascii-ci) + `(line-starts-with-ascii-ci? #f ,(cadr expression) source-bytes + ',(string->utf8 (cadr expression)) start end)) + ((line-prefix-boundary-ascii-ci line-marker-ascii-ci) + `(fold-line-prefix-boundary? #f ,(cadr expression) + ,(eq? (car expression) 'line-marker-ascii-ci) + source-bytes ',(string->utf8 (cadr expression)) start end)) + ((line-blank?) '(fold-line-blank? #f source-bytes start end)) + ((line-has-word-after-prefix?) + `(fold-line-has-word-after-prefix? #f ,(cadr expression) source-bytes + ',(string->utf8 (cadr expression)) start end)) + ((line-has-key-after-prefix?) + `(fold-line-has-key-after-prefix? #f ,(cadr expression) source-bytes + ',(string->utf8 (cadr expression)) start end)) + ((line-byte-equal?) + (let (proof (assq '%validated-loop-byte indices)) + (if (and proof (equal? (cadr expression) (list 'line-index (cadr proof)))) + `(fx= ,(cddr proof) ,(caddr expression)) + `(let ((at ,(offset (cadr expression))) (bytes source-bytes)) + (and (< at end) (= (u8vector-ref bytes at) ,(caddr expression))))))) + ((line-bytes-all-in? line-bytes-any-in? line-bytes-in-set?) + (let* ((all? (eq? (car expression) 'line-bytes-all-in?)) + (names? (eq? (car expression) 'line-bytes-in-set?)) + ;; The admitted byte-at-cursor idiom denotes zero or one byte. + ;; Do not turn every inline character predicate into a scan loop. + ;; Keep the same range admission and empty-span truth values. + (single-byte? (equal? (caddr expression) + (list 'line-step (cadr expression)))) + (proof (assq '%validated-loop-byte indices)) + (validated-byte? (and single-byte? proof (not names?) + (equal? (cadr expression) + (list 'line-index (cadr proof))))) + (members (cadddr expression)) + (byte-set (and (not names?) (fold-byte-set-for members expression)))) + (if validated-byte? + ;; The loop already admitted start <= cursor < limit <= end. Its + ;; line-step span contains exactly one byte, so no scan, offset + ;; arithmetic, range admission or second source read is necessary. + `(fx= 1 (u8vector-ref ',byte-set ,(cddr proof))) + `(let ((from ,(offset (cadr expression))) (until ,(offset (caddr expression))) + (bytes source-bytes)) + (unless (<= start from until end) (error "native fold range outside source line")) + ,(cond + (names? + `(fold-line-name-set-contains? bytes from until + ',(list->vector members))) + (single-byte? + `(,(if all? 'or 'and) + (,(if all? 'fx= 'fx<) from until) + (fx= 1 (u8vector-ref ',byte-set (u8vector-ref bytes from))))) + (else + `(let loop ((cursor from)) + ,(if all? + `(or (fx= cursor until) + (and (= 1 (u8vector-ref ',byte-set + (u8vector-ref bytes cursor))) + (loop (fx+ cursor 1)))) + `(and (fx< cursor until) + (or (= 1 (u8vector-ref ',byte-set + (u8vector-ref bytes cursor))) + (loop (fx+ cursor 1)))))))))))) + ((source-slices-equal? source-slices-equal-ascii-ci?) + `(fold-source-slices-equal? source-bytes + ,@(map offset (cdr expression)) + ,(eq? (car expression) 'source-slices-equal-ascii-ci?))) + ((future-line-marker-before-boundary?) + `(fold-future-marker-before-boundary? + source-bytes end ',(string->utf8 (cadr expression)) + ,(and (> (string-length (caddr expression)) 0) + `(quote ,(string->utf8 (caddr expression)))) + ,(fold-marker-byte (cadddr expression)) + ,(fold-marker-byte (list-ref expression 4)) + ,(list-ref expression 5) ,(list-ref expression 6) + ,(and (> (string-length (list-ref expression 7)) 0) + (fold-marker-byte (list-ref expression 7))) + (fold-future-cache-for ',expression))) + ((future-heading-title?) + (let (level (list-ref expression 3)) + `(let (minimum (max 1 ,(if (pair? level) (uint level) level))) + (fold-future-heading-title? + source-bytes end ,(fold-marker-byte (cadr expression)) + ,(fold-marker-byte (caddr expression)) minimum ,(list-ref expression 4) + (fold-future-cache-for (list ',expression minimum)))))) + ((future-named-line-marker-before-boundary?) + (let ((parent? (= (length expression) 16)) + (boundary (list-ref expression 5))) + `(let* ((name-from ,(offset (cadr expression))) + (name-until ,(offset (caddr expression))) + (stop-from ,(and parent? (offset (list-ref expression 11)))) + (stop-until ,(and parent? (offset (list-ref expression 12)))) + (index + (fold-future-index-for + '(future-named-index ,expression) + (lambda () + (fold-future-named-index + source-bytes ,(list-ref expression 3) + ,(if parent? (list-ref expression 13) "") + ,(and (> (string-length boundary) 0) boundary) + ,(fold-marker-byte (list-ref expression 6)) + ,(fold-marker-byte (list-ref expression 7)) + ,(list-ref expression 8) ,(list-ref expression 9) + ,(list-ref expression 10) ,parent? + ,(and parent? (list-ref expression 15))))))) + (fold-future-named-index-result + index source-bytes end name-from name-until + ,(list-ref expression 3) ,(list-ref expression 4) ,(list-ref expression 10) + stop-from stop-until + ,(if parent? (list-ref expression 13) "") + ,(if parent? (list-ref expression 14) "") + ,(and parent? (list-ref expression 15)))))) + (else (error "unadmitted native fold predicate" expression)))) diff --git a/src/compiler/event-fold-scheme-modules.ss b/src/compiler/event-fold-scheme-modules.ss new file mode 100644 index 00000000..6199f5ed --- /dev/null +++ b/src/compiler/event-fold-scheme-modules.ss @@ -0,0 +1,115 @@ +;;; -*- Gerbil -*- +;;; Bound native compilation units without changing the admitted strategy. +(import (only-in ./event-fold-scheme event-fold-scheme-source)) +(export event-fold-scheme-module-sources) + +(def (private-definition? form) + (and (pair? form) (eq? (car form) 'def) (pair? (cadr form)) + (let (name (symbol->string (caadr form))) + (and (>= (string-length name) 12) + (string=? (substring name 0 12) "%event-fold-"))))) + +(def (references form directory) + (let ((seen (make-table test: eq?)) (names '())) + (def (walk value) + (cond + ((symbol? value) + (when (and (table-ref directory value #f) (not (table-ref seen value #f))) + (table-set! seen value #t) + (set! names (cons value names)))) + ((and (pair? value) (not (eq? (car value) 'quote))) + (walk (car value)) (walk (cdr value))))) + (walk form) + (reverse names))) + +(def (ordered-definitions definitions directory) + (let ((active (make-table test: eq?)) (done (make-table test: eq?)) (ordered '())) + (def (visit name) + (when (table-ref active name #f) (error "cyclic native fold procedure graph" name)) + (unless (table-ref done name #f) + (let (form (table-ref directory name)) + (table-set! active name #t) + ;; Only bodies contain references; the definition's own name is a binder. + (for-each visit (references (cddr form) directory)) + (table-set! active name) + (table-set! done name #t) + (set! ordered (cons form ordered))))) + (for-each (lambda (form) (visit (caadr form))) definitions) + (reverse ordered))) + +(def (chunks definitions width) + (let loop ((rest definitions) (part '()) (count 0) (parts '())) + (cond + ((null? rest) (reverse (if (null? part) parts (cons (reverse part) parts)))) + ((= count width) (loop rest '() 0 (cons (reverse part) parts))) + (else (loop (cdr rest) (cons (car rest) part) (+ count 1) parts))))) + +(def (forms-source forms) + (call-with-output-string + (lambda (port) + (display ";;; Engine-owned bounded native EventFold unit.\n" port) + (for-each (lambda (form) (write form port) (newline port)) forms)))) + +(def (safe-component? value) + (and (> (string-length value) 0) + (every (lambda (char) + (or (char<=? #\a char #\z) (char<=? #\A char #\Z) + (char<=? #\0 char #\9) (memv char '(#\- #\_)))) + (string->list value)))) + +(def (event-fold-scheme-module-sources name grammar root initial line finish + (helpers '()) (parameters '()) (width 64) (consumer 'events)) + (unless (and (symbol? name) (safe-component? (symbol->string name)) + (exact-integer? width) (<= 1 width 64)) + (error "invalid native fold compilation namespace or unit width")) + (let* ((source (event-fold-scheme-source name grammar root initial line finish helpers parameters consumer)) + (forms (call-with-input-string source + (lambda (port) + (let read-forms ((forms '())) + (let (form (read port)) + (if (eof-object? form) (reverse forms) + (read-forms (cons form forms)))))))) + (definitions (filter private-definition? forms)) + (directory (make-table test: eq?)) + (owners (make-table test: eq?)) + (base (symbol->string name))) + (for-each (lambda (form) (table-set! directory (caadr form) form)) definitions) + (let* ((parts (chunks (ordered-definitions definitions directory) width)) + (names (map (lambda (ordinal) (string-append base "-part-" (number->string ordinal))) + (iota (length parts))))) + (for-each + (lambda (part module) + (for-each (lambda (form) (table-set! owners (caadr form) module)) part)) + parts names) + (def (imports body current) + (let ((groups (make-table test: equal?)) (order '())) + (for-each + (lambda (ref) + (let (owner (table-ref owners ref)) + (unless (equal? current owner) + (unless (table-ref groups owner #f) + (set! order (cons owner order)) + (table-set! groups owner '())) + (table-set! groups owner (cons ref (table-ref groups owner)))))) + (references body directory)) + (map (lambda (owner) + `(only-in ,(string-append "./" owner ".ss") + ,@(reverse (table-ref groups owner)))) + (reverse order)))) + (def (import-forms body current) + (let (specifications (imports body current)) + (if (null? specifications) '() (list `(import ,@specifications))))) + (append + (map + (lambda (part module) + (cons module + (forms-source + `(,(car forms) + ,@(import-forms (map cddr part) module) + (export ,@(map caadr part)) + ,@part)))) + parts names) + (let (public (filter (lambda (form) (not (private-definition? form))) (cdr forms))) + (list (cons base + (forms-source + `(,(car forms) ,@(import-forms public #f) ,@public))))))))) diff --git a/src/compiler/event-fold-scheme-runs.ss b/src/compiler/event-fold-scheme-runs.ss new file mode 100644 index 00000000..8cb7814b --- /dev/null +++ b/src/compiler/event-fold-scheme-runs.ss @@ -0,0 +1,89 @@ +;;; Lower admitted overwrite-only runs into the ordinary request-owned loop. +(import (only-in ./event-fold-scheme-context fold-scheme-fresh-name current-fold-scheme-diagnostics) + (only-in ./event-fold-scheme-expressions fold-scheme-condition)) +(export fold-scheme-run-loop) + +(def (fold-scheme-run-loop form plan from until cursor limit byte bindings indices lower-body) + (let ((members (make-u8vector 256 0)) + (next (fold-scheme-fresh-name 'run-next)) + (last-byte (fold-scheme-fresh-name 'run-last)) + (summaries (vector-ref plan 4)) + (narrow (and (> (vector-length plan) 5) (vector-ref plan 5))) + (selected-members (fold-scheme-fresh-name 'consumer-members))) + (def accumulators (map (lambda (_) (fold-scheme-fresh-name 'run-count)) summaries)) + (def classes + (map (lambda (summary) + (let (table (make-u8vector 256 0)) + (for-each (lambda (b) (u8vector-set! table b 1)) (cdr summary)) + table)) summaries)) + (def (advance summary table value input) + `(if (fx= 1 (u8vector-ref ',table ,input)) (+ ,value 1) 0)) + (def (publish summary value) + (let (target (cdr (assq (car summary) bindings))) + (if (symbol? target) `(set! ,target ,value) + `(vector-set! slots ,(caddr target) ,value)))) + (for-each (lambda (member) (u8vector-set! members member 1)) (vector-ref plan 1)) + (def narrow-members + (if narrow + (let (table (make-u8vector 256 0)) + (for-each (lambda (b) (u8vector-set! table b 1)) (vector-ref narrow 1)) table) + members)) + (def (ordinary? byte) `(fx= 0 (u8vector-ref ,selected-members ,byte))) + (def (facts position) + (cons (cons '%validated-loop-byte (cons (cadr form) byte)) + (cons (cons '%loop-byte-value byte) + (cons (cons (cadr form) position) + (if (equal? (cadddr form) '(line-content-end)) + (cons (cons '%shared-content-end limit) indices) indices))))) + `(let ((from ,from) (,limit ,until)) + (unless (<= start from ,limit end) + (error "native fold iteration outside source line")) + (let iterate ((,cursor from)) + (unless (= ,cursor ,limit) + (let* ((,byte (u8vector-ref source-bytes ,cursor)) + (,selected-members + ,(if narrow + `(if (and (fx= 0 (u8vector-ref ',narrow-members ,byte)) + ,@(map (lambda (name) + (fold-scheme-condition + `(uint-equal? (state ,name) (uint 0)) + bindings (facts cursor))) + (vector-ref narrow 0))) + ',narrow-members ',members) + `',members))) + (if (and ,(ordinary? byte) + ,@(map (lambda (name) + (fold-scheme-condition `(uint-equal? (state ,name) (uint 0)) + bindings (facts cursor))) + (vector-ref plan 0)) + ,(fold-scheme-condition (vector-ref plan 2) bindings (facts cursor))) + (let scan ((,next (+ ,cursor 1)) + ,@(map (lambda (summary table value) + (list value (advance summary table + (cdr (assq (car summary) bindings)) byte))) + summaries classes accumulators)) + (if (and (< ,next ,limit) ,(ordinary? `(u8vector-ref source-bytes ,next))) + (scan (+ ,next 1) + ,@(map (lambda (summary table value) + (advance summary table value `(u8vector-ref source-bytes ,next))) + summaries classes accumulators)) + (let (,last-byte (- ,next 1)) + ,@(if (eq? (current-fold-scheme-diagnostics) 'counts) + `((fold-cost-count! 'ordinary-run-spans) + (fold-cost-count! 'ordinary-run-scanned-bytes (- ,next ,cursor)) + (fold-cost-count! 'ordinary-run-summary-byte-visits + (* (- ,next ,cursor) ,(length summaries))) + (fold-cost-count! (cond + ((= (- ,next ,cursor) 1) 'ordinary-run-length-1) + ((<= (- ,next ,cursor) 4) 'ordinary-run-length-2-4) + ((<= (- ,next ,cursor) 16) 'ordinary-run-length-5-16) + (else 'ordinary-run-length-over-16)))) '()) + ,@(map publish summaries accumulators) + (let (,byte (u8vector-ref source-bytes ,last-byte)) + ,@(lower-body (vector-ref plan 3) bindings (facts last-byte))) + (iterate ,next)))) + (begin + ,@(if (eq? (current-fold-scheme-diagnostics) 'counts) + '((fold-cost-count! 'ordinary-run-fallback-bytes)) '()) + ,@(lower-body (list-ref form 4) bindings (facts cursor)) + (iterate (+ ,cursor 1)))))))))) diff --git a/src/compiler/event-fold-scheme-statements.ss b/src/compiler/event-fold-scheme-statements.ss new file mode 100644 index 00000000..8ea32379 --- /dev/null +++ b/src/compiler/event-fold-scheme-statements.ss @@ -0,0 +1,253 @@ +;;; -*- Gerbil -*- +;;; Direct transitions over request-local slots and a reversed event chain. +(import (only-in ./event-fold-scheme-context + current-fold-scheme-counter fold-scheme-fresh-name fold-scheme-share-statements + fold-scheme-event current-fold-scheme-tape-kinds current-fold-scheme-idle-plan + current-fold-scheme-effect-only) + (only-in ./event-fold-zero-state fold-zero-state-plan fold-ordinary-state-plan fold-consumer-state-body) + (only-in ./event-fold-consumer-state fold-consumer-selector-body) + (only-in ./event-fold-ordinary-run fold-ordinary-run-plan) + (only-in ./event-fold-scheme-runs fold-scheme-run-loop) + (only-in ./event-fold-scheme-expressions + fold-scheme-offset fold-scheme-uint fold-scheme-predicate fold-scheme-condition) + (only-in ./event-fold-byte-dispatch fold-byte-dispatch fold-byte-prefilter fold-line-negative-region) + (only-in ./event-fold-shared-probes fold-fuse-byte-probes) + (only-in ./event-fold-prefix-family fold-scheme-negative-prefix-test) + (only-in ./event-fold-analysis fold-eliminate-inert-branches fold-share-statement-scans fold-rewrite-scans)) +(export fold-scheme-statements fold-scheme-frame-statements fold-scheme-bindings fold-scheme-initial + current-fold-scheme-counter fold-scheme-fresh-name) + +(def (fold-scheme-bindings initial) + (let loop ((rest initial) (ordinal 0) (bindings '())) + (if (null? rest) (reverse bindings) + (loop (cdr rest) (+ ordinal 1) + (cons (cons (caar rest) `(vector-ref slots ,ordinal)) bindings))))) + +(def (fold-scheme-initial initial) + `(vector ,@(map (lambda (entry) + (if (equal? (cadr entry) '(uint-stack)) ''() (cadr entry))) + initial))) + +(def (lvalue bindings name) + (let (entry (assq name bindings)) + (unless entry (error "unbound native fold transition slot" name)) + (cdr entry))) + +(def (assign target value) + (if (symbol? target) `(set! ,target ,value) + `(vector-set! slots ,(caddr target) ,value))) + +;; Source and frame extents are immutable within generated transitions. Hoist +;; only the total start-anchored horizontal scan, never state-dependent scans, +;; queries or scans across a nested frame. Lexical facts have no cache lookup, +;; allocation, invalidation table or request lifetime beyond their frame. +(def (fold-scheme-frame-statements forms bindings indices helpers) + (let (plan (and (current-fold-scheme-tape-kinds) + (fold-line-negative-region forms))) + (if plan + (let* ((paths (vector-ref plan 0)) + (at-start (map cdr (filter (lambda (path) (eq? (car path) 'start)) paths))) + (at-horizontal (map cdr (filter (lambda (path) (eq? (car path) 'horizontal)) paths))) + (horizontal (and (pair? at-horizontal) + (or (cond ((assq '%frame-horizontal indices) => cdr) (else #f)) + (fold-scheme-fresh-name 'frame-horizontal)))) + (hit (fold-scheme-fresh-name 'frame-prefix-hit)) + (facts (if horizontal (cons (cons '%frame-horizontal horizontal) indices) indices))) + (list `(let* (,@(if (and horizontal (not (assq '%frame-horizontal indices))) + `((,horizontal ,(fold-scheme-offset '(line-skip-horizontal start) bindings indices))) '()) + (,hit (or ,(fold-scheme-negative-prefix-test at-start) + ,(fold-scheme-negative-prefix-test at-horizontal horizontal)))) + ,@(fold-scheme-frame-body (vector-ref plan 1) bindings + (cons (cons '%line-prefix-hit hit) facts) helpers)))) + (fold-scheme-frame-body forms bindings indices helpers)))) + +(def (fold-scheme-frame-body forms bindings indices helpers) + (def (count value) + (cond ((equal? value '(line-skip-horizontal start)) 1) + ((not (pair? value)) 0) + ((memq (car value) '(with-source-bounds quote)) 0) + (else (foldl + 0 (map count value))))) + (if (and (not (assq '%frame-horizontal indices)) (> (count forms) 1)) + (let (horizontal (fold-scheme-fresh-name 'frame-horizontal)) + (list `(let ((,horizontal ,(fold-scheme-offset '(line-skip-horizontal start) bindings indices))) + ,@(fold-scheme-statements forms bindings + (cons (cons '%frame-horizontal horizontal) indices) helpers)))) + (fold-scheme-statements forms bindings indices helpers))) + +(def (fold-scheme-statements forms bindings indices helpers) + (def (offset value) (fold-scheme-offset value bindings indices)) + (def (uint value) (fold-scheme-uint value bindings indices)) + (def (predicate value) (fold-scheme-predicate value bindings indices)) + (def (condition value) (fold-scheme-condition value bindings indices)) + (def (body statements (locals bindings) (cursors indices)) + (if (null? statements) '((void)) + (fold-scheme-statements statements locals cursors helpers))) + (def (finishes count) + (map (lambda (_) (fold-scheme-event 'finish)) (iota count))) + (fold-scheme-share-statements + (map + (lambda (form) + (case (car form) + ((%shared-scans) + (let* ((scans (cadr form)) + (names (map (lambda (_) (fold-scheme-fresh-name 'consumer-scan)) scans)) + (replacements (map cons scans names))) + `(let ,(map (lambda (scan name) (list name (offset scan))) scans names) + ,@(body (map (cut fold-rewrite-scans <> replacements) (caddr form)) bindings + (append (map cons names names) indices))))) + ((%shared-byte-probes) + (let* ((first (cadr form)) (second (caddr form)) (probe (cadr first)) + (mask (fold-scheme-fresh-name 'probe-mask)) + (cursor (fold-scheme-fresh-name 'probe-cursor)) + (limit (fold-scheme-fresh-name 'probe-limit)) + (facts (if (equal? (caddr probe) '(line-content-end)) + (cons (cons '%shared-content-end limit) indices) indices))) + `(let ((from ,(offset (cadr probe))) (,limit ,(offset (caddr probe)))) + (unless (<= start from ,limit end) + (error "native fold range outside source line")) + (let (,mask + (let scan ((,cursor from) (found 0)) + (if (or (fx= ,cursor ,limit) (fx= found ,(list-ref form 4))) found + (scan (fx+ ,cursor 1) + (fxior found (u8vector-ref ',(list-ref form 3) + (u8vector-ref source-bytes ,cursor))))))) + (if (fx= 0 (fxand ,mask 1)) (begin ,@(body (cadddr first) bindings facts)) + (begin ,@(body (caddr first) bindings facts))) + (if (fx= 0 (fxand ,mask 2)) (begin ,@(body (cadddr second) bindings facts)) + (begin ,@(body (caddr second) bindings facts))))))) + ((set-bool set-uint) + (assign (lvalue bindings (cadr form)) + (if (eq? (car form) 'set-bool) + (predicate (caddr form)) (uint (caddr form))))) + ((start-node) (fold-scheme-event 'start (cadr form))) + ((finish-node) (fold-scheme-event 'finish)) + ((token) + `(let ((from ,(offset (caddr form))) (until ,(offset (cadddr form)))) + (unless (= from until) + ,(fold-scheme-event 'token (cadr form) 'from 'until)))) + ((if) + (let* ((proof (assq '%validated-loop-byte indices)) + (plan (and proof (fold-byte-dispatch form (cadr proof))))) + (if plan + `(case ,(cddr proof) + ,@(map (lambda (entry) `((,(car entry)) ,@(body (cdr entry)))) + (vector-ref plan 0)) + (else ,@(body (vector-ref plan 1)))) + `(if ,(condition (or (and proof (fold-byte-prefilter (cadr form) (cadr proof))) + (cadr form))) + (begin ,@(body (caddr form))) (begin ,@(body (cadddr form))))))) + ((join-once) + (let* ((handled (fold-scheme-fresh-name 'handled)) + (locals (cons (cons (cadr form) handled) bindings))) + `(let (,handled #f) + ,@(body (caddr form) locals) + (unless ,handled ,@(body (cadddr form) locals))))) + ((for-line-bytes) + (let* ((form (if (current-fold-scheme-tape-kinds) + (append (take form 4) + (list (fold-consumer-selector-body (fold-consumer-state-body (list-ref form 4))))) form)) + (cursor (fold-scheme-fresh-name 'cursor)) (limit (fold-scheme-fresh-name 'limit)) + (byte (fold-scheme-fresh-name 'byte)) + (run (and (current-fold-scheme-tape-kinds) + (eq? (current-fold-scheme-idle-plan) 'ordinary-run) + (fold-ordinary-run-plan (list-ref form 4) (cadr form))))) + (if run + (parameterize ((current-fold-scheme-idle-plan #f)) + (fold-scheme-run-loop form run (offset (caddr form)) (offset (cadddr form)) + cursor limit byte bindings indices body)) + `(let ((from ,(offset (caddr form))) (,limit ,(offset (cadddr form)))) + (unless (<= start from ,limit end) + (error "native fold iteration outside source line")) + (let iterate ((,cursor from)) + (unless (= ,cursor ,limit) + ,@(if (current-fold-scheme-tape-kinds) + `((let (,byte (u8vector-ref source-bytes ,cursor)) + ,@(let* ((ordinary? (memq (current-fold-scheme-idle-plan) '(ordinary ordinary-run))) + (plan (and (current-fold-scheme-idle-plan) + (if ordinary? + (fold-ordinary-state-plan (list-ref form 4) (cadr form)) + (fold-zero-state-plan (list-ref form 4))))) + (prepared (if plan + (list (list 'if (cons 'and + (append (if ordinary? + (list `(not (line-bytes-any-in? (line-index ,(cadr form)) + (line-step (line-index ,(cadr form))) ,(vector-ref plan 1)))) '()) + (map (lambda (name) `(uint-equal? (state ,name) (uint 0))) (vector-ref plan 0)))) + (vector-ref plan (if ordinary? 2 1)) (list-ref form 4))) + (list-ref form 4)))) + (parameterize ((current-fold-scheme-idle-plan #f)) + (body prepared bindings + (cons (cons '%validated-loop-byte (cons (cadr form) byte)) + (cons (cons '%loop-byte-value byte) + (cons (cons (cadr form) cursor) + (if (equal? (cadddr form) '(line-content-end)) + (cons (cons '%shared-content-end limit) indices) + indices))))))))) + (body (list-ref form 4) bindings (cons (cons (cadr form) cursor) indices))) + (iterate (+ ,cursor 1)))))))) + ((with-source-bounds) + `(let ((from ,(offset (cadr form))) (until ,(offset (caddr form)))) + (fold-source-frame-boundaries! source-bytes from until) + (let ((start from) (end until)) + ,@(fold-scheme-frame-statements (cadddr form) bindings + (filter (lambda (entry) + (not (memq (car entry) '(%validated-loop-byte %loop-byte-value %shared-content-end %frame-horizontal %line-prefix-hit)))) indices) + helpers)))) + ((call-source-helper) + (let (helper (assq (cadr form) helpers)) + (unless helper (error "unknown native fold helper" form)) + `(set! events + (,(cdr helper) source-bytes ,(offset (caddr form)) ,(offset (cadddr form)) + events ,@(map uint (if (= (length form) 5) + (list-ref form 4) '())))))) + ((scan-list-marker) + `(let (marker (scan-event-list-marker-bytes source-bytes start end + ',(string->utf8 (cadr form)) ,(caddr form) ,(cadddr form))) + ,(assign (lvalue bindings (list-ref form 4)) '(if marker #t #f)) + (when marker + ,@(map (lambda (name ordinal) + (assign (lvalue bindings name) `(vector-ref marker ,ordinal))) + (list-tail form 5) '(0 1 2 3 4))))) + ((push-frame open-level) + (let (target (lvalue bindings (cadr form))) + `(begin + ,(assign target `(cons ,(uint (caddr form)) ,target)) + ,@(if (eq? (car form) 'open-level) + (list (fold-scheme-event 'start (cadddr form))) '())))) + ((pop-frame close-frame) + (let (target (lvalue bindings (cadr form))) + `(when (pair? ,target) + ,(assign target `(cdr ,target)) + ,@(if (eq? (car form) 'close-frame) (finishes (caddr form)) '())))) + ((close-through) + (let (target (lvalue bindings (cadr form))) + `(let (level ,(uint (caddr form))) + (let close () + (when (and (pair? ,target) (>= (car ,target) level)) + ,(assign target `(cdr ,target)) + ,(fold-scheme-event 'finish) + (close)))))) + ((close-frames-while) + (let (target (lvalue bindings (cadr form))) + `(let close () + (when (and (pair? ,target) ,(condition (caddr form))) + ,(assign target `(cdr ,target)) + ,@(finishes (list-ref form 3)) + (close))))) + ((close-all close-all-frames) + (let ((target (lvalue bindings (cadr form))) + (count (if (eq? (car form) 'close-all) 1 (caddr form)))) + `(begin + (let close () + (when (pair? ,target) + ,(assign target `(cdr ,target)) + ,@(finishes count) + (close))) + ,(assign target ''())))) + (else (error "unadmitted native fold transition" form)))) + (if (current-fold-scheme-tape-kinds) + (fold-share-statement-scans (fold-fuse-byte-probes + (if (eq? (current-fold-scheme-effect-only) 'pruned) + (let (proof (assq '%validated-loop-byte indices)) + (fold-eliminate-inert-branches forms (and proof (cadr proof)))) + forms)) #t) forms)) bindings indices)) diff --git a/src/compiler/event-fold-scheme.ss b/src/compiler/event-fold-scheme.ss new file mode 100644 index 00000000..4556d4d6 --- /dev/null +++ b/src/compiler/event-fold-scheme.ss @@ -0,0 +1,167 @@ +;;; -*- Gerbil -*- +;;; Engine-owned native strategy source. No eval or runtime opcode fallback. +(import (only-in ./event-fold-ir event-fold-ir) + (only-in ./event-fold-call-plan fold-fuse-single-use-calls) + (only-in ../language/descriptor language-grammar-ir language-grammar-machine) + (only-in ./machine parser-machine-grammar-digest) + (only-in ./event-fold-program snapshot-fold-data) + (only-in ./event-fold-scheme-context + current-fold-scheme-prefix current-fold-scheme-expressions + current-fold-scheme-definitions + current-fold-scheme-diagnostics fold-scheme-observed + current-fold-scheme-tape-kinds) + (only-in ./event-fold-scheme-statements + fold-scheme-statements fold-scheme-frame-statements fold-scheme-bindings fold-scheme-initial + current-fold-scheme-counter fold-scheme-fresh-name)) +(export event-fold-scheme-source) + +(def (validate-helper-graph helpers line finish) + (def visited (make-table test: eq?)) + (def (visit name active) + (let (helper (assq name helpers)) + (unless helper (error "unknown native fold helper" name)) + (when (memq name active) (error "recursive native fold helper" name)) + (unless (table-ref visited name #f) + (walk (caddr helper) (cons name active)) + (table-set! visited name #t)))) + (def (walk forms active) + (for-each + (lambda (form) + (case (car form) + ((call-source-helper) + (visit (cadr form) active)) + ((if join-once) (walk (caddr form) active) (walk (cadddr form) active)) + ((for-line-bytes) (walk (list-ref form 4) active)) + ((with-source-bounds) (walk (cadddr form) active)))) + forms)) + (walk line '()) + (walk finish '()) + ;; Reject dormant cycles too: generated products must be closed. + (for-each (lambda (helper) (visit (car helper) '())) helpers)) + +(def (helper-definition helper names) + (let* ((initial (cadr helper)) + (bindings (fold-scheme-bindings initial)) + (parameters (if (= (length helper) 4) (cadddr helper) '())) + (arguments (map (lambda (_) (fold-scheme-fresh-name 'argument)) parameters))) + `(def (,(cdr (assq (car helper) names)) source-bytes from until events ,@arguments) + ,(fold-scheme-observed (car helper) + `(begin (fold-source-frame-boundaries! source-bytes from until) + (let ((slots ,(fold-scheme-initial initial)) + (start from) (end until)) + ,@(map (lambda (parameter argument) + `(vector-set! slots ,(caddr (cdr (assq parameter bindings))) ,argument)) + parameters arguments) + ,@(fold-scheme-frame-statements (caddr helper) bindings '() names) + events)))))) + +(def (event-fold-scheme-source name grammar root initial line finish + (helpers '()) (parameters '()) (consumer 'events)) + (unless (memq consumer '(events tape)) (error "invalid Fold consumer" consumer)) + (parameterize ((current-fold-scheme-counter 0) + (current-fold-scheme-prefix name) + (current-fold-scheme-expressions (make-table test: equal?)) + (current-fold-scheme-definitions '()) + (current-fold-scheme-tape-kinds + (and (eq? consumer 'tape) + (let loop ((kinds (cdr (assq 'syntax-kinds (language-grammar-ir grammar)))) + (index 0)) + (if (null? kinds) '() + (cons (list (caar kinds) index) + (loop (cdr kinds) (+ index 1)))))))) + (apply fold-scheme-source-owned grammar + (snapshot-fold-data (list name root initial line finish helpers parameters))))) + +(def (fold-scheme-source-owned grammar name root initial line finish helpers parameters) + ;; Snapshot/cycle admission precedes the shared typed lowering. Both products + ;; use the same grammar-kind checks, state types, helper arity and digest. + (let* ((ir (event-fold-ir name grammar root initial line finish helpers parameters)) + (digest (hash-get ir "parser_digest")) + (tape? (current-fold-scheme-tape-kinds)) + (binder (string->symbol (string-append "bind-" (symbol->string name)))) + (bindings (fold-scheme-bindings initial)) + (names (map (lambda (helper) (cons (car helper) (fold-scheme-fresh-name 'helper))) helpers))) + (validate-helper-graph helpers line finish) + (call-with-output-string + (lambda (port) + (display ";;; Generated by the engine native EventFold compiler.\n" port) + (let (forms + `((import (only-in :gerbil-parser/src/runtime/source-lines + fold-source-byte-spans fold-source-frame-boundaries! fold-source-input-bytes) + ,@(if tape? + '((only-in :gerbil-parser/src/runtime/fold-tape + make-fold-tape fold-tape-start! fold-tape-token! + fold-tape-finish! fold-tape-publish!)) '()) + (only-in :gerbil-parser/src/compiler/event-list-marker + event-byte-indent-column scan-event-list-marker-bytes) + (only-in :gerbil-parser/src/runtime/event-fold-lines + fold-key-byte? fold-byte-prefix? fold-byte-marker-level + line-starts-with-ascii-ci? fold-line-prefix-boundary? + fold-line-blank? fold-line-has-word-after-prefix? + fold-line-has-key-after-prefix? fold-line-name-set-contains?) + (only-in :gerbil-parser/src/compiler/event-fold-future + ascii-lower-byte current-future-scan-cache fold-future-cache-for + fold-future-index-for fold-future-named-index + fold-future-named-index-result fold-future-heading-title? + fold-future-marker-before-boundary? fold-source-slices-equal?) + ,@(if (current-fold-scheme-diagnostics) + '((only-in :gerbil-parser/src/runtime/fold-cost call-with-fold-cost fold-cost-count!)) '())) + ,@(map (lambda (helper) (helper-definition helper names)) helpers) + (export ,name ,binder) + (def (,binder expected-digest) + (unless (equal? expected-digest ,digest) + (error "native EventFold product identity mismatch")) + ,(if (current-fold-scheme-diagnostics) + '(error "diagnostic Fold cannot bind a production performance entry") name)) + (def (,name source (overrides '())) + (parameterize ((current-future-scan-cache #f)) + (let ((source-bytes ,(fold-scheme-observed 'source-encoding + '(fold-source-input-bytes source))) + (slots ,(fold-scheme-initial initial))) + (for-each + (lambda (override) + (unless (and (exact-integer? (cdr override)) (>= (cdr override) 0)) + (error "invalid native fold parameter override" override)) + (case (car override) + ,@(map + (lambda (entry) + `((,(car entry)) + ,(if (and (exact-integer? (cadr entry)) (>= (cadr entry) 0)) + `(vector-set! slots ,(caddr (cdr (assq (car entry) bindings))) + (cdr override)) + `(error "native fold parameter is not unsigned" override)))) + initial) + (else (error "undeclared native fold parameter" override)))) + overrides) + (let* ((events + (fold-source-byte-spans source-bytes + ,(if tape? + `(fold-tape-start! + (make-fold-tape ',(string->utf8 + (string-append "OEV1" (parser-machine-grammar-digest + (language-grammar-machine grammar))))) + ,(cadr (assq root tape?))) + `(list '(start ,root))) + (lambda (start end events) + (begin + ,@(fold-scheme-frame-statements line bindings '() names) + events)))) + (start 0) (end 0)) + (begin + (void) + ,@(fold-scheme-statements finish bindings '() names)) + ;; Every outer event cell is allocated by this request's + ;; root, transitions or closed helpers. No callback can + ;; retain that private spine; publication consumes it once. + ;; Event payloads (including quoted markers) stay untouched. + ,(fold-scheme-observed 'event-publication + (if tape? + '(begin (fold-tape-finish! events) + (fold-tape-publish! events)) + '(reverse! (cons '(finish) events)))))))))) + (for-each + (lambda (form) (write form port) (newline port)) + (let (complete (cons (car forms) + (append (reverse (current-fold-scheme-definitions)) (cdr forms)))) + (if (and tape? (not (current-fold-scheme-diagnostics))) + (fold-fuse-single-use-calls complete) complete)))))))) diff --git a/src/compiler/event-fold-shared-probes.ss b/src/compiler/event-fold-shared-probes.ss new file mode 100644 index 00000000..4db3e379 --- /dev/null +++ b/src/compiler/event-fold-shared-probes.ss @@ -0,0 +1,41 @@ +;;; Fuse adjacent admitted source probes, not their ordered state transitions. +(import (only-in ./event-fold-analysis fold-expression-analysis fold-statement-analysis)) +(export fold-fuse-byte-probes) + +(def (probe form) + (and (pair? form) (eq? (car form) 'if) + (pair? (cadr form)) (eq? (caadr form) 'line-bytes-any-in?) + (list? (list-ref (cadr form) 3)) (cadr form))) + +(def (compatible? first second) + (let* ((left (probe first)) (right (probe second)) + (bounds (list (cadr left) (caddr left))) + (dependencies (fold-expression-analysis (cons 'offset-less? bounds))) + (effects (fold-statement-analysis (append (caddr first) (cadddr first))))) + (and (equal? bounds (list (cadr right) (caddr right))) + (null? (vector-ref dependencies 2)) + (not (any (lambda (read) (memq read (vector-ref effects 1))) + (vector-ref dependencies 0))) + ;; No opaque mutation, helper handoff or source-frame change may + ;; invalidate a probe. Errors/events keep their original order below. + (not (any (cut memq <> '(opaque helper bounds)) (vector-ref effects 2)))))) + +(def (fold-fuse-byte-probes forms) + (let loop ((rest forms)) + (cond ((null? rest) '()) + ((and (pair? (cdr rest)) (probe (car rest)) (probe (cadr rest)) + (compatible? (car rest) (cadr rest))) + (let ((classes (make-u8vector 256 0)) + (first (car rest)) (second (cadr rest))) + (for-each (lambda (form bit) + (for-each (lambda (byte) + (u8vector-set! classes byte + (fxior (u8vector-ref classes byte) bit))) + (list-ref (cadr form) 3))) + (list first second) '(1 2)) + ;; Private compiler plan only; public IR admission is unchanged. + (cons (list '%shared-byte-probes first second classes + (+ (if (null? (list-ref (cadr first) 3)) 0 1) + (if (null? (list-ref (cadr second) 3)) 0 2))) + (loop (cddr rest))))) + (else (cons (car rest) (loop (cdr rest))))))) diff --git a/src/compiler/event-fold-state-frame.ss b/src/compiler/event-fold-state-frame.ss new file mode 100644 index 00000000..643cdfa0 --- /dev/null +++ b/src/compiler/event-fold-state-frame.ss @@ -0,0 +1,79 @@ +;;; -*- Gerbil -*- +;;; Private request-owned execution slots; never escape as parser results. +(export prepare-fold-state-layout instantiate-fold-state-frame + prepare-fold-state-frame fold-state-frame? + fold-frame-ref fold-frame-update! fold-frame-bound? + fold-frame-bind! fold-frame-unbind!) + +(defstruct fold-state-layout (index defaults) final: #t) +(defstruct fold-state-frame (layout values temporaries) final: #t) +(def missing-fold-slot (gensym 'missing-fold-slot)) + +(def (prepare-fold-state-layout bindings) + (let ((index (make-table test: eq?)) (defaults '()) (width 0)) + ;; First binding owns reads. Updating its cell represents updating every + ;; duplicate name, since binding enumeration never leaves this frame. + (for-each (lambda (entry) + (unless (or (boolean? (cdr entry)) (null? (cdr entry)) + (and (exact-integer? (cdr entry)) (>= (cdr entry) 0))) + (error "mutable or untyped event fold initial value" entry)) + (unless (table-ref index (car entry) #f) + (table-set! index (car entry) width) + (set! defaults (cons (cdr entry) defaults)) + (set! width (+ width 1)))) + bindings) + (make-fold-state-layout index (list->vector (reverse defaults))))) + +(def (instantiate-fold-state-frame layout) + (make-fold-state-frame layout (vector-copy (fold-state-layout-defaults layout)) #f)) + +(def (prepare-fold-state-frame bindings) + (instantiate-fold-state-frame (prepare-fold-state-layout bindings))) + +(def (fold-frame-slot frame name) + (table-ref (fold-state-layout-index (fold-state-frame-layout frame)) name #f)) + +(def (fold-frame-value frame name) + (let (slot (fold-frame-slot frame name)) + (if slot (vector-ref (fold-state-frame-values frame) slot) + (let (temporaries (fold-state-frame-temporaries frame)) + (if temporaries (table-ref temporaries name missing-fold-slot) + missing-fold-slot))))) + +(def (fold-frame-ref frame name) + (let (value (fold-frame-value frame name)) + (when (eq? value missing-fold-slot) (error "undeclared event fold state" name)) + value)) + +(def (fold-frame-bound? frame name) + (not (eq? (fold-frame-value frame name) missing-fold-slot))) + +(def (fold-frame-update! frame name value) + (let (slot (fold-frame-slot frame name)) + (if slot + (unless (eq? (vector-ref (fold-state-frame-values frame) slot) missing-fold-slot) + (vector-set! (fold-state-frame-values frame) slot value)) + (let (temporaries (fold-state-frame-temporaries frame)) + (when (and temporaries + (not (eq? (table-ref temporaries name missing-fold-slot) missing-fold-slot))) + (table-set! temporaries name value))))) + frame) + +(def (fold-frame-bind! frame name value) + (when (fold-frame-bound? frame name) + (error "event fold temporary state already bound" name)) + (let (slot (fold-frame-slot frame name)) + (if slot (vector-set! (fold-state-frame-values frame) slot value) + (let (temporaries (or (fold-state-frame-temporaries frame) + (let (table (make-table test: eq?)) + (fold-state-frame-temporaries-set! frame table) + table))) + (table-set! temporaries name value)))) + frame) + +(def (fold-frame-unbind! frame name) + (let (slot (fold-frame-slot frame name)) + (if slot (vector-set! (fold-state-frame-values frame) slot missing-fold-slot) + (let (temporaries (fold-state-frame-temporaries frame)) + (when temporaries (table-set! temporaries name))))) + frame) diff --git a/src/compiler/event-fold-zero-state.ss b/src/compiler/event-fold-zero-state.ss new file mode 100644 index 00000000..042f2111 --- /dev/null +++ b/src/compiler/event-fold-zero-state.ss @@ -0,0 +1,158 @@ +;;; Guarded idle-state specialization over admitted IR, not value memoization. +(import (only-in ./event-fold-analysis fold-statement-analysis fold-expression-analysis) + (only-in ./event-fold-byte-dispatch fold-byte-regions)) +(export fold-zero-state-plan fold-zero-state-support fold-ordinary-state-plan fold-consumer-state-body) + +;; Minimize the *external* guard by an exact cofactor witness. A payload +;; predicate in an eliminated consumer branch must not make another consumer +;; unavailable. Dropping a cell is admitted only when rerunning specialization +;; without that premise produces the identical ordered body. Build-time only. +(def (fold-zero-state-support forms (complete-zero? #f)) + (let (plan (fold-zero-state-plan forms complete-zero?)) + (and plan + (let* ((body (vector-ref plan 1)) + (support + (foldl (lambda (name current) + (let* ((remaining (filter (lambda (cell) (not (eq? cell name))) current)) + (witness (fold-zero-state-plan forms complete-zero? remaining))) + (if (equal? body (if witness (vector-ref witness 1) forms)) remaining current))) + (vector-ref plan 0) (vector-ref plan 0)))) + (vector support body))))) + +;; Facts come from each consumer's executed selector, not an all-machine +;; idle guard. They are compile-time control-flow facts, never retained state. +(def (fold-consumer-state-body forms) + (let (plan (fold-zero-state-plan forms #t '() #t)) + (if plan (vector-ref plan 1) forms))) + +;;; A single common-byte kernel, not one generated kernel per byte region. +;;; The original complete body remains the fallback for every exceptional byte. +(def (fold-ordinary-state-plan forms cursor) + (let* ((idle (fold-zero-state-plan forms)) + (body (if idle (vector-ref idle 1) forms)) + (regions (fold-byte-regions body cursor))) + (and regions + (vector (if idle (vector-ref idle 0) '()) + (foldr append '() (map car (vector-ref regions 0))) + (vector-ref regions 1))))) + +(def (fold-zero-state-plan forms (complete-zero? #f) (guarded-cells #f) (consumer-local? #f)) + (let ((seen (make-hash-table-eq)) (names '())) + (def (collect value) + (when (pair? value) + (when (and (memq (car value) '(uint-positive? uint-equal?)) + (pair? (cadr value)) (eq? (caadr value) 'state) + (or (eq? (car value) 'uint-positive?) + (equal? (caddr value) '(uint 0)))) + (let (name (cadadr value)) + (unless (hash-get seen name) + (hash-put! seen name #t) (set! names (cons name names))))) + (for-each collect (if (symbol? (car value)) (cdr value) value)))) + (for-each collect forms) + ;; A read-only configuration is not an idle machine state. Requiring its + ;; default nonzero value to be zero makes the fast branch unreachable. + ;; Specialize only candidate cells actually written inside this loop; + ;; all other predicates retain their ordinary evaluation. + (let (writes (vector-ref (fold-statement-analysis forms) 1)) + (set! names (if guarded-cells guarded-cells (filter (cut memq <> writes) names)))) + ;; The admitted IR has distinct slot cells and an event-tape owner. An + ;; append, query-cache update or possible range error is observable, but + ;; is not an arbitrary slot write. Keep its execution/order while dropping + ;; facts only for declared writes or unknown/helper/frame barriers. + (def (barrier? effects) (any (cut memq <> '(opaque helper bounds)) effects)) + (def (constant predicate facts) + (cond + ((and (pair? predicate) (eq? (car predicate) 'uint-positive?) + (pair? (cadr predicate)) (eq? (caadr predicate) 'state) + (memq (cadadr predicate) facts)) '(bool #f)) + ((and (pair? predicate) (eq? (car predicate) 'uint-equal?) + (pair? (cadr predicate)) (eq? (caadr predicate) 'state) + (memq (cadadr predicate) facts) (equal? (caddr predicate) '(uint 0))) '(bool #t)) + ((and complete-zero? (pair? predicate) + (memq (car predicate) '(uint-equal? uint-not-equal? uint-greater?)) + (pair? (cadr predicate)) (eq? (caadr predicate) 'state) + (memq (cadadr predicate) facts) + (pair? (caddr predicate)) (eq? (caaddr predicate) 'uint)) + (let (literal (cadr (caddr predicate))) + (list 'bool (case (car predicate) + ((uint-equal?) (= 0 literal)) + ((uint-not-equal?) (not (= 0 literal))) + (else (> 0 literal)))))) + ((and (pair? predicate) (eq? (car predicate) 'not)) + (let (value (constant (cadr predicate) facts)) + (cond ((equal? value '(bool #t)) '(bool #f)) + ((equal? value '(bool #f)) '(bool #t)) + (else (list 'not value))))) + ((and (pair? predicate) (memq (car predicate) '(and or))) + (let* ((op (car predicate)) (identity (eq? op 'and))) + (let loop ((rest (cdr predicate)) (kept '()) (facts facts)) + (if (null? rest) + (cond ((null? kept) (list 'bool identity)) + ((null? (cdr kept)) (car kept)) + (else (cons op (reverse kept)))) + (let (value (constant (car rest) facts)) + (cond ((equal? value (list 'bool identity)) (loop (cdr rest) kept facts)) + ((equal? value (list 'bool (not identity))) + ;; Earlier unknown leaves retain queries/errors/order. + (if (null? kept) value (cons op (append (reverse kept) (list value))))) + (else (loop (cdr rest) (cons value kept) + (if (barrier? (vector-ref (fold-expression-analysis value) 2)) '() facts))))))))) + (else predicate))) + (def (intersection left right) (filter (cut memq <> right) left)) + (def (selector-facts predicate truth facts) + ;; Unknown predicate effects discard knowledge before any deduction. + ;; Only admitted unsigned cells justify not-positive => exactly zero. + (if (or (not consumer-local?) + (barrier? (vector-ref (fold-expression-analysis predicate) 2))) facts + (case (car predicate) + ((not) (selector-facts (cadr predicate) (not truth) facts)) + ((and or) + (if (eq? truth (eq? (car predicate) 'and)) + (foldl (lambda (part known) (selector-facts part truth known)) facts (cdr predicate)) + facts)) + ((uint-positive? uint-equal? uint-not-equal?) + (if (and (pair? (cadr predicate)) (eq? (caadr predicate) 'state) + (or (and (eq? (car predicate) 'uint-positive?) (not truth)) + (and (not (eq? (car predicate) 'uint-positive?)) + (equal? (caddr predicate) '(uint 0)) + (eq? truth (eq? (car predicate) 'uint-equal?))))) + (let (name (cadadr predicate)) (if (memq name facts) facts (cons name facts))) + facts)) + (else facts)))) + (def (walk body facts) + (let loop ((rest body) (facts facts) (out '())) + (if (null? rest) (vector (reverse out) facts) + (let (form (car rest)) + (case (car form) + ((if) + (let (predicate (constant (cadr form) facts)) + (if (or (equal? predicate '(bool #t)) (equal? predicate '(bool #f))) + (let (chosen (walk (if (cadr predicate) (caddr form) (cadddr form)) facts)) + (loop (cdr rest) (vector-ref chosen 1) + (append (reverse (vector-ref chosen 0)) out))) + (let* ((branch-facts (if (barrier? (vector-ref (fold-expression-analysis predicate) 2)) '() facts)) + (yes (walk (caddr form) (selector-facts predicate #t branch-facts))) + (no (walk (cadddr form) (selector-facts predicate #f branch-facts)))) + (loop (cdr rest) (intersection (vector-ref yes 1) (vector-ref no 1)) + (cons (list 'if predicate (vector-ref yes 0) (vector-ref no 0)) out)))))) + ((join-once) + (let* ((outer (filter (lambda (name) (not (eq? name (cadr form)))) facts)) + (first (walk (caddr form) outer)) + (fallback (walk (cadddr form) (vector-ref first 1)))) + (loop (cdr rest) (intersection (vector-ref first 1) (vector-ref fallback 1)) + (cons (list 'join-once (cadr form) (vector-ref first 0) (vector-ref fallback 0)) out)))) + (else + (let* ((effect (fold-statement-analysis (list form))) + (remaining (if (not (barrier? (vector-ref effect 2))) + (filter (lambda (name) (not (memq name (vector-ref effect 1)))) facts) + '()))) + ;; Only a literal zero write establishes a new local fact; + ;; all other writes invalidate just their declared support. + (loop (cdr rest) + (if (and consumer-local? (eq? (car form) 'set-uint) + (equal? (caddr form) '(uint 0))) + (cons (cadr form) remaining) remaining) + (cons form out))))))))) + (and (or consumer-local? (pair? names)) (<= (length names) 32) + (let (body (vector-ref (walk forms names) 0)) + (and (not (equal? body forms)) (vector (reverse names) body)))))) diff --git a/src/compiler/event-list-marker.ss b/src/compiler/event-list-marker.ss index bef3d819..58c4a794 100644 --- a/src/compiler/event-list-marker.ss +++ b/src/compiler/event-list-marker.ss @@ -1,20 +1,22 @@ ;;; -*- Gerbil -*- ;;; Generic source-backed list-marker lexing for a bounded Scheme event fold. +(import (only-in ../runtime/byte-spans byte-span-skip)) -(export event-line-indent-column scan-event-list-marker) +(export event-line-indent-column scan-event-list-marker + event-byte-indent-column scan-event-list-marker-bytes) (def (horizontal? byte) (memv byte '(9 32))) -(def (content-end bytes) - (let loop ((cursor (u8vector-length bytes))) - (if (and (> cursor 0) (memv (u8vector-ref bytes (- cursor 1)) '(10 13))) +(def (content-end bytes (floor 0) (until (u8vector-length bytes))) + (let loop ((cursor until)) + (if (and (> cursor floor) (memv (u8vector-ref bytes (- cursor 1)) '(10 13))) (loop (- cursor 1)) cursor))) -(def (line-indent bytes limit tab-width) +(def (line-indent bytes limit tab-width (from 0)) (unless (and (exact-integer? tab-width) (> tab-width 0)) (error "event list tab width must be positive" tab-width)) - (let loop ((cursor 0) (column 0)) + (let loop ((cursor from) (column 0)) (if (and (< cursor limit) (horizontal? (u8vector-ref bytes cursor))) (loop (+ cursor 1) (if (= (u8vector-ref bytes cursor) 9) @@ -23,10 +25,12 @@ (values cursor column)))) (def (event-line-indent-column line tab-width) - (let* ((bytes (string->utf8 line)) - (limit (content-end bytes))) - (let-values (((_ column) (line-indent bytes limit tab-width))) - column))) + (let (bytes (string->utf8 line)) + (event-byte-indent-column bytes 0 (u8vector-length bytes) tab-width))) + +(def (event-byte-indent-column bytes from until tab-width) + (let-values (((_ column) (line-indent bytes (content-end bytes from until) tab-width from))) + column)) (def (ascii-digit? byte) (<= 48 byte 57)) @@ -38,11 +42,9 @@ (let (byte (u8vector-ref bytes first)) (cond ((ascii-digit? byte) - (let loop ((cursor (+ first 1))) - (if (and (< cursor limit) (ascii-digit? (u8vector-ref bytes cursor))) - (loop (+ cursor 1)) - (and (< cursor limit) (memv (u8vector-ref bytes cursor) '(41 46)) - (+ cursor 1))))) + (let (cursor (byte-span-skip bytes (+ first 1) limit ascii-digit?)) + (and (< cursor limit) (memv (u8vector-ref bytes cursor) '(41 46)) + (+ cursor 1)))) ((ascii-alpha? byte) (and (< (+ first 1) limit) (memv (u8vector-ref bytes (+ first 1)) '(41 46)) @@ -55,9 +57,17 @@ (string->list unordered))) (error "event list markers must be nonempty ASCII" unordered)) (let* ((bytes (string->utf8 line)) - (limit (content-end bytes)) - (unordered-bytes (string->utf8 unordered))) - (let-values (((first column) (line-indent bytes limit tab-width))) + (marker (scan-event-list-marker-bytes bytes 0 (u8vector-length bytes) + (string->utf8 unordered) ordered? tab-width))) + (when marker + (for-each (lambda (index) (vector-set! marker index (+ start (vector-ref marker index)))) '(2 3 4))) + marker)) + +;; The compiler admits the ASCII marker constant. Byte positions are absolute +;; within the borrowed request buffer; indentation columns stay frame-relative. +(def (scan-event-list-marker-bytes bytes from until unordered-bytes ordered? tab-width) + (let (limit (content-end bytes from until)) + (let-values (((first column) (line-indent bytes limit tab-width from))) (and (< first limit) (let* ((byte (u8vector-ref bytes first)) (unordered? (let loop ((index 0)) @@ -70,10 +80,5 @@ (and bullet-end (or (= bullet-end limit) (horizontal? (u8vector-ref bytes bullet-end))) - (let skip ((cursor bullet-end)) - (if (and (< cursor limit) - (horizontal? (u8vector-ref bytes cursor))) - (skip (+ cursor 1)) - (vector column (not unordered?) - (+ start first) (+ start bullet-end) - (+ start cursor)))))))))) + (vector column (not unordered?) first bullet-end + (byte-span-skip bytes bullet-end limit horizontal?)))))))) diff --git a/src/compiler/event-strategy-aot.ss b/src/compiler/event-strategy-aot.ss index d7bb2b5f..577370f5 100644 --- a/src/compiler/event-strategy-aot.ss +++ b/src/compiler/event-strategy-aot.ss @@ -1,7 +1,9 @@ ;;; -*- Gerbil -*- ;;; One pure Scheme line-event algorithm, executable here and lowered to Rust. -(import (only-in ../language/descriptor language-grammar-ir language-grammar-machine) +(import (only-in ../runtime/source-lines + fold-source-lines source-line-events line-starts-with?) + (only-in ../language/descriptor language-grammar-ir language-grammar-machine) (only-in ../runtime/identity sha256-text) (only-in ./machine parser-machine-grammar-digest) (only-in ./rust-syntax @@ -10,7 +12,7 @@ rust-module write-rust-module)) (export define-line-event-parser event-node event-token line-starts-with? compile-line-event-parser generate-line-event-module - source-line-events kind-index) + source-line-events fold-source-lines kind-index) (defrules event-node () ((_ kind child ...) @@ -20,40 +22,6 @@ ((_ kind start end) (list (list 'token 'kind start end)))) -(def (line-starts-with? line prefix) - (and (<= (string-length prefix) (string-length line)) - (string=? (substring line 0 (string-length prefix)) prefix))) - -(def (source-line-events source root visit) - (let (size (string-length source)) - (let loop ((start 0) (cursor 0) (byte-start 0) - (reversed (list (list 'start root)))) - (cond - ((= cursor size) - (if (= start size) - (reverse (cons '(finish) reversed)) - (let* ((line (substring source start size)) - (byte-end (+ byte-start - (u8vector-length (string->utf8 line))))) - (reverse (cons '(finish) - (append (reverse (visit line byte-start byte-end)) - reversed)))))) - ((or (char=? (string-ref source cursor) #\newline) - (char=? (string-ref source cursor) #\return)) - (let* ((after (+ cursor - (if (and (char=? (string-ref source cursor) #\return) - (< (+ cursor 1) size) - (char=? (string-ref source (+ cursor 1)) - #\newline)) - 2 1))) - (line (substring source start after)) - (byte-end (+ byte-start - (u8vector-length (string->utf8 line))))) - (loop after after byte-end - (append (reverse (visit line byte-start byte-end)) - reversed)))) - (else (loop start (+ cursor 1) byte-start reversed)))))) - (def (kind-index grammar name category) (let (kinds (cdr (assq 'syntax-kinds (language-grammar-ir grammar)))) (let loop ((rest kinds) (index 0)) diff --git a/src/compiler/funcs.ss b/src/compiler/funcs.ss index dd5861ba..faa661ae 100644 --- a/src/compiler/funcs.ss +++ b/src/compiler/funcs.ss @@ -1,15 +1,24 @@ ;;; -*- Gerbil -*- -;;; Allocation-free indexed-set algorithms shared by parser compilation. +;;; Shared indexed-set and source-position algorithms for parser compilation. -(import (only-in :std/list/list-builder with-list-builder)) -(export compiler-index-set-add +(import (only-in :std/list/list-builder with-list-builder) + (prefix-in :std/struct/queue stdq-)) +(export ascii-lower-byte + compiler-index-set-add compiler-index-set-difference compiler-index-set-empty? compiler-index-set-member? compiler-index-set-for-each + compiler-index-set-reachable + compiler-index-set-reachable/memo + compiler-index-partition-refine compiler-index-set-singleton compiler-index-set-union - compiler-index-set->ordered-values) + compiler-index-set->ordered-values + compiler-position-index-add! + compiler-position-index-freeze! + compiler-position-vector-next + compiler-u8vector-prefix?) ;; Indexed identities are dense non-negative integers within one compiler ;; domain, so one arbitrary-precision integer is a compact immutable set. @@ -37,16 +46,145 @@ (def (compiler-index-set-member? set index) (not (zero? (bitwise-and set (arithmetic-shift 1 index))))) -;; Gambit's optimized first-set-bit visits only present indexes. Clearing the -;; lowest bit makes sparse traversal O(cardinality), not O(maximum index), -;; and allocates no intermediate list on the propagation hot path. +;; The public set stays an immutable integer. Wide traversal extracts one +;; positive fixnum-sized field at a time; clearing a field bit never copies +;; the full bigint. Small and extremely sparse sets retain direct traversal. +;; Empty fields count as work: use the direct path when there are fewer set +;; bits than fields spanning the first and last bit. This prevents a long +;; empty gap from turning two callbacks into a scan of every field. +(def compiler-index-field-width (integer-length (##greatest-fixnum))) + ;; : (-> Integer Procedure Void) (def (compiler-index-set-for-each set procedure) - (let loop ((rest set)) - (unless (zero? rest) - (let (index (first-set-bit rest)) - (procedure index) - (loop (bitwise-and rest (- rest 1))))))) + (def (direct) + (let loop ((rest set)) + (unless (zero? rest) + (let (index (first-set-bit rest)) + (procedure index) + (loop (bitwise-and rest (- rest 1))))))) + (def (visit field offset) + (let loop ((rest field)) + (unless (zero? rest) + (let (index (first-set-bit rest)) + (procedure (+ offset index)) + (loop (bitwise-and rest (- rest 1))))))) + (if (fixnum? set) + (let loop ((rest set)) + (unless (zero? rest) + (let (index (first-set-bit rest)) + (procedure index) + (loop (bitwise-and rest (- rest 1)))))) + (let* ((limit (integer-length set)) + (width compiler-index-field-width) + (start (* width (quotient (first-set-bit set) width))) + (fields (quotient (+ (- limit start) (- width 1)) width))) + (if (< (bit-count set) fields) + (direct) + (let loop ((offset start)) + (when (< offset limit) + (visit (extract-bit-field width offset set) offset) + (loop (+ offset width)))))))) + +;; Reflexive transitive reachability over a vector of immutable index masks. +;; Mark the complete outgoing delta before queuing it: every node is expanded +;; at most once, including self edges, cycles and overlapping initial roots. +;; Only new neighbors are enumerated; membership and union operate once per +;; expanded nonempty row instead of once per outgoing edge. +(def (compiler-index-set-reachable initial edges) + (let ((reached initial) (queue (stdq-make-Queue))) + (compiler-index-set-for-each initial (lambda (node) (stdq-enqueue! queue node))) + (let drain () + (unless (stdq-queue-empty? queue) + (let (neighbors (vector-ref edges (stdq-dequeue! queue))) + (unless (zero? neighbors) + (let (delta (compiler-index-set-difference neighbors reached)) + (unless (zero? delta) + (set! reached (compiler-index-set-union reached delta)) + (compiler-index-set-for-each delta (lambda (node) (stdq-enqueue! queue node))))))) + (drain))) + reached)) + +;; Memoized reflexive transitive closure in one immutable graph. The caller +;; owns a graph-sized cache (#f means absent) and must not reuse it with another +;; graph. Every cached entry is an exact singleton closure; cache and output +;; lifetime are local to the compiler invocation. +(def (compiler-index-set-reachable/memo initial epsilon-edges cache) + (let (reached 0) + (compiler-index-set-for-each + initial + (lambda (block) + (let (closure (vector-ref cache block)) + (unless closure + (set! closure + (compiler-index-set-reachable + (compiler-index-set-singleton block) epsilon-edges)) + (vector-set! cache block closure)) + (set! reached (compiler-index-set-union reached closure))))) + reached)) + +;; Refine dense indexed partitions with label/set predecessor signatures. +;; Signature equality must be invariant under bijective group renaming. +;; Only splitting a source group can change signature equality at its targets. +;; Global renumbering of unsplit groups is injective and cannot change equality. +;; visit-successors must enumerate every signature dependency, including self +;; edges; observe receives completed group/node/signature counts each round. +;; Input IDs are never mutated. Final IDs follow first-seen node order. +;; Signature callbacks read the current IDs only during the call: internal +;; round buffers are reused. The returned vector is owned by this invocation. +(def (compiler-index-partition-refine initial initial-count signature visit-successors observe) + (let* ((count (vector-length initial)) + (sizes (make-vector count 0)) + ;; -1: unseen, -2: split, otherwise the first new group identity. + (first-child (make-vector count -1)) + (next-a (make-vector count 0)) (next-b (make-vector count 0)) + (dirty-a (make-vector count #t)) (dirty-b (make-vector count #f))) + (let refine ((ids initial) (group-count initial-count) + (next next-a) (spare next-b) (dirty dirty-a) (pending dirty-b)) + (vector-fill! sizes 0) + (vector-fill! first-child -1) + (let ((keys (make-table test: equal?)) + (next-count 0) (examined 0)) + (let count-loop ((node 0)) + (when (< node count) + (let (group (vector-ref ids node)) + (vector-set! sizes group (+ 1 (vector-ref sizes group)))) + (count-loop (+ node 1)))) + (let node-loop ((node 0)) + (when (< node count) + (let* ((group (vector-ref ids node)) + (singleton? (= (vector-ref sizes group) 1)) + (affected? (vector-ref dirty group)) + (key (and (not singleton?) affected? + (begin (set! examined (+ examined 1)) + (cons group (signature node ids))))) + (found (cond (singleton? #f) + (affected? (table-ref keys key #f)) + (else (let (first (vector-ref first-child group)) + (and (>= first 0) first)))))) + (unless found + (set! found next-count) + (set! next-count (+ next-count 1)) + (when (and (not singleton?) affected?) + (table-set! keys key found))) + (vector-set! next node found) + (let (first (vector-ref first-child group)) + (cond ((= first -1) (vector-set! first-child group found)) + ((and (>= first 0) (not (= first found))) + (vector-set! first-child group -2))))) + (node-loop (+ node 1)))) + (observe next-count count examined) + (if (and (> next-count group-count) (< next-count count)) + (begin + (vector-fill! pending #f) + (let source-loop ((source 0)) + (when (< source count) + (when (= (vector-ref first-child (vector-ref ids source)) -2) + (visit-successors source + (lambda (target) + (vector-set! pending (vector-ref next target) #t)))) + (source-loop (+ source 1)))) + (refine next next-count spare next pending dirty)) + (values next next-count)))))) ;; Materialize in canonical catalog order through the standard library's ;; linear list builder. @@ -57,3 +195,42 @@ set (lambda (index) (collect! (vector-ref values index)))))) + +;; A source-order scan conses offsets into descending lists. Freeze once so +;; repeated forward queries can use binary search without copying any suffix. +(def (compiler-position-index-add! table keys key position) + (let (positions (hash-get table key)) + (unless positions (set! keys (cons key keys))) + (hash-put! table key (cons position (or positions '()))) + keys)) + +(def (compiler-position-index-freeze! table keys) + (for-each (lambda (key) + (hash-put! table key (list->vector (hash-get table key)))) + keys)) + +;; Vectors are descending; return the first source position at or after from. +(def (compiler-position-vector-next positions from) + (and positions + (let search ((low 0) (high (vector-length positions))) + (if (< low high) + (let (middle (quotient (+ low high) 2)) + (if (>= (vector-ref positions middle) from) + (search (+ middle 1) high) + (search low middle))) + (and (> low 0) (vector-ref positions (- low 1))))))) + +(def (ascii-lower-byte value) + (if (and (<= 65 value) (<= value 90)) (+ value 32) value)) + +;; Compare a prefix in place so noncandidate source lines allocate no key. +(def (compiler-u8vector-prefix? bytes from until prefix ascii-ci?) + (let (size (u8vector-length prefix)) + (and (<= (+ from size) until) + (let compare ((index 0)) + (or (= index size) + (and (let ((actual (u8vector-ref bytes (+ from index))) + (expected (u8vector-ref prefix index))) + (= (if ascii-ci? (ascii-lower-byte actual) actual) + (if ascii-ci? (ascii-lower-byte expected) expected))) + (compare (+ index 1)))))))) diff --git a/src/compiler/fused-reduction.ss b/src/compiler/fused-reduction.ss new file mode 100644 index 00000000..c29b9318 --- /dev/null +++ b/src/compiler/fused-reduction.ss @@ -0,0 +1,217 @@ +;;; -*- Gerbil -*- +;;; Descriptor-bound fused reductions; no builtin language imports or dispatch. +(import (only-in ./lr-table validate-lr-tables) + (only-in :clan/poo/object .o .cc .ref .slot? object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ../language/descriptor language-grammar? language-grammar-ir language-grammar-machine) + (only-in ./machine parser-machine-grammar-digest parser-machine-ir) + (only-in ./build-strategy BuildStrategy. build-strategy-common-shape? + make-bound-build-strategy declare-build-strategy-provider) + (only-in ./lr lr-spec-ref production-table production-id production-lhs + production-rhs production-action operand-actions validate-production-semantics + compute-nullable base-symbol nonterminal-symbol? nonterminal-name)) +(export FusedReductionStrategy. FusedReductionStrategyContract + make-fused-reduction-strategy fused-production-expression) + +(def (strategy-productions candidate) + (lr-spec-ref (cdr (assq 'lr-spec (language-grammar-ir (.ref candidate 'descriptor)))) 'productions)) + +(def (strategy-table candidate) + (production-table (strategy-productions candidate))) + +(def (strategy-name? name) + (and (symbol? name) + (not (memq name '(begin import export def quote lambda let let* if case else + car cdr cons values error token-start event-program-append + make-recognition-fragment recognition-children-field recognition-children-alias + event-children-field event-children-alias association-row-index-ref + recognition-sequence-for-action recognition-sequence-append recognition-sequence->list + recognition-sequence-start))) + (let (text (symbol->string name)) + (and (> (string-length text) 0) (char-alphabetic? (string-ref text 0)) + (andmap (lambda (ch) (or (char-alphabetic? ch) (char-numeric? ch) (memq ch '(#\- #\_)))) + (string->list text)))))) + +;;; Admit the entire action algebra before any form is emitted. Both semantic +;;; backends use the same ordered field/alias declarations and production IDs. +(def (strategy-shape? candidate) + (with-catch (lambda (_) #f) + (lambda () + (and (build-strategy-common-shape? candidate) + (eq? (.ref candidate 'provider) +fused-reduction-provider+) + (andmap (lambda (slot) (.slot? candidate slot)) '(step-name event-name digest-name)) + (let (names (map (lambda (slot) (.ref candidate slot)) '(step-name event-name digest-name))) + (and (andmap strategy-name? names) + (not (eq? (car names) (cadr names))) + (not (eq? (car names) (caddr names))) + (not (eq? (cadr names) (caddr names))))) + (let (productions (validate-production-semantics (strategy-productions candidate))) + (let (spec (cdr (assq 'lr-spec (language-grammar-ir (.ref candidate 'descriptor))))) + (validate-lr-tables productions (lr-spec-ref spec 'actions) (lr-spec-ref spec 'gotos))) + (and (pair? productions) + (every (lambda (production) + (memq (production-action production) '(pass concat))) + productions) + #t)))))) + +(define-type (FusedReductionStrategyContract @ PooFlowContract.) + identity: 'gerbil-parser/fused-reduction-strategy + .classify: (lambda (candidate context) + (let (accepted? (strategy-shape? candidate)) + (poo-flow-classification-evidence + 'gerbil-parser/fused-reduction-strategy candidate accepted? + (if accepted? '() '((expected gerbil-parser/fused-reduction-strategy))) context)))) + +(def (stack-tail-name prefix index) + (string->symbol (string-append prefix (number->string index)))) + +(def (nth-tail name count) + (let loop ((n count) (value name)) + (if (zero? n) value (loop (fx- n 1) `(cdr ,value))))) + +;; Share each stack suffix in generated let* bindings. Repeating an nth-tail +;; expression for every operand makes wide production source quadratic. +(def (stack-value-bindings count) + (let loop ((remaining count) (tail 'semantic-values) (bindings '())) + (if (zero? remaining) + (values (reverse bindings) tail) + (let* ((index (fx- remaining 1)) + (operand (stack-tail-name "v" index)) + (next (stack-tail-name "tail" index)) + (bound (cons (list operand `(car ,tail)) bindings))) + (if (zero? index) + (values (reverse bound) `(cdr ,tail)) + (loop index next (cons (list next `(cdr ,tail)) bound))))))) + +(def (operand-expression value operand offset) + (foldl + (lambda (action current) + (case (car action) + ((field) + `(recognition-children-field ',(cadr action) + (recognition-sequence-for-action ,current) ,offset + make-recognition-fragment)) + ((alias) + `(recognition-children-alias ',(cadr action) + (recognition-sequence-for-action ,current) ,offset)) + (else (error "unsupported fused reduction operand action" action)))) + value (operand-actions operand))) + +(def (nullable-operand? operand nullable) + (let (base (base-symbol operand)) + (and (nonterminal-symbol? base) (table-ref nullable (nonterminal-name base) #f)))) + +;;; Share each required suffix bound once. Only aliases of nullable operands +;;; need a later sibling's origin; nonnullable productions emit no extra work. +(def (fused-production-expression production nullable) + (unless (memq (production-action production) '(pass concat)) + (error "unsupported fused reduction semantic action" (production-action production))) + (let* ((rhs (production-rhs production)) + (count (length rhs)) + (operands (list->vector rhs)) + (boundaries (make-vector count #f)) + (bindings '())) + (def (following-bound index) + (if (fx>= index count) 'offset + (or (vector-ref boundaries index) + (let* ((following (if (nullable-operand? (vector-ref operands index) nullable) + (following-bound (fx+ index 1)) 'offset)) + (name (stack-tail-name "source-offset" index))) + (set! bindings + (cons `(,name (recognition-sequence-start ,(stack-tail-name "v" index) ,following)) bindings)) + (vector-set! boundaries index name) + name)))) + (let (value + (let loop ((remaining rhs) (index 0) (combined ''())) + (if (null? remaining) combined + (let* ((operand (car remaining)) + (offset (if (and (nullable-operand? operand nullable) + (any (lambda (action) (eq? (car action) 'alias)) (operand-actions operand))) + (following-bound (fx+ index 1)) 'offset)) + (value (operand-expression (stack-tail-name "v" index) operand offset))) + (loop (cdr remaining) (fx+ index 1) + (if (and (eq? (production-action production) 'pass) (= count 1)) value + `(recognition-sequence-append ,combined ,value))))))) + (if (null? bindings) value `(let* ,(reverse bindings) ,value))))) + +(def (step-clause table nullable production-id) + (let* ((production (vector-ref table production-id)) + (count (length (production-rhs production)))) + (let-values (((bindings remaining-values) (stack-value-bindings count))) + `((,production-id) + (let* ((remaining-states ,(nth-tail 'states count)) + ,@bindings + (remaining-values ,remaining-values) + (offset (if (pair? rest) (token-start (car rest)) + input-end-offset)) + (value ,(fused-production-expression production nullable)) + (entry (and (pair? remaining-states) + (association-row-index-ref + goto-index (car remaining-states) + ',(production-lhs production)))) + (target (and entry (cdr entry)))) + (if target + (values target (cons target remaining-states) + (cons value remaining-values)) + (values #f #f #f))))))) + +(def (step-definition table nullable name) + `(def (,name production-id states semantic-values rest + input-end-offset goto-index) + (case production-id + ,@(let loop ((i 0) (acc '())) + (if (= i (vector-length table)) (reverse acc) + (loop (fx+ i 1) (cons (step-clause table nullable i) acc)))) + (else (error "unknown generated reduction" production-id))))) +(def (event-step-definition form step-name event-name) + (cond + ((eq? form step-name) event-name) + ((eq? form 'recognition-sequence-append) 'event-program-append) + ((eq? form 'recognition-children-field) 'event-children-field) + ((eq? form 'recognition-children-alias) 'event-children-alias) + ((not (pair? form)) form) + ((eq? (car form) 'recognition-sequence-for-action) + (event-step-definition (cadr form) step-name event-name)) + ((eq? (car form) 'quote) form) + (else (cons (event-step-definition (car form) step-name event-name) + (event-step-definition (cdr form) step-name event-name))))) + +(def (fused-reduction-module strategy) + (validate FusedReductionStrategyContract strategy) + (let-values (((_names nullable) (compute-nullable (strategy-productions strategy)))) + (let* ((table (strategy-table strategy)) (name (.ref strategy 'step-name)) + (event-name (.ref strategy 'event-name)) (digest-name (.ref strategy 'digest-name)) + (step (step-definition table nullable name))) + `(begin + (import (only-in :gerbil-parser/src/runtime/event-program event-program-append) + (only-in :gerbil-parser/src/runtime/recognition make-recognition-fragment) + (only-in :gerbil-parser/src/runtime/reduce recognition-children-field recognition-children-alias) + (only-in :gerbil-parser/src/runtime/event-reduce event-children-field event-children-alias) + (only-in :gerbil-parser/src/runtime/funcs association-row-index-ref recognition-sequence->list + recognition-sequence-for-action recognition-sequence-append recognition-sequence-start) + (only-in :gerbil-parser/src/runtime/token token-start)) + (export ,name ,event-name ,digest-name) + (def ,digest-name ,(.ref strategy 'digest)) + ,step ,(event-step-definition step name event-name))))) + +(def (emit-fused-reduction-module strategy port) + ;; Validate and materialize before writing even the first output byte. + (let (form (fused-reduction-module strategy)) + (display ";;; Generated by the engine FusedReductionStrategy from Parser IR.\n" port) + (write form port) (newline port))) + +(def +fused-reduction-provider+ + (declare-build-strategy-provider 'fused-reductions 'scheme strategy-shape? emit-fused-reduction-module)) + +(def FusedReductionStrategy. + (.o (:: self BuildStrategy.) + provider: +fused-reduction-provider+ + step-name: 'direct-step + event-name: 'direct-event-step + digest-name: 'direct-grammar-digest)) + +(def (make-fused-reduction-strategy descriptor (prototype FusedReductionStrategy.)) + (let (strategy (make-bound-build-strategy descriptor prototype)) + (validate FusedReductionStrategyContract strategy) + strategy)) diff --git a/src/compiler/graph-projection-rowan.ss b/src/compiler/graph-projection-runtime.ss similarity index 82% rename from src/compiler/graph-projection-rowan.ss rename to src/compiler/graph-projection-runtime.ss index 55ab339e..3467af5d 100644 --- a/src/compiler/graph-projection-rowan.ss +++ b/src/compiler/graph-projection-runtime.ss @@ -1,5 +1,5 @@ ;;; -*- Gerbil -*- -;;; Checked POO graph rules to an immutable Rowan projection table. +;;; Checked POO graph rules to an immutable syntax projection table. (import (only-in ../language/descriptor language-grammar-ir language-grammar-machine) @@ -10,8 +10,8 @@ graph-node-syntax-kind graph-node-category graph-node-label graph-node-fields graph-field-token graph-field-name graph-field-mode)) -(export graph-projection-digest graph-projection-rowan-source - generate-graph-projection-rowan-module) +(export graph-projection-digest graph-projection-runtime-source + generate-graph-projection-runtime-module) (def (kind-index kinds name category) (let loop ((rest kinds) (index 0)) @@ -49,7 +49,7 @@ (language-grammar-machine language-grammar)) (graph-projection-nodes projection))) -(def (graph-projection-rowan-source language-grammar projection) +(def (graph-projection-runtime-source language-grammar projection) (unless (graph-projection? projection) (error "graph projection AOT requires a checked POO value" projection)) (let* ((ir (language-grammar-ir language-grammar)) @@ -60,8 +60,8 @@ (projection-digest (graph-projection-digest language-grammar projection))) (call-with-output-string (lambda (port) - (display "// @generated by gerbil-parser/src/compiler/graph-projection-rowan.ss\n" port) - (display "use gerbil_parser_rowan::{GraphFieldMode, GraphFieldRule, GraphNodeRule, GraphProjectionSpec};\n\n" port) + (display "// @generated by gerbil-parser/src/compiler/graph-projection-runtime.ss\n" port) + (display "use gerbil_parser_runtime::{GraphFieldMode, GraphFieldRule, GraphNodeRule, GraphProjectionSpec};\n\n" port) (display "pub static GRAPH: GraphProjectionSpec = GraphProjectionSpec {\n" port) (display " grammar_digest: " port) (write digest port) @@ -80,13 +80,17 @@ (for-each (lambda (field) (display "GraphFieldRule { token_kind: " port) - (display (kind-index kinds (graph-field-token field) 'token) port) + (display (kind-index kinds (graph-field-token field) + (if (memq (graph-field-mode field) '(node-text each-node-text)) + 'node 'token)) port) (display ", name: " port) (write (graph-field-name field) port) (display ", mode: GraphFieldMode::" port) (display (case (graph-field-mode field) ((each) "Each") ((append-or-empty) "AppendOrEmpty") + ((node-text) "NodeText") + ((each-node-text) "EachNodeText") (else "Append")) port) (display " }, " port)) (graph-node-fields node)) @@ -94,7 +98,7 @@ nodes) (display " ],\n};\n" port))))) -(def (generate-graph-projection-rowan-module output-path language-grammar projection) - (let (source (graph-projection-rowan-source language-grammar projection)) +(def (generate-graph-projection-runtime-module output-path language-grammar projection) + (let (source (graph-projection-runtime-source language-grammar projection)) (call-with-output-file output-path (lambda (port) (display source port))))) diff --git a/src/compiler/hcl-reductions.ss b/src/compiler/hcl-reductions.ss new file mode 100644 index 00000000..3f3bbbd5 --- /dev/null +++ b/src/compiler/hcl-reductions.ss @@ -0,0 +1,2 @@ +;;; Generated by the engine FusedReductionStrategy from Parser IR. +(begin (import (only-in :gerbil-parser/src/runtime/event-program event-program-append) (only-in :gerbil-parser/src/runtime/recognition make-recognition-fragment) (only-in :gerbil-parser/src/runtime/reduce recognition-children-field recognition-children-alias) (only-in :gerbil-parser/src/runtime/event-reduce event-children-field event-children-alias) (only-in :gerbil-parser/src/runtime/funcs association-row-index-ref recognition-sequence->list recognition-sequence-for-action recognition-sequence-append recognition-sequence-start) (only-in :gerbil-parser/src/runtime/token token-start)) (export direct-step direct-event-step direct-grammar-digest) (def direct-grammar-digest "sha256:7fa44a0c1fccad8bfc8c10096afd4b11fd7d945dacb88ebc79f494afee851648") (def (direct-step production-id states semantic-values rest input-end-offset goto-index) (case production-id ((0) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$accept))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((1) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$config-file.0))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((2) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$config-file.1))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((3) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-field 'item (recognition-sequence-for-action v0) offset make-recognition-fragment)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$config-file.1))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((4) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$config-file.0))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((5) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'HclFile (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'config-file))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((6) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$body.2))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((7) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$body.3))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((8) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-field 'item (recognition-sequence-for-action v0) offset make-recognition-fragment)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$body.3))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((9) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$body.2))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((10) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Body (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'body))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((11) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'body-item))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((12) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'body-item))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((13) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'name (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1) (recognition-children-field 'value (recognition-sequence-for-action v2) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$attribute.4))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((14) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Attribute (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'attribute))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((15) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.6))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((16) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.7))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((17) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.7))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((18) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'label (recognition-sequence-for-action v1) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.6))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((19) (let* ((remaining-states (cdr (cdr (cdr (cdr (cdr states)))))) (v4 (car semantic-values)) (tail4 (cdr semantic-values)) (v3 (car tail4)) (tail3 (cdr tail4)) (v2 (car tail3)) (tail2 (cdr tail3)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'type (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1) v2) (recognition-children-field 'body (recognition-sequence-for-action v3) offset make-recognition-fragment)) v4)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.5))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((20) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Block (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'block))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((21) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((22) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$conditional-expression.9))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((23) (let* ((remaining-states (cdr (cdr (cdr (cdr states))))) (v3 (car semantic-values)) (tail3 (cdr semantic-values)) (v2 (car tail3)) (tail2 (cdr tail3)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'consequent (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2) (recognition-children-field 'alternative (recognition-sequence-for-action v3) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$conditional-expression.10))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((24) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$conditional-expression.9))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((25) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'condition (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$conditional-expression.8))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((26) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'ConditionalExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'conditional-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((27) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$binary-expression.12))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((28) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'operator (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v1) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$binary-expression.13))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((29) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$binary-expression.12))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((30) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$binary-expression.11))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((31) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'BinaryExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((32) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((33) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((34) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((35) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((36) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((37) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((38) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((39) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((40) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((41) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((42) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((43) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((44) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((45) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.15))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((46) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.16))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((47) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.16))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((48) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.16))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((49) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.15))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((50) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'operand (recognition-sequence-for-action v1) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.14))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((51) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'UnaryExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'unary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((52) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-expression.18))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((53) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'step (recognition-sequence-for-action v1) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-expression.18))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((54) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'root (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-expression.17))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((55) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'TraversalExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'postfix-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((56) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.19))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((57) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.19))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((58) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.19))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((59) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'postfix-suffix))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((60) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.21))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((61) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.21))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((62) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'key (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.20))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((63) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'IndexExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'postfix-suffix))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((64) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.23))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((65) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.25))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((66) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'argument (recognition-sequence-for-action v1) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.26))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((67) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.25))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((68) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'argument (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.24))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((69) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.23))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((70) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) v1) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.22))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((71) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'CallExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'postfix-suffix))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((72) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((73) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((74) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((75) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((76) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((77) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'LiteralExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((78) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) v1) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((79) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$traversal-expression.28))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((80) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'step (recognition-sequence-for-action v1) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$traversal-expression.29))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((81) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$traversal-expression.28))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((82) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'root (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$traversal-expression.27))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((83) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'TraversalExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'traversal-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((84) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'StringExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'string-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((85) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'HeredocExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'heredoc-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((86) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NumberExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'number-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((87) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.31))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((88) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.31))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((89) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.32))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((90) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.32))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((91) (let* ((remaining-states (cdr (cdr (cdr (cdr states))))) (v3 (car semantic-values)) (tail3 (cdr semantic-values)) (v2 (car tail3)) (tail2 (cdr tail3)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) v1) v2) v3)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.30))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((92) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'TupleExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'tuple-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((93) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-elements.33))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((94) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-elements.33))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((95) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'element (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'tuple-elements))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((96) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.34))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((97) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.34))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((98) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.35))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((99) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.35))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((100) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) v1) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'tuple-tail))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((101) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.36))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((102) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.36))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((103) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.37))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((104) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.37))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((105) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'tuple-tail))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((106) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.39))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((107) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.40))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((108) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-field 'entry (recognition-sequence-for-action v0) offset make-recognition-fragment)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.40))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((109) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.39))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((110) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) v1) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.38))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((111) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'ObjectExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'object-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((112) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.42))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((113) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.42))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((114) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.43))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((115) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.43))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((116) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.44))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((117) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.44))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((118) (let* ((remaining-states (cdr (cdr (cdr (cdr states))))) (v3 (car semantic-values)) (tail3 (cdr semantic-values)) (v2 (car tail3)) (tail2 (cdr tail3)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'key (recognition-sequence-for-action v0) offset make-recognition-fragment)) v1) (recognition-children-field 'value (recognition-sequence-for-action v2) offset make-recognition-fragment)) v3)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.41))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((119) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'ObjectEntry (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'object-entry))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) (else (error "unknown generated reduction" production-id)))) (def (direct-event-step production-id states semantic-values rest input-end-offset goto-index) (case production-id ((0) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$accept))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((1) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$config-file.0))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((2) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$config-file.1))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((3) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-field 'item v0 offset make-recognition-fragment)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$config-file.1))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((4) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$config-file.0))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((5) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'HclFile v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'config-file))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((6) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$body.2))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((7) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$body.3))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((8) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-field 'item v0 offset make-recognition-fragment)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$body.3))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((9) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$body.2))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((10) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'Body v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'body))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((11) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'body-item))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((12) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'body-item))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((13) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() (event-children-field 'name v0 offset make-recognition-fragment)) v1) (event-children-field 'value v2 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$attribute.4))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((14) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'Attribute v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'attribute))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((15) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.6))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((16) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.7))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((17) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.7))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((18) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) (event-children-field 'label v1 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.6))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((19) (let* ((remaining-states (cdr (cdr (cdr (cdr (cdr states)))))) (v4 (car semantic-values)) (tail4 (cdr semantic-values)) (v3 (car tail4)) (tail3 (cdr tail4)) (v2 (car tail3)) (tail2 (cdr tail3)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append (event-program-append (event-program-append '() (event-children-field 'type v0 offset make-recognition-fragment)) v1) v2) (event-children-field 'body v3 offset make-recognition-fragment)) v4)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$block.5))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((20) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'Block v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'block))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((21) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((22) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$conditional-expression.9))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((23) (let* ((remaining-states (cdr (cdr (cdr (cdr states))))) (v3 (car semantic-values)) (tail3 (cdr semantic-values)) (v2 (car tail3)) (tail2 (cdr tail3)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append (event-program-append '() v0) (event-children-field 'consequent v1 offset make-recognition-fragment)) v2) (event-children-field 'alternative v3 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$conditional-expression.10))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((24) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$conditional-expression.9))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((25) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() (event-children-field 'condition v0 offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$conditional-expression.8))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((26) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'ConditionalExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'conditional-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((27) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$binary-expression.12))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((28) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() (event-children-field 'operator v0 offset make-recognition-fragment)) (event-children-field 'right v1 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$binary-expression.13))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((29) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$binary-expression.12))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((30) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() (event-children-field 'left v0 offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$binary-expression.11))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((31) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'BinaryExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((32) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((33) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((34) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((35) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((36) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((37) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((38) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((39) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((40) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((41) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((42) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((43) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((44) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'binary-operator))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((45) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.15))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((46) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.16))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((47) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.16))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((48) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.16))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((49) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) (event-children-field 'operator v1 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.15))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((50) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) (event-children-field 'operand v1 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$unary-expression.14))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((51) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'UnaryExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'unary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((52) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-expression.18))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((53) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) (event-children-field 'step v1 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-expression.18))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((54) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() (event-children-field 'root v0 offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-expression.17))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((55) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'TraversalExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'postfix-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((56) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.19))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((57) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.19))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((58) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.19))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((59) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'postfix-suffix))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((60) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.21))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((61) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.21))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((62) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() v0) (event-children-field 'key v1 offset make-recognition-fragment)) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.20))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((63) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'IndexExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'postfix-suffix))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((64) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.23))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((65) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.25))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((66) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) (event-children-field 'argument v1 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.26))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((67) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.25))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((68) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() (event-children-field 'argument v0 offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.24))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((69) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.23))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((70) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() v0) v1) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$postfix-suffix.22))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((71) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'CallExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'postfix-suffix))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((72) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((73) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((74) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((75) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((76) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((77) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'LiteralExpression (event-children-field 'value v0 offset make-recognition-fragment) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((78) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() v0) v1) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'primary-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((79) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$traversal-expression.28))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((80) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) (event-children-field 'step v1 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$traversal-expression.29))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((81) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$traversal-expression.28))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((82) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() (event-children-field 'root v0 offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$traversal-expression.27))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((83) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'TraversalExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'traversal-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((84) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'StringExpression (event-children-field 'value v0 offset make-recognition-fragment) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'string-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((85) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'HeredocExpression (event-children-field 'value v0 offset make-recognition-fragment) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'heredoc-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((86) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'NumberExpression (event-children-field 'value v0 offset make-recognition-fragment) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'number-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((87) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.31))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((88) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.31))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((89) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.32))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((90) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.32))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((91) (let* ((remaining-states (cdr (cdr (cdr (cdr states))))) (v3 (car semantic-values)) (tail3 (cdr semantic-values)) (v2 (car tail3)) (tail2 (cdr tail3)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append (event-program-append '() v0) v1) v2) v3)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-expression.30))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((92) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'TupleExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'tuple-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((93) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-elements.33))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((94) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-elements.33))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((95) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() (event-children-field 'element v0 offset make-recognition-fragment)) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'tuple-elements))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((96) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.34))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((97) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.34))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((98) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.35))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((99) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.35))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((100) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() v0) v1) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'tuple-tail))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((101) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.36))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((102) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.36))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((103) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.37))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((104) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$tuple-tail.37))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((105) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'tuple-tail))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((106) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.39))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((107) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.40))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((108) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-field 'entry v0 offset make-recognition-fragment)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.40))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((109) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() v0) v1)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.39))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((110) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() v0) v1) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-expression.38))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((111) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'ObjectExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'object-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((112) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.42))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((113) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.42))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((114) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.43))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((115) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.43))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((116) (let* ((remaining-states states) (remaining-values semantic-values) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value '()) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.44))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((117) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.44))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((118) (let* ((remaining-states (cdr (cdr (cdr (cdr states))))) (v3 (car semantic-values)) (tail3 (cdr semantic-values)) (v2 (car tail3)) (tail2 (cdr tail3)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append (event-program-append '() (event-children-field 'key v0 offset make-recognition-fragment)) v1) (event-children-field 'value v2 offset make-recognition-fragment)) v3)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$object-entry.41))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((119) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'ObjectEntry v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'object-entry))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) (else (error "unknown generated reduction" production-id))))) diff --git a/src/compiler/hcl-source.ss b/src/compiler/hcl-source.ss new file mode 100644 index 00000000..04921d6e --- /dev/null +++ b/src/compiler/hcl-source.ss @@ -0,0 +1,930 @@ +;;; -*- Gerbil -*- +;;; Generated by src/compiler/recursive-source.ss from Grammar IR. +;;; Edit the generator, then regenerate this module. + +(import + (only-in :gerbil-parser/src/compiler/machine parser-machine-trivia) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/scan scan-quoted-string scan-line-comment scan-block-comment scan-number-literal) + (only-in :gerbil-parser/src/runtime/significant parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/token make-token token-kind token-lexeme token-start token-end) + (only-in :gerbil-parser/src/runtime/artifact make-success-parse-artifact/raw-event-tape make-success-parse-artifact/canonical-events)) + +(export direct-parse-hcl direct-lex-hcl direct-hcl-grammar-digest) + +(def direct-hcl-grammar-digest "sha256:7fa44a0c1fccad8bfc8c10096afd4b11fd7d945dacb88ebc79f494afee851648") + +(def (ascii-alpha? ch) (let (code (char->integer ch)) (or (<= 65 code 90) (<= 97 code 122) (= code 95)))) + +(def (ascii-digit? ch) (let (code (char->integer ch)) (<= 48 code 57))) + +(def (ascii-ident-rest? ch) (or (ascii-alpha? ch) (ascii-digit? ch) (char=? ch #\-))) + +(def (direct-lex-hcl source) + (let (length (string-length source)) + (def (ascii-range-end start end) (and end (let check ((index start)) (if (= index end) end (and (< (char->integer (string-ref source index)) 128) (check (fx+ index 1))))))) + (let loop + ((start 0) (tokens '())) + (if (= start length) + (reverse tokens) + (let* ((ch (string-ref source start)) + (kind + (cond + ((ascii-alpha? ch) 'identifier) + ((ascii-digit? ch) 'number) + ((or (char=? ch #\space) (char=? ch #\tab)) 'horizontal-whitespace) + ((or (char=? ch #\newline) (char=? ch #\return)) 'newline) + ((or (char=? ch #\=) (char=? ch #\{) (char=? ch #\})) 'punctuation) + ((char=? ch #\") 'string) + ((char=? ch #\#) 'comment) + ((and (char=? ch #\/) (< (fx+ start 1) length) (char=? (string-ref source (fx+ start 1)) #\/)) 'comment) + ((and (char=? ch #\/) (< (fx+ start 1) length) (char=? (string-ref source (fx+ start 1)) #\*)) 'block-comment-token) + (else #f)))) + (if (not kind) #f + (let (first-end + (case kind + ((string) (ascii-range-end start (scan-quoted-string source start "\""))) + ((comment) (ascii-range-end start (scan-line-comment source start '("#" "//")))) + ((block-comment-token) (ascii-range-end start (scan-block-comment source start "/*" "*/"))) + (else (fx+ start 1)))) + (and first-end + (let scan + ((end first-end)) + (if (and + (< end length) + (let (next (string-ref source end)) + (case kind ((identifier) (ascii-ident-rest? next)) ((number) (ascii-digit? next)) ((horizontal-whitespace) (or (char=? next #\space) (char=? next #\tab))) ((newline) (or (char=? next #\newline) (char=? next #\return))) (else #f)))) + (scan (fx+ end 1)) + (let (token-end (if (and (eq? kind 'number) (< end length) (let (next (string-ref source end)) (or (char=? next #\.) (char=? next #\e) (char=? next #\E)))) (ascii-range-end start (scan-number-literal source start)) end)) + (and token-end (if (and (eq? kind 'punctuation) (< token-end length) (char=? (string-ref source token-end) #\=)) #f (loop token-end (cons (make-token kind (substring source start token-end) start token-end) tokens))))))))))))))) + +(def (direct-parse-hcl machine source (use-simple? #t) (use-fast-lex? #t) (use-canonical? #t)) + (let* ((source-bytes (string->utf8 source)) + (byte-length (u8vector-length source-bytes)) + (tokens (or (and use-fast-lex? (= byte-length (string-length source)) (direct-lex-hcl source)) (lex-source machine source))) + (significant (list->vector (parser-significant-tokens machine tokens))) + (limit (vector-length significant)) + (events #f) + (event-count 0)) + (def (offset pos) (if (< pos limit) (token-start (vector-ref significant pos)) byte-length)) + (def (range-end start next) (if (= start next) (offset start) (token-end (vector-ref significant (fx- next 1))))) + (def (emit-event! operation name byte-offset) + (unless events (set! events (make-vector (* 3 (max 64 (* 6 (length tokens)))) #f))) + (when (= (* 3 event-count) (vector-length events)) (let* ((old events) (grown (make-vector (* 2 (vector-length old)) #f))) (let loop ((i 0)) (when (< i (* 3 event-count)) (vector-set! grown i (vector-ref old i)) (loop (fx+ i 1)))) (set! events grown))) + (let (base (* 3 event-count)) (vector-set! events base operation) (vector-set! events (fx+ base 1) name) (vector-set! events (fx+ base 2) byte-offset)) + (set! event-count (fx+ event-count 1))) + (def (rollback! mark) (set! event-count mark)) + (def (emit-simple-attribute! name equals value) + (let* ((start (token-start name)) (name-end (token-end name)) (value-start (token-start value)) (value-end (token-end value)) (value-node (case (token-kind value) ((number) 'NumberExpression) ((string) 'StringExpression) ((identifier) 'LiteralExpression)))) + (emit-event! 'open-field 'item start) + (emit-event! 'open-node 'Attribute start) + (emit-event! 'open-field 'name start) + (emit-event! 'token name 0) + (emit-event! 'close-field 'name name-end) + (emit-event! 'token equals 0) + (emit-event! 'open-field 'value value-start) + (emit-event! 'open-node 'ConditionalExpression value-start) + (emit-event! 'open-field 'condition value-start) + (emit-event! 'open-node 'BinaryExpression value-start) + (emit-event! 'open-field 'left value-start) + (emit-event! 'open-node 'UnaryExpression value-start) + (emit-event! 'open-field 'operand value-start) + (emit-event! 'open-node 'TraversalExpression value-start) + (emit-event! 'open-field 'root value-start) + (emit-event! 'open-node value-node value-start) + (emit-event! 'open-field 'value value-start) + (emit-event! 'token value 0) + (emit-event! 'close-field 'value value-end) + (emit-event! 'close-node value-node value-end) + (emit-event! 'close-field 'root value-end) + (emit-event! 'close-node 'TraversalExpression value-end) + (emit-event! 'close-field 'operand value-end) + (emit-event! 'close-node 'UnaryExpression value-end) + (emit-event! 'close-field 'left value-end) + (emit-event! 'close-node 'BinaryExpression value-end) + (emit-event! 'close-field 'condition value-end) + (emit-event! 'close-node 'ConditionalExpression value-end) + (emit-event! 'close-field 'value value-end) + (emit-event! 'close-node 'Attribute value-end) + (emit-event! 'close-field 'item value-end))) + (def (simple-attributes?) + (let loop + ((pos 0)) + (if (= pos limit) #t + (let (input (vector-ref significant pos)) + (cond + ((eq? (token-kind input) 'newline) (loop (fx+ pos 1))) + ((and (< (fx+ pos 2) limit) (eq? (token-kind input) 'identifier) (equal? (token-lexeme (vector-ref significant (fx+ pos 1))) "=") (memq (token-kind (vector-ref significant (fx+ pos 2))) '(number string identifier))) (loop (fx+ pos 3))) + (else #f)))))) + (def (simple-value-at? pos) + (and (< (fx+ pos 2) limit) (eq? (token-kind (vector-ref significant pos)) 'identifier) (equal? (token-lexeme (vector-ref significant (fx+ pos 1))) "=") (memq (token-kind (vector-ref significant (fx+ pos 2))) '(number string identifier)))) + (def (brace-at? pos brace) (and (< pos limit) (equal? (token-lexeme (vector-ref significant pos)) brace))) + (def (block-opening-at pos) + (and + (< pos limit) + (eq? (token-kind (vector-ref significant pos)) 'identifier) + (let labels ((cursor (fx+ pos 1))) (cond ((= cursor limit) #f) ((brace-at? cursor "{") cursor) ((memq (token-kind (vector-ref significant cursor)) '(string identifier)) (labels (fx+ cursor 1))) (else #f))))) + (def (closed-blocks?) + (def (scan-block pos) + (let (opening-pos (block-opening-at pos)) + (and opening-pos + (let body + ((cursor (fx+ opening-pos 1))) + (cond + ((= cursor limit) #f) + ((brace-at? cursor "}") (fx+ cursor 1)) + ((eq? (token-kind (vector-ref significant cursor)) 'newline) (body (fx+ cursor 1))) + ((simple-value-at? cursor) (body (fx+ cursor 3))) + (else (let (next (scan-block cursor)) (and next (body next))))))))) + (let root ((pos 0) (saw-block? #f)) (cond ((= pos limit) saw-block?) ((eq? (token-kind (vector-ref significant pos)) 'newline) (root (fx+ pos 1) saw-block?)) (else (let (next (scan-block pos)) (and next (root next #t))))))) + (def (emit-simple-attributes!) + (emit-event! 'open-node 'HclFile 0) + (let loop + ((pos 0)) + (if (= pos limit) + (begin (emit-event! 'close-node 'HclFile byte-length) (void)) + (let (input (vector-ref significant pos)) + (if (eq? (token-kind input) 'newline) (begin (emit-event! 'token input 0) (loop (fx+ pos 1))) (begin (emit-simple-attribute! input (vector-ref significant (fx+ pos 1)) (vector-ref significant (fx+ pos 2))) (loop (fx+ pos 3)))))))) + (def (make-closed-canonical-artifact shape) + (let ((remaining tokens) (trivia? (parser-machine-trivia machine)) (canonical-events (cons #f '())) (tail #f) (next-token-id 0) (next-node-id 0)) + (set! tail canonical-events) + (def (emit! event) (let (cell (cons event '())) (set-cdr! tail cell) (set! tail cell))) + (def (emit-token! input) + (unless (and (pair? remaining) (eq? input (car remaining))) (error "simple HCL token order changed" input)) + (emit! (vector 'token next-token-id (token-kind input) (token-lexeme input) (token-start input) (token-end input))) + (set! next-token-id (fx+ next-token-id 1)) + (set! remaining (cdr remaining))) + (def (emit-trivia-until! boundary) (let loop () (when (and (pair? remaining) (<= (token-end (car remaining)) boundary)) (unless (trivia? (car remaining)) (error "unclaimed simple HCL token" (car remaining))) (emit-token! (car remaining)) (loop)))) + (def (open-node! name start) (emit-trivia-until! start) (let (id next-node-id) (emit! (vector 'start-node id name start)) (set! next-node-id (fx+ next-node-id 1)) id)) + (def (close-node! id name end) (emit-trivia-until! end) (emit! (vector 'finish-node id name end))) + (def (open-field! name start) (emit-trivia-until! start) (emit! (vector 'start-field name start))) + (def (close-field! name end) (emit! (vector 'finish-field name end))) + (def (emit-attribute! name equals value) + (let* ((start (token-start name)) + (name-end (token-end name)) + (value-start (token-start value)) + (value-end (token-end value)) + (value-node (case (token-kind value) ((number) 'NumberExpression) ((string) 'StringExpression) ((identifier) 'LiteralExpression))) + (base next-node-id)) + (set! next-node-id (fx+ base 6)) + (emit-trivia-until! start) + (emit! (vector 'start-field 'item start)) + (emit! (vector 'start-node base 'Attribute start)) + (emit! (vector 'start-field 'name start)) + (emit-token! name) + (emit! (vector 'finish-field 'name name-end)) + (emit-trivia-until! (token-start equals)) + (emit-token! equals) + (emit-trivia-until! value-start) + (emit! (vector 'start-field 'value value-start)) + (emit! (vector 'start-node (fx+ base 1) 'ConditionalExpression value-start)) + (emit! (vector 'start-field 'condition value-start)) + (emit! (vector 'start-node (fx+ base 2) 'BinaryExpression value-start)) + (emit! (vector 'start-field 'left value-start)) + (emit! (vector 'start-node (fx+ base 3) 'UnaryExpression value-start)) + (emit! (vector 'start-field 'operand value-start)) + (emit! (vector 'start-node (fx+ base 4) 'TraversalExpression value-start)) + (emit! (vector 'start-field 'root value-start)) + (emit! (vector 'start-node (fx+ base 5) value-node value-start)) + (emit! (vector 'start-field 'value value-start)) + (emit-token! value) + (emit! (vector 'finish-field 'value value-end)) + (emit! (vector 'finish-node (fx+ base 5) value-node value-end)) + (emit! (vector 'finish-field 'root value-end)) + (emit! (vector 'finish-node (fx+ base 4) 'TraversalExpression value-end)) + (emit! (vector 'finish-field 'operand value-end)) + (emit! (vector 'finish-node (fx+ base 3) 'UnaryExpression value-end)) + (emit! (vector 'finish-field 'left value-end)) + (emit! (vector 'finish-node (fx+ base 2) 'BinaryExpression value-end)) + (emit! (vector 'finish-field 'condition value-end)) + (emit! (vector 'finish-node (fx+ base 1) 'ConditionalExpression value-end)) + (emit! (vector 'finish-field 'value value-end)) + (emit! (vector 'finish-node base 'Attribute value-end)) + (emit! (vector 'finish-field 'item value-end)))) + (def (emit-closed-block! pos) + (let* ((type (vector-ref significant pos)) (opening-pos (block-opening-at pos)) (opening (vector-ref significant opening-pos)) (start (token-start type)) (type-end (token-end type)) (body-start (token-start (vector-ref significant (fx+ opening-pos 1))))) + (open-field! 'item start) + (let (block-id (open-node! 'Block start)) + (open-field! 'type start) + (emit-trivia-until! start) + (emit-token! type) + (close-field! 'type type-end) + (let labels + ((cursor (fx+ pos 1))) + (when + (< cursor opening-pos) + (let* ((label (vector-ref significant cursor)) (label-start (token-start label))) (emit-trivia-until! label-start) (open-field! 'label label-start) (emit-token! label) (close-field! 'label (token-end label)) (labels (fx+ cursor 1))))) + (emit-trivia-until! (token-start opening)) + (emit-token! opening) + (open-field! 'body body-start) + (let (body-id (open-node! 'Body body-start)) + (let body + ((cursor (fx+ opening-pos 1)) (body-end body-start)) + (let (input (vector-ref significant cursor)) + (cond + ((equal? (token-lexeme input) "}") + (close-node! body-id 'Body body-end) + (close-field! 'body body-end) + (emit-trivia-until! (token-start input)) + (emit-token! input) + (close-node! block-id 'Block (token-end input)) + (close-field! 'item (token-end input)) + (fx+ cursor 1)) + ((eq? (token-kind input) 'newline) (emit-trivia-until! (token-start input)) (emit-token! input) (body (fx+ cursor 1) (token-end input))) + ((simple-value-at? cursor) (let (value (vector-ref significant (fx+ cursor 2))) (emit-attribute! input (vector-ref significant (fx+ cursor 1)) value) (body (fx+ cursor 3) (token-end value)))) + (else (let (next (emit-closed-block! cursor)) (body next (token-end (vector-ref significant (fx- next 1))))))))))))) + (let (root-id (open-node! 'HclFile 0)) + (let loop + ((pos 0)) + (when + (< pos limit) + (let (input (vector-ref significant pos)) + (if (eq? (token-kind input) 'newline) + (begin (emit-trivia-until! (token-start input)) (emit-token! input) (loop (fx+ pos 1))) + (if (eq? shape 'flat-blocks) (loop (emit-closed-block! pos)) (begin (emit-attribute! input (vector-ref significant (fx+ pos 1)) (vector-ref significant (fx+ pos 2))) (loop (fx+ pos 3)))))))) + (close-node! root-id 'HclFile byte-length)) + (unless (null? remaining) (error "simple HCL source tokens remain" remaining)) + (make-success-parse-artifact/canonical-events direct-hcl-grammar-digest source (cdr canonical-events) source-bytes))) + (let* ((simple? (and use-simple? (simple-attributes?))) (blocks? (and use-simple? (not simple?) (closed-blocks?)))) + (if (and use-canonical? (or simple? blocks?)) + (make-closed-canonical-artifact (if simple? 'attributes 'flat-blocks)) + (letrec ((events-config-file + (lambda (pos) + (let (mark0 event-count) + (emit-event! 'open-node 'HclFile (offset pos)) + (let (open1 event-count) + (let (next2 + (let repeat-loop3 + ((next4 pos) (count5 0)) + (let (mark6 event-count) + (let (after7 + (let (mark8 event-count) + (let (next9 (if (< next4 limit) (let (input (vector-ref significant next4)) (if (eq? (token-kind input) 'newline) (begin (emit-event! 'token input 0) (fx+ next4 1)) #f)) #f)) + (if next9 next9 + (begin + (rollback! mark8) + (let (next10 + (let (mark11 event-count) + (emit-event! 'open-field 'item (offset next4)) + (let (open12 event-count) + (let (next13 (events-body-item next4)) (if next13 (begin (if (= event-count open12) (rollback! mark11) (emit-event! 'close-field 'item (range-end next4 next13))) next13) (begin (rollback! mark11) #f)))))) + (if next10 next10 (begin (rollback! mark8) #f)))))))) + (if after7 (begin (when (= after7 next4) (error "zero-width repeated grammar expression")) (repeat-loop3 after7 (fx+ count5 1))) (begin (rollback! mark6) next4)))))) + (if next2 (begin (emit-event! 'close-node 'HclFile (range-end pos next2)) next2) (begin (rollback! mark0) #f))))))) + (events-body + (lambda (pos) + (let (mark14 event-count) + (emit-event! 'open-node 'Body (offset pos)) + (let (open15 event-count) + (let (next16 + (let repeat-loop17 + ((next18 pos) (count19 0)) + (let (mark20 event-count) + (let (after21 + (let (mark22 event-count) + (let (next23 (if (< next18 limit) (let (input (vector-ref significant next18)) (if (eq? (token-kind input) 'newline) (begin (emit-event! 'token input 0) (fx+ next18 1)) #f)) #f)) + (if next23 next23 + (begin + (rollback! mark22) + (let (next24 + (let (mark25 event-count) + (emit-event! 'open-field 'item (offset next18)) + (let (open26 event-count) + (let (next27 (events-body-item next18)) (if next27 (begin (if (= event-count open26) (rollback! mark25) (emit-event! 'close-field 'item (range-end next18 next27))) next27) (begin (rollback! mark25) #f)))))) + (if next24 next24 (begin (rollback! mark22) #f)))))))) + (if after21 (begin (when (= after21 next18) (error "zero-width repeated grammar expression")) (repeat-loop17 after21 (fx+ count19 1))) (begin (rollback! mark20) next18)))))) + (if next16 (begin (emit-event! 'close-node 'Body (range-end pos next16)) next16) (begin (rollback! mark14) #f))))))) + (events-body-item (lambda (pos) (let (mark28 event-count) (let (next29 (events-block pos)) (if next29 next29 (begin (rollback! mark28) (let (next30 (events-attribute pos)) (if next30 next30 (begin (rollback! mark28) #f))))))))) + (events-attribute + (lambda (pos) + (let (mark31 event-count) + (emit-event! 'open-node 'Attribute (offset pos)) + (let (open32 event-count) + (let (next33 + (let (mark34 event-count) + (let (next35 + (let (mark36 event-count) + (emit-event! 'open-field 'name (offset pos)) + (let (open37 event-count) + (let (next38 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'identifier) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next38 (begin (if (= event-count open37) (rollback! mark36) (emit-event! 'close-field 'name (range-end pos next38))) next38) (begin (rollback! mark36) #f)))))) + (if next35 + (let (next39 (if (< next35 limit) (let (input (vector-ref significant next35)) (if (equal? (token-lexeme input) "=") (begin (emit-event! 'token input 0) (fx+ next35 1)) #f)) #f)) + (if next39 + (let (next40 + (let (mark41 event-count) + (emit-event! 'open-field 'value (offset next39)) + (let (open42 event-count) + (let (next43 (events-expression next39)) (if next43 (begin (if (= event-count open42) (rollback! mark41) (emit-event! 'close-field 'value (range-end next39 next43))) next43) (begin (rollback! mark41) #f)))))) + (if next40 next40 (begin (rollback! mark34) #f))) + (begin (rollback! mark34) #f))) + (begin (rollback! mark34) #f))))) + (if next33 (begin (emit-event! 'close-node 'Attribute (range-end pos next33)) next33) (begin (rollback! mark31) #f))))))) + (events-block + (lambda (pos) + (let (mark44 event-count) + (emit-event! 'open-node 'Block (offset pos)) + (let (open45 event-count) + (let (next46 + (let (mark47 event-count) + (let (next48 + (let (mark49 event-count) + (emit-event! 'open-field 'type (offset pos)) + (let (open50 event-count) + (let (next51 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'identifier) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next51 (begin (if (= event-count open50) (rollback! mark49) (emit-event! 'close-field 'type (range-end pos next51))) next51) (begin (rollback! mark49) #f)))))) + (if next48 + (let (next52 + (let repeat-loop53 + ((next54 next48) (count55 0)) + (let (mark56 event-count) + (let (after57 + (let (mark58 event-count) + (emit-event! 'open-field 'label (offset next54)) + (let (open59 event-count) + (let (next60 + (let (mark61 event-count) + (let (next62 (if (< next54 limit) (let (input (vector-ref significant next54)) (if (eq? (token-kind input) 'string) (begin (emit-event! 'token input 0) (fx+ next54 1)) #f)) #f)) + (if next62 next62 + (begin + (rollback! mark61) + (let (next63 (if (< next54 limit) (let (input (vector-ref significant next54)) (if (eq? (token-kind input) 'identifier) (begin (emit-event! 'token input 0) (fx+ next54 1)) #f)) #f)) + (if next63 next63 (begin (rollback! mark61) #f)))))))) + (if next60 (begin (if (= event-count open59) (rollback! mark58) (emit-event! 'close-field 'label (range-end next54 next60))) next60) (begin (rollback! mark58) #f)))))) + (if after57 (begin (when (= after57 next54) (error "zero-width repeated grammar expression")) (repeat-loop53 after57 (fx+ count55 1))) (begin (rollback! mark56) next54)))))) + (if next52 + (let (next64 (if (< next52 limit) (let (input (vector-ref significant next52)) (if (equal? (token-lexeme input) "{") (begin (emit-event! 'token input 0) (fx+ next52 1)) #f)) #f)) + (if next64 + (let (next65 + (let (mark66 event-count) + (emit-event! 'open-field 'body (offset next64)) + (let (open67 event-count) + (let (next68 (events-body next64)) (if next68 (begin (if (= event-count open67) (rollback! mark66) (emit-event! 'close-field 'body (range-end next64 next68))) next68) (begin (rollback! mark66) #f)))))) + (if next65 + (let (next69 (if (< next65 limit) (let (input (vector-ref significant next65)) (if (equal? (token-lexeme input) "}") (begin (emit-event! 'token input 0) (fx+ next65 1)) #f)) #f)) + (if next69 next69 (begin (rollback! mark47) #f))) + (begin (rollback! mark47) #f))) + (begin (rollback! mark47) #f))) + (begin (rollback! mark47) #f))) + (begin (rollback! mark47) #f))))) + (if next46 (begin (emit-event! 'close-node 'Block (range-end pos next46)) next46) (begin (rollback! mark44) #f))))))) + (events-expression (lambda (pos) (events-conditional-expression pos))) + (events-conditional-expression + (lambda (pos) + (let (mark70 event-count) + (emit-event! 'open-node 'ConditionalExpression (offset pos)) + (let (open71 event-count) + (let (next72 + (let (mark73 event-count) + (let (next74 + (let (mark75 event-count) + (emit-event! 'open-field 'condition (offset pos)) + (let (open76 event-count) + (let (next77 (events-binary-expression pos)) (if next77 (begin (if (= event-count open76) (rollback! mark75) (emit-event! 'close-field 'condition (range-end pos next77))) next77) (begin (rollback! mark75) #f)))))) + (if next74 + (let (next78 + (let (mark79 event-count) + (let (next80 + (let (mark81 event-count) + (let (next82 (if (< next74 limit) (let (input (vector-ref significant next74)) (if (equal? (token-lexeme input) "?") (begin (emit-event! 'token input 0) (fx+ next74 1)) #f)) #f)) + (if next82 + (let (next83 + (let (mark84 event-count) + (emit-event! 'open-field 'consequent (offset next82)) + (let (open85 event-count) + (let (next86 (events-expression next82)) + (if next86 (begin (if (= event-count open85) (rollback! mark84) (emit-event! 'close-field 'consequent (range-end next82 next86))) next86) (begin (rollback! mark84) #f)))))) + (if next83 + (let (next87 (if (< next83 limit) (let (input (vector-ref significant next83)) (if (equal? (token-lexeme input) ":") (begin (emit-event! 'token input 0) (fx+ next83 1)) #f)) #f)) + (if next87 + (let (next88 + (let (mark89 event-count) + (emit-event! 'open-field 'alternative (offset next87)) + (let (open90 event-count) + (let (next91 (events-expression next87)) + (if next91 (begin (if (= event-count open90) (rollback! mark89) (emit-event! 'close-field 'alternative (range-end next87 next91))) next91) (begin (rollback! mark89) #f)))))) + (if next88 next88 (begin (rollback! mark81) #f))) + (begin (rollback! mark81) #f))) + (begin (rollback! mark81) #f))) + (begin (rollback! mark81) #f))))) + (if next80 next80 (begin (rollback! mark79) next74))))) + (if next78 next78 (begin (rollback! mark73) #f))) + (begin (rollback! mark73) #f))))) + (if next72 (begin (emit-event! 'close-node 'ConditionalExpression (range-end pos next72)) next72) (begin (rollback! mark70) #f))))))) + (events-binary-expression + (lambda (pos) + (let (mark92 event-count) + (emit-event! 'open-node 'BinaryExpression (offset pos)) + (let (open93 event-count) + (let (next94 + (let (mark95 event-count) + (let (next96 + (let (mark97 event-count) + (emit-event! 'open-field 'left (offset pos)) + (let (open98 event-count) + (let (next99 (events-unary-expression pos)) (if next99 (begin (if (= event-count open98) (rollback! mark97) (emit-event! 'close-field 'left (range-end pos next99))) next99) (begin (rollback! mark97) #f)))))) + (if next96 + (let (next100 + (let repeat-loop101 + ((next102 next96) (count103 0)) + (let (mark104 event-count) + (let (after105 + (let (mark106 event-count) + (let (next107 + (let (mark108 event-count) + (emit-event! 'open-field 'operator (offset next102)) + (let (open109 event-count) + (let (next110 (events-binary-operator next102)) + (if next110 (begin (if (= event-count open109) (rollback! mark108) (emit-event! 'close-field 'operator (range-end next102 next110))) next110) (begin (rollback! mark108) #f)))))) + (if next107 + (let (next111 + (let (mark112 event-count) + (emit-event! 'open-field 'right (offset next107)) + (let (open113 event-count) + (let (next114 (events-unary-expression next107)) + (if next114 (begin (if (= event-count open113) (rollback! mark112) (emit-event! 'close-field 'right (range-end next107 next114))) next114) (begin (rollback! mark112) #f)))))) + (if next111 next111 (begin (rollback! mark106) #f))) + (begin (rollback! mark106) #f))))) + (if after105 (begin (when (= after105 next102) (error "zero-width repeated grammar expression")) (repeat-loop101 after105 (fx+ count103 1))) (begin (rollback! mark104) next102)))))) + (if next100 next100 (begin (rollback! mark95) #f))) + (begin (rollback! mark95) #f))))) + (if next94 (begin (emit-event! 'close-node 'BinaryExpression (range-end pos next94)) next94) (begin (rollback! mark92) #f))))))) + (events-binary-operator + (lambda (pos) + (let (mark115 event-count) + (let (next116 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "==") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next116 next116 + (begin + (rollback! mark115) + (let (next117 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "!=") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next117 next117 + (begin + (rollback! mark115) + (let (next118 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "<=") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next118 next118 + (begin + (rollback! mark115) + (let (next119 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) ">=") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next119 next119 + (begin + (rollback! mark115) + (let (next120 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "&&") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next120 next120 + (begin + (rollback! mark115) + (let (next121 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "||") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next121 next121 + (begin + (rollback! mark115) + (let (next122 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "+") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next122 next122 + (begin + (rollback! mark115) + (let (next123 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "-") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next123 next123 + (begin + (rollback! mark115) + (let (next124 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "*") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next124 next124 + (begin + (rollback! mark115) + (let (next125 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "/") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next125 next125 + (begin + (rollback! mark115) + (let (next126 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "%") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next126 next126 + (begin + (rollback! mark115) + (let (next127 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "<") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next127 next127 + (begin + (rollback! mark115) + (let (next128 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) ">") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next128 next128 (begin (rollback! mark115) #f)))))))))))))))))))))))))))))))))))))))))) + (events-unary-expression + (lambda (pos) + (let (mark129 event-count) + (emit-event! 'open-node 'UnaryExpression (offset pos)) + (let (open130 event-count) + (let (next131 + (let (mark132 event-count) + (let (next133 + (let repeat-loop134 + ((next135 pos) (count136 0)) + (let (mark137 event-count) + (let (after138 + (let (mark139 event-count) + (emit-event! 'open-field 'operator (offset next135)) + (let (open140 event-count) + (let (next141 + (let (mark142 event-count) + (let (next143 (if (< next135 limit) (let (input (vector-ref significant next135)) (if (equal? (token-lexeme input) "!") (begin (emit-event! 'token input 0) (fx+ next135 1)) #f)) #f)) + (if next143 next143 + (begin + (rollback! mark142) + (let (next144 (if (< next135 limit) (let (input (vector-ref significant next135)) (if (equal? (token-lexeme input) "-") (begin (emit-event! 'token input 0) (fx+ next135 1)) #f)) #f)) + (if next144 next144 + (begin + (rollback! mark142) + (let (next145 (if (< next135 limit) (let (input (vector-ref significant next135)) (if (equal? (token-lexeme input) "+") (begin (emit-event! 'token input 0) (fx+ next135 1)) #f)) #f)) + (if next145 next145 (begin (rollback! mark142) #f))))))))))) + (if next141 (begin (if (= event-count open140) (rollback! mark139) (emit-event! 'close-field 'operator (range-end next135 next141))) next141) (begin (rollback! mark139) #f)))))) + (if after138 (begin (when (= after138 next135) (error "zero-width repeated grammar expression")) (repeat-loop134 after138 (fx+ count136 1))) (begin (rollback! mark137) next135)))))) + (if next133 + (let (next146 + (let (mark147 event-count) + (emit-event! 'open-field 'operand (offset next133)) + (let (open148 event-count) + (let (next149 (events-postfix-expression next133)) + (if next149 (begin (if (= event-count open148) (rollback! mark147) (emit-event! 'close-field 'operand (range-end next133 next149))) next149) (begin (rollback! mark147) #f)))))) + (if next146 next146 (begin (rollback! mark132) #f))) + (begin (rollback! mark132) #f))))) + (if next131 (begin (emit-event! 'close-node 'UnaryExpression (range-end pos next131)) next131) (begin (rollback! mark129) #f))))))) + (events-postfix-expression + (lambda (pos) + (let (mark150 event-count) + (emit-event! 'open-node 'TraversalExpression (offset pos)) + (let (open151 event-count) + (let (next152 + (let (mark153 event-count) + (let (next154 + (let (mark155 event-count) + (emit-event! 'open-field 'root (offset pos)) + (let (open156 event-count) + (let (next157 (events-primary-expression pos)) (if next157 (begin (if (= event-count open156) (rollback! mark155) (emit-event! 'close-field 'root (range-end pos next157))) next157) (begin (rollback! mark155) #f)))))) + (if next154 + (let (next158 + (let repeat-loop159 + ((next160 next154) (count161 0)) + (let (mark162 event-count) + (let (after163 + (let (mark164 event-count) + (emit-event! 'open-field 'step (offset next160)) + (let (open165 event-count) + (let (next166 (events-postfix-suffix next160)) + (if next166 (begin (if (= event-count open165) (rollback! mark164) (emit-event! 'close-field 'step (range-end next160 next166))) next166) (begin (rollback! mark164) #f)))))) + (if after163 (begin (when (= after163 next160) (error "zero-width repeated grammar expression")) (repeat-loop159 after163 (fx+ count161 1))) (begin (rollback! mark162) next160)))))) + (if next158 next158 (begin (rollback! mark153) #f))) + (begin (rollback! mark153) #f))))) + (if next152 (begin (emit-event! 'close-node 'TraversalExpression (range-end pos next152)) next152) (begin (rollback! mark150) #f))))))) + (events-postfix-suffix + (lambda (pos) + (let (mark167 event-count) + (let (next168 + (let (mark169 event-count) + (let (next170 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) ".") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next170 + (let (next171 + (let (mark172 event-count) + (let (next173 (if (< next170 limit) (let (input (vector-ref significant next170)) (if (eq? (token-kind input) 'identifier) (begin (emit-event! 'token input 0) (fx+ next170 1)) #f)) #f)) + (if next173 next173 + (begin + (rollback! mark172) + (let (next174 (if (< next170 limit) (let (input (vector-ref significant next170)) (if (eq? (token-kind input) 'number) (begin (emit-event! 'token input 0) (fx+ next170 1)) #f)) #f)) + (if next174 next174 + (begin + (rollback! mark172) + (let (next175 (if (< next170 limit) (let (input (vector-ref significant next170)) (if (equal? (token-lexeme input) "*") (begin (emit-event! 'token input 0) (fx+ next170 1)) #f)) #f)) + (if next175 next175 (begin (rollback! mark172) #f))))))))))) + (if next171 next171 (begin (rollback! mark169) #f))) + (begin (rollback! mark169) #f))))) + (if next168 next168 + (begin + (rollback! mark167) + (let (next176 + (let (mark177 event-count) + (emit-event! 'open-node 'IndexExpression (offset pos)) + (let (open178 event-count) + (let (next179 + (let (mark180 event-count) + (let (next181 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "[") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next181 + (let (next182 + (let (mark183 event-count) + (emit-event! 'open-field 'key (offset next181)) + (let (open184 event-count) + (let (next185 + (let (mark186 event-count) + (let (next187 (if (< next181 limit) (let (input (vector-ref significant next181)) (if (equal? (token-lexeme input) "*") (begin (emit-event! 'token input 0) (fx+ next181 1)) #f)) #f)) + (if next187 next187 (begin (rollback! mark186) (let (next188 (events-expression next181)) (if next188 next188 (begin (rollback! mark186) #f)))))))) + (if next185 (begin (if (= event-count open184) (rollback! mark183) (emit-event! 'close-field 'key (range-end next181 next185))) next185) (begin (rollback! mark183) #f)))))) + (if next182 + (let (next189 (if (< next182 limit) (let (input (vector-ref significant next182)) (if (equal? (token-lexeme input) "]") (begin (emit-event! 'token input 0) (fx+ next182 1)) #f)) #f)) + (if next189 next189 (begin (rollback! mark180) #f))) + (begin (rollback! mark180) #f))) + (begin (rollback! mark180) #f))))) + (if next179 (begin (emit-event! 'close-node 'IndexExpression (range-end pos next179)) next179) (begin (rollback! mark177) #f)))))) + (if next176 next176 + (begin + (rollback! mark167) + (let (next190 + (let (mark191 event-count) + (emit-event! 'open-node 'CallExpression (offset pos)) + (let (open192 event-count) + (let (next193 + (let (mark194 event-count) + (let (next195 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "(") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next195 + (let (next196 + (let (mark197 event-count) + (let (next198 + (let (mark199 event-count) + (let (next200 + (let (mark201 event-count) + (emit-event! 'open-field 'argument (offset next195)) + (let (open202 event-count) + (let (next203 (events-expression next195)) + (if next203 (begin (if (= event-count open202) (rollback! mark201) (emit-event! 'close-field 'argument (range-end next195 next203))) next203) (begin (rollback! mark201) #f)))))) + (if next200 + (let (next204 + (let repeat-loop205 + ((next206 next200) (count207 0)) + (let (mark208 event-count) + (let (after209 + (let (mark210 event-count) + (let (next211 (if (< next206 limit) (let (input (vector-ref significant next206)) (if (equal? (token-lexeme input) ",") (begin (emit-event! 'token input 0) (fx+ next206 1)) #f)) #f)) + (if next211 + (let (next212 + (let (mark213 event-count) + (emit-event! 'open-field 'argument (offset next211)) + (let (open214 event-count) + (let (next215 (events-expression next211)) + (if next215 + (begin (if (= event-count open214) (rollback! mark213) (emit-event! 'close-field 'argument (range-end next211 next215))) next215) + (begin (rollback! mark213) #f)))))) + (if next212 next212 (begin (rollback! mark210) #f))) + (begin (rollback! mark210) #f))))) + (if after209 (begin (when (= after209 next206) (error "zero-width repeated grammar expression")) (repeat-loop205 after209 (fx+ count207 1))) (begin (rollback! mark208) next206)))))) + (if next204 next204 (begin (rollback! mark199) #f))) + (begin (rollback! mark199) #f))))) + (if next198 next198 (begin (rollback! mark197) next195))))) + (if next196 + (let (next216 (if (< next196 limit) (let (input (vector-ref significant next196)) (if (equal? (token-lexeme input) ")") (begin (emit-event! 'token input 0) (fx+ next196 1)) #f)) #f)) + (if next216 next216 (begin (rollback! mark194) #f))) + (begin (rollback! mark194) #f))) + (begin (rollback! mark194) #f))))) + (if next193 (begin (emit-event! 'close-node 'CallExpression (range-end pos next193)) next193) (begin (rollback! mark191) #f)))))) + (if next190 next190 (begin (rollback! mark167) #f)))))))))))) + (events-primary-expression + (lambda (pos) + (let (mark217 event-count) + (let (next218 (events-object-expression pos)) + (if next218 next218 + (begin + (rollback! mark217) + (let (next219 (events-tuple-expression pos)) + (if next219 next219 + (begin + (rollback! mark217) + (let (next220 (events-heredoc-expression pos)) + (if next220 next220 + (begin + (rollback! mark217) + (let (next221 (events-string-expression pos)) + (if next221 next221 + (begin + (rollback! mark217) + (let (next222 (events-number-expression pos)) + (if next222 next222 + (begin + (rollback! mark217) + (let (next223 + (let (mark224 event-count) + (emit-event! 'open-node 'LiteralExpression (offset pos)) + (let (open225 event-count) + (let (next226 + (let (mark227 event-count) + (emit-event! 'open-field 'value (offset pos)) + (let (open228 event-count) + (let (next229 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'identifier) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next229 (begin (if (= event-count open228) (rollback! mark227) (emit-event! 'close-field 'value (range-end pos next229))) next229) (begin (rollback! mark227) #f)))))) + (if next226 (begin (emit-event! 'close-node 'LiteralExpression (range-end pos next226)) next226) (begin (rollback! mark224) #f)))))) + (if next223 next223 + (begin + (rollback! mark217) + (let (next230 + (let (mark231 event-count) + (let (next232 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "(") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next232 + (let (next233 (events-expression next232)) + (if next233 + (let (next234 (if (< next233 limit) (let (input (vector-ref significant next233)) (if (equal? (token-lexeme input) ")") (begin (emit-event! 'token input 0) (fx+ next233 1)) #f)) #f)) + (if next234 next234 (begin (rollback! mark231) #f))) + (begin (rollback! mark231) #f))) + (begin (rollback! mark231) #f))))) + (if next230 next230 (begin (rollback! mark217) #f)))))))))))))))))))))))) + (events-traversal-expression + (lambda (pos) + (let (mark235 event-count) + (emit-event! 'open-node 'TraversalExpression (offset pos)) + (let (open236 event-count) + (let (next237 + (let (mark238 event-count) + (let (next239 + (let (mark240 event-count) + (emit-event! 'open-field 'root (offset pos)) + (let (open241 event-count) + (let (next242 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'identifier) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next242 (begin (if (= event-count open241) (rollback! mark240) (emit-event! 'close-field 'root (range-end pos next242))) next242) (begin (rollback! mark240) #f)))))) + (if next239 + (let (next243 + (let repeat-loop244 + ((next245 next239) (count246 0)) + (let (mark247 event-count) + (let (after248 + (let (mark249 event-count) + (let (next250 (if (< next245 limit) (let (input (vector-ref significant next245)) (if (equal? (token-lexeme input) ".") (begin (emit-event! 'token input 0) (fx+ next245 1)) #f)) #f)) + (if next250 + (let (next251 + (let (mark252 event-count) + (emit-event! 'open-field 'step (offset next250)) + (let (open253 event-count) + (let (next254 (if (< next250 limit) (let (input (vector-ref significant next250)) (if (eq? (token-kind input) 'identifier) (begin (emit-event! 'token input 0) (fx+ next250 1)) #f)) #f)) + (if next254 (begin (if (= event-count open253) (rollback! mark252) (emit-event! 'close-field 'step (range-end next250 next254))) next254) (begin (rollback! mark252) #f)))))) + (if next251 next251 (begin (rollback! mark249) #f))) + (begin (rollback! mark249) #f))))) + (if after248 (begin (when (= after248 next245) (error "zero-width repeated grammar expression")) (repeat-loop244 after248 (fx+ count246 1))) (begin (rollback! mark247) next245)))))) + (if next243 next243 (begin (rollback! mark238) #f))) + (begin (rollback! mark238) #f))))) + (if next237 (begin (emit-event! 'close-node 'TraversalExpression (range-end pos next237)) next237) (begin (rollback! mark235) #f))))))) + (events-string-expression + (lambda (pos) + (let (mark255 event-count) + (emit-event! 'open-node 'StringExpression (offset pos)) + (let (open256 event-count) + (let (next257 + (let (mark258 event-count) + (emit-event! 'open-field 'value (offset pos)) + (let (open259 event-count) + (let (next260 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'string) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next260 (begin (if (= event-count open259) (rollback! mark258) (emit-event! 'close-field 'value (range-end pos next260))) next260) (begin (rollback! mark258) #f)))))) + (if next257 (begin (emit-event! 'close-node 'StringExpression (range-end pos next257)) next257) (begin (rollback! mark255) #f))))))) + (events-heredoc-expression + (lambda (pos) + (let (mark261 event-count) + (emit-event! 'open-node 'HeredocExpression (offset pos)) + (let (open262 event-count) + (let (next263 + (let (mark264 event-count) + (emit-event! 'open-field 'value (offset pos)) + (let (open265 event-count) + (let (next266 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'heredoc) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next266 (begin (if (= event-count open265) (rollback! mark264) (emit-event! 'close-field 'value (range-end pos next266))) next266) (begin (rollback! mark264) #f)))))) + (if next263 (begin (emit-event! 'close-node 'HeredocExpression (range-end pos next263)) next263) (begin (rollback! mark261) #f))))))) + (events-number-expression + (lambda (pos) + (let (mark267 event-count) + (emit-event! 'open-node 'NumberExpression (offset pos)) + (let (open268 event-count) + (let (next269 + (let (mark270 event-count) + (emit-event! 'open-field 'value (offset pos)) + (let (open271 event-count) + (let (next272 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'number) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next272 (begin (if (= event-count open271) (rollback! mark270) (emit-event! 'close-field 'value (range-end pos next272))) next272) (begin (rollback! mark270) #f)))))) + (if next269 (begin (emit-event! 'close-node 'NumberExpression (range-end pos next269)) next269) (begin (rollback! mark267) #f))))))) + (events-tuple-expression + (lambda (pos) + (let (mark273 event-count) + (emit-event! 'open-node 'TupleExpression (offset pos)) + (let (open274 event-count) + (let (next275 + (let (mark276 event-count) + (let (next277 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "[") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next277 + (let (next278 + (let repeat-loop279 + ((next280 next277) (count281 0)) + (let (mark282 event-count) + (let (after283 (if (< next280 limit) (let (input (vector-ref significant next280)) (if (eq? (token-kind input) 'newline) (begin (emit-event! 'token input 0) (fx+ next280 1)) #f)) #f)) + (if after283 (begin (when (= after283 next280) (error "zero-width repeated grammar expression")) (repeat-loop279 after283 (fx+ count281 1))) (begin (rollback! mark282) next280)))))) + (if next278 + (let (next284 (let (mark285 event-count) (let (next286 (events-tuple-elements next278)) (if next286 next286 (begin (rollback! mark285) next278))))) + (if next284 + (let (next287 (if (< next284 limit) (let (input (vector-ref significant next284)) (if (equal? (token-lexeme input) "]") (begin (emit-event! 'token input 0) (fx+ next284 1)) #f)) #f)) + (if next287 next287 (begin (rollback! mark276) #f))) + (begin (rollback! mark276) #f))) + (begin (rollback! mark276) #f))) + (begin (rollback! mark276) #f))))) + (if next275 (begin (emit-event! 'close-node 'TupleExpression (range-end pos next275)) next275) (begin (rollback! mark273) #f))))))) + (events-tuple-elements + (lambda (pos) + (let (mark288 event-count) + (let (next289 + (let (mark290 event-count) + (emit-event! 'open-field 'element (offset pos)) + (let (open291 event-count) (let (next292 (events-expression pos)) (if next292 (begin (if (= event-count open291) (rollback! mark290) (emit-event! 'close-field 'element (range-end pos next292))) next292) (begin (rollback! mark290) #f)))))) + (if next289 (let (next293 (let (mark294 event-count) (let (next295 (events-tuple-tail next289)) (if next295 next295 (begin (rollback! mark294) next289))))) (if next293 next293 (begin (rollback! mark288) #f))) (begin (rollback! mark288) #f)))))) + (events-tuple-tail + (lambda (pos) + (let (mark296 event-count) + (let (next297 + (let (mark298 event-count) + (let (next299 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) ",") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next299 + (let (next300 + (let repeat-loop301 + ((next302 next299) (count303 0)) + (let (mark304 event-count) + (let (after305 (if (< next302 limit) (let (input (vector-ref significant next302)) (if (eq? (token-kind input) 'newline) (begin (emit-event! 'token input 0) (fx+ next302 1)) #f)) #f)) + (if after305 (begin (when (= after305 next302) (error "zero-width repeated grammar expression")) (repeat-loop301 after305 (fx+ count303 1))) (begin (rollback! mark304) next302)))))) + (if next300 + (let (next306 (let (mark307 event-count) (let (next308 (events-tuple-elements next300)) (if next308 next308 (begin (rollback! mark307) next300))))) (if next306 next306 (begin (rollback! mark298) #f))) + (begin (rollback! mark298) #f))) + (begin (rollback! mark298) #f))))) + (if next297 next297 + (begin + (rollback! mark296) + (let (next309 + (let (mark310 event-count) + (let (next311 + (let repeat-loop312 + ((next313 pos) (count314 0)) + (let (mark315 event-count) + (let (after316 (if (< next313 limit) (let (input (vector-ref significant next313)) (if (eq? (token-kind input) 'newline) (begin (emit-event! 'token input 0) (fx+ next313 1)) #f)) #f)) + (if after316 (begin (when (= after316 next313) (error "zero-width repeated grammar expression")) (repeat-loop312 after316 (fx+ count314 1))) (begin (rollback! mark315) (if (> count314 0) next313 #f))))))) + (if next311 + (let (next317 (let (mark318 event-count) (let (next319 (events-tuple-elements next311)) (if next319 next319 (begin (rollback! mark318) next311))))) (if next317 next317 (begin (rollback! mark310) #f))) + (begin (rollback! mark310) #f))))) + (if next309 next309 (begin (rollback! mark296) #f))))))))) + (events-object-expression + (lambda (pos) + (let (mark320 event-count) + (emit-event! 'open-node 'ObjectExpression (offset pos)) + (let (open321 event-count) + (let (next322 + (let (mark323 event-count) + (let (next324 (if (< pos limit) (let (input (vector-ref significant pos)) (if (equal? (token-lexeme input) "{") (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next324 + (let (next325 + (let repeat-loop326 + ((next327 next324) (count328 0)) + (let (mark329 event-count) + (let (after330 + (let (mark331 event-count) + (let (next332 (if (< next327 limit) (let (input (vector-ref significant next327)) (if (eq? (token-kind input) 'newline) (begin (emit-event! 'token input 0) (fx+ next327 1)) #f)) #f)) + (if next332 next332 + (begin + (rollback! mark331) + (let (next333 + (let (mark334 event-count) + (emit-event! 'open-field 'entry (offset next327)) + (let (open335 event-count) + (let (next336 (events-object-entry next327)) + (if next336 (begin (if (= event-count open335) (rollback! mark334) (emit-event! 'close-field 'entry (range-end next327 next336))) next336) (begin (rollback! mark334) #f)))))) + (if next333 next333 (begin (rollback! mark331) #f)))))))) + (if after330 (begin (when (= after330 next327) (error "zero-width repeated grammar expression")) (repeat-loop326 after330 (fx+ count328 1))) (begin (rollback! mark329) next327)))))) + (if next325 + (let (next337 (if (< next325 limit) (let (input (vector-ref significant next325)) (if (equal? (token-lexeme input) "}") (begin (emit-event! 'token input 0) (fx+ next325 1)) #f)) #f)) + (if next337 next337 (begin (rollback! mark323) #f))) + (begin (rollback! mark323) #f))) + (begin (rollback! mark323) #f))))) + (if next322 (begin (emit-event! 'close-node 'ObjectExpression (range-end pos next322)) next322) (begin (rollback! mark320) #f))))))) + (events-object-entry + (lambda (pos) + (let (mark338 event-count) + (emit-event! 'open-node 'ObjectEntry (offset pos)) + (let (open339 event-count) + (let (next340 + (let (mark341 event-count) + (let (next342 + (let (mark343 event-count) + (emit-event! 'open-field 'key (offset pos)) + (let (open344 event-count) + (let (next345 + (let (mark346 event-count) + (let (next347 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'identifier) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next347 next347 + (begin + (rollback! mark346) + (let (next348 (if (< pos limit) (let (input (vector-ref significant pos)) (if (eq? (token-kind input) 'string) (begin (emit-event! 'token input 0) (fx+ pos 1)) #f)) #f)) + (if next348 next348 (begin (rollback! mark346) #f)))))))) + (if next345 (begin (if (= event-count open344) (rollback! mark343) (emit-event! 'close-field 'key (range-end pos next345))) next345) (begin (rollback! mark343) #f)))))) + (if next342 + (let (next349 + (let (mark350 event-count) + (let (next351 (if (< next342 limit) (let (input (vector-ref significant next342)) (if (equal? (token-lexeme input) "=") (begin (emit-event! 'token input 0) (fx+ next342 1)) #f)) #f)) + (if next351 next351 + (begin + (rollback! mark350) + (let (next352 (if (< next342 limit) (let (input (vector-ref significant next342)) (if (equal? (token-lexeme input) ":") (begin (emit-event! 'token input 0) (fx+ next342 1)) #f)) #f)) + (if next352 next352 (begin (rollback! mark350) #f)))))))) + (if next349 + (let (next353 + (let (mark354 event-count) + (emit-event! 'open-field 'value (offset next349)) + (let (open355 event-count) + (let (next356 (events-expression next349)) (if next356 (begin (if (= event-count open355) (rollback! mark354) (emit-event! 'close-field 'value (range-end next349 next356))) next356) (begin (rollback! mark354) #f)))))) + (if next353 + (let (next357 + (let (mark358 event-count) + (let (next359 (if (< next353 limit) (let (input (vector-ref significant next353)) (if (equal? (token-lexeme input) ",") (begin (emit-event! 'token input 0) (fx+ next353 1)) #f)) #f)) + (if next359 next359 (begin (rollback! mark358) next353))))) + (if next357 next357 (begin (rollback! mark341) #f))) + (begin (rollback! mark341) #f))) + (begin (rollback! mark341) #f))) + (begin (rollback! mark341) #f))))) + (if next340 (begin (emit-event! 'close-node 'ObjectEntry (range-end pos next340)) next340) (begin (rollback! mark338) #f)))))))) + (let (end (if simple? (begin (emit-simple-attributes!) limit) (events-config-file 0))) + (if (and end (= end limit)) (make-success-parse-artifact/raw-event-tape direct-hcl-grammar-digest source tokens events event-count (parser-machine-trivia machine) source-bytes) #f))))))) diff --git a/src/compiler/language-artifact.ss b/src/compiler/language-artifact.ss index b31959e2..79aaf9e5 100644 --- a/src/compiler/language-artifact.ss +++ b/src/compiler/language-artifact.ss @@ -1,17 +1,33 @@ ;;; -*- Gerbil -*- -;;; Build-time storage boundary for materialized language IR. +;;; Build-time declaration expansion and immutable language IR storage. -(import (only-in ../runtime/identity sha256-text) +(import (only-in ./lr current-grammar-source-map validate-resolved-repetitions lr-spec-ref) + (only-in ./lr-contextual-program compile-lr-contextual-program) + (only-in ../grammar/algebra grammar-expression-header?) + (only-in :gerbil/core/expander + with-syntax syntax-case syntax datum->syntax syntax->datum syntax->list + identifier? stx-source stx-map stx-for-each stx-list? stx-pair? stx-car stx-cdr raise-syntax-error) + (only-in :std/list/list delete-duplicates/hash take drop) + (only-in :std/list/list-builder with-list-builder) + (only-in ../runtime/identity sha256-text sha256-bytes) (only-in ../runtime/language-artifact compiled-language-artifact-relative-path load-compiled-language-artifact/roots sha256-identity-filename) (only-in :gerbil/compiler/base current-compile-output-dir) (only-in :std/misc/ports read-all-as-u8vector) - (only-in :std/encoding/base64 base64-encode) - (only-in :std/string/utf8 utf8->string) + (only-in ../runtime/embedded-image pack-language-artifact-image) (only-in :std/encoding/zlib compress uncompress)) -(export materialize-compiled-language-artifact +(export expand-language-grammar-syntax expand-language-declaration-syntax + expand-admitted-language-declaration-syntax + make-language-declaration make-admitted-language-declaration + compile-admitted-language-declaration + compiled-language-declaration-grammar + compiled-language-declaration-grammar-locator + compiled-language-declaration-bound-locator + compiled-language-declaration-parser-locator + compiled-language-declaration-program-locator + project-language-catalog materialize-compiled-language-artifact materialize-compiled-language-artifact/output-dirs compile-language-parser-artifact compile-language-parser-artifact/output-dirs @@ -22,35 +38,48 @@ (def +parser-artifact-cache-schema+ "gerbil-parser.parser-artifact-cache.v1") (def +parser-artifact-generator-contract+ - "gerbil-parser.lalr1-generator.v1") + "gerbil-parser.lalr1-generator.v3") (def +language-declaration-cache-schema+ "gerbil-parser.language-declaration-cache.v1") (def +language-declaration-generator-contract+ - "gerbil-parser.language-declaration-generator.v1") + "gerbil-parser.language-declaration-generator.v3") + +;; v3 derives aliased entry roots independently of syntax catalog order. +;; Both cache layers must regenerate when compiler semantics change. ;; : (-> Datum String) (def (serialize value) (call-with-output-string (lambda (port) (write value port)))) -;; : (-> String String Boolean) -(def (materialized-content-matches? path serialized) +;; Artifact identity, compression, and immutable verification share these +;; owned bytes. Text cache receipts retain their character-port writer. +;; : (-> Datum U8Vector) +(def (serialize-artifact-bytes value) + (call-with-output-u8vector '(char-encoding: UTF-8 eol-encoding: lf) + (lambda (port) (write value port)))) + +;; : (forall (a) (-> String a (-> InputPort a) Boolean)) +(def (content-matches? path expected read-content) (and (file-exists? path) (with-exception-catcher (lambda (_) #f) (lambda () - (equal? serialized - (utf8->string - (uncompress - (call-with-input-file path read-all-as-u8vector)))))))) - -;; A content-addressed target is immutable. Existing corrupt content is never -;; silently replaced; a competing writer is accepted only after the winner's -;; complete bytes have been independently verified. -;; : (-> String U8Vector String Void) -(def (publish-materialized-content! path bytes serialized) + (equal? expected + (call-with-input-file path read-content)))))) + +(def (materialized-content-matches? path bytes) + (content-matches? + path bytes + (lambda (port) + (uncompress (read-all-as-u8vector port))))) + +;; Both sidecars and cache receipts are immutable. A competing writer is +;; accepted only after its complete content has been independently verified. +(def (publish-immutable-content! path expected matches? write-content! + conflict-message temporary-message) (if (file-exists? path) - (unless (materialized-content-matches? path serialized) - (error "compiled language artifact target contains different bytes" path)) + (unless (matches? path expected) + (error conflict-message path)) (let make-temporary () (let (temporary (string-append path ".tmp." @@ -61,24 +90,29 @@ (lambda (exception) (when (file-exists? temporary) (delete-file temporary)) - (if (materialized-content-matches? path serialized) + (if (matches? path expected) (void) (raise exception))) (lambda () - (call-with-output-file - temporary - (lambda (port) - (write-subu8vector bytes 0 (u8vector-length bytes) port))) - (unless (materialized-content-matches? temporary serialized) - (error "compiled language artifact temporary write is invalid" - temporary)) + (call-with-output-file temporary write-content!) + (unless (matches? temporary expected) + (error temporary-message temporary)) (rename-file temporary path #f)))))))) +;; : (-> String U8Vector U8Vector Void) +(def (publish-materialized-content! path compressed bytes) + (publish-immutable-content! + path bytes materialized-content-matches? + (lambda (port) + (write-subu8vector compressed 0 (u8vector-length compressed) port)) + "compiled language artifact target contains different bytes" + "compiled language artifact temporary write is invalid")) + ;; : (forall (a) (-> a [String] [String])) ;; : (-> Datum List List) (def (materialize-compiled-language-artifact/output-dirs value output-dirs) - (let* ((serialized (serialize value)) - (digest (sha256-text serialized)) + (let* ((bytes (serialize-artifact-bytes value)) + (digest (sha256-bytes bytes)) (relative-path (compiled-language-artifact-relative-path digest)) (paths (map (lambda (output-dir) @@ -91,66 +125,50 @@ (for-each (lambda (path) (when (file-exists? path) - (unless (materialized-content-matches? path serialized) + (unless (materialized-content-matches? path bytes) (error "compiled language artifact target contains different bytes" path)))) paths) (unless (null? missing) - (let (bytes (compress (string->utf8 serialized) compression: 9)) + (let (compressed (compress bytes compression: 9)) (for-each (lambda (path) (create-directory* (path-directory path)) - (publish-materialized-content! path bytes serialized)) + (publish-materialized-content! path compressed bytes)) missing))) (list relative-path digest))) ;; : (-> String String Boolean) (def (text-content-matches? path serialized) - (and (file-exists? path) - (with-exception-catcher - (lambda (_) #f) - (lambda () - (equal? serialized - (call-with-input-file - path (lambda (port) (read-line port #f)))))))) + (content-matches? + path serialized + (lambda (port) (read-line port #f)))) ;; : (-> String String Void) (def (publish-text-content! path serialized) - (if (file-exists? path) - (unless (text-content-matches? path serialized) - (error "compiled language cache receipt contains different bytes" path)) - (let make-temporary () - (let (temporary - (string-append path ".tmp." - (number->string (random-integer 1073741824)))) - (if (file-exists? temporary) - (make-temporary) - (with-exception-catcher - (lambda (exception) - (when (file-exists? temporary) - (delete-file temporary)) - (if (text-content-matches? path serialized) - (void) - (raise exception))) - (lambda () - (call-with-output-file - temporary (lambda (port) (display serialized port))) - (unless (text-content-matches? temporary serialized) - (error "compiled language cache temporary write is invalid" - temporary)) - (rename-file temporary path #f)))))))) + (publish-immutable-content! + path serialized text-content-matches? + (lambda (port) (display serialized port)) + "compiled language cache receipt contains different bytes" + "compiled language cache temporary write is invalid")) ;; : (-> String String) (def (parser-cache-relative-path key) (string-append "gerbil-parser/compiled-language-parser-cache/" (sha256-identity-filename key) ".scm")) -;; : (-> String List (Maybe List)) -(def (read-parser-cache-receipt key output-dirs) - (let* ((relative-path (parser-cache-relative-path key)) - (path (find file-exists? - (map (lambda (root) (path-expand relative-path root)) - output-dirs)))) +(def (cache-receipt-field receipt field) + (let (row (assq field receipt)) + (and row (cdr row)))) + +;; A receipt must contain exactly one datum with the expected identity and +;; all fields needed by its caller. Cache kinds differ only in their schema +;; and required locator fields. +(def (read-cache-receipt relative-path output-dirs schema key required-fields + invalid-message) + (let (path (find file-exists? + (map (lambda (root) (path-expand relative-path root)) + output-dirs))) (and path (call-with-input-file path @@ -158,17 +176,21 @@ (let ((receipt (read port)) (trailing (read port))) (unless (and (eof-object? trailing) (list? receipt) - (let (row (assq 'schema receipt)) - (and row - (equal? (cdr row) - +parser-artifact-cache-schema+))) - (let (row (assq 'key receipt)) - (and row (equal? (cdr row) key))) - (assq 'artifact receipt)) - (error "invalid compiled language parser cache receipt" - path receipt)) + (equal? (cache-receipt-field receipt 'schema) + schema) + (equal? (cache-receipt-field receipt 'key) key) + (andmap (lambda (field) (assq field receipt)) + required-fields)) + (error invalid-message path receipt)) receipt)))))) +;; : (-> String List (Maybe List)) +(def (read-parser-cache-receipt key output-dirs) + (read-cache-receipt + (parser-cache-relative-path key) output-dirs + +parser-artifact-cache-schema+ key '(artifact) + "invalid compiled language parser cache receipt")) + ;; : (-> Alist Alist) (def (parser-artifact-with-materialization value) (if (assq 'materialization value) @@ -187,7 +209,7 @@ (list +parser-artifact-generator-contract+ grammar)))) (receipt (read-parser-cache-receipt key output-dirs))) (if receipt - (let (locator (cdr (assq 'artifact receipt))) + (let (locator (cache-receipt-field receipt 'artifact)) (values (load-compiled-language-artifact/roots "gerbil-parser.parser-ir.v1" locator output-dirs) @@ -234,7 +256,7 @@ ;;; The sidecar remains the build cache; only its compressed bytes cross into ;;; the runtime module, so the uncompressed Grammar/LR datum is never emitted ;;; as per-character generated C. -;; : (-> List String) +;; : (-> List U64Vector) (def (encode-compiled-language-artifact locator) (let* ((relative-path (car locator)) (path @@ -243,7 +265,7 @@ (current-artifact-output-dirs))) (error "compiled language artifact sidecar is unavailable" relative-path)))) - (base64-encode + (pack-language-artifact-image (call-with-input-file path read-all-as-u8vector)))) ;; : (-> Datum (-> Datum) (values Datum List Symbol)) @@ -258,29 +280,10 @@ ;; : (-> String List (Maybe List)) (def (read-declaration-cache-receipt key output-dirs) - (let* ((relative-path (declaration-cache-relative-path key)) - (path (find file-exists? - (map (lambda (root) (path-expand relative-path root)) - output-dirs)))) - (and path - (call-with-input-file - path - (lambda (port) - (let ((receipt (read port)) (trailing (read port))) - (unless (and (eof-object? trailing) - (list? receipt) - (let (row (assq 'schema receipt)) - (and row - (equal? (cdr row) - +language-declaration-cache-schema+))) - (let (row (assq 'key receipt)) - (and row (equal? (cdr row) key))) - (assq 'grammar receipt) - (assq 'bound receipt) - (assq 'parser receipt)) - (error "invalid compiled language declaration cache receipt" - path receipt)) - receipt)))))) + (read-cache-receipt + (declaration-cache-relative-path key) output-dirs + +language-declaration-cache-schema+ key '(grammar bound parser) + "invalid compiled language declaration cache receipt")) ;;; Caches the complete declaration projection, not just the final parser IR. ;;; On a hit expansion receives receipt-bound content-addressed locators and avoids @@ -296,9 +299,9 @@ declaration-identity grammar)))) (receipt (read-declaration-cache-receipt key output-dirs))) (if receipt - (values (cdr (assq 'grammar receipt)) - (cdr (assq 'bound receipt)) - (cdr (assq 'parser receipt)) + (values (cache-receipt-field receipt 'grammar) + (cache-receipt-field receipt 'bound) + (cache-receipt-field receipt 'parser) 'hit) (let* ((grammar-locator (materialize-compiled-language-artifact/output-dirs @@ -343,3 +346,352 @@ (def (materialize-compiled-language-artifact value) (materialize-compiled-language-artifact/output-dirs value (current-artifact-output-dirs))) + +;;; Validate a published ABI projection over already inferred concise syntax. +;;; Unused token kinds require an explicit reservation rather than lexer remaps. +(def (project-language-catalog inferred-kinds inferred-terminals catalog) + (def (require condition message) (unless condition (error message))) + (def (unique rows) (delete-duplicates/hash rows)) + (if (not catalog) + (values inferred-kinds inferred-terminals) + (begin + (require (and (list? catalog) (memv (length catalog) '(3 4)) + (every (lambda (row) (and (list? row) (pair? row))) (cdr catalog)) + (equal? (map car (cdr catalog)) + (if (= (length catalog) 3) '(syntax-kinds terminals) + '(syntax-kinds terminals reserved-token-kinds)))) + "catalog requires syntax-kinds and terminals sections") + (let ((kinds (cdadr catalog)) (terminals (cdaddr catalog)) + (reserved (if (= (length catalog) 4) (cdr (list-ref catalog 3)) '()))) + (require (every (lambda (row) + (and (list? row) (= (length row) 3) + (symbol? (car row)) (memq (cadr row) '(node token)) + (list? (caddr row)) (every symbol? (caddr row)) + (= (length (caddr row)) (length (unique (caddr row)))))) kinds) + "catalog has an invalid syntax-kind row") + (require (and (pair? kinds) (eq? (cadar kinds) 'node)) + "catalog must start with a node kind") + (require (= (length kinds) (length (unique (map car kinds)))) + "catalog has duplicate syntax kinds") + (for-each + (lambda (inferred) + (let (published (assq (car inferred) kinds)) + (require (and published (eq? (cadr published) (cadr inferred)) + (every (lambda (field) (memq field (caddr published))) (caddr inferred))) + "catalog must retain every inferred kind, category and field"))) inferred-kinds) + (require (and (every symbol? reserved) (= (length reserved) (length (unique reserved))) + (every (lambda (name) + (let (row (assq name kinds)) + (and row (eq? (cadr row) 'token) + (not (assq name inferred-kinds))))) reserved)) + "catalog reservations must name unique unused token kinds") + (require (every (lambda (row) + (or (assq (car row) inferred-kinds) (eq? (cadr row) 'node) + (memq (car row) reserved))) kinds) + "catalog may reserve nodes but cannot invent token kinds") + (require (every (lambda (row) + (and (list? row) (= (length row) 2) (every symbol? row))) terminals) + "catalog has an invalid terminal row") + (require (and (= (length terminals) (length inferred-terminals)) + (= (length terminals) (length (unique (map car terminals)))) + (every (lambda (row) (equal? (assq (car row) terminals) row)) inferred-terminals)) + "catalog must retain the exact inferred terminal mapping") + (values kinds terminals))))) + +;;; This record exists only during expansion. Sections hold syntax lists until +;;; canonical admission; runtime descriptors receive immutable artifacts only. +(defstruct language-declaration + (syntax prefix identities sections conflict-policy case-insensitive? lineage source-map)) + +;;; One result ties runtime emission to the grammar and all published products. +(defstruct compiled-language-declaration + (grammar grammar-locator bound-locator parser-locator program-locator)) + +(def (read-language-declaration-syntax stx) + ;; Admit exactly the six existing low-level clause shapes, then normalize + ;; defaults once. This avoids recursive macro replay and duplicate assembly. + (def (reject) (raise-syntax-error #f "invalid language grammar declaration" stx)) + (def forms (and (stx-list? stx) (syntax->list stx))) + (unless (and forms (>= (length forms) 2) (identifier? (cadr forms))) (reject)) + (def prefix (cadr forms)) + (def clauses (cddr forms)) + (def names + (map (lambda (row) + (unless (and (stx-list? row) (stx-pair? row)) (reject)) + (syntax->datum (stx-car row))) clauses)) + (def common '(identity syntax-kinds terminals lexical-rules rules extras keywords parser-entrypoints)) + (unless (and (>= (length names) 10) (equal? (take names 8) common) + (member (drop names 8) + '((recoveries conflicts case-insensitive lineage flow) + (recoveries lineage flow) + (recoveries conflicts case-insensitive source-ownership lineage flow) + (recoveries source-ownership lineage flow) + (recoveries conflicts case-insensitive flow) (recoveries flow)))) (reject)) + (def rows (map cons names clauses)) + (def (section name) (stx-cdr (cdr (assq name rows)))) + (def (scalar name default) + (let (entry (assq name rows)) + (if entry + (let (values (syntax->list (stx-cdr (cdr entry)))) + (unless (= (length values) 1) (reject)) (car values)) + (datum->syntax prefix default)))) + (def identities (syntax->list (section 'identity))) + (unless (= (length identities) 3) (reject)) + (def lineage + (if (assq 'lineage rows) (syntax->datum (section 'lineage)) '(canonical-language-compiler))) + (make-language-declaration + stx prefix identities + (map (lambda (name) (cons name (section name))) + '(syntax-kinds terminals lexical-rules rules extras keywords + parser-entrypoints recoveries flow)) + (syntax->datum (scalar 'conflicts 'reject)) + (syntax->datum (scalar 'case-insensitive #f)) + lineage (syntax->datum (scalar 'source-ownership '())))) + +(def (expand-language-grammar-syntax stx compile-parser bind-grammar-ir + assembly-binding descriptor-binding begin-binding define-binding) + (expand-language-declaration-syntax + (read-language-declaration-syntax stx) compile-parser bind-grammar-ir + assembly-binding descriptor-binding begin-binding define-binding)) + +;;; Admission retains contextual facts together with the exact canonical grammar. +(defstruct admitted-language-declaration (grammar origin lineage source-map)) +(defstruct language-declaration-bindings (grammar bound parser machine language)) + +(def (derive-language-declaration-bindings declaration) + (def prefix (language-declaration-prefix declaration)) + (def (binding suffix) + (datum->syntax prefix (string->symbol (string-append (symbol->string (syntax->datum prefix)) suffix)))) + (make-language-declaration-bindings + (binding "-grammar") (binding "-bound-grammar-ir") (binding "-parser-ir") + (binding "-parser") (binding (if (every (lambda (value) (syntax->datum value)) (language-declaration-identities declaration)) "-language-grammar" "-syntax")))) + +;;; Construction and syntax admission do not publish artifacts. +(def (admit-language-declaration declaration grammar-name) + (def stx (language-declaration-syntax declaration)) + (def lineage (language-declaration-lineage declaration)) + (def (section name) (cdr (assq name (language-declaration-sections declaration)))) + (def origin + (let (source (stx-source stx)) + (if source (call-with-output-string (lambda (port) (display source port))) ""))) + (def (grammar-expression-datum expression) + (let* ((datum (syntax->datum expression)) + (head (and (pair? datum) + (case (car datum) + ((seq) 'sequence) + ((prec) 'precedence) + (else (car datum))))) + (canonical (and head (cons head (cdr datum))))) + (unless (grammar-expression-header? canonical) + (raise-syntax-error #f "invalid GrammarExpr declaration" expression stx)) + (let (children (cdr (syntax->list expression))) + (case head + ((empty literal layout-start layout-next layout-end token reference) + canonical) + ((sequence choice) + (cons head (map grammar-expression-datum children))) + ((optional repeat repeat1) + (list head (grammar-expression-datum (car children)))) + ((field alias) + (list head (cadr canonical) + (grammar-expression-datum (cadr children)))) + ((precedence) + (list head (cadr canonical) (caddr canonical) + (grammar-expression-datum (caddr children)))))))) + (def syntax-rows (section 'syntax-kinds)) + (def terminal-rows (section 'terminals)) + (def lexical-rows (section 'lexical-rules)) + (def rule-rows (section 'rules)) + (def extra-names (section 'extras)) + (def keyword-rows (section 'keywords)) + (def entry-rows (section 'parser-entrypoints)) + (def recovery-rows (section 'recoveries)) + (def flow-rows (section 'flow)) + (def conflict-policy (language-declaration-conflict-policy declaration)) + (def case-insensitive? (language-declaration-case-insensitive? declaration)) + (def explicit-source-map (language-declaration-source-map declaration)) + (def (source-location value) + (let (source (stx-source value)) + (list + (cons 'path origin) + (cons 'location + (if source + (call-with-output-string + (lambda (port) (display source port))) + origin)) + (cons 'generated? #f)))) + (def (section-sources namespace rows) + (cons + namespace + (stx-map + (lambda (row) + (syntax-case row () + ((name . _) + (cons (syntax->datum #'name) (source-location #'name))) + (_ (raise-syntax-error #f "invalid bound declaration row" row)))) + rows))) + (def (field-sources rows) + (cons + 'field + (with-list-builder (emit) + (stx-for-each + (lambda (row) + (syntax-case row () + ((kind _ (field ...)) + (stx-for-each + (lambda (field) + (emit (cons (list (syntax->datum #'kind) (syntax->datum field)) + (source-location field)))) + #'(field ...))) + (_ (raise-syntax-error #f "invalid bound syntax-kind row" row)))) + rows)))) + (let* ((source-map + (append + explicit-source-map + (list + (section-sources 'syntax-kind syntax-rows) + (section-sources 'terminal terminal-rows) + (section-sources 'lexical-rule lexical-rows) + (section-sources 'rule rule-rows) + (field-sources syntax-rows)))) + (grammar + (list + (cons 'schema "gerbil-parser.grammar-ir.v1") + (cons 'grammar (syntax->datum grammar-name)) + (cons 'syntax-kinds (syntax->datum syntax-rows)) + (cons 'terminals (syntax->datum terminal-rows)) + (cons 'lexical-rules (syntax->datum lexical-rows)) + (cons 'rules + (stx-map + (lambda (row) + (syntax-case row () + ((name expression) + (list (syntax->datum #'name) + (grammar-expression-datum #'expression))) + (_ (raise-syntax-error #f "invalid grammar rule declaration" row stx)))) + rule-rows)) + (cons 'extras + (map list (syntax->datum extra-names))) + (cons 'keywords (syntax->datum keyword-rows)) + (cons 'parser-entrypoints (syntax->datum entry-rows)) + (cons 'recoveries (syntax->datum recovery-rows)) + (cons 'conflict-policy conflict-policy) + (cons 'case-insensitive? case-insensitive?) + (cons 'flow (syntax->datum flow-rows))))) + (make-admitted-language-declaration grammar origin lineage source-map))) + +;;; The common LR product is admitted during expansion, using the existing +;;; immutable artifact publisher. A cache hit returns its locator without +;;; loading, copying, validating or hashing the backend instruction datum. +(def (compile-lr-program-artifact parser-locator output-dirs) + (let* ((schema "gerbil-parser.lr-program-cache.v1") + (key (sha256-text (serialize (list schema parser-locator)))) + (relative-path (string-append "gerbil-parser/compiled-language-program-cache/" + (sha256-identity-filename key) ".scm")) + (receipt (read-cache-receipt relative-path output-dirs schema key '(program) + "invalid compiled LR program cache receipt"))) + (if receipt + (cache-receipt-field receipt 'program) + (let* ((parser-ir (load-compiled-language-artifact/roots + "gerbil-parser.parser-ir.v1" parser-locator output-dirs)) + (program (compile-lr-contextual-program parser-ir (cadr parser-locator))) + (locator (materialize-compiled-language-artifact/output-dirs program output-dirs)) + (serialized (serialize `((schema . ,schema) (key . ,key) (program . ,locator))))) + (for-each (lambda (root) + (let (path (path-expand relative-path root)) + (create-directory* (path-directory path)) + (publish-text-content! path serialized))) output-dirs) + locator)))) + +(def (validate-declaration-parser-admission grammar parser-locator source-map output-dirs) + ;; A complete declaration cache can skip compiler invocation. Recheck the + ;; current admission against its canonical nullable facts before publication; + ;; this needs no lowering, fixed point or LR state reconstruction. + (let* ((parser (load-compiled-language-artifact/roots + "gerbil-parser.parser-ir.v1" parser-locator output-dirs)) + (spec (cdr (assq 'lr-spec parser))) + (nullable (make-table test: eq?))) + (for-each (lambda (name) (table-set! nullable name #t)) (lr-spec-ref spec 'nullable)) + (parameterize ((current-grammar-source-map source-map)) + (validate-resolved-repetitions (cdr (assq 'rules grammar)) nullable)))) + +;;; Publication consumes one admitted value; cache and compiler authority stay here. +(def (compile-admitted-language-declaration admitted compile-parser bind-grammar-ir + output-dirs: (output-dirs (current-artifact-output-dirs))) + (def grammar (admitted-language-declaration-grammar admitted)) + (def origin (admitted-language-declaration-origin admitted)) + (def lineage (admitted-language-declaration-lineage admitted)) + (def source-map (admitted-language-declaration-source-map admitted)) + (let-values (((grammar-locator bound-locator parser-locator _status) + (compile-language-declaration-artifacts/output-dirs + (list origin lineage source-map) grammar + (lambda () (bind-grammar-ir grammar origin lineage source-map)) + (lambda () (parameterize ((current-grammar-source-map source-map)) + (compile-parser grammar))) output-dirs))) + (validate-declaration-parser-admission grammar parser-locator source-map output-dirs) + (make-compiled-language-declaration + grammar grammar-locator bound-locator parser-locator + (compile-lr-program-artifact parser-locator output-dirs)))) + +;;; Emission reads admitted products and hygienic bindings, with no compiler calls. +(def (emit-compiled-language-declaration declaration compiled bindings + assembly-binding descriptor-binding begin-binding define-binding) + (def prefix (language-declaration-prefix declaration)) + (def identities (language-declaration-identities declaration)) + (def grammar-binding (language-declaration-bindings-grammar bindings)) + (def bound-binding (language-declaration-bindings-bound bindings)) + (def ir-binding (language-declaration-bindings-parser bindings)) + (def machine-binding (language-declaration-bindings-machine bindings)) + (def language-binding (language-declaration-bindings-language bindings)) + (def admitted-grammar (compiled-language-declaration-grammar compiled)) + (def grammar-encoded (compiled-language-declaration-grammar-locator compiled)) + (def bound-encoded (compiled-language-declaration-bound-locator compiled)) + (def grammar-payload (encode-compiled-language-artifact grammar-encoded)) + (def bound-payload (encode-compiled-language-artifact bound-encoded)) + (def program-encoded (compiled-language-declaration-program-locator compiled)) + (def program-payload (encode-compiled-language-artifact program-encoded)) + ;; Runtime assembly receives sections from the same canonical grammar that + ;; was bound, compiled and published, rather than replaying author sections. + (def (emission-section name) + (let (rows (cdr (assq name admitted-grammar))) + (cons name (syntax->list + (datum->syntax prefix + (if (eq? name 'extras) (map car rows) rows)))))) + (datum->syntax prefix + (list begin-binding + (list assembly-binding grammar-binding bound-binding ir-binding machine-binding + grammar-encoded grammar-payload bound-encoded bound-payload program-encoded program-payload + (emission-section 'syntax-kinds) + (emission-section 'lexical-rules) + (emission-section 'rules) + (emission-section 'extras) + (emission-section 'parser-entrypoints)) + (list define-binding language-binding + (append (list descriptor-binding "gerbil-parser.language-grammar.v1") + identities (list grammar-binding ir-binding machine-binding #f)))))) + +;;; Native and imported author surfaces share canonical admission and publication. +(def (expand-language-declaration-syntax declaration compile-parser bind-grammar-ir + assembly-binding descriptor-binding begin-binding define-binding) + (def bindings (derive-language-declaration-bindings declaration)) + (def admitted + (admit-language-declaration declaration (language-declaration-bindings-grammar bindings))) + (expand-admitted-language-declaration/bindings + declaration admitted bindings compile-parser bind-grammar-ir + assembly-binding descriptor-binding begin-binding define-binding)) + +;;; POO normalization already produced canonical IR and effective context. This +;;; internal entry never reconstructs sections or invokes declaration admission. +(def (expand-admitted-language-declaration-syntax declaration admitted compile-parser bind-grammar-ir + assembly-binding descriptor-binding begin-binding define-binding) + (expand-admitted-language-declaration/bindings + declaration admitted (derive-language-declaration-bindings declaration) + compile-parser bind-grammar-ir + assembly-binding descriptor-binding begin-binding define-binding)) + +(def (expand-admitted-language-declaration/bindings declaration admitted bindings + compile-parser bind-grammar-ir + assembly-binding descriptor-binding begin-binding define-binding) + (def compiled (compile-admitted-language-declaration admitted compile-parser bind-grammar-ir)) + (emit-compiled-language-declaration + declaration compiled bindings assembly-binding descriptor-binding begin-binding define-binding)) diff --git a/src/compiler/language-expander.ss b/src/compiler/language-expander.ss new file mode 100644 index 00000000..abc14f04 --- /dev/null +++ b/src/compiler/language-expander.ss @@ -0,0 +1,14 @@ +;;; -*- Gerbil -*- +;;; Engine-only expansion of admitted canonical declarations. +;;; Author grammars use language-support/grammar; this module owns IR fixtures +;;; and source-format compiler products, not a second author grammar interface. +(import (for-syntax (only-in ./parser-ir compile-parser) + (only-in ./bound-ir bind-grammar-ir) + (only-in ./language-artifact expand-language-grammar-syntax)) + (only-in ../language/descriptor make-language-grammar) + (only-in ../language/assembly assemble-language-parser)) +(export compile-language) +(defsyntax (compile-language stx) + (expand-language-grammar-syntax + stx compile-parser bind-grammar-ir + #'assemble-language-parser #'make-language-grammar #'begin #'def)) diff --git a/src/compiler/language-pack.ss b/src/compiler/language-pack.ss new file mode 100644 index 00000000..6c315da7 --- /dev/null +++ b/src/compiler/language-pack.ss @@ -0,0 +1,46 @@ +;;; -*- Gerbil -*- +;;; Language pack authoring generates a public C entry without core dispatch edits. +(export generate-language-pack) +(def (module-name? value) + (and (string? value) (> (string-length value) 0) + (andmap (lambda (ch) (or (char-alphabetic? ch) (char-numeric? ch) + (memv ch '(#\- #\_ #\/)))) (string->list value)))) +(def (c-name? value) + (and (string? value) (> (string-length value) 0) + (let (first (string-ref value 0)) + (or (char=? first #\_) (char<=? #\a first #\z) (char<=? #\A first #\Z))) + (andmap (lambda (ch) (or (char=? ch #\_) (char<=? #\a ch #\z) + (char<=? #\A ch #\Z) (char<=? #\0 ch #\9))) + (string->list value)))) +(def (generate-language-pack source-path header-path module-id grammar-module descriptor-export c-prefix + (contextual-export #f)) + (unless (and (module-name? module-id) (module-name? grammar-module) + (symbol? descriptor-export) (module-name? (symbol->string descriptor-export)) + (or (not contextual-export) (and (symbol? contextual-export) + (module-name? (symbol->string contextual-export)))) + (c-name? c-prefix)) + (error "invalid language pack identity")) + (let* ((source + (string-append + ";;; @generated language C entry; compile through the normal package builder.\n" + "(import :gerbil-parser/src/ffi/language-abi\n" + " (only-in :gerbil-parser/src/ffi/language-handles register-language!)\n" + " (only-in :" grammar-module " " (symbol->string descriptor-export) + (if contextual-export (string-append " " (symbol->string contextual-export)) "") "))\n" + "(export create-language-handle)\n" + "(def (create-language-handle) (register-language! " (symbol->string descriptor-export) + (if contextual-export (string-append " " (symbol->string contextual-export)) "") "))\n" + "(begin-foreign\n (namespace (\"" module-id "#\" create-callback))\n" + " (c-declare \"#include \\n___U64 " c-prefix "_language_create_impl(void);\\nuint64_t " c-prefix "_language_create(void) { return gerbil_parser_language_is_owner_thread() ? " c-prefix "_language_create_impl() : 0; }\")\n" + " (c-define (create-callback) () unsigned-int64 \"" c-prefix "_language_create_impl\" \"extern\"\n" + " (with-exception-catcher (lambda (_) 0) (lambda () (" module-id "#create-language-handle)))))\n")) + (header + (string-append "/* @generated language entry; runtime must already be initialized on its owner thread. */\n" + "#pragma once\n#include \n#ifdef __cplusplus\nextern \"C\" {\n#endif\n" + "/* Creates a distinct handle. Zero signals admission failure. Release with the generic API. */\n" + "gerbil_parser_language " c-prefix "_language_create(void);\n#ifdef __cplusplus\n}\n#endif\n"))) + (create-directory* (path-directory source-path)) + (create-directory* (path-directory header-path)) + (call-with-output-file source-path (lambda (port) (display source port))) + (call-with-output-file header-path (lambda (port) (display header port))) + source-path)) diff --git a/src/compiler/lexical-expression.ss b/src/compiler/lexical-expression.ss new file mode 100644 index 00000000..3f78d8a3 --- /dev/null +++ b/src/compiler/lexical-expression.ss @@ -0,0 +1,436 @@ +;;; -*- Gerbil -*- +;;; Closed lexical-expression expansion and ordered candidate selection. + +(import (only-in ../runtime/lexical-source + scan-module-text scan-header-delimiter scan-header-data) + (only-in ../grammar/lexical-algebra + text-profile-data) + (only-in ../runtime/scan + scan-block-comment scan-decimal-digits scan-heredoc scan-horizontal-whitespace + scan-identifier scan-line scan-line-comment scan-character-run + make-text-profile-scanner scan-until-delimiters make-literal-end-scanner + scan-longest-literal scan-nested-block-comment scan-newline + scan-number-literal scan-number-literal/profile scan-escaped-quoted-strings + scan-quoted-strings scan-quoted-string/profile scan-whitespace)) +(export lexical-end lexical-choice lexical-dispatch lexical-dispatch/ranked + generated-lexical-rule lexical-rule-certificate-template prefer-generated-match) + +;;; Expands one closed lexical algebra case into its ordinary scanner call. +;;; The templates preserve source/offset bindings; runtime behavior stays in scan.ss. +;; lexical-end +;; : (-> Syntax Syntax) +;; | doc m% +;; `lexical-end` expands a declared lexical expression. +;; +;; # Examples +;; +;; ```scheme +;; (lexical-end source offset (identifier)) +;; ;; => (scan-identifier source offset) +;; ``` +;; % +(defrules lexical-end/primitive + (whitespace+ horizontal-whitespace+ newline+ line decimal-digit+ number identifier + heredoc number-literal + quoted-string escaped-quoted-string quoted-string-profile until-delimiters header-delimiter header-data module-text + line-comment block-comment nested-block-comment + choice literals fallback precedence external character-run) + ((_ source offset (whitespace+)) + (scan-whitespace source offset)) + ((_ source offset (horizontal-whitespace+)) + (scan-horizontal-whitespace source offset)) + ((_ source offset (newline+)) + (scan-newline source offset)) + ((_ source offset (line)) + (scan-line source offset)) + ((_ source offset (character-run character minimum)) + (scan-character-run source offset character minimum)) + ((_ source offset (decimal-digit+)) + (scan-decimal-digits source offset)) + ((_ source offset (number)) + (scan-number-literal source offset)) + ((_ source offset + (number-literal (prefix ...) separator (suffix ...) + leading-period? trailing-period?)) + (scan-number-literal/profile + source offset (list prefix ...) separator (list suffix ...) + leading-period? trailing-period?)) + ((_ source offset (identifier)) + (scan-identifier source offset)) + ((_ source offset (quoted-string delimiter ...)) + (scan-quoted-strings source offset (list delimiter ...))) + ((_ source offset (escaped-quoted-string delimiter ...)) + (scan-escaped-quoted-strings source offset (list delimiter ...))) + ((_ source offset (quoted-string-profile delimiter escapes unicode-width)) + (scan-quoted-string/profile source offset delimiter escapes unicode-width)) + ((_ source offset (module-text border word end-border open close comment extra)) + (scan-module-text source offset '(module-text border word end-border open close comment extra))) + ((_ source offset (header-delimiter prefix count index)) + (scan-header-delimiter source offset prefix count index)) + ((_ source offset (header-data prefix count stops)) + (scan-header-data source offset prefix count stops)) + ((_ source offset (until-delimiters characters)) + (scan-until-delimiters source offset characters)) + ((_ source offset (heredoc)) + (scan-heredoc source offset)) + ((_ source offset (line-comment start ...)) + (scan-line-comment source offset (list start ...))) + ((_ source offset (block-comment opening closing)) + (scan-block-comment source offset opening closing)) + ((_ source offset (nested-block-comment opening closing)) + (scan-nested-block-comment source offset opening closing)) + ((_ source offset (choice expression ...)) + (lexical-choice source offset (expression ...))) + ((_ source offset (precedence _rank expression)) + (lexical-end source offset expression)) + ((_ source offset (external _version scanner)) + (scanner source offset)) + ((_ source offset (literals value ...)) + (let (matched (scan-longest-literal source offset (list value ...))) + (and matched (+ offset (string-length matched))))) + ((_ source offset (fallback)) + (+ offset 1))) + +(defrules lexical-end (text-profile) + ((_ source offset (text-profile expression)) + ((make-text-profile-scanner (text-profile-data expression)) source offset)) + ((_ source offset expression) + (lexical-end/primitive source offset expression))) + +;; : (-> (OrFalse Fixnum) (OrFalse Fixnum) (OrFalse Fixnum)) +(def (prefer-longest-end current candidate) + (cond + ((not current) candidate) + ((not candidate) current) + ((>= current candidate) current) + (else candidate))) + +;;; Chooses the longest lexical alternative; declaration order breaks ties. +;; lexical-choice +;; : (-> Syntax Syntax) +;; | doc m% +;; `lexical-choice` expands an ordered lexical alternative. +;; +;; # Examples +;; +;; ```scheme +;; (lexical-choice source offset ((identifier) (fallback))) +;; ;; => first matching end offset +;; ``` +;; % +(defrules lexical-choice + () + ((_ source offset ()) #f) + ((_ source offset (expression rest ...)) + (prefer-longest-end + (lexical-end source offset expression) + (lexical-choice source offset (rest ...))))) + +;;; Projects optional lexical precedence without inspecting scanner results. +;; lexical-expression-rank +;; : (-> Syntax Syntax) +;; | doc m% +;; Expands the precedence rank carried by one lexical expression. +;; +;; # Examples +;; +;; ```scheme +;; (lexical-expression-rank (precedence 10 (identifier))) +;; ;; => 10 +;; ``` +;; % +(defrules lexical-expression-rank (precedence) + ((_ (precedence rank _expression)) rank) + ((_ _expression) 0)) + +;;; Materializes one scanner per declared lexical rule. Static literal +;;; catalogs compile to a trie here, while the parser machine is initialized, +;;; instead of linearly probing every literal for every source token. +(defrules lexical-scanner (literals precedence text-profile choice) + ((_ (text-profile expression)) + (make-text-profile-scanner (text-profile-data expression))) + ((_ (choice expression ...)) + (let (scanners (list (lexical-scanner expression) ...)) + (lambda (source offset) + (foldl (lambda (scanner end) + (prefer-longest-end end (scanner source offset))) + #f scanners)))) + ((_ (literals value ...)) + (make-literal-end-scanner '(value ...))) + ((_ (precedence _rank expression)) + (lexical-scanner expression)) + ((_ expression) + (lambda (source offset) + (lexical-end source offset expression)))) + +;; Only a complete literal catalog can join the shared mode trie. Choices and +;; external scanners retain their ordinary generated procedures. +(defrules lexical-static-literals (literals precedence) + ((_ (literals value ...)) '(value ...)) + ((_ (precedence _rank expression)) + (lexical-static-literals expression)) + ((_ _expression) #f)) + +;; Closed run primitives join the shared mode DFA. The number rule adds +;; non-accepting fraction/exponent states; other expressions retain scanners. +(defrules lexical-regular-kind + (whitespace+ horizontal-whitespace+ newline+ decimal-digit+ + number identifier precedence) + ((_ (whitespace+)) 'whitespace+) + ((_ (horizontal-whitespace+)) 'horizontal-whitespace+) + ((_ (newline+)) 'newline+) + ((_ (decimal-digit+)) 'decimal-digit+) + ((_ (number)) 'number) + ((_ (identifier)) 'identifier) + ((_ (precedence _rank expression)) + (lexical-regular-kind expression)) + ((_ _expression) #f)) + +;;; The first transition of the closed regular scanners can be decided once +;;; for each ASCII character and LR lexical mode. Unknown/nonregular forms +;;; remain candidates, so this filter cannot change maximal-munch decisions. +(defrules lexical-first-character? + (whitespace+ horizontal-whitespace+ newline+ decimal-digit+ number + identifier number-literal quoted-string escaped-quoted-string quoted-string-profile heredoc + line-comment block-comment + nested-block-comment precedence choice character-run literals) + ((_ ch (whitespace+)) (char-whitespace? ch)) + ((_ ch (horizontal-whitespace+)) + (or (char=? ch #\space) (char=? ch #\tab))) + ((_ ch (newline+)) + (or (char=? ch #\newline) (char=? ch #\return))) + ((_ ch (decimal-digit+)) (char-numeric? ch)) + ((_ ch (number)) (char-numeric? ch)) + ((_ ch (identifier)) + (or (char-alphabetic? ch) (char=? ch #\_))) + ((_ ch (character-run character _minimum)) + (char=? ch (string-ref character 0))) + ((_ ch (number-literal (prefix ...) _separator _suffixes + leading-period? _trailing-period?)) + (or (and (char>=? ch #\0) (char<=? ch #\9)) + (and leading-period? (char=? ch #\.)) + (or (zero? (string-length prefix)) + (char=? ch (string-ref prefix 0))) ...)) + ((_ ch (quoted-string delimiter ...)) + (or (or (zero? (string-length delimiter)) + (char=? ch (string-ref delimiter 0))) ...)) + ((_ ch (escaped-quoted-string delimiter ...)) + (or (or (zero? (string-length delimiter)) + (char=? ch (string-ref delimiter 0))) ...)) + ((_ ch (quoted-string-profile delimiter _escapes _unicode-width)) + (char=? ch (string-ref delimiter 0))) + ((_ ch (literals value ...)) + (or (or (zero? (string-length value)) + (char-ci=? ch (string-ref value 0))) ...)) + ((_ ch (heredoc)) (char=? ch #\<)) + ((_ ch (line-comment prefix ...)) + (or (char=? ch (string-ref prefix 0)) ...)) + ((_ ch (block-comment opening _closing)) + (char=? ch (string-ref opening 0))) + ((_ ch (nested-block-comment opening _closing)) + (char=? ch (string-ref opening 0))) + ((_ ch (precedence _rank expression)) + (lexical-first-character? ch expression)) + ((_ ch (choice expression ...)) + (or (lexical-first-character? ch expression) ...)) + ((_ _ch _expression) #t)) + +(defrules lexical-pure-fallback? + (fallback precedence) + ((_ (fallback)) #t) + ((_ (precedence _rank expression)) (lexical-pure-fallback? expression)) + ((_ _expression) #f)) + +;;; This predicate is a lower bound on scanner acceptance; the first-character +;;; filter above is an upper bound. Do not conflate the two for incomplete +;;; strings, comments, numeric prefixes, or unknown/external expressions. +(defrules lexical-guaranteed-first-character? + (whitespace+ horizontal-whitespace+ newline+ decimal-digit+ number identifier + literals choice precedence) + ((_ ch (whitespace+)) (lexical-first-character? ch (whitespace+))) + ((_ ch (horizontal-whitespace+)) (lexical-first-character? ch (horizontal-whitespace+))) + ((_ ch (newline+)) (lexical-first-character? ch (newline+))) + ((_ ch (decimal-digit+)) (lexical-first-character? ch (decimal-digit+))) + ((_ ch (number)) (lexical-first-character? ch (number))) + ((_ ch (identifier)) (lexical-first-character? ch (identifier))) + ((_ ch (literals value ...)) + (or (and (= (string-length value) 1) (char=? ch (string-ref value 0))) ...)) + ((_ ch (choice expression ...)) + (or (lexical-guaranteed-first-character? ch expression) ...)) + ((_ ch (precedence _rank expression)) (lexical-guaranteed-first-character? ch expression)) + ((_ _ch _expression) #f)) + +;; prefer-ranked-match +;; : (-> (OrFalse List) (OrFalse List) (OrFalse List)) +;; | doc m% +;; Chooses one scanner match by length, precedence, then declaration. +;; +;; # Examples +;; +;; ```scheme +;; (prefer-ranked-match '(left 4 1) '(right 4 2)) +;; ;; => (right 4 2) +;; ``` +;; % +(def (prefer-ranked-match current candidate) + (cond + ((not current) candidate) + ((not candidate) current) + ((> (cadr current) (cadr candidate)) current) + ((< (cadr current) (cadr candidate)) candidate) + ((>= (caddr current) (caddr candidate)) current) + (else candidate))) + +(def (prefer-generated-match current candidate) + (cond + ((not current) candidate) + ((not candidate) current) + ((> (cadr current) (cadr candidate)) current) + ((< (cadr current) (cadr candidate)) candidate) + ((> (caddr current) (caddr candidate)) current) + ((< (caddr current) (caddr candidate)) candidate) + ((< (cadddr current) (cadddr candidate)) current) + (else candidate))) + +;;; Keeps large literal catalogs as immutable data instead of expanding one C +;;; branch per literal. Capability checks run only while preparing interned LR +;;; lexical modes, so SRFI-1's maintained list search avoids code-size growth +;;; without entering the source-scanning hot path. +(def (lexical-literal-admitted? literal values case-insensitive?) + (member literal values (if case-insensitive? string-ci=? string=?))) + +;;; AOT capability predicate for literal-producing lexical algebra. Runtime +;;; mode selection never probes scanners with synthetic input: closed literal +;;; and choice forms lower to ordinary comparisons, while opaque external +;;; scanners are conservatively admitted. +(defrules lexical-expression-admits-literal? + (choice literals precedence external) + ((_ literal case-insensitive? (literals value ...)) + (lexical-literal-admitted? literal '(value ...) case-insensitive?)) + ((_ literal case-insensitive? (choice expression ...)) + (or (lexical-expression-admits-literal? + literal case-insensitive? expression) ...)) + ((_ literal case-insensitive? (precedence _rank expression)) + (lexical-expression-admits-literal? + literal case-insensitive? expression)) + ((_ _literal _case-insensitive? (external _version _scanner)) #t) + ((_ _literal _case-insensitive? _expression) #f)) + +;;; A lexical mode admits complete rules, not already-produced lexemes. This +;;; preserves each admitted rule's ordinary longest-match behavior on source +;;; (for example a rule containing both "*" and "**"). +(defrules lexical-rule-admitted? + () + ((_ terminals name expression extras case-insensitive?) + (or (not terminals) + (memq 'name extras) + (any + (lambda (terminal) + (and (pair? terminal) + (eq? (car terminal) 'terminal) + (case (cadr terminal) + ((token) (eq? (caddr terminal) 'name)) + ((literal layout-start layout-next) + (lexical-expression-admits-literal? + (caddr terminal) case-insensitive? expression)) + (else #f)))) + terminals)))) + +;;; One AOT-generated capability/scanner pair. Capability checks run once per +;;; interned LR mode; the scanner remains the ordinary generated lexical rule. +;;; Literal rules share construction code; their trie scanning remains unchanged. +;;; These predicates run during mode preparation, outside token traversal. +(def (make-generated-literal-rule name literals rank extras case-insensitive?) + (vector + (lambda (terminals) + (or (not terminals) (memq name extras) + (any (lambda (terminal) + (and (pair? terminal) (eq? (car terminal) 'terminal) + (case (cadr terminal) + ((token) (eq? (caddr terminal) name)) + ((literal layout-start layout-next) + (lexical-literal-admitted? (caddr terminal) literals case-insensitive?)) + (else #f)))) terminals))) + (lambda (_source _offset) #f) + literals name rank + (lambda (ch) + (any (lambda (literal) + (or (zero? (string-length literal)) + (char-ci=? ch (string-ref literal 0)))) literals)) + #f)) + +(defrules generated-lexical-rule + (literals precedence) + ((_ (name (literals value ...)) extras case-insensitive?) + (make-generated-literal-rule 'name '(value ...) 0 extras case-insensitive?)) + ((_ (name (precedence rank (literals value ...))) extras case-insensitive?) + (make-generated-literal-rule 'name '(value ...) rank extras case-insensitive?)) + ((_ (name expression) extras case-insensitive?) + (let* ((literals (lexical-static-literals expression)) + (regular-kind (lexical-regular-kind expression)) + (scanner (and (not literals) (not regular-kind) + (lexical-scanner expression)))) + (vector + (lambda (terminals) + (lexical-rule-admitted? + terminals name expression extras case-insensitive?)) + (lambda (source offset) + (let (end (and scanner (scanner source offset))) + (and end + (list 'name end (lexical-expression-rank expression))))) + literals + 'name + (lexical-expression-rank expression) + (lambda (ch) (lexical-first-character? ch expression)) + regular-kind)))) + +(defrules lexical-rule-certificate-template + () + ((_ (name expression)) + (vector (lexical-pure-fallback? expression) + (lambda (ch) (lexical-guaranteed-first-character? ch expression))))) + +;;; Returns name, end offset, and precedence for generated-lexer. Longest +;;; consumption wins globally; lexical precedence breaks equal-length ties. +;; lexical-dispatch/ranked +;; : (-> Syntax Syntax) +;; | doc m% +;; Expands global longest-match dispatch with lexical tie precedence. +;; +;; # Examples +;; +;; ```scheme +;; (lexical-dispatch/ranked source offset rows) +;; ;; => (token-name end-offset precedence) or #f +;; ``` +;; % +(defrules lexical-dispatch/ranked + () + ((_ source offset ()) #f) + ((_ source offset ((name expression) row ...)) + (let ((end (lexical-end source offset expression)) + (rest (lexical-dispatch/ranked source offset (row ...)))) + (prefer-ranked-match + (and end (list 'name end (lexical-expression-rank expression))) + rest)))) + +;;; Binds the matched declaration name to its end offset without runtime macro state. +;; lexical-dispatch +;; : (-> Syntax Syntax) +;; | doc m% +;; `lexical-dispatch` expands the ordered token-rule dispatch. +;; +;; # Examples +;; +;; ```scheme +;; (lexical-dispatch source offset ((Identifier (identifier)))) +;; ;; => (Identifier . end-offset) +;; ``` +;; % +(defrules lexical-dispatch + () + ((_ source offset ()) #f) + ((_ source offset rows) + (let (match (lexical-dispatch/ranked source offset rows)) + (and match (cons (car match) (cadr match)))))) + diff --git a/src/compiler/lexical-lexer.ss b/src/compiler/lexical-lexer.ss new file mode 100644 index 00000000..bfdb3dbf --- /dev/null +++ b/src/compiler/lexical-lexer.ss @@ -0,0 +1,306 @@ +;;; -*- Gerbil -*- +;;; Generated lexer plans and conservative mode certificates. + +(import (only-in ../runtime/funcs + vector-intern-map make-value-interner value-interner-intern) + (only-in :std/vector/vector + vector-map/index) + (only-in ../runtime/lr-parser + lr-lexical-mode-id lr-lexical-mode-terminals) + (only-in ../runtime/scan + make-ranked-lexical-scanner-factory make-ranked-regular-scanner scan-emit) + (only-in ../runtime/token + token-end)) +(import (only-in ./lexical-expression generated-lexical-rule + lexical-rule-certificate-template prefer-generated-match)) +(export generated-lexer prepare-generated-lexer current-lexical-plan-sharing-enabled?) + +(def current-lexical-plan-sharing-enabled? (make-parameter #t)) + +;;; Quotient conservative first-transition masks, then full admission masks. +;;; A certificate retains every admitted rule that could match the input's +;;; first character. Rule ordinal preserves kind, precedence, and tie order; +;;; equal candidate sets and the common all-rule fallback prove equal scans. +;;; Non-ASCII and unknown lexical expressions keep their conservative path. +(def (prepare-lexical-certificates rules mode-keys all-key metadata) + (let ((interner (make-value-interner)) (rule-vector (list->vector rules)) + (certificate-vector (list->vector metadata))) + (def (normalize key mask) + (let* ((firsts (vector-ref mask 0)) (guarantees (vector-ref mask 1)) + (candidates (vector-map/index + (lambda (ordinal admitted) (and admitted (vector-ref firsts ordinal))) key)) + ;; Empty projections fall through to the common all-rule scanner. + (effective (if (any (lambda (candidate) candidate) (vector->list candidates)) candidates + (vector-map/index (lambda (ordinal admitted) (and admitted (vector-ref firsts ordinal))) all-key))) + (dominator + (let loop ((ordinal 0) (best #f)) + (if (= ordinal (vector-length effective)) best + (let* ((rule (vector-ref rule-vector ordinal)) + (rank (vector-ref rule 4)) + (eligible (and (vector-ref effective ordinal) + (vector-ref guarantees ordinal) + (not (vector-ref (vector-ref certificate-vector ordinal) 0))))) + (loop (+ ordinal 1) + (if (and eligible + (or (not best) (> rank (car best)))) + (cons rank ordinal) best))))))) + ;; A pure fallback consumes exactly one character. The best guaranteed + ;; nonfallback rule consumes at least one, and can make it unable to + ;; win by priority and declaration order. Keep all other competitors. + (vector-map/index + (lambda (ordinal admitted) + (and admitted + (let (rule (vector-ref rule-vector ordinal)) + (not (and dominator (vector-ref (vector-ref certificate-vector ordinal) 0) + (or (> (car dominator) (vector-ref rule 4)) + (and (= (car dominator) (vector-ref rule 4)) + (< (cdr dominator) ordinal)))))))) effective))) + (let-values (((classes masks) + (vector-intern-map + (vector-map/index (lambda (code _) code) (make-vector 128 #f)) + (lambda (code) + (let (ch (integer->char code)) + (vector + (list->vector (map (lambda (rule) ((vector-ref rule 5) ch)) rules)) + (list->vector (map (lambda (certificate) ((vector-ref certificate 1) ch)) metadata))))) + (lambda (key id) (cons id key))))) + (let-values (((modes ignored) + (vector-intern-map mode-keys (lambda (key) key) + (lambda (key _id) + (vector-map/index + (lambda (_index mask) + (let (candidate (normalize key (cdr mask))) + (value-interner-intern interner candidate (lambda () candidate)))) masks))))) + (vector (vector-map/index (lambda (_index entry) (car entry)) classes) modes))))) + + +;;; Generates only the source traversal shell; token construction remains scan-owned. +;;; Declaration order is observable only after length and precedence tie. +;; generated-lexer +;; : (-> Syntax Syntax) +;; | doc m% +;; `generated-lexer` expands lexical rows into a source-to-token procedure. +;; +;; # Examples +;; +;; ```scheme +;; (generated-lexer (lexical-rules (Identifier (identifier)))) +;; ;; => source-to-token procedure +;; ``` +;; % +(defrules generated-lexer + (lexical-rules extras) + ((_ (lexical-rules row ...) (extras extra-name ...) case-insensitive? + mode-catalog) + (generated-lexer (lexical-rules row ...) (extras extra-name ...) + case-insensitive? mode-catalog make-ranked-lexical-scanner-factory)) + ((_ (lexical-rules row ...) (extras extra-name ...) case-insensitive? + mode-catalog ranked-scanner-factory) + (prepare-generated-lexer + (list (generated-lexical-rule row '(extra-name ...) case-insensitive?) ...) + (lambda () (list (lexical-rule-certificate-template row) ...)) + mode-catalog ranked-scanner-factory))) + +;;; Plan construction and scanning are shared native engine code. Language +;;; expansion contributes only rule scanners and conservative certificates. +(def (prepare-generated-lexer bare-rules metadata-thunk mode-catalog + (ranked-scanner-factory make-ranked-lexical-scanner-factory)) + (let* ((rules + (let loop ((remaining bare-rules) (ordinal 0) (found '())) + (if (null? remaining) + (reverse found) + (let* ((rule (car remaining)) + (scanner (vector-ref rule 1)) + (ranked-scanner + (and scanner + (lambda (source offset) + (let (match (scanner source offset)) + (and match + (list (car match) (cadr match) + (caddr match) ordinal))))))) + (loop (cdr remaining) (+ ordinal 1) + (cons + (vector (vector-ref rule 0) ranked-scanner + (vector-ref rule 2) (vector-ref rule 3) + (vector-ref rule 4) (vector-ref rule 5) + (vector-ref rule 6)) + found)))))) + (literal-entries + (let loop ((remaining rules) (ordinal 0) (entries '())) + (if (null? remaining) + entries + (let ((rule (car remaining))) + (loop + (cdr remaining) (+ ordinal 1) + (if (vector-ref rule 2) + (fold + (lambda (literal found) + (cons + (list literal (vector-ref rule 3) + (vector-ref rule 4) ordinal) + found)) + entries (vector-ref rule 2)) + entries)))))) + (literal-ascii-starts + (let (starts (make-vector 128 #f)) + (for-each + (lambda (entry) + (let* ((literal (car entry)) + (code (and (positive? (string-length literal)) + (char->integer (string-ref literal 0))))) + (when (and code (< code 128)) + (vector-set! starts code #t)))) + literal-entries) + starts)) + (literal-scanner + (and (pair? literal-entries) + (ranked-scanner-factory literal-entries))) + (prepare-scanners + (lambda (admissions) + (let ((admitted-literals (make-vector (length rules) #f))) + (let loop ((remaining rules) (ordinal 0) + (has-literals? #f) (scanners '()) + (regular-entries '()) + (regular-first-predicates '())) + (if (null? remaining) + (let* ((regular-scanner + (and (not literal-scanner) (pair? regular-entries) + (make-ranked-regular-scanner regular-entries))) + (regular-ascii-starts + (and (pair? regular-entries) + (vector-map/index + (lambda (index _) + (any (lambda (predicate) + (predicate (integer->char index))) + regular-first-predicates)) + (make-vector 128)))) + (ranked-scanner (and literal-scanner + (literal-scanner regular-entries admitted-literals))) + (ascii-scanners + (vector-map/index + (lambda (index _) + (filter + (lambda (entry) + ((cdr entry) (integer->char index))) + scanners)) + (make-vector 128)))) + (lambda (source offset) + (let* ((ch (string-ref source offset)) + (code (char->integer ch)) + (candidates + (if (< code 128) + (vector-ref ascii-scanners code) + scanners)) + (selected + ;; Candidate order is prepared. Carry the winner + ;; through one traversal without allocating a + ;; source/offset callback for every token. + (let scan-candidates ((remaining candidates) (selected #f)) + (if (null? remaining) selected + (scan-candidates + (cdr remaining) + (prefer-generated-match + selected ((caar remaining) source offset))))))) + (cond + ((and ranked-scanner + (or (and has-literals? + (or (>= code 128) (vector-ref literal-ascii-starts code))) + (and (pair? regular-entries) + (or (>= code 128) (vector-ref regular-ascii-starts code))))) + (prefer-generated-match selected (ranked-scanner source offset))) + ((and regular-scanner + (or (>= code 128) (vector-ref regular-ascii-starts code))) + (prefer-generated-match selected (regular-scanner source offset))) + (else selected))))) + (let* ((rule (car remaining)) + (admitted? (vector-ref admissions ordinal)) + (literals (and admitted? (vector-ref rule 2))) + (regular-kind (and admitted? (vector-ref rule 6)))) + (cond + ((pair? literals) + (vector-set! admitted-literals ordinal #t) + (loop + (cdr remaining) (+ ordinal 1) #t scanners + regular-entries regular-first-predicates)) + (literals + (loop (cdr remaining) (+ ordinal 1) + has-literals? scanners regular-entries + regular-first-predicates)) + (regular-kind + (loop (cdr remaining) (+ ordinal 1) + has-literals? scanners + (cons (list regular-kind (vector-ref rule 3) + (vector-ref rule 4) ordinal) + regular-entries) + (cons (vector-ref rule 5) + regular-first-predicates))) + (admitted? + (loop + (cdr remaining) (+ ordinal 1) has-literals? + (cons (cons (vector-ref rule 1) + (vector-ref rule 5)) + scanners) + regular-entries regular-first-predicates)) + (else + (loop (cdr remaining) (+ ordinal 1) + has-literals? scanners regular-entries + regular-first-predicates))))))))) + (admission-key + (lambda (terminals) + (list->vector (map (lambda (rule) (and ((vector-ref rule 0) terminals) #t)) rules)))) + (all-scanners (prepare-scanners (admission-key #f))) + (mode-keys (vector-map/index + (lambda (_index mode) (admission-key (lr-lexical-mode-terminals mode))) mode-catalog)) + ;; Normal parsing uses full plans only. Materialize ASCII certificates + ;; once, on the first request that needs the stronger quotient. + (certificates (delay (prepare-lexical-certificates rules mode-keys (admission-key #f) + (metadata-thunk)))) + (mode-scanners + (if (current-lexical-plan-sharing-enabled?) + (let-values (((plans unique) + (vector-intern-map mode-keys + (lambda (key) key) + (lambda (key _id) (prepare-scanners key))))) plans) + (vector-map/index + (lambda (_index key) (prepare-scanners key)) + mode-keys)))) + (letrec + ((scan-one + (lambda (source offset byte-offset mode) + (let (match + (or (if mode + ((vector-ref mode-scanners + (lr-lexical-mode-id mode)) + source offset) + (all-scanners source offset)) + ;; A mode miss must still materialize the offending token + ;; for the LR failure frontier and lossless diagnostics. + ;; Successful directed scans never enter this cold path. + (and mode + (all-scanners source offset)))) + (unless match + (error "no lexical rule matched parser-directed source" + offset mode)) + (let (output-token + (scan-emit source (car match) offset (cadr match) + byte-offset)) + (values output-token (cadr match)))))) + (scan-from + (lambda (source initial-offset initial-byte-offset) + (let (length (string-length source)) + (let loop ((offset initial-offset) + (byte-offset initial-byte-offset) + (tokens '())) + (if (= offset length) + (reverse tokens) + (let-values (((output-token end) + (scan-one source offset byte-offset #f))) + (loop end (token-end output-token) + (cons output-token tokens))))))))) + (values (case-lambda + ((source) (scan-from source 0 0)) + ((source offset byte-offset) + (scan-from source offset byte-offset)) + ((source offset byte-offset mode) + (scan-one source offset byte-offset mode))) mode-scanners certificates)))) + diff --git a/src/compiler/line-structure-rowan.ss b/src/compiler/line-structure-runtime.ss similarity index 96% rename from src/compiler/line-structure-rowan.ss rename to src/compiler/line-structure-runtime.ss index 6d17e43f..60869cc7 100644 --- a/src/compiler/line-structure-rowan.ss +++ b/src/compiler/line-structure-runtime.ss @@ -1,5 +1,5 @@ ;;; -*- Gerbil -*- -;;; POO line strategy -> typed Rust syntax -> one immutable Rowan table. +;;; POO line strategy -> typed Rust syntax -> one immutable syntax table. (import (only-in ../language/descriptor language-grammar-ir language-grammar-machine) @@ -43,9 +43,9 @@ key-line-repeated key-line-node key-line-key-token key-line-value-token key-line-trivia-token)) (export line-structure-parser-digest - line-structure-rowan-syntax - line-structure-rowan-source - generate-line-structure-rowan-module) + line-structure-runtime-syntax + line-structure-runtime-source + generate-line-structure-runtime-module) (def (kind-index kinds name category) (let loop ((rest kinds) (index 0)) @@ -285,7 +285,7 @@ (line-structure-key-lines structure))) ;; Public input is one validated POO contract; output is a bounded Rust AST. -(def (line-structure-rowan-syntax language-grammar structure) +(def (line-structure-runtime-syntax language-grammar structure) (unless (line-structure? structure) (error "line-structure AOT requires a POO parser declaration" structure)) (let* ((ir (language-grammar-ir language-grammar)) @@ -321,12 +321,13 @@ (text_token (kind-value kinds (text-line-token text) 'token)) (inline_link (optional-value (text-line-inline-link text) - (lambda (value) (inline-link-value kinds value))))))))) + (lambda (value) (inline-link-value kinds value)))))) + "line-structure-runtime.ss"))) -(def (line-structure-rowan-source language-grammar structure) - (rust-render (line-structure-rowan-syntax language-grammar structure))) +(def (line-structure-runtime-source language-grammar structure) + (rust-render (line-structure-runtime-syntax language-grammar structure))) -(def (generate-line-structure-rowan-module output-path language-grammar structure) - (let (source (line-structure-rowan-source language-grammar structure)) +(def (generate-line-structure-runtime-module output-path language-grammar structure) + (let (source (line-structure-runtime-source language-grammar structure)) (call-with-output-file output-path (lambda (port) (display source port))))) diff --git a/src/compiler/lr-automaton.ss b/src/compiler/lr-automaton.ss index 9a854daf..304222d3 100644 --- a/src/compiler/lr-automaton.ss +++ b/src/compiler/lr-automaton.ss @@ -1,7 +1,10 @@ ;;; -*- Gerbil -*- ;;; Packed LR items, transition indexes, and deterministic LR(0) interning. -(import (only-in :std/struct/queue +(import (only-in ./funcs compiler-index-set-add + compiler-index-set-difference compiler-index-set-for-each + compiler-index-set-union) + (only-in :std/struct/queue dequeue! enqueue! make-Queue queue-empty?) (only-in ./lr base-symbol nonterminal-name nonterminal-symbol? @@ -10,23 +13,38 @@ core-item-after-dot core-item-tail-after-next item-after-dot item-complete? item-lookahead item-production-id item Vector Vector (Pair Fixnum Fixnum)) +;;; Dense dotted positions preserve production/dot ordering without padding +;;; every rule to the widest RHS. The inverse catalog stages decoding once; +;;; all construction routes share this private representation. +(defstruct lr-item-layout (terminal-count offsets owners) final: #t) + (def (make-item-layout table terminal-values) - (cons (vector-length terminal-values) - (foldl (lambda (production dot-width) - (max dot-width (+ (length (production-rhs production)) 1))) - 1 - (vector->list table)))) - -(def (item-lookahead item layout) (modulo item (car layout))) -(def (item-body item layout) (quotient item (car layout))) + (let* ((count (vector-length table)) (offsets (make-vector (+ count 1)))) + (let index ((id 0) (offset 0)) + (vector-set! offsets id offset) + (if (= id count) + (let (owners (make-vector offset)) + (let production-loop ((id 0)) + (when (< id count) + (let fill ((item (vector-ref offsets id)) + (end (vector-ref offsets (+ id 1)))) + (when (< item end) + (vector-set! owners item id) + (fill (+ item 1) end))) + (production-loop (+ id 1)))) + (make-lr-item-layout (vector-length terminal-values) offsets owners)) + (index (+ id 1) + (+ offset 1 (length (production-rhs (vector-ref table id))))))))) + +(def (item-lookahead item layout) (modulo item (lr-item-layout-terminal-count layout))) +(def (item-body item layout) (quotient item (lr-item-layout-terminal-count layout))) (def (item-production-id item layout) - (quotient (item-body item layout) (cdr layout))) -(def (item-dot item layout) (modulo (item-body item layout) (cdr layout))) + (core-item-production-id (item-body item layout) layout)) +(def (item-dot item layout) (core-item-dot (item-body item layout) layout)) (def (item-after-dot item layout table) (let* ((production (vector-ref table (item-production-id item layout))) @@ -39,16 +57,6 @@ (def (item Vector Pair Vector) +;; : (-> Vector lr-item-layout Vector) (def (make-core-symbol-catalog table layout) - (let ((catalog (make-vector (* (vector-length table) (cdr layout)) #f))) + (let ((catalog (make-vector (core-item-count layout) #f))) (let production-loop ((production-id 0)) (when (< production-id (vector-length table)) (let symbol-loop @@ -106,40 +117,190 @@ (rhs (production-rhs production))) (list-tail rhs (+ (core-item-dot item layout) 1)))) -(def (lr0-closure seed productions-by-lhs core-symbols layout) - (let (seen (make-table test: eq?)) - (for-each (lambda (item) (table-set! seen item #t)) seed) - (let loop ((pending seed) (items '())) - (if (null? pending) - (list-sort < (reverse items)) - (let* ((item (car pending)) - (symbol (vector-ref core-symbols item)) - (expanded (cdr pending))) +;; LR(0) prediction is a grammar-only reachability relation. Solve it once +;; over dense production-id masks, then apply it directly to every goto kernel. +(def (make-core-prediction-catalog core-symbols productions-by-lhs layout) + (let ((name-index (make-table test: eq?)) + (names '()) + (node-count 0) + (item-nodes (make-vector (vector-length core-symbols) #f))) + (let index-loop ((item 0)) + (when (< item (vector-length core-symbols)) + (let (symbol (vector-ref core-symbols item)) (when (and symbol (nonterminal-symbol? symbol)) + (let* ((name (nonterminal-name symbol)) + (known (table-ref name-index name #f)) + (node (or known node-count))) + (unless known + (table-set! name-index name node) + (set! names (cons (cons name node) names)) + (set! node-count (+ node-count 1))) + (vector-set! item-nodes item node)))) + (index-loop (+ item 1)))) + (let ((masks (make-vector node-count 0)) + (pending (make-vector node-count 0)) + (parents (make-vector node-count '())) + (queued (make-vector node-count #f)) + (queue (make-Queue))) + (for-each + (lambda (entry) + (let ((node (cdr entry)) (own 0) (children 0)) + (for-each + (lambda (production) + (let* ((id (production-id production)) + (child (vector-ref item-nodes + (make-core-item id 0 layout)))) + (set! own (compiler-index-set-add own id)) + (when child + (set! children (compiler-index-set-add children child))))) + (table-ref productions-by-lhs (car entry) '())) + (vector-set! masks node own) + (vector-set! pending node own) + (unless (zero? own) + (vector-set! queued node #t) + (enqueue! queue node)) + (compiler-index-set-for-each + children + (lambda (child) + (vector-set! parents child + (cons node (vector-ref parents child))))))) + names) + ;; Reverse dependency edges carry only new production bits. Recursive + ;; grammar components converge before any LR state is constructed. + (let propagate () + (unless (queue-empty? queue) + (let* ((node (dequeue! queue)) + (delta (vector-ref pending node))) + (vector-set! pending node 0) + (vector-set! queued node #f) (for-each - (lambda (production) - (let (candidate - (make-core-item (production-id production) 0 layout)) - (unless (table-ref seen candidate #f) - (table-set! seen candidate #t) - (set! expanded (cons candidate expanded))))) - (table-ref productions-by-lhs (nonterminal-name symbol) '()))) - (loop expanded (cons item items))))))) - -(def (lr0-state-kernels state core-symbols) - (let ((kernels (make-table test: equal?)) (symbol-order '())) - (for-each + (lambda (parent) + (let (novel + (compiler-index-set-difference + delta (vector-ref masks parent))) + (unless (zero? novel) + (vector-set! masks parent + (compiler-index-set-union + (vector-ref masks parent) novel)) + (vector-set! pending parent + (compiler-index-set-union + (vector-ref pending parent) novel)) + (unless (vector-ref queued parent) + (vector-set! queued parent #t) + (enqueue! queue parent))))) + (vector-ref parents node))) + (propagate))) + (let (catalog (make-vector (vector-length core-symbols) 0)) + (let materialize ((item 0)) + (when (< item (vector-length item-nodes)) + (let (node (vector-ref item-nodes item)) + (when node + (vector-set! catalog item (vector-ref masks node)))) + (materialize (+ item 1)))) + catalog)))) + +;; Symbol IDs are local to one construction, including template scratch. +(def (make-lr0-symbol-catalog core-symbols) + (let ((index (make-table test: equal?)) + (ids (make-vector (vector-length core-symbols) #f)) + (count 0)) + (let loop ((item 0)) + (when (< item (vector-length core-symbols)) + (let (symbol (vector-ref core-symbols item)) + (when symbol + (let (id (table-ref index symbol #f)) + (unless id + (set! id count) + (set! count (+ count 1)) + (table-set! index symbol id)) + (vector-set! ids item id)))) + (loop (+ item 1)))) + (let (symbols (make-vector count #f)) + (hash-for-each (lambda (symbol id) (vector-set! symbols id symbol)) index) + (values symbols ids)))) + +;; Each distinct prediction mask owns one immutable template: sorted dot-zero +;; items, goto groups ordered by their earliest item, and a sparse group index. +;; A row is #(first-item symbol-id advanced-kernel). Templates share across +;; states; their group masks never enter mutable scratch by reference mutation. +(def (make-prediction-template mask layout symbol-ids) + (let ((items '()) (order '()) (groups (make-table test: eq?))) + (compiler-index-set-for-each + mask + (lambda (production-id) + (let* ((item (make-core-item production-id 0 layout)) + (id (vector-ref symbol-ids item))) + (set! items (cons item items)) + (when id + (let (row (table-ref groups id #f)) + (unless row + (set! row (vector item id 0)) + (set! order (cons id order)) + (table-set! groups id row)) + (vector-set! row 2 + (compiler-index-set-add (vector-ref row 2) (+ item 1)))))))) + (vector (reverse items) + (map (lambda (id) (table-ref groups id)) (reverse order)) + groups))) + +(def (lr0-closure kernel-mask template) + (let (seed '()) + (compiler-index-set-for-each + kernel-mask (lambda (item) (set! seed (cons item seed)))) + ;; Retain the immutable template suffix when the kernel is exhausted. + (let merge ((left (reverse seed)) (right (vector-ref template 0)) + (found '())) + (cond + ((null? left) (foldl cons right found)) + ((null? right) (foldl cons left found)) + ((= (car left) (car right)) + (merge (cdr left) (cdr right) (cons (car left) found))) + ((< (car left) (car right)) + (merge (cdr left) right (cons (car left) found))) + (else (merge left (cdr right) (cons (car right) found))))))) + +(def (lr0-state-kernels kernel-mask template symbol-ids masks first-items) + (let ((order '()) (groups (vector-ref template 2))) + ;; Only advanced kernel items contribute new groups or alter template + ;; groups. The full closure is never hashed or grouped again per state. + (compiler-index-set-for-each + kernel-mask (lambda (item) - (let (symbol (vector-ref core-symbols item)) - (when symbol - (unless (table-ref kernels symbol #f) - (set! symbol-order (cons symbol symbol-order))) - (table-set! kernels symbol - (cons (+ item 1) - (or (table-ref kernels symbol #f) '())))))) - state) - (map (lambda (symbol) (cons symbol (table-ref kernels symbol))) - (reverse symbol-order)))) + (let (id (vector-ref symbol-ids item)) + (when id + (when (zero? (vector-ref masks id)) + (set! order (cons id order)) + (vector-set! first-items id item)) + (vector-set! masks id + (compiler-index-set-add (vector-ref masks id) (+ item 1))))))) + (let (own + (list-sort + (lambda (left right) (< (vector-ref left 0) (vector-ref right 0))) + (map + (lambda (id) + (let ((shared (table-ref groups id #f)) + (first (vector-ref first-items id)) + (mask (vector-ref masks id))) + (if shared + (vector (min first (vector-ref shared 0)) id + (compiler-index-set-union mask (vector-ref shared 2))) + (vector first id mask)))) + order))) + ;; Earliest-item order reproduces the original sorted-closure symbol + ;; discovery order, including kernel symbols that precede predictions. + (let (rows + (let merge ((left own) (right (vector-ref template 1)) (found '())) + (cond + ((and (pair? right) + (not (zero? (vector-ref masks (vector-ref (car right) 1))))) + (merge left (cdr right) found)) + ((null? right) (foldl cons left found)) + ((null? left) (merge left (cdr right) (cons (car right) found))) + ((< (vector-ref (car left) 0) (vector-ref (car right) 0)) + (merge (cdr left) right (cons (car left) found))) + (else (merge left (cdr right) (cons (car right) found)))))) + (for-each (lambda (id) (vector-set! masks id 0)) order) + rows)))) ;;; Owns LR(0) state interning and transition discovery as one work queue. ;;; Each state is published once; vector growth preserves assigned indices. @@ -159,33 +320,56 @@ ;; % (def (build-lr0-automaton productions table layout core-symbols) (let* ((productions-by-lhs (production-index-by-lhs productions)) - (initial (lr0-closure (list (make-core-item 0 0 layout)) - productions-by-lhs core-symbols layout)) + (prediction-catalog + (make-core-prediction-catalog core-symbols productions-by-lhs layout)) + (initial-kernel (compiler-index-set-add 0 (make-core-item 0 0 layout))) + (templates (make-table test: equal?)) (states (make-vector 128 #f)) (state-transitions (make-vector 128 '())) + (kernel-plans (make-vector 128 #f)) (state-index (make-table test: equal?)) (state-count 1) (processed-count 0) (trace? (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1"))) - (vector-set! states 0 initial) - (table-set! state-index initial 0) + (let-values (((symbols symbol-ids) (make-lr0-symbol-catalog core-symbols))) + (let ((masks (make-vector (vector-length symbols) 0)) + (first-items (make-vector (vector-length symbols) 0))) + (def (template-for kernel) + (let (mask 0) + (compiler-index-set-for-each + kernel + (lambda (item) + (set! mask (compiler-index-set-union + mask (vector-ref prediction-catalog item))))) + (or (table-ref templates mask #f) + (let (template (make-prediction-template mask layout symbol-ids)) + (table-set! templates mask template) + template)))) + (let (template (template-for initial-kernel)) + (vector-set! states 0 (lr0-closure initial-kernel template)) + (vector-set! kernel-plans 0 (cons initial-kernel template))) + (table-set! state-index initial-kernel 0) (def (grow!) (when (= state-count (vector-length states)) (let ((next-states (make-vector (* 2 state-count) #f)) - (next-transitions (make-vector (* 2 state-count) '()))) + (next-transitions (make-vector (* 2 state-count) '())) + (next-plans (make-vector (* 2 state-count) #f))) (let copy ((index 0)) (when (< index state-count) (vector-set! next-states index (vector-ref states index)) (vector-set! next-transitions index (vector-ref state-transitions index)) + (vector-set! next-plans index (vector-ref kernel-plans index)) (copy (+ index 1)))) (set! states next-states) - (set! state-transitions next-transitions)))) - (def (append-state! state) + (set! state-transitions next-transitions) + (set! kernel-plans next-plans)))) + (def (append-state! kernel) (grow!) - (let (index state-count) - (vector-set! states index state) - (table-set! state-index state index) + (let ((index state-count) (template (template-for kernel))) + (vector-set! states index (lr0-closure kernel template)) + (vector-set! kernel-plans index (cons kernel template)) + (table-set! state-index kernel index) (set! state-count (+ state-count 1)) index)) (let (queue (make-Queue)) @@ -202,19 +386,21 @@ (display " processed=") (displayln processed-count) (force-output)) - (for-each - (lambda (kernel) - (let* ((symbol (car kernel)) - (target (lr0-closure (cdr kernel) - productions-by-lhs core-symbols - layout)) - (existing (table-ref state-index target #f)) - (target-index (or existing (append-state! target)))) - (vector-set! - state-transitions index - (transition-set (vector-ref state-transitions index) - symbol target-index)) - (unless existing - (enqueue! queue target-index)))) - (lr0-state-kernels (vector-ref states index) core-symbols)) - (loop))))))) + ;; Closure adds only dot-zero items. Every noninitial state is + ;; therefore identified exactly by its advanced-item kernel mask; + ;; the augmented dot-zero initial kernel has a disjoint key. + (let ((rows '()) (plan (vector-ref kernel-plans index))) + (for-each + (lambda (kernel) + (let* ((symbol (vector-ref symbols (vector-ref kernel 1))) + (mask (vector-ref kernel 2)) + (existing (table-ref state-index mask #f)) + (target-index + (or existing + (append-state! mask)))) + (set! rows (cons (cons symbol target-index) rows)) + (unless existing (enqueue! queue target-index)))) + (lr0-state-kernels (car plan) (cdr plan) symbol-ids masks first-items)) + ;; There is exactly one kernel per symbol; publish the row once. + (vector-set! state-transitions index (reverse rows))) + (loop))))))))) diff --git a/src/compiler/lr-compiler.ss b/src/compiler/lr-compiler.ss index e82a3ac1..c9399d15 100644 --- a/src/compiler/lr-compiler.ss +++ b/src/compiler/lr-compiler.ss @@ -2,15 +2,21 @@ ;;; LR conflict resolution and canonical parser-table publication. (import (only-in :std/iter for in-range) + (only-in ./lr-table validate-lr-tables) + (only-in ./lr-automaton core-item-production-id core-item-dot) (only-in ./funcs compiler-index-set-for-each) (only-in ./lr - compute-first compute-nullable - lower-rules nonterminal-name nonterminal-symbol? - production-id production-precedence production-table + compute-first compute-completion + validate-resolved-start validate-resolved-repetitions + lower-rules current-grammar-source-map lr-spec-ref nonterminal-name nonterminal-symbol? + production-action production-id production-precedence production-table + layout-end-action? validate-production-semantics terminal-symbol? union-values) - (only-in ./lr-automaton - transition-index transition-target) - (only-in ./lr-lookahead build-states-via-lr0)) + (only-in ./lr-lookahead + build-states-via-lr0 build-states-via-canonical-lr1) + (only-in ./lr-partition build-states-via-partitioned-lr1) + (only-in ./lr-follow-automaton + build-states-via-follow-partition-lr1)) (export compile-lr-spec) ;; fork-action @@ -26,7 +32,7 @@ ;; : (-> Symbol List List String Datum List) (def (fork-or-reject conflict-policy shift reduce message evidence) - (if (eq? conflict-policy 'selective-glr) + (if (memq conflict-policy '(selective-glr probe)) (fork-action shift reduce) (error message evidence))) @@ -79,7 +85,8 @@ (list terminal left-production right-production))) ((> left-rank right-rank) left) ((< left-rank right-rank) right) - ((eq? conflict-policy 'selective-glr) (fork-action left right)) + ((memq conflict-policy '(selective-glr probe)) + (fork-action left right)) (else (error "unresolved reduce/reduce conflict" terminal left-production right-production))))) @@ -113,6 +120,56 @@ (else (error "unresolved LR action conflict" state terminal current action)))) +(def (resolve-action/diagnosed state terminal current action table policy + reject-conflict items) + (with-catch + (lambda (condition) + (reject-conflict condition state items terminal current action)) + (lambda () (resolve-action state terminal current action table policy)))) + +;; Reconstruct occurrence evidence only after admission rejects an action pair. +;; This sidecar never enters the canonical table, serialized IR or runtime. +(def (reject-lr-conflict condition rules root policy table layout core-symbols + state items terminal left right) + (def origins (make-vector (vector-length table) #f)) + (def sources + (let (entry (assq 'rule (current-grammar-source-map))) + (if entry (cdr entry) '()))) + (lower-rules + rules root (eq? policy 'selective-glr) + (lambda (production owner path expression) + (let (source (assq owner sources)) + (vector-set! origins (production-id production) + (list (cons 'rule owner) (cons 'expressionPath path) + (cons 'loweringOperation (car expression)) + (cons 'source (if source (cdr source) '()))))))) + (vector-set! origins 0 + (list (cons 'rule root) (cons 'expressionPath '()) + (cons 'loweringOperation 'augmented-start) + (cons 'source '()))) + (def (action-evidence action) + (cons (cons 'action action) + (if (eq? (car action) 'reduce) + (list (cons 'expressionOrigin (vector-ref origins (cadr action)))) '()))) + ;; A shift is shared by all matching state items, not owned by one arbitrary + ;; production. Keep each occurrence and dot separately, including helpers. + (def shift-items + (if (or (eq? (car left) 'shift) (eq? (car right) 'shift)) + (filter-map + (lambda (item) + (and (equal? terminal (vector-ref core-symbols item)) + (list (cons 'dot (core-item-dot item layout)) + (cons 'expressionOrigin + (vector-ref origins (core-item-production-id item layout)))))) + items) + '())) + (apply error (error-message condition) + (append (error-irritants condition) + (list (list (cons 'lrConflict + (list (cons 'state state) (cons 'terminal terminal) + (cons 'actions (map action-evidence (list left right))) + (cons 'shiftItems shift-items)))))))) + ;; index-state-rows ;; : (-> List Fixnum Procedure Procedure Vector) (def (index-state-rows rows state-count key-procedure value-procedure) @@ -134,14 +191,34 @@ ;; : (-> Vector Fixnum Vector Vector List Vector Vector Pair Vector Symbol ;; (values Vector Fixnum)) (def (build-actions states state-count lookaheads lookahead-offsets transitions table - terminal-values layout core-symbols conflict-policy) + terminal-values layout core-symbols conflict-policy reject-conflict) (let ((actions (make-vector state-count '())) - (transitions (transition-index transitions)) + (transition-rows + (index-state-rows transitions state-count cadr caddr)) + (terminal-index (make-table test: equal?)) + (state-actions (make-vector (vector-length terminal-values) #f)) + (raw-reductions (make-vector (vector-length terminal-values) '())) + (shift-targets (make-vector (vector-length terminal-values) #f)) + (core-terminal-ids (make-vector (vector-length core-symbols) #f)) (trace? (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1")) (started (##current-time-point)) + (layout? (vector-any (lambda (production) + (layout-end-action? (production-action production))) table)) (processed-items 0) (published-actions 0) (publication-count 0)) + (let index-terminals ((id 0)) + (when (< id (vector-length terminal-values)) + (table-set! terminal-index (vector-ref terminal-values id) id) + (index-terminals (+ id 1)))) + (let index-cores ((item 0)) + (when (< item (vector-length core-symbols)) + (let (symbol (vector-ref core-symbols item)) + (when (and symbol (terminal-symbol? symbol)) + (let (id (table-ref terminal-index symbol #f)) + (unless id (error "LR terminal is not indexed" item symbol)) + (vector-set! core-terminal-ids item id)))) + (index-cores (+ item 1)))) ;; Publish exactly one action row per state. Keeping publication outside the ;; item traversal avoids rebuilding the growing row on recursive returns. (for (state-id (in-range state-count)) @@ -149,29 +226,45 @@ (unless (list? state-items) (error "LR action state is not an item list" state-id state-items)) - (let ((state-actions (make-table test: equal?)) - (terminal-order '()) + (let ((terminal-order '()) + (shift-order '()) (lookahead-node (vector-ref lookahead-offsets state-id))) - (def (install! terminal action) - (let (current (table-ref state-actions terminal #f)) + (def (install! terminal-id action) + (when (and layout? (eq? (car action) 'reduce)) + (vector-set! raw-reductions terminal-id + (cons action (vector-ref raw-reductions terminal-id)))) + (let (current (vector-ref state-actions terminal-id)) (if current - (table-set! state-actions terminal - (resolve-action state-id terminal current action - table conflict-policy)) + (vector-set! state-actions terminal-id + (resolve-action/diagnosed + state-id (vector-ref terminal-values terminal-id) + current action table conflict-policy reject-conflict state-items)) (begin - (table-set! state-actions terminal action) - (set! terminal-order (cons terminal terminal-order)))))) + (vector-set! state-actions terminal-id action) + (set! terminal-order (cons terminal-id terminal-order)))))) + (for-each + (lambda (row) + (when (terminal-symbol? (car row)) + (let (id (table-ref terminal-index (car row) #f)) + (unless id (error "LR shift terminal is not indexed" state-id row)) + (vector-set! shift-targets id (cdr row)) + (set! shift-order (cons id shift-order))))) + (vector-ref transition-rows state-id)) (for-each (lambda (item) (set! processed-items (+ processed-items 1)) - (let* ((production-id (quotient item (cdr layout))) - (symbol (vector-ref core-symbols item))) + (let* ((production-id (core-item-production-id item layout)) + (symbol (vector-ref core-symbols item)) + (terminal-id (vector-ref core-terminal-ids item))) (cond - ((and symbol (terminal-symbol? symbol)) + (terminal-id (install! - symbol + terminal-id (list 'shift - (transition-target transitions state-id symbol) + (let (target (vector-ref shift-targets terminal-id)) + (unless target (error "LR shift transition is missing" + state-id symbol)) + target) (production-precedence (vector-ref table production-id))))) ((not symbol) @@ -182,18 +275,29 @@ (compiler-index-set-for-each (vector-ref lookaheads lookahead-node) (lambda (lookahead) - (install! (vector-ref terminal-values lookahead) - action))))))) + (install! lookahead action))))))) (set! lookahead-node (+ lookahead-node 1))) state-items) (vector-set! actions state-id - (map (lambda (terminal) - (cons terminal (table-ref state-actions terminal))) + (map (lambda (terminal-id) + (let ((action (vector-ref state-actions terminal-id)) + (reductions (vector-ref raw-reductions terminal-id))) + (cons (vector-ref terminal-values terminal-id) + (if (and layout? (eq? (car action) 'shift) (pair? reductions)) + (list 'layout-guard action + (if (null? (cdr reductions)) (car reductions) + (cons 'fork (reverse reductions)))) action)))) (reverse terminal-order))) (set! publication-count (+ publication-count 1)) (set! published-actions - (+ published-actions (length terminal-order)))) + (+ published-actions (length terminal-order))) + ;; Rows own copied immutable cells; clear only the touched scratch + ;; slots instead of allocating state-count times the terminal domain. + (for-each (lambda (id) + (vector-set! state-actions id #f) + (vector-set! raw-reductions id '())) terminal-order) + (for-each (lambda (id) (vector-set! shift-targets id #f)) shift-order)) (when (and trace? (zero? (modulo (+ state-id 1) 100))) (display "[gerbil-parser-lr] action-states=") (display (+ state-id 1)) @@ -234,45 +338,114 @@ ;;; resolution, and publication into the canonical immutable LR spec v1. ;; compile-lr-spec ;; : (-> List Symbol Symbol Boolean List) -(def (compile-lr-spec rules root (conflict-policy 'reject) - (case-insensitive? #f)) +(def (compile-lr-spec/selected rules root conflict-policy + case-insensitive? construction) + (unless (memq construction + '(lalr canonical-lr1 partitioned-lr1 follow-partition-lr1)) + (error "unknown LR construction" construction)) (let* ((started (##current-time-point)) (productions - (lower-rules rules root (eq? conflict-policy 'selective-glr))) + (validate-production-semantics + (lower-rules rules root (eq? conflict-policy 'selective-glr)))) (table (production-table productions))) (trace-lr-phase 'lowered (length productions) started) - (let-values (((nullable nullable-index) - (compute-nullable productions))) + (let-values (((nullable nullable-index productive productive-index) + (compute-completion productions))) + (validate-resolved-repetitions rules nullable-index) (trace-lr-phase 'nullable (length nullable) started) + (validate-resolved-start rules root productive-index) + (trace-lr-phase 'productive (length productive) started) (let-values (((first first-index) (compute-first productions nullable-index))) (trace-lr-phase 'first (length first) started) (let-values (((states state-count lookaheads lookahead-offsets transitions terminal-values layout core-symbols lr0-state-visit-count lookahead-item-visit-count) - (build-states-via-lr0 + ((case construction + ((canonical-lr1) build-states-via-canonical-lr1) + ((partitioned-lr1) build-states-via-partitioned-lr1) + ((follow-partition-lr1) + build-states-via-follow-partition-lr1) + (else build-states-via-lr0)) productions table first-index nullable-index))) (trace-lr-phase 'states state-count started) (let-values (((actions action-state-publication-count) (build-actions states state-count lookaheads lookahead-offsets transitions table terminal-values layout core-symbols - conflict-policy))) + conflict-policy + (lambda (condition state items terminal left right) + (reject-lr-conflict condition rules root conflict-policy + table layout core-symbols + state items terminal left right))))) (unless (= action-state-publication-count state-count) (error "LR action rows were not published exactly once per state" state-count action-state-publication-count)) (trace-lr-phase 'actions (vector-length actions) started) (let (gotos (build-gotos transitions state-count)) + (validate-lr-tables productions actions gotos) (trace-lr-phase 'gotos (vector-length gotos) started) - (list - (cons 'schema "gerbil-parser.lr-spec.v1") - (cons 'case-insensitive? case-insensitive?) - (cons 'productions productions) - (cons 'nullable nullable) - (cons 'first first) - (cons 'algorithm 'lalr1-lr0-fixed-point-v1) - (cons 'state-count state-count) - (cons 'lr0-state-visit-count lr0-state-visit-count) - (cons 'lookahead-item-visit-count lookahead-item-visit-count) - (cons 'actions actions) - (cons 'gotos gotos))))))))) + (let (spec + (list + (cons 'schema "gerbil-parser.lr-spec.v1") + (cons 'case-insensitive? case-insensitive?) + (cons 'productions productions) + (cons 'nullable nullable) + (cons 'first first) + (cons 'algorithm + (case construction + ((canonical-lr1) 'canonical-lr1-reference-v1) + ((partitioned-lr1) + 'conflict-partitioned-lr1-v1) + ((follow-partition-lr1) + 'follow-partition-lr1-v1) + (else 'lalr1-lr0-fixed-point-v1))) + (cons 'state-count state-count) + (cons 'lr0-state-visit-count + (and (eq? construction 'lalr) + lr0-state-visit-count)) + (cons 'lookahead-item-visit-count + (and (eq? construction 'lalr) + lookahead-item-visit-count)) + (cons 'actions actions) + (cons 'gotos gotos))) + (case construction + ((lalr) spec) + ((follow-partition-lr1) + (cons (cons 'follow-block-count + lr0-state-visit-count) + (cons (cons 'output-item-count + lookahead-item-visit-count) + spec))) + (else + (cons (cons 'canonical-state-count + lr0-state-visit-count) + (cons (cons 'output-item-count + lookahead-item-visit-count) + spec)))))))))))) + +;;; The ordinary path stays LALR. Probe publishes unresolved cells as forks +;;; without admitting GLR lowering; only those cells trigger the LR(1) route. +;;; This avoids using exception construction as routine algorithm selection. +(def (lr-spec-has-fork? spec) + (vector-any + (lambda (row) + (any (lambda (entry) (eq? (car (cdr entry)) 'fork)) row)) + (lr-spec-ref spec 'actions))) + +(def (compile-lr-spec rules root (conflict-policy 'reject) + (case-insensitive? #f) (construction 'lalr)) + (if (eq? construction 'lalr-then-follow) + (if (eq? conflict-policy 'reject) + (let (probe + (compile-lr-spec/selected + rules root 'probe case-insensitive? 'lalr)) + (if (lr-spec-has-fork? probe) + (compile-lr-spec/selected + rules root conflict-policy case-insensitive? + 'follow-partition-lr1) + probe)) + (compile-lr-spec/selected + rules root conflict-policy case-insensitive? 'lalr)) + (compile-lr-spec/selected + rules root conflict-policy case-insensitive? construction))) diff --git a/src/compiler/lr-conflict-candidates.ss b/src/compiler/lr-conflict-candidates.ss new file mode 100644 index 00000000..dd513044 --- /dev/null +++ b/src/compiler/lr-conflict-candidates.ss @@ -0,0 +1,275 @@ +;;; -*- Gerbil -*- +;;; Conservative raw LR action overlap on the LR(0) propagation graph. + +(import (prefix-in :std/struct/queue stdq-) + (only-in ./funcs + compiler-index-set-add compiler-index-set-difference + compiler-index-set-for-each + compiler-index-set-singleton compiler-index-set-union) + (only-in ./lr + +lr-eof+ base-symbol nonterminal-name nonterminal-symbol? + production-id production-index-by-lhs production-rhs + production-terminal-catalog + terminal-symbol?) + (only-in ./lr-automaton make-item-layout make-core-symbol-catalog make-core-item + core-item-production-id core-item-count) + (only-in ./lr-lookahead build-states-via-lr0 make-core-suffix-catalog)) +(export forward-reachable-follow-masks + initial-backward-follow-partitions + initial-backward-follow-partitions/from-lr0 + lr0-conflict-candidates raw-conflict-cells) + +;; Definition 3.22 specialized to k=1: carry one terminal mask per production +;; and propagate FIRST(tail follow) through nonterminal occurrences. Static +;; FIRST(tail) bits travel once when a production first becomes reachable; +;; nullable tails propagate only newly discovered follow bits after that. +;; This is independent of canonical LR(1) state construction. +(def (forward-reachable-follow-masks productions table first nullable) + (let-values (((terminal-values terminal-index) + (production-terminal-catalog productions))) + (let* ((count (vector-length table)) + (by-lhs (production-index-by-lhs productions)) + (edges (make-vector count '())) + (layout (make-item-layout table terminal-values)) + (core-symbols (make-core-symbol-catalog table layout))) + (let-values (((suffix-first nullable-suffixes) + (make-core-suffix-catalog table layout core-symbols first nullable terminal-index))) + (let production-loop ((parent 0)) + (when (< parent count) + (let occurrence-loop ((rest (production-rhs (vector-ref table parent))) (dot 0)) + (unless (null? rest) + (let (symbol (base-symbol (car rest))) + (when (nonterminal-symbol? symbol) + (let ((suffix (make-core-item parent (+ dot 1) layout))) + (for-each + (lambda (child) + (vector-set! + edges parent + (cons (vector (production-id child) + (vector-ref suffix-first suffix) + (vector-ref nullable-suffixes suffix)) + (vector-ref edges parent)))) + (table-ref by-lhs (nonterminal-name symbol) '()))))) + (occurrence-loop (cdr rest) (+ dot 1)))) + (production-loop (+ parent 1)))) + (let ((follows (make-vector count 0)) + (deltas (make-vector count 0)) + (activated (make-vector count #f)) + (queued (make-vector count #f)) + (pending (stdq-make-Queue))) + (let (start-follow + (compiler-index-set-singleton + (table-ref terminal-index +lr-eof+))) + (vector-set! follows 0 start-follow) + (vector-set! deltas 0 start-follow)) + (vector-set! queued 0 #t) + (stdq-enqueue! pending 0) + (let drain () + (unless (stdq-queue-empty? pending) + (let* ((parent (stdq-dequeue! pending)) + (delta (vector-ref deltas parent)) + (first-visit? (not (vector-ref activated parent)))) + (vector-set! queued parent #f) + (vector-set! deltas parent 0) + (vector-set! activated parent #t) + (for-each + (lambda (edge) + (let* ((child (vector-ref edge 0)) + (known (vector-ref follows child)) + (static (if first-visit? (vector-ref edge 1) 0)) + (propagated (if (vector-ref edge 2) delta 0)) + (evidence (compiler-index-set-union + static propagated)) + (new (compiler-index-set-difference evidence known))) + (unless (zero? new) + (vector-set! follows child + (compiler-index-set-union known new)) + (vector-set! deltas child + (compiler-index-set-union + (vector-ref deltas child) new)) + (unless (vector-ref queued child) + (vector-set! queued child #t) + (stdq-enqueue! pending child))))) + (vector-ref edges parent)) + (drain)))) + (values follows terminal-values)))))) + +;; Keep the first raw action at each terminal and mark a conflict when a +;; distinct action appears. Shift is action 0, accept is 1, and reduction by +;; production n is n+1. This avoids allocating a production-sized integer +;; bitset for every completed item. Precedence and GLR do not erase this +;; evidence: it is intended for partition construction, not parser execution. +(def (raw-conflict-cells states state-count lookaheads offsets + terminal-values layout core-symbols) + (let ((conflicts (make-vector state-count 0)) + (terminal-count (vector-length terminal-values)) + (terminal-index (make-table test: equal?))) + (let index-loop ((index 0)) + (when (< index terminal-count) + (table-set! terminal-index (vector-ref terminal-values index) index) + (index-loop (+ index 1)))) + (let state-loop ((state 0)) + (when (< state state-count) + (let ((actions (make-vector terminal-count #f)) + (node (vector-ref offsets state)) + (mask 0)) + (def (record-action! terminal action) + (let (prior (vector-ref actions terminal)) + (cond + ((not prior) (vector-set! actions terminal action)) + ((not (= prior action)) + (set! mask (compiler-index-set-add mask terminal)))))) + (for-each + (lambda (core) + (let ((symbol (vector-ref core-symbols core)) + (production-id (core-item-production-id core layout))) + (cond + ((and symbol (terminal-symbol? symbol)) + (let (index (table-ref terminal-index symbol #f)) + (unless index + (error "LR terminal missing from catalogue" symbol)) + (record-action! index 0))) + ((not symbol) + (let (action + (if (zero? production-id) + 1 + (+ production-id 1))) + (compiler-index-set-for-each + (vector-ref lookaheads node) + (lambda (terminal) + (record-action! terminal action)))))) + (set! node (+ node 1)))) + (vector-ref states state)) + (vector-set! conflicts state mask)) + (state-loop (+ state 1)))) + conflicts)) + +;; A raw overlap here is a *candidate* for follow-string partitioning. LALR +;; propagation unions contexts, so its overlap can be spurious; conversely it +;; contains every canonical LR(1) conflict after projection to an LR(0) state. +(def (lr0-conflict-candidates productions table first nullable) + (let-values (((states state-count lookaheads offsets transitions + terminal-values layout core-symbols + state-visits item-visits) + (build-states-via-lr0 productions table first nullable))) + (values states state-count terminal-values + (raw-conflict-cells states state-count lookaheads offsets + terminal-values layout core-symbols)))) + +;; Definition 3.25's initial backward partition for k=1, conservatively +;; refining against every LR(0)/LALR candidate terminal. The output slot for +;; each valid dotted core contains (compatibility-mask . follow-mask) blocks. +;; This is only the seed: backward and forward edge refinements still follow. +(def (initial-backward-follow-partitions/from-lr0 + productions table first nullable states count terminals candidates + lookaheads offsets) + (let-values (((follows follow-terminals) + (forward-reachable-follow-masks + productions table first nullable))) + (unless (equal? terminals follow-terminals) + (error "LR terminal catalogues disagree")) + (let* ((terminal-index (make-table test: equal?)) + (layout (make-item-layout table terminals)) + (core-symbols (make-core-symbol-catalog table layout)) + (core-count (core-item-count layout)) + (action-by-core (make-vector core-count 0)) + (completed-core (make-vector core-count #f)) + (candidate-by-core (make-vector core-count 0)) + (partitions (make-vector core-count #f))) + (let terminal-loop ((terminal 0)) + (when (< terminal (vector-length terminals)) + (table-set! terminal-index + (vector-ref terminals terminal) terminal) + (terminal-loop (+ terminal 1)))) + (let-values (((suffix-first nullable-suffixes) + (make-core-suffix-catalog table layout core-symbols first nullable terminal-index))) + ;; Definition 3.24 requires each item to have an action on the + ;; potentially conflicting lookahead. Nonterminal-dot items have no + ;; raw action; terminal-dot items shift only their own terminal, and + ;; completed items reduce only on reachable production follows. + (let production-loop ((id 0)) + (when (< id (vector-length table)) + (let dot-loop ((tail (production-rhs (vector-ref table id))) + (dot 0)) + (let* ((core (make-core-item id dot layout)) + (symbol (and (pair? tail) (base-symbol (car tail)))) + (action-mask + (cond + ((not symbol) (vector-ref follows id)) + ((terminal-symbol? symbol) + (compiler-index-set-singleton + (table-ref terminal-index symbol))) + (else 0)))) + (vector-set! action-by-core core action-mask) + (unless symbol + (vector-set! completed-core core #t)) + (unless (null? tail) + (dot-loop (cdr tail) (+ dot 1))))) + (production-loop (+ id 1)))) + (let state-loop ((state 0)) + (when (< state count) + (let (node (vector-ref offsets state)) + (for-each + (lambda (core) + ;; A completed item's state-local LALR mask includes every + ;; canonical follow at this LR(0) item. The production-wide + ;; follow mask can mark an action that is absent here. + ;; Shift actions remain independent of item lookahead. + (let (action-mask + (if (vector-ref completed-core core) + (vector-ref lookaheads node) + (vector-ref action-by-core core))) + (vector-set! + candidate-by-core core + (compiler-index-set-union + (vector-ref candidate-by-core core) + (bitwise-and (vector-ref candidates state) + action-mask)))) + (set! node (+ node 1))) + (vector-ref states state))) + (state-loop (+ state 1)))) + (let production-loop ((id 0)) + (when (< id (vector-length table)) + (let dot-loop ((tail (production-rhs (vector-ref table id))) + (dot 0)) + (let* ((core (make-core-item id dot layout)) + (first-mask (vector-ref suffix-first core)) + (tail-nullable? (vector-ref nullable-suffixes core)) + (candidate-mask (vector-ref candidate-by-core core)) + (blocks (make-table test: eq?)) + (signatures '())) + (compiler-index-set-for-each + (vector-ref follows id) + (lambda (lookahead) + (let* ((compatible + (if tail-nullable? + (compiler-index-set-add first-mask lookahead) + first-mask)) + (signature + (bitwise-and compatible candidate-mask)) + (known (table-ref blocks signature #f))) + (unless known + (set! signatures (cons signature signatures))) + (table-set! + blocks signature + (compiler-index-set-add (or known 0) lookahead))))) + (vector-set! + partitions core + (map (lambda (signature) + (cons signature (table-ref blocks signature))) + (reverse signatures))) + (unless (null? tail) + (dot-loop (cdr tail) (+ dot 1))))) + (production-loop (+ id 1)))) + (values partitions candidates terminals))))) + +(def (initial-backward-follow-partitions productions table first nullable) + (let-values (((states count lookaheads offsets transitions terminals + layout core-symbols state-visits item-visits) + (build-states-via-lr0 productions table first nullable))) + (let (candidates + (raw-conflict-cells states count lookaheads offsets terminals + layout core-symbols)) + (initial-backward-follow-partitions/from-lr0 + productions table first nullable states count terminals candidates + lookaheads offsets)))) diff --git a/src/compiler/lr-contextual-program.ss b/src/compiler/lr-contextual-program.ss new file mode 100644 index 00000000..57a97c26 --- /dev/null +++ b/src/compiler/lr-contextual-program.ss @@ -0,0 +1,20 @@ +;;; Ordinary LR admission shares the closed recognition/context/construction owner. +(import (only-in ../runtime/contextual-ir make-contextual-parser-ir) + (only-in ../runtime/parser-ir-data parser-ir-ref) + (only-in ../runtime/identity sha256-text)) +(export compile-lr-contextual-program) +(def (compile-lr-contextual-program parser-ir grammar-digest) + (make-contextual-parser-ir + (list (cons 'dialect 'lr) (cons 'program parser-ir)) + (list (cons 'dialect 'lexical) + (cons 'rules (parser-ir-ref parser-ir 'lexical-rules)) + (cons 'extras (parser-ir-ref parser-ir 'extras)) + (cons 'case-insensitive? (parser-ir-ref parser-ir 'case-insensitive?))) + (list (cons 'entries (parser-ir-ref parser-ir 'parser-entrypoints))) + (list (cons 'syntax-kinds (parser-ir-ref parser-ir 'syntax-kinds)) + (cons 'terminals (parser-ir-ref parser-ir 'terminals)) + (cons 'projections (parser-ir-ref parser-ir 'rules))) + (parser-ir-ref parser-ir 'root-kind) + (list (cons 'base-grammar-digest grammar-digest) + (cons 'parser-ir-digest + (sha256-text (call-with-output-string (lambda (port) (write parser-ir port)))))))) diff --git a/src/compiler/lr-follow-automaton.ss b/src/compiler/lr-follow-automaton.ss new file mode 100644 index 00000000..b31a18f9 --- /dev/null +++ b/src/compiler/lr-follow-automaton.ss @@ -0,0 +1,745 @@ +;;; -*- Gerbil -*- +;;; Direct LR(0)-derived LR(1) NFA partition and determinization. + +(import (prefix-in :std/struct/queue stdq-) + (only-in :std/vector/extensible + list->ExtensibleVector ExtensibleVector->vector + ExtensibleVector-fill-pointer ExtensibleVector-push! + ExtensibleVector-ref ExtensibleVector-set!) + (only-in ./funcs + compiler-index-set-add compiler-index-set-for-each + compiler-index-set-reachable compiler-index-set-reachable/memo + compiler-index-partition-refine + compiler-index-set-singleton compiler-index-set-union) + (only-in ./lr + +lr-eof+ base-symbol nonterminal-name nonterminal-symbol? + production-id production-index-by-lhs production-rhs + production-terminal-catalog) + (only-in ./lr-automaton + make-core-item make-core-symbol-catalog make-item-layout + materialize-transitions) + (only-in ./lr-conflict-candidates + initial-backward-follow-partitions/from-lr0 + raw-conflict-cells) + (only-in ./lr-lookahead + build-states-via-lr0 build-states-via-canonical-lr1 + make-core-suffix-catalog)) +(export build-states-via-follow-partition-lr1) + +;; Report completed compiler work only. The existing LR trace switch owns +;; observability; there is no timer/heartbeat or changed qualification deadline. +(def (trace-follow-work phase count (total #f)) + (when (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1") + (displayln "[gerbil-parser-follow] phase=" phase " count=" count + (if total (string-append " total=" (number->string total)) "")) + (force-output))) + +(def (make-follow-core-metadata productions table first nullable + layout core-symbols terminal-index) + (let ((result (make-vector (vector-length core-symbols) #f)) + (by-lhs (production-index-by-lhs productions))) + (let-values (((suffix-first nullable-suffixes) + (make-core-suffix-catalog table layout core-symbols first nullable terminal-index))) + (let production-loop ((id 0)) + (when (< id (vector-length table)) + (let dot-loop ((tail (production-rhs (vector-ref table id))) + (dot 0)) + (let* ((core (make-core-item id dot layout)) + (symbol (vector-ref core-symbols core))) + (if (and symbol (nonterminal-symbol? symbol)) + (vector-set! + result core + (vector + symbol + (vector-ref suffix-first (+ core 1)) + (vector-ref nullable-suffixes (+ core 1)) + (map (lambda (child) + (make-core-item (production-id child) 0 layout)) + (table-ref by-lhs (nonterminal-name symbol) '())))) + (vector-set! result core (vector symbol 0 #f '()))) + (unless (null? tail) + (dot-loop (cdr tail) (+ dot 1))))) + (production-loop (+ id 1)))) + result))) + +;; Each block is (core . follow-mask). The index maps one (core, follow) +;; vertex to its current block; no canonical LR(1) DFA state is constructed. +(def (index-follow-blocks partitions terminal-count) + (let ((blocks (list->ExtensibleVector '())) + (index (make-vector (vector-length partitions) #f))) + (let core-loop ((core 0)) + (when (< core (vector-length partitions)) + (let (parts (vector-ref partitions core)) + (when parts + (let (row (make-vector terminal-count #f)) + (for-each + (lambda (part) + (let ((id (ExtensibleVector-push! + blocks (cons core (cdr part))))) + (compiler-index-set-for-each + (cdr part) + (lambda (lookahead) + (vector-set! row lookahead id))))) + parts) + (vector-set! index core row)))) + (core-loop (+ core 1)))) + (values (ExtensibleVector->vector blocks) index))) + +(def (follow-block-ref index core lookahead) + (let ((row (vector-ref index core))) + (unless row + (error "missing dotted core in follow partition" core)) + (let (block (vector-ref row lookahead)) + (unless block + (error "missing follow vertex in partition" core lookahead)) + block))) + +;; Return the labeled shift target and epsilon target set of one NFA vertex. +;; FIRST(tail follow) distributes into fixed FIRST(tail) plus the follow bit +;; when nullable. Translate the fixed targets once per core in this round's +;; index; a split changes IDs, so the cache must never survive the round. +(def (follow-successors core lookahead metadata index fixed-epsilons) + (let* ((item (vector-ref metadata core)) + (symbol (vector-ref item 0)) + (shift (and symbol (follow-block-ref index (+ core 1) lookahead))) + (children (vector-ref item 3)) + (closure-mask 0)) + (unless (null? children) + (let (fixed (vector-ref fixed-epsilons core)) + (unless fixed + (set! fixed + (foldl + (lambda (child mask) + (let (next mask) + (compiler-index-set-for-each + (vector-ref item 1) + (lambda (terminal) + (set! next (compiler-index-set-add + next (follow-block-ref index child terminal))))) + next)) + 0 children)) + (vector-set! fixed-epsilons core fixed)) + (set! closure-mask + (if (vector-ref item 2) + (foldl (lambda (child mask) + (compiler-index-set-add + mask (follow-block-ref index child lookahead))) + fixed children) + fixed)))) + (cons shift closure-mask))) + +;; Backward refinement: vertices in a block must agree on which target blocks +;; they can reach by a shift or epsilon edge. Splitting is monotone and stops +;; because there are finitely many reachable (core, follow) pairs. +(def (refine-follow-blocks partitions metadata terminal-count) + ;; A split can affect only cores whose shift or closure points to it. Keep + ;; that reverse dependency graph at dotted-core granularity: the follow + ;; block IDs are rebuilt each round, but unrelated cores need no new + ;; successor signatures. + (let ((dependents (make-vector (vector-length partitions) '()))) + (let core-loop ((core 0)) + (when (< core (vector-length partitions)) + (when (vector-ref partitions core) + (let (item (vector-ref metadata core)) + (when (vector-ref item 0) + (let (target (+ core 1)) + (vector-set! dependents target + (cons core (vector-ref dependents target))))) + (for-each + (lambda (target) + (vector-set! dependents target + (cons core (vector-ref dependents target)))) + (vector-ref item 3)))) + (core-loop (+ core 1)))) + (let refine ((current partitions) + (active (make-vector (vector-length partitions) #t)) + (rounds 0)) + (let-values (((blocks index) + (index-follow-blocks current terminal-count))) + (let ((fixed-epsilons (make-vector (vector-length metadata) #f)) + (next (make-vector (vector-length current) #f)) + (next-active (make-vector (vector-length current) #f)) + (changed? #f)) + (let core-loop ((core 0)) + (when (< core (vector-length current)) + (let (parts (vector-ref current core)) + (if (and parts (vector-ref active core)) + (let (new-parts '()) + (for-each + (lambda (part) + (if (and (positive? (cdr part)) + (zero? (bitwise-and (cdr part) + (- (cdr part) 1)))) + ;; A one-lookahead block cannot split, even when a + ;; successor block changed in the previous round. + (set! new-parts (cons part new-parts)) + (let ((groups (make-table test: equal?)) + (keys '())) + (compiler-index-set-for-each + (cdr part) + (lambda (lookahead) + (let* ((key + (follow-successors + core lookahead metadata index fixed-epsilons)) + (known (table-ref groups key #f))) + (unless known (set! keys (cons key keys))) + (table-set! + groups key + (compiler-index-set-add + (or known 0) lookahead))))) + (when (pair? (cdr keys)) + (set! changed? #t) + (for-each + (lambda (dependent) + (vector-set! next-active dependent #t)) + (vector-ref dependents core))) + (for-each + (lambda (key) + (set! new-parts + (cons (cons key (table-ref groups key)) + new-parts))) + (reverse keys))))) + parts) + (vector-set! next core (reverse new-parts))) + (vector-set! next core parts))) + (core-loop (+ core 1)))) + (if changed? + (refine next next-active (+ rounds 1)) + ;; A stable round preserves every block and its numeric position. + ;; Only this round's active nonsingleton keys use the final index; + ;; inactive/singleton keys may refer to earlier round IDs. Recompute + ;; one representative for those blocks, never reuse a stale key. + (let* ((count (vector-length blocks)) + (shifts (make-vector count #f)) + (epsilons (make-vector count 0)) + (block 0) (reused 0)) + (let publish ((core 0)) + (when (< core (vector-length next)) + (let (parts (vector-ref next core)) + (when parts + (for-each + (lambda (part) + (let* ((mask (cdr part)) + (reuse? (and (vector-ref active core) + (not (zero? (bitwise-and mask (- mask 1)))))) + (edge + (if reuse? + (begin (set! reused (+ reused 1)) (car part)) + (follow-successors + core (- (integer-length (bitwise-and mask (- mask))) 1) + metadata index fixed-epsilons)))) + (vector-set! shifts block (car edge)) + (vector-set! epsilons block (cdr edge)) + (set! block (+ block 1)))) + parts))) + (publish (+ core 1)))) + (trace-follow-work 'stable-edge-reuse reused count) + (values next blocks index rounds shifts epsilons)))))))) + +;; The backward blocks are the atomic vertices for the forward pass. Begin +;; with one group per dotted core (and a singleton start), then split by the +;; predecessor groups that can reach each vertex on each edge label. +(def (forward-refine-blocks blocks shifts epsilons start core-symbols) + (let* ((count (vector-length blocks)) + (predecessors (make-vector count '()))) + (let ((label-index (make-table test: equal?)) + (next-label 1)) + (let source-loop ((source 0)) + (when (< source count) + (let ((shift (vector-ref shifts source)) + (symbol (vector-ref + core-symbols (car (vector-ref blocks source))))) + (when shift + (let (label (table-ref label-index symbol #f)) + (unless label + (set! label next-label) + (set! next-label (+ next-label 1)) + (table-set! label-index symbol label)) + (vector-set! + predecessors shift + (cons (cons label source) + (vector-ref predecessors shift))))) + (compiler-index-set-for-each + (vector-ref epsilons source) + (lambda (target) + (vector-set! + predecessors target + (cons (cons 0 source) + (vector-ref predecessors target))))) + (source-loop (+ source 1)))))) + (def (group-ids key-procedure) + (let ((ids (make-vector count 0)) + (keys (make-table test: equal?)) + (group-count 0)) + (let block-loop ((block 0)) + (when (< block count) + (let* ((key (key-procedure block)) + (found (table-ref keys key #f))) + (unless found + (set! found group-count) + (table-set! keys key found) + (set! group-count (+ group-count 1))) + (vector-set! ids block found)) + (block-loop (+ block 1)))) + (values ids group-count))) + (def (predecessor-signature block ids) + (let ((sources (make-table test: eq?)) + (labels '())) + (for-each + (lambda (edge) + (let* ((label (car edge)) + (source-group (vector-ref ids (cdr edge))) + (known (table-ref sources label #f))) + (unless known + (set! labels (cons label labels))) + (table-set! + sources label + (compiler-index-set-add (or known 0) source-group)))) + (vector-ref predecessors block)) + (map (lambda (label) + (cons label (table-ref sources label))) + (list-sort < labels)))) + (let-values (((initial initial-count) + (group-ids + (lambda (block) + (cons (car (vector-ref blocks block)) + (= block start)))))) + (let-values (((next next-count) + (compiler-index-partition-refine + initial initial-count predecessor-signature + (lambda (source visit) + (let (shift (vector-ref shifts source)) + (when shift (visit shift))) + (compiler-index-set-for-each + (vector-ref epsilons source) visit)) + (lambda (groups nodes examined) + (trace-follow-work 'forward-refinement groups nodes))))) + (let ((merged (make-vector next-count #f)) + (merged-shifts (make-vector next-count 0)) + (merged-epsilons (make-vector next-count 0))) + (let block-loop ((block 0)) + (when (< block count) + (let* ((group (vector-ref next block)) + (entry (vector-ref blocks block)) + (prior (vector-ref merged group)) + (shift (vector-ref shifts block))) + (if prior + (vector-set! + merged group + (cons (car prior) + (compiler-index-set-union + (cdr prior) (cdr entry)))) + (vector-set! merged group entry)) + (when shift + (vector-set! + merged-shifts group + (compiler-index-set-add + (vector-ref merged-shifts group) + (vector-ref next shift)))) + (compiler-index-set-for-each + (vector-ref epsilons block) + (lambda (target) + (vector-set! + merged-epsilons group + (compiler-index-set-add + (vector-ref merged-epsilons group) + (vector-ref next target))))) + (block-loop (+ block 1))))) + (values merged merged-shifts merged-epsilons + (vector-ref next start))))))) + +(def (determinize-follow-blocks blocks index shift-edges epsilon-edges + terminal-values layout core-symbols) + (let* ((block-count (vector-length blocks)) + (start #f)) + (let ((terminal-index (make-table test: equal?))) + (let loop ((i 0)) + (when (< i (vector-length terminal-values)) + (table-set! terminal-index (vector-ref terminal-values i) i) + (loop (+ i 1)))) + (set! start + (follow-block-ref + index 0 (table-ref terminal-index +lr-eof+)))) + (trace-follow-work 'nfa-edges block-count) + (let-values (((merged merged-shifts merged-epsilons merged-start) + (forward-refine-blocks + blocks shift-edges epsilon-edges start core-symbols))) + (trace-follow-work 'forward-complete (vector-length merged) block-count) + (set! blocks merged) + (set! shift-edges merged-shifts) + (set! epsilon-edges merged-epsilons) + (set! start merged-start) + (set! block-count (vector-length merged))) + (let ((shift-sources 0) + (shift-labels (make-vector block-count #f)) + (label-index (make-table test: equal?)) + (label-values (list->ExtensibleVector '()))) + (let index-shifts ((block 0)) + (when (< block block-count) + (unless (zero? (vector-ref shift-edges block)) + (let* ((core (car (vector-ref blocks block))) + (symbol (vector-ref core-symbols core)) + (label (table-ref label-index symbol #f))) + (unless label + (set! label (ExtensibleVector-push! label-values symbol)) + (table-set! label-index symbol label)) + (vector-set! shift-labels block label) + (set! shift-sources (compiler-index-set-add shift-sources block)))) + (index-shifts (+ block 1)))) + (let ((states (list->ExtensibleVector '())) + (symbol-targets (make-vector (ExtensibleVector-fill-pointer label-values) 0)) + (transitions (list->ExtensibleVector '())) + (state-index (make-table test: equal?)) + (queue (stdq-make-Queue)) + ;; Epsilon closure distributes over union. Cache singleton closures + ;; only for large quotients, where repeated traversals dominate. + (closure-cache (and (>= block-count 3500) + (make-vector block-count #f))) + ;; Seed subsets recur across transitions. Retain only a graph-sized + ;; number of keys, mapped to canonical IDs already owned by states. + (subset-index (and (>= block-count 3500) (make-table test: equal?))) + (subset-count 0) (subset-hits 0)) + (def (intern! closure) + (let (existing (table-ref state-index closure #f)) + (if existing + existing + (let (id (ExtensibleVector-push! states closure)) + (ExtensibleVector-push! transitions '()) + (table-set! state-index closure id) + (stdq-enqueue! queue id) + id)))) + (intern! (if closure-cache + (compiler-index-set-reachable/memo + (compiler-index-set-singleton start) + epsilon-edges closure-cache) + (compiler-index-set-reachable + (compiler-index-set-singleton start) epsilon-edges))) + (let drain () + (unless (stdq-queue-empty? queue) + (let* ((state (stdq-dequeue! queue)) + (closure (ExtensibleVector-ref states state)) + (symbols '())) + ;; Completed/nonshifting items remain in state identity and output, + ;; but cannot contribute a labeled successor. Visit only shift sources. + (compiler-index-set-for-each + (bitwise-and closure shift-sources) + (lambda (block) + (let* ((label (vector-ref shift-labels block)) + (known (vector-ref symbol-targets label))) + (when (zero? known) (set! symbols (cons label symbols))) + (vector-set! symbol-targets label + (compiler-index-set-union known (vector-ref shift-edges block)))))) + (for-each + (lambda (label) + (let* ((symbol (ExtensibleVector-ref label-values label)) + (targets (vector-ref symbol-targets label)) + (known (and subset-index (table-ref subset-index targets #f))) + (target + (if known + (begin (set! subset-hits (+ subset-hits 1)) known) + (let (id (intern! + (if closure-cache + (compiler-index-set-reachable/memo targets epsilon-edges closure-cache) + (compiler-index-set-reachable targets epsilon-edges)))) + (when (and subset-index (< subset-count block-count)) + (table-set! subset-index targets id) + (set! subset-count (+ subset-count 1))) + id)))) + ;; Clear only touched labels before the next state's scan. + (vector-set! symbol-targets label 0) + (ExtensibleVector-set! + transitions state + (cons (cons symbol target) + (ExtensibleVector-ref transitions state))))) + (reverse symbols)) + (ExtensibleVector-set! + transitions state + (reverse (ExtensibleVector-ref transitions state))) + (when (zero? (modulo (+ state 1) 128)) + (trace-follow-work 'dfa-states (+ state 1) (ExtensibleVector-fill-pointer states))) + (drain)))) + (when subset-index + (trace-follow-work 'subset-cache subset-hits subset-count)) + (trace-follow-work 'dfa-complete (ExtensibleVector-fill-pointer states)) + (let* ((count (ExtensibleVector-fill-pointer states)) + (output-states (make-vector count '())) + (offsets (make-vector (+ count 1) 0)) + (core-masks (make-vector (vector-length core-symbols) 0)) + (item-count 0)) + (let state-loop ((state 0)) + (when (< state count) + (let ((cores '())) + (compiler-index-set-for-each + (ExtensibleVector-ref states state) + (lambda (block) + (let* ((entry (vector-ref blocks block)) + (core (car entry)) + (known (vector-ref core-masks core))) + (when (zero? known) + (set! cores (cons core cores))) + (vector-set! core-masks core + (compiler-index-set-union + known (cdr entry)))))) + (let (ordered (list-sort < cores)) + (vector-set! output-states state + (cons ordered + (map (lambda (core) + (let (mask (vector-ref core-masks core)) + (vector-set! core-masks core 0) + mask)) + ordered))) + (vector-set! offsets state item-count) + (set! item-count (+ item-count (length ordered))))) + (when (zero? (modulo (+ state 1) 128)) + (trace-follow-work 'state-items (+ state 1) count)) + (state-loop (+ state 1)))) + (vector-set! offsets count item-count) + (let ((lookaheads (make-vector item-count 0)) + (state-cores (make-vector count '())) + (node 0)) + (let fill ((state 0)) + (when (< state count) + (let ((entry (vector-ref output-states state))) + (vector-set! state-cores state (car entry)) + (for-each + (lambda (mask) + (vector-set! lookaheads node mask) + (set! node (+ node 1))) + (cdr entry))) + (fill (+ state 1)))) + (values state-cores count lookaheads offsets + (materialize-transitions + (ExtensibleVector->vector transitions) count) + terminal-values layout core-symbols + block-count item-count))))))) + +(def (canonical-at-conflict-lower-bound? trial) + (let* ((states (list-ref trial 0)) + (count (list-ref trial 1)) + (lookaheads (list-ref trial 2)) + (offsets (list-ref trial 3)) + (terminal-values (list-ref trial 5)) + (layout (list-ref trial 6)) + (core-symbols (list-ref trial 7)) + (groups (make-table test: equal?)) + (group-cores (list->ExtensibleVector '())) + (group-masks (list->ExtensibleVector '()))) + (let state-loop ((state 0)) + (when (< state count) + (let* ((cores (vector-ref states state)) + (known (table-ref groups cores #f)) + (group + (if known known + (let (id (ExtensibleVector-push! group-cores cores)) + (ExtensibleVector-push! + group-masks (make-vector (length cores) 0)) + (table-set! groups cores id) + id))) + (row (ExtensibleVector-ref group-masks group))) + (let item-loop ((index 0) (node (vector-ref offsets state)) + (remaining cores)) + (unless (null? remaining) + (vector-set! + row index + (compiler-index-set-union + (vector-ref row index) (vector-ref lookaheads node))) + (item-loop (+ index 1) (+ node 1) (cdr remaining))))) + (state-loop (+ state 1)))) + (let* ((group-count (ExtensibleVector-fill-pointer group-cores)) + (merged-states (ExtensibleVector->vector group-cores)) + (merged-offsets (make-vector (+ group-count 1) 0)) + (item-count 0)) + (let group-loop ((group 0)) + (when (< group group-count) + (vector-set! merged-offsets group item-count) + (set! item-count + (+ item-count + (vector-length (ExtensibleVector-ref group-masks group)))) + (group-loop (+ group 1)))) + (vector-set! merged-offsets group-count item-count) + (let ((merged-lookaheads (make-vector item-count 0)) + (node 0)) + (let group-loop ((group 0)) + (when (< group group-count) + (let (row (ExtensibleVector-ref group-masks group)) + (let item-loop ((index 0)) + (when (< index (vector-length row)) + (vector-set! merged-lookaheads node + (vector-ref row index)) + (set! node (+ node 1)) + (item-loop (+ index 1))))) + (group-loop (+ group 1)))) + (let (candidates + (raw-conflict-cells + merged-states group-count merged-lookaheads merged-offsets + terminal-values layout core-symbols)) + (= count + (+ group-count + (let loop ((group 0) (total 0)) + (if (= group group-count) + total + (loop (+ group 1) + (if (zero? (vector-ref candidates group)) + total (+ total 1)))))))))))) + +;; A canonical lower-bound trial is most promising when no grammar +;; nonterminal is shared by many productions. The state lower bound below +;; remains the exact admission test; this only avoids costly failed trials. +(def (low-sharing-productions? table) + (let ((uses (make-table test: equal?)) + (eligible? #t)) + (let production-loop ((id (- (vector-length table) 1))) + (when (and eligible? (>= id 0)) + (for-each + (lambda (symbol) + (when (nonterminal-symbol? symbol) + (let* ((name (nonterminal-name symbol)) + (count (+ 1 (table-ref uses name 0)))) + (table-set! uses name count) + (when (> count 2) (set! eligible? #f))))) + (production-rhs (vector-ref table id))) + (production-loop (- id 1)))) + eligible?)) + +(def (build-states-via-follow-partition-lr1/from-lr0 + productions table first nullable (canonical-trial? #t)) + (let-values (((states count lookaheads offsets transitions + terminal-values layout core-symbols + state-visits item-visits) + (build-states-via-lr0 productions table first nullable))) + (let (candidates + (raw-conflict-cells + states count lookaheads offsets terminal-values layout core-symbols)) + (def (build-direct) + (let-values (((catalogue terminal-index) + (production-terminal-catalog productions))) + (unless (equal? terminal-values catalogue) + (error "LR terminal catalogues disagree")) + (let (metadata + (make-follow-core-metadata + productions table first nullable layout core-symbols + terminal-index)) + (let-values (((initial seed-candidates seed-terminals) + (initial-backward-follow-partitions/from-lr0 + productions table first nullable + states count terminal-values candidates + lookaheads offsets))) + (let-values (((refined blocks index rounds shifts epsilons) + (refine-follow-blocks + initial metadata (vector-length terminal-values)))) + (trace-follow-work 'backward-complete (vector-length blocks) rounds) + (determinize-follow-blocks + blocks index shifts epsilons terminal-values layout + core-symbols)))))) + (let (conflict-count + (let loop ((state 0) (total 0)) + (if (= state count) + total + (loop (+ state 1) + (if (zero? (vector-ref candidates state)) + total (+ total 1)))))) + (if (zero? conflict-count) + ;; A conflict-free LALR table is already adequate. Reuse the same + ;; LR(0) graph and propagated lookaheads instead of determinizing an + ;; equivalent follow NFA. + (values states count lookaheads offsets transitions + terminal-values layout core-symbols 0 + (vector-length lookaheads)) + (if (and canonical-trial? + (or (>= conflict-count 64) + (and (>= conflict-count 8) + (>= (vector-length table) 64) + (low-sharing-productions? table)))) + ;; Every conflicting LR(0) state needs at least one split in an + ;; LR(1) construction. Stop the canonical trial at that lower bound; + ;; an exact match leaves no state compression for follow refinement. + (let* ((lower-bound (+ count conflict-count)) + (trial + (call-with-values + (lambda () + (build-states-via-canonical-lr1 + productions table first nullable lower-bound)) + list))) + (if (and (= (length trial) 10) + (= (cadr trial) lower-bound)) + (apply values + (append (take trial 8) + (list 0 (list-ref trial 9)))) + (build-direct))) + (build-direct))))))) + +;; Repeating one nonterminal within a reachable production is evidence +;; that canonical LR(1) may duplicate contexts which follow partitioning can +;; share. An unreachable rule must not force the expensive direct route. +(def (repeated-nonterminal-rhs? rhs) + (let symbol-loop ((remaining rhs) (first #f) (others '())) + (and (pair? remaining) + (let (symbol (car remaining)) + (if (nonterminal-symbol? symbol) + (let (name (nonterminal-name symbol)) + (cond + ((or (eq? name first) (memq name others)) #t) + (first (symbol-loop (cdr remaining) first + (cons name others))) + (else (symbol-loop (cdr remaining) name others)))) + (symbol-loop (cdr remaining) first others)))))) + +(def (reachable-repeated-nonterminal? productions table) + (let ((candidates (make-vector (vector-length table) #f)) + (candidate? #f)) + (let production-loop ((id (- (vector-length table) 1))) + (when (>= id 0) + (when (repeated-nonterminal-rhs? + (production-rhs (vector-ref table id))) + (vector-set! candidates id #t) + (set! candidate? #t)) + (production-loop (- id 1)))) + (and candidate? + (let ((by-lhs (production-index-by-lhs productions)) + (seen (make-vector (vector-length table) #f)) + (pending (stdq-make-Queue))) + (vector-set! seen 0 #t) + (stdq-enqueue! pending 0) + (let search () + (and (not (stdq-queue-empty? pending)) + (let (id (stdq-dequeue! pending)) + (or (vector-ref candidates id) + (begin + (for-each + (lambda (entry) + (let (symbol (base-symbol entry)) + (when (nonterminal-symbol? symbol) + (for-each + (lambda (child) + (let (next (production-id child)) + (unless (vector-ref seen next) + (vector-set! seen next #t) + (stdq-enqueue! pending next)))) + (table-ref by-lhs + (nonterminal-name symbol) '()))))) + (production-rhs (vector-ref table id))) + (search)))))))))) + +(def (build-states-via-follow-partition-lr1 productions table first nullable) + ;; A reachable repeated nonterminal is a cheap signal that canonical LR(1) + ;; can duplicate contexts. Build the follow partition directly in that case. + (if (>= (vector-length table) 256) + (if (reachable-repeated-nonterminal? productions table) + (build-states-via-follow-partition-lr1/from-lr0 + productions table first nullable #f) + (let (trial + (call-with-values + (lambda () + (build-states-via-canonical-lr1 + productions table first nullable + (* 3 (vector-length table)))) + list)) + (if (and (= (length trial) 10) + (canonical-at-conflict-lower-bound? trial)) + (apply values + (append (take trial 8) + (list 0 (list-ref trial 9)))) + (build-states-via-follow-partition-lr1/from-lr0 + productions table first nullable #f)))) + (build-states-via-follow-partition-lr1/from-lr0 + productions table first nullable))) diff --git a/src/compiler/lr-lookahead.ss b/src/compiler/lr-lookahead.ss index d48b31ea..79a5b267 100644 --- a/src/compiler/lr-lookahead.ss +++ b/src/compiler/lr-lookahead.ss @@ -2,20 +2,25 @@ ;;; LALR lookahead fixed-point propagation over an interned LR(0) graph. (import (prefix-in :std/struct/queue stdq-) + (only-in :std/vector/extensible + list->ExtensibleVector + ExtensibleVector->vector ExtensibleVector-push! + ExtensibleVector-ref ExtensibleVector-set!) (only-in ./funcs compiler-index-set-add compiler-index-set-difference compiler-index-set-empty? compiler-index-set-singleton compiler-index-set-union) (only-in ./lr +lr-eof+ nonterminal-name nonterminal-symbol? production-id - production-rhs production-terminal-catalog sequence-first - sequence-nullable?) + production-index-by-lhs + production-rhs production-terminal-catalog) (only-in ./lr-automaton build-lr0-automaton - make-core-item + make-core-item core-item-count make-core-symbol-catalog make-item-layout materialize-transitions)) -(export build-states-via-lr0) +(export build-states-via-lr0 build-states-via-canonical-lr1 + make-core-suffix-catalog) (def (trace-lookahead-phase phase count started trace?) (when trace? @@ -38,38 +43,49 @@ mask (table-ref terminal-index terminal))) 0 terminals)) -;; FIRST(tail) and tail-nullability depend only on the dotted core item. Build -;; them once for every production position before the fixed-point queue drains. -;; : (-> Vector Pair Vector Table Table Table (values Vector Vector)) -(def (make-core-lookahead-catalog table layout core-symbols first nullable - terminal-index) - (let* ((size (* (vector-length table) (cdr layout))) +;;; Completed FIRST/nullable facts for every valid suffix, including the +;; empty suffix at the completed core. Consumers select core or core+1; +;; padding remains zero/false. Marks were unwrapped by the symbol catalog. +;; : (-> Vector lr-item-layout Vector Table Table Table (values Vector Vector)) +(def (make-core-suffix-catalog table layout core-symbols first nullable + terminal-index) + (let* ((size (core-item-count layout)) (first-masks (make-vector size 0)) - (nullable-tails (make-vector size #f))) + (nullable-suffixes (make-vector size #f)) + (nonterminal-masks (make-table test: eq?))) + (def (first-mask name) + (let (known (table-ref nonterminal-masks name #f)) + (or known + (let (mask (terminal-list->mask (table-ref first name '()) terminal-index)) + (table-set! nonterminal-masks name mask) + mask)))) (let production-loop ((production-id 0)) (when (< production-id (vector-length table)) - (let item-loop - ((rest (production-rhs (vector-ref table production-id))) - (dot 0)) - (unless (null? rest) - (let (item (make-core-item production-id dot layout)) - (when (nonterminal-symbol? (vector-ref core-symbols item)) - (let (tail (cdr rest)) - (vector-set! - first-masks item - (terminal-list->mask - (sequence-first tail first nullable) terminal-index)) - (vector-set! nullable-tails item - (sequence-nullable? tail nullable))))) - (item-loop (cdr rest) (+ dot 1)))) + (let (width (length (production-rhs (vector-ref table production-id)))) + (vector-set! nullable-suffixes (make-core-item production-id width layout) #t) + (let item-loop ((dot (- width 1)) (tail-mask 0) (tail-nullable? #t)) + (when (>= dot 0) + (let* ((item (make-core-item production-id dot layout)) + (symbol (vector-ref core-symbols item)) + (nonterminal? (nonterminal-symbol? symbol)) + (head-nullable? (and nonterminal? + (table-ref nullable (nonterminal-name symbol) #f))) + (head-mask (if nonterminal? + (first-mask (nonterminal-name symbol)) + (compiler-index-set-singleton (table-ref terminal-index symbol)))) + (mask (if head-nullable? + (compiler-index-set-union head-mask tail-mask) head-mask)) + (nullable? (and head-nullable? tail-nullable?))) + (vector-set! first-masks item mask) + (vector-set! nullable-suffixes item nullable?) + (item-loop (- dot 1) mask nullable?))))) (production-loop (+ production-id 1)))) - (values first-masks nullable-tails))) + (values first-masks nullable-suffixes))) ;; Assign every admitted (state, core-item) pair one dense node id. The fixed -;; point then uses vectors for its mutable state; the packed-key table is paid -;; only while constructing the immutable propagation graph. -;; : (-> Vector Fixnum Fixnum (values Vector Vector Vector Table Fixnum)) -(def (index-lr0-items states state-count item-space) +;; point and immutable graph construction both use these contiguous slices. +;; : (-> Vector Fixnum (values Vector Vector Fixnum)) +(def (index-lr0-items states state-count) (let ((offsets (make-vector (+ state-count 1) 0)) (node-count 0)) (let count-states ((state 0)) @@ -79,93 +95,129 @@ (+ node-count (length (vector-ref states state)))) (count-states (+ state 1)))) (vector-set! offsets state-count node-count) - (let ((node-items (make-vector node-count 0)) - (node-states (make-vector node-count 0)) - (node-index (make-table test: eq?))) + (let (node-items (make-vector node-count 0)) (let index-states ((state 0)) (when (< state state-count) (let ((node (vector-ref offsets state))) (for-each (lambda (item) (vector-set! node-items node item) - (vector-set! node-states node state) - (table-set! node-index (+ item (* item-space state)) node) (set! node (+ node 1))) (vector-ref states state))) (index-states (+ state 1)))) - (values offsets node-items node-states node-index node-count)))) + (values offsets node-items node-count)))) -;; Resolve the immutable propagation graph once through the packed-key index. + +;; State item rows are sorted contiguous slices. Binary search locates a node +;; without allocating or hashing a packed (state, item) key. +(def (lookahead-node-index offsets node-items state item) + (let search ((low (vector-ref offsets state)) + (high (vector-ref offsets (+ state 1)))) + (if (= low high) + #f + (let* ((middle (quotient (+ low high) 2)) + (found (vector-ref node-items middle))) + (cond + ((= found item) middle) + ((< found item) (search (+ middle 1) high)) + (else (search low middle))))))) + +;; Resolve the immutable propagation graph through sorted state item slices. ;; Fixed-point updates then touch vectors only. (def (make-lookahead-propagation-graph - state-transitions productions-by-lhs core-symbols layout item-space - node-items node-states node-index) + state-transitions productions-by-lhs core-symbols layout + node-items offsets) (let* ((node-count (vector-length node-items)) (closure-targets (make-vector node-count '())) - (shift-targets (make-vector node-count #f))) - (let node-loop ((node 0)) - (when (< node node-count) - (let* ((state (vector-ref node-states node)) - (item (vector-ref node-items node)) - (symbol (vector-ref core-symbols item))) - (when (and symbol (nonterminal-symbol? symbol)) - (vector-set! - closure-targets node - (map (lambda (production) - (let* ((target-item - (make-core-item - (production-id production) 0 layout)) - (target - (table-ref node-index - (+ target-item (* item-space state)) #f))) - (unless target - (error "LR(0) closure target is not indexed" - state item target-item)) - target)) - (table-ref productions-by-lhs - (nonterminal-name symbol) '())))) + (shift-targets (make-vector node-count #f)) + (symbol-index (make-table test: equal?)) + (symbol-count 0) + (core-symbol-ids (make-vector (vector-length core-symbols) #f))) + (let index-symbols ((item 0)) + (when (< item (vector-length core-symbols)) + (let (symbol (vector-ref core-symbols item)) (when symbol - (let (transition - (assoc symbol (vector-ref state-transitions state))) - (unless transition - (error "LR(0) transition missing during lookahead indexing" - state item symbol)) - (let (target - (table-ref node-index - (+ (+ item 1) (* item-space (cdr transition))) - #f)) - (unless target - (error "LR(0) shift target is not indexed" - state item symbol (cdr transition))) - (vector-set! shift-targets node target)))) - (node-loop (+ node 1))))) + (let* ((known (table-ref symbol-index symbol #f)) + (id (or known symbol-count))) + (unless known + (table-set! symbol-index symbol id) + (set! symbol-count (+ symbol-count 1))) + (vector-set! core-symbol-ids item id)))) + (index-symbols (+ item 1)))) + (let ((targets (make-vector symbol-count #f)) + (prediction-targets (make-vector symbol-count #f))) + (let state-loop ((state 0)) + (when (< state (- (vector-length offsets) 1)) + (let (prediction-order '()) + (for-each + (lambda (row) + (let (id (table-ref symbol-index (car row) #f)) + (unless id (error "LR transition symbol is not indexed" state row)) + (vector-set! targets id (cdr row)))) + (vector-ref state-transitions state)) + (let node-loop ((node (vector-ref offsets state))) + (when (< node (vector-ref offsets (+ state 1))) + (let* ((item (vector-ref node-items node)) + (symbol (vector-ref core-symbols item))) + (when (and symbol (nonterminal-symbol? symbol)) + (let* ((id (vector-ref core-symbol-ids item)) + (known (vector-ref prediction-targets id)) + (predictions + (or known + (map + (lambda (production) + (let* ((target-item (make-core-item (production-id production) 0 layout)) + (target (lookahead-node-index offsets node-items state target-item))) + (unless target + (error "LR(0) closure target is not indexed" state item target-item)) + target)) + (table-ref productions-by-lhs (nonterminal-name symbol) '()))))) + (unless known + (vector-set! prediction-targets id predictions) + (set! prediction-order (cons id prediction-order))) + (vector-set! closure-targets node predictions))) + (when symbol + (let* ((target-state (vector-ref targets (vector-ref core-symbol-ids item))) + (target (and target-state + (lookahead-node-index offsets node-items target-state (+ item 1))))) + (unless target + (error "LR(0) shift target is not indexed" state item symbol target-state)) + (vector-set! shift-targets node target)))) + (node-loop (+ node 1)))) + ;; Stored adjacency lists are immutable; discard only scratch + ;; references so memory grows with edges, not states times symbols. + (for-each (lambda (id) (vector-set! prediction-targets id #f)) prediction-order) + (for-each + (lambda (row) + (vector-set! targets (table-ref symbol-index (car row)) #f)) + (vector-ref state-transitions state))) + (state-loop (+ state 1))))) (values closure-targets shift-targets))) ;;; Propagates terminal masks to a fixed point over the interned LR(0) graph. -;;; Dense vectors replace three hash tables and quotient/modulo decoding. A -;;; single packed-key index is retained only to resolve immutable graph edges. +;;; Dense vectors own both graph identity and fixed-point mutable state. ;; propagate-lalr-lookaheads ;; : (-> Vector Vector Fixnum Table Vector Vector Table Table Vector Table Pair List) (def (propagate-lalr-lookaheads states state-transitions state-count productions-by-lhs table core-symbols - tail-first-masks nullable-tails + suffix-first-masks nullable-suffixes terminal-index layout) - (let* ((item-space (* (cdr layout) (vector-length table))) - (queue (stdq-make-Queue)) + (let* ((queue (stdq-make-Queue)) (processed-count 0) (started (##current-time-point)) (trace? (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1"))) - (let-values (((offsets node-items node-states node-index node-count) - (index-lr0-items states state-count item-space))) + (let-values (((offsets node-items node-count) + (index-lr0-items states state-count))) (trace-lookahead-phase 'lookahead-index node-count started trace?) (let-values (((closure-targets shift-targets) (make-lookahead-propagation-graph state-transitions productions-by-lhs core-symbols layout - item-space node-items node-states node-index))) + node-items offsets))) (trace-lookahead-phase 'lookahead-graph node-count started trace?) (let ((lookaheads (make-vector node-count 0)) (pending (make-vector node-count 0)) - (queued (make-vector node-count #f))) + (queued (make-vector node-count #f)) + (activated (make-vector node-count #f))) (def (enqueue-mask! node evidence) (let* ((known (vector-ref lookaheads node)) (novel @@ -180,7 +232,7 @@ (vector-set! queued node #t) (stdq-enqueue! queue node))))) (enqueue-mask! - (table-ref node-index (make-core-item 0 0 layout)) + (lookahead-node-index offsets node-items 0 (make-core-item 0 0 layout)) (compiler-index-set-singleton (table-ref terminal-index +lr-eof+))) (let loop () @@ -196,14 +248,24 @@ (displayln processed-count) (force-output)) (unless (null? (vector-ref closure-targets node)) - (let (evidence - (if (vector-ref nullable-tails item) - (compiler-index-set-union - (vector-ref tail-first-masks item) delta) - (vector-ref tail-first-masks item))) - (for-each - (lambda (target) (enqueue-mask! target evidence)) - (vector-ref closure-targets node)))) + (let* ((first? (not (vector-ref activated node))) + (nullable? (vector-ref nullable-suffixes (+ item 1))) + (evidence + (cond + (first? + (if nullable? + (compiler-index-set-union + (vector-ref suffix-first-masks (+ item 1)) delta) + (vector-ref suffix-first-masks (+ item 1)))) + (nullable? delta) + (else 0)))) + ;; FIRST(tail) is static; subsequent activations propagate + ;; only new lookahead bits, and only through nullable tails. + (vector-set! activated node #t) + (unless (zero? evidence) + (for-each + (lambda (target) (enqueue-mask! target evidence)) + (vector-ref closure-targets node))))) (let (target (vector-ref shift-targets node)) (when target (enqueue-mask! target delta))) (loop)))) @@ -221,8 +283,8 @@ (core-symbols (make-core-symbol-catalog table layout)) (started (##current-time-point)) (trace? (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1"))) - (let-values (((tail-first-masks nullable-tails) - (make-core-lookahead-catalog + (let-values (((suffix-first-masks nullable-suffixes) + (make-core-suffix-catalog table layout core-symbols first nullable terminal-index))) (let-values (((states state-transitions state-count productions-by-lhs lr0-state-visit-count) @@ -239,7 +301,7 @@ (let-values (((lookaheads lookahead-offsets lookahead-item-visit-count) (propagate-lalr-lookaheads states state-transitions state-count productions-by-lhs - table core-symbols tail-first-masks nullable-tails + table core-symbols suffix-first-masks nullable-suffixes terminal-index layout))) (when trace? (display "[gerbil-parser-lr] phase=lookahead count=") @@ -253,3 +315,162 @@ (materialize-transitions state-transitions state-count) terminal-values layout core-symbols lr0-state-visit-count lookahead-item-visit-count))))))) + +;;; Canonical LR(1) is the semantic reference for conflict-directed state +;;; partitioning. A state contains one mask per LR(0) core; closure propagates +;;; FIRST(tail) and the incoming follow mask until stable. State identity +;;; includes those masks, so contexts that LALR merges remain distinguishable. +;;; The representation deliberately matches build-states-via-lr0's result. +(def (build-states-via-canonical-lr1 productions table first nullable + (state-limit #f)) + (call-with-current-continuation + (lambda (abort) + (let-values (((terminal-values terminal-index) + (production-terminal-catalog productions))) + (let* ((layout (make-item-layout table terminal-values)) + (core-symbols (make-core-symbol-catalog table layout)) + (productions-by-lhs (production-index-by-lhs productions)) + (states (list->ExtensibleVector '())) + (state-transitions (list->ExtensibleVector '())) + (state-index (make-table test: equal?)) + (closure-masks (make-vector (vector-length core-symbols) 0)) + (state-count 0)) + (let-values (((suffix-first-masks nullable-suffixes) + (make-core-suffix-catalog + table layout core-symbols first nullable terminal-index))) + (def (canonical-closure kernels) + (let ((cores (list->ExtensibleVector '())) + (pending (stdq-make-Queue))) + (def (admit! core mask) + (let* ((old (vector-ref closure-masks core)) + (next (compiler-index-set-union old mask))) + (unless (= old next) + (when (zero? old) + (ExtensibleVector-push! cores core)) + (vector-set! closure-masks core next) + (stdq-enqueue! pending core)))) + (for-each (lambda (entry) (admit! (car entry) (cdr entry))) + kernels) + (let loop () + (unless (stdq-queue-empty? pending) + (let* ((core (stdq-dequeue! pending)) + (mask (vector-ref closure-masks core)) + (symbol (vector-ref core-symbols core))) + (when (and symbol (nonterminal-symbol? symbol)) + (let (evidence + (if (vector-ref nullable-suffixes (+ core 1)) + (compiler-index-set-union + (vector-ref suffix-first-masks (+ core 1)) mask) + (vector-ref suffix-first-masks (+ core 1)))) + (for-each + (lambda (production) + (admit! + (make-core-item (production-id production) 0 layout) + evidence)) + (table-ref productions-by-lhs + (nonterminal-name symbol) '())))) + (loop)))) + (let (result + (list-sort + (lambda (left right) (< (car left) (car right))) + (map (lambda (core) + (let (mask (vector-ref closure-masks core)) + (vector-set! closure-masks core 0) + (cons core mask))) + (vector->list (ExtensibleVector->vector cores))))) + result))) + (def (intern! state) + (let (existing (table-ref state-index state #f)) + (if existing + (values existing #f) + (let (index + (begin + (when (and state-limit (>= state-count state-limit)) + (abort #f)) + (ExtensibleVector-push! states state))) + (ExtensibleVector-push! state-transitions '()) + (table-set! state-index state index) + (set! state-count (+ state-count 1)) + (values index #t))))) + (let* ((eof-mask + (compiler-index-set-singleton + (table-ref terminal-index +lr-eof+))) + (initial + (canonical-closure + (list (cons (make-core-item 0 0 layout) eof-mask)))) + (queue (stdq-make-Queue))) + (intern! initial) + (stdq-enqueue! queue 0) + (let drain () + (unless (stdq-queue-empty? queue) + (let* ((index (stdq-dequeue! queue)) + (state (ExtensibleVector-ref states index)) + (kernels (make-table test: equal?)) + (kernel-cores (make-table test: equal?)) + (symbol-order '())) + (for-each + (lambda (entry) + (let ((symbol (vector-ref core-symbols (car entry)))) + (when symbol + (let (items (table-ref kernels symbol #f)) + (unless items + (set! items (make-table test: eq?)) + (table-set! kernels symbol items) + (set! symbol-order (cons symbol symbol-order))) + (let* ((next-core (+ (car entry) 1)) + (old (table-ref items next-core 0))) + (when (zero? old) + (table-set! kernel-cores symbol + (cons next-core + (table-ref kernel-cores symbol '())))) + (table-set! + items next-core + (compiler-index-set-union old (cdr entry)))))))) + state) + (for-each + (lambda (symbol) + (let* ((items (table-ref kernels symbol)) + (target + (canonical-closure + (map (lambda (core) + (cons core (table-ref items core 0))) + (table-ref kernel-cores symbol '()))))) + (let-values (((target-index novel?) (intern! target))) + (ExtensibleVector-set! + state-transitions index + (cons (cons symbol target-index) + (ExtensibleVector-ref state-transitions index))) + (when novel? + (stdq-enqueue! queue target-index))))) + (reverse symbol-order)) + (ExtensibleVector-set! + state-transitions index + (reverse (ExtensibleVector-ref state-transitions index))) + (drain)))) + (let ((core-states (make-vector state-count #f)) + (offsets (make-vector (+ state-count 1) 0)) + (node-count 0)) + (let state-loop ((index 0)) + (when (< index state-count) + (let (state (ExtensibleVector-ref states index)) + (vector-set! offsets index node-count) + (vector-set! core-states index (map car state)) + (set! node-count (+ node-count (length state)))) + (state-loop (+ index 1)))) + (vector-set! offsets state-count node-count) + (let ((lookaheads (make-vector node-count 0)) + (node 0)) + (let fill ((index 0)) + (when (< index state-count) + (for-each + (lambda (entry) + (vector-set! lookaheads node (cdr entry)) + (set! node (+ node 1))) + (ExtensibleVector-ref states index)) + (fill (+ index 1)))) + (values core-states state-count lookaheads offsets + (materialize-transitions + (ExtensibleVector->vector state-transitions) + state-count) + terminal-values layout core-symbols + state-count node-count)))))))))) diff --git a/src/compiler/lr-partition.ss b/src/compiler/lr-partition.ss new file mode 100644 index 00000000..6a12c3e2 --- /dev/null +++ b/src/compiler/lr-partition.ss @@ -0,0 +1,252 @@ +;;; -*- Gerbil -*- +;;; Conflict-preserving LR(1) quotient using standard tables and growable vectors. + +(import (only-in :std/hash/misc hash-ensure-ref) + (only-in :std/vector/extensible + list->ExtensibleVector ExtensibleVector-fill-pointer + ExtensibleVector-push! ExtensibleVector-ref) + (only-in ./funcs + compiler-index-set-for-each compiler-index-set-union) + (only-in ./lr terminal-symbol?) + (only-in ./lr-automaton core-item-production-id) + (only-in ./lr-lookahead build-states-via-canonical-lr1)) +(export build-states-via-partitioned-lr1) + +;; One bit names each raw parser action. Shift destinations are tested by the +;; transition-congruence pass; states with the same LR(0) core have identical +;; shift terminals and precedence-bearing shift productions. +(def (raw-action-masks states lookaheads offsets state-count terminal-values + terminal-index layout core-symbols) + (let (result (make-vector state-count #f)) + (let state-loop ((state 0)) + (when (< state state-count) + (let ((masks (make-vector (vector-length terminal-values) 0)) + (node (vector-ref offsets state))) + (for-each + (lambda (core) + (let ((symbol (vector-ref core-symbols core)) + (production-id (core-item-production-id core layout))) + (cond + ((and symbol (terminal-symbol? symbol)) + (let (terminal-id (table-ref terminal-index symbol)) + (vector-set! masks terminal-id + (bitwise-ior + 1 (vector-ref masks terminal-id))))) + ((not symbol) + (let (action-bit + (if (zero? production-id) + 2 + (arithmetic-shift 1 (+ production-id 2)))) + (compiler-index-set-for-each + (vector-ref lookaheads node) + (lambda (terminal-id) + (vector-set! + masks terminal-id + (bitwise-ior action-bit + (vector-ref masks terminal-id)))))))) + (set! node (+ node 1)))) + (vector-ref states state)) + (vector-set! result state masks)) + (state-loop (+ state 1)))) + result)) + +(def (single-action? mask) + (zero? (bitwise-and mask (- mask 1)))) + +;; A merged cell with multiple actions must be an action set that one of its +;; canonical members already had. Thus packing introduces no new conflict. +(def (safe-to-add? members union-actions candidate action-masks terminal-count) + (let terminal-loop ((terminal 0)) + (if (= terminal terminal-count) + #t + (let* ((candidate-action + (vector-ref (vector-ref action-masks candidate) terminal)) + (combined + (bitwise-ior (vector-ref union-actions terminal) + candidate-action))) + (and (or (single-action? combined) + (= candidate-action combined) + (let member-loop ((index 0)) + (and (< index (ExtensibleVector-fill-pointer members)) + (or (= combined + (vector-ref + (vector-ref + action-masks + (ExtensibleVector-ref members index)) + terminal)) + (member-loop (+ index 1)))))) + (terminal-loop (+ terminal 1))))))) + +(def (make-group state action-masks) + (let (members (list->ExtensibleVector '())) + (ExtensibleVector-push! members state) + (vector members + (vector-copy (vector-ref action-masks state))))) + +(def (pack-canonical-states states action-masks state-count terminal-count) + (let ((groups (list->ExtensibleVector '())) + (core-groups (make-table test: equal?)) + (ids (make-vector state-count 0))) + (let state-loop ((state 0)) + (when (< state state-count) + (let* ((candidates + (hash-ensure-ref + core-groups (vector-ref states state) + (lambda () (list->ExtensibleVector '())))) + (admitted #f)) + (let candidate-loop ((index 0)) + (when (and (not admitted) + (< index (ExtensibleVector-fill-pointer candidates))) + (let* ((group-id (ExtensibleVector-ref candidates index)) + (group (ExtensibleVector-ref groups group-id)) + (members (vector-ref group 0)) + (union-actions (vector-ref group 1))) + (if (safe-to-add? members union-actions state action-masks + terminal-count) + (begin + (set! admitted group-id) + (ExtensibleVector-push! members state) + (let union-loop ((terminal 0)) + (when (< terminal terminal-count) + (vector-set! + union-actions terminal + (bitwise-ior + (vector-ref union-actions terminal) + (vector-ref (vector-ref action-masks state) + terminal))) + (union-loop (+ terminal 1))))) + (candidate-loop (+ index 1))))) + (unless admitted + (set! admitted + (ExtensibleVector-push! + groups (make-group state action-masks))) + (ExtensibleVector-push! candidates admitted)) + (vector-set! ids state admitted)) + (state-loop (+ state 1)))) + (values groups ids)))) + +(def (index-transitions transitions state-count) + (let (rows (make-vector state-count '())) + (for-each + (lambda (row) + (let (state (car row)) + (vector-set! rows state + (cons (cons (cadr row) (caddr row)) + (vector-ref rows state))))) + transitions) + (let loop ((state 0)) + (when (< state state-count) + (vector-set! rows state (reverse (vector-ref rows state))) + (loop (+ state 1)))) + rows)) + +(def (transition-signature rows state ids) + (map (lambda (row) (cons (car row) (vector-ref ids (cdr row)))) + (vector-ref rows state))) + +;; Refine by transition targets until every quotient edge has one destination. +;; The lookup table and ExtensibleVector handle grouping without list copying. +(def (refine-partition ids rows state-count) + (let ((next (list->ExtensibleVector '())) + (next-ids (make-vector state-count 0)) + (index (make-table test: equal?))) + (let state-loop ((state 0)) + (when (< state state-count) + (let* ((key (cons (vector-ref ids state) + (transition-signature rows state ids))) + (group-id (table-ref index key #f))) + (unless group-id + (set! group-id + (ExtensibleVector-push! + next (list->ExtensibleVector '()))) + (table-set! index key group-id)) + (ExtensibleVector-push! + (ExtensibleVector-ref next group-id) state) + (vector-set! next-ids state group-id)) + (state-loop (+ state 1)))) + (values next next-ids))) + +(def (materialize-partition groups ids states lookaheads offsets rows + terminal-values layout core-symbols + canonical-visits) + (let* ((count (ExtensibleVector-fill-pointer groups)) + (merged-states (make-vector count '())) + (merged-offsets (make-vector (+ count 1) 0)) + (merged-transitions '()) + (node-count 0)) + (let group-loop ((group-id 0)) + (when (< group-id count) + (let* ((members (ExtensibleVector-ref groups group-id)) + (first (ExtensibleVector-ref members 0)) + (cores (vector-ref states first))) + (vector-set! merged-states group-id cores) + (vector-set! merged-offsets group-id node-count) + (set! node-count (+ node-count (length cores))) + (for-each + (lambda (row) + (set! merged-transitions + (cons (list group-id (car row) + (vector-ref ids (cdr row))) + merged-transitions))) + (vector-ref rows first))) + (group-loop (+ group-id 1)))) + (vector-set! merged-offsets count node-count) + (let (merged-lookaheads (make-vector node-count 0)) + (let group-loop ((group-id 0)) + (when (< group-id count) + (let* ((members (ExtensibleVector-ref groups group-id)) + (first (ExtensibleVector-ref members 0)) + (item-count (length (vector-ref states first)))) + (let item-loop ((item 0)) + (when (< item item-count) + (let ((target (+ (vector-ref merged-offsets group-id) + item)) + (mask 0)) + (let member-loop ((member 0)) + (when (< member + (ExtensibleVector-fill-pointer members)) + (let (state (ExtensibleVector-ref members member)) + (set! mask + (compiler-index-set-union + mask + (vector-ref + lookaheads + (+ (vector-ref offsets state) item))))) + (member-loop (+ member 1)))) + (vector-set! merged-lookaheads target mask)) + (item-loop (+ item 1))))) + (group-loop (+ group-id 1)))) + (values merged-states count merged-lookaheads merged-offsets + (reverse merged-transitions) terminal-values layout core-symbols + canonical-visits node-count)))) + +(def (build-states-via-partitioned-lr1 productions table first nullable) + (let-values (((states state-count lookaheads offsets transitions + terminal-values layout core-symbols + canonical-visits item-visits) + (build-states-via-canonical-lr1 + productions table first nullable))) + (let ((terminal-index (make-table test: equal?)) + (rows (index-transitions transitions state-count))) + (let index-loop ((terminal 0)) + (when (< terminal (vector-length terminal-values)) + (table-set! terminal-index + (vector-ref terminal-values terminal) terminal) + (index-loop (+ terminal 1)))) + (let (action-masks + (raw-action-masks + states lookaheads offsets state-count terminal-values + terminal-index layout core-symbols)) + (let-values (((initial-groups initial-ids) + (pack-canonical-states + states action-masks state-count + (vector-length terminal-values)))) + (let refine ((groups initial-groups) (ids initial-ids)) + (let-values (((next next-ids) + (refine-partition ids rows state-count))) + (if (= (ExtensibleVector-fill-pointer groups) + (ExtensibleVector-fill-pointer next)) + (materialize-partition + next next-ids states lookaheads offsets rows terminal-values + layout core-symbols canonical-visits) + (refine next next-ids))))))))) diff --git a/src/compiler/lr-table.ss b/src/compiler/lr-table.ss new file mode 100644 index 00000000..c8800a20 --- /dev/null +++ b/src/compiler/lr-table.ss @@ -0,0 +1,152 @@ +;;; -*- Gerbil -*- +;;; Canonical LR table admission before indexes or executable products exist. +(import (only-in :std/iter for in-range) + (prefix-in :std/struct/queue stdq-) + (only-in ./lr +lr-eof+ canonical-base-symbol? + production-action production-lhs production-rhs base-symbol layout-end-action? + production-terminal-catalog)) +(export validate-lr-tables layout-productions?) + +;;; Derive capability from admitted declarations, never from an author flag. +(def (layout-productions? productions) + (any (lambda (production) + (or (layout-end-action? (production-action production)) + (any (lambda (operand) + (match (base-symbol operand) + (['terminal (or 'layout-start 'layout-next) _] #t) + (_ #f))) + (production-rhs production)))) + productions)) + +(def (bounded-index? value count) + (and (fixnum? value) (fx>= value 0) (fx< value count))) + +(def (precedence? value) + (or (not value) + (match value + ([(or 'none 'left 'right 'dynamic) (? integer?)] #t) + (_ #f)))) + +(def (terminal? value) + (or (equal? value +lr-eof+) + (and (pair? value) (eq? (car value) 'terminal) + (canonical-base-symbol? value)))) + +(def (make-action-validator states productions layout?) + (def (reduce? value) + (match value + (['reduce id] (bounded-index? id productions)) + (_ #f))) + (def (shift? value) + (match value + (['shift target precedence] + (and (bounded-index? target states) (precedence? precedence))) + (_ #f))) + (def (atomic? value terminal) + (or (shift? value) (reduce? value) + (match value + (['accept] (equal? terminal +lr-eof+)) + (['reject-nonassoc key (? integer?)] (equal? key terminal)) + (_ #f)))) + (def (fork? value branch?) + ;; The producer flattens forks. Admit two or more atomic alternatives; + ;; recursive wrappers are neither a table format nor a request protocol. + (match value + (['fork first second rest ...] + (and (branch? first) (branch? second) (every branch? rest))) + (_ #f))) + (lambda (action terminal) + (or (atomic? action terminal) + (fork? action (lambda (branch) (atomic? branch terminal))) + (match action + (['layout-guard primary fallback] + (and layout? (shift? primary) (or (reduce? fallback) (fork? fallback reduce?)))) + (_ #f))))) + +(def (validate-rows rows key? value? kind) + (for (state (in-range (vector-length rows))) + (let ((row (vector-ref rows state)) (seen (make-table test: equal?))) + (unless (list? row) (error "invalid LR table row" kind state row)) + (for-each + (lambda (entry) + (unless (and (pair? entry) (key? (car entry)) + (not (table-ref seen (car entry) #f)) + (value? (cdr entry) (car entry))) + (error "invalid LR table entry" kind state entry)) + (table-set! seen (car entry) #t)) + row))) + rows) + +;;; Tables name only the vocabulary of their admitted production IR. Operand +;;; annotations are semantic metadata; membership uses the canonical base. +(def (production-vocabulary productions) + (def nonterminals (make-table test: eq?)) + (for-each + (lambda (production) + (table-set! nonterminals (production-lhs production) #t)) + productions) + (let-values (((_ terminals) (production-terminal-catalog productions))) + (values terminals nonterminals))) + +(def (for-each-shift-target visit action) + (match action + (['shift target _] (visit target)) + (['fork branches ...] + (for-each (lambda (branch) (for-each-shift-target visit branch)) branches)) + (['layout-guard primary fallback] + (for-each-shift-target visit primary) + (for-each-shift-target visit fallback)) + (_ (void)))) + +(def (accepting-action? action) + (match action + (['accept] #t) + (['fork branches ...] (any accepting-action? branches)) + (_ #f))) + +;;; Conflict resolution can disconnect states of the original LR automaton. +;;; Retain their IDs, but require a reachable accepting state. This graph check +;;; is necessary, not proof that reductions have a viable parse stack. +(def (validate-accept-reachability actions gotos) + (def states (vector-length actions)) + (def seen (make-vector states #f)) + (def pending (stdq-make-Queue)) + (def accepted? #f) + (def (enqueue! state) + (unless (vector-ref seen state) + (vector-set! seen state #t) + (stdq-enqueue! pending state))) + (enqueue! 0) + (let loop () + (unless (or accepted? (stdq-queue-empty? pending)) + (let (state (stdq-dequeue! pending)) + (set! accepted? (any (lambda (entry) (accepting-action? (cdr entry))) + (vector-ref actions state))) + (for-each (lambda (entry) (for-each-shift-target enqueue! (cdr entry))) + (vector-ref actions state)) + (for-each (lambda (entry) (enqueue! (cdr entry))) (vector-ref gotos state))) + (loop))) + (unless accepted? (error "invalid LR table no reachable acceptance"))) + +(def (validate-lr-tables productions actions gotos) + ;; Production semantics are admitted by each publishing owner first. + (def production-count (length productions)) + (def layout? (layout-productions? productions)) + (unless (and (fixnum? production-count) (fx>= production-count 0) + (vector? actions) (vector? gotos) + (positive? (vector-length actions)) + (= (vector-length actions) (vector-length gotos))) + (error "invalid LR table dimensions" production-count actions gotos)) + (let-values (((terminals nonterminals) (production-vocabulary productions))) + (def states (vector-length actions)) + (validate-rows actions + (lambda (key) + (and (terminal? key) + (table-ref terminals key #f) + (or layout? (not (memq (cadr key) '(layout-start layout-next)))))) + (make-action-validator states production-count layout?) 'actions) + (validate-rows gotos + (lambda (key) (and (symbol? key) (table-ref nonterminals key #f))) + (lambda (target _) (bounded-index? target states)) 'gotos)) + (validate-accept-reachability actions gotos) + (values actions gotos)) diff --git a/src/compiler/lr.ss b/src/compiler/lr.ss index 711c354c..de296f0b 100644 --- a/src/compiler/lr.ss +++ b/src/compiler/lr.ss @@ -3,17 +3,23 @@ (import (only-in ./funcs compiler-index-set->ordered-values + compiler-index-set-difference compiler-index-set-singleton compiler-index-set-union)) (export +lr-eof+ compute-first - compute-nullable + compute-nullable compute-completion + compute-productive validate-resolved-start + validate-resolved-repetitions current-grammar-source-map lr-spec-ref lower-rules base-symbol nonterminal-name nonterminal-symbol? operand-actions + operand-actions-valid? + layout-end-action? validate-production-semantics + canonical-base-symbol? production-action production-id production-lhs @@ -83,11 +89,95 @@ (def (operand-actions value) (if (marked-symbol? value) (caddr value) '())) +;;; Canonical semantic names have one contract for prepared and generated +;;; execution. Validate a proper chain before either owner publishes a product. +(def (operand-actions-valid? actions) + (and (list? actions) + (every (lambda (action) + (match action + (['field (? symbol?)] #t) + (['alias (? symbol?)] #t) + (_ #f))) + actions))) + (def (operand-add-action value action) (if (marked-symbol? value) (list 'marked (cadr value) (append (caddr value) (list action))) (list 'marked value (list action)))) +;;; Validate the producer's closed algebra before indexing or selecting an +;;; executor. These checks belong to canonical IR, not a backend policy. +(def (layout-end-action? action) + ;; Classification after admission is constant work in the request path. + (or (eq? action 'layout-end) + (and (pair? action) (eq? (car action) 'layout-end)))) + +(def (canonical-layout-end-action? action) + (or (eq? action 'layout-end) + (and (list? action) (pair? action) (pair? (cdr action)) + (eq? (car action) 'layout-end) + (every (lambda (boundary) + (and (string? boundary) (positive? (string-length boundary)))) + (cdr action))))) + +(def (canonical-base-symbol? value) + (match value + (['nonterminal (? symbol?)] #t) + (['terminal 'token (? symbol?)] #t) + (['terminal (or 'literal 'layout-start 'layout-next) (? string? text)] + (positive? (string-length text))) + (_ #f))) + +(def (validate-canonical-operand operand) + (match operand + (['marked base actions] + (unless (canonical-base-symbol? base) + (error "invalid canonical LR operand" operand)) + (unless (operand-actions-valid? actions) + (error "invalid LR operand actions" actions))) + (_ (unless (canonical-base-symbol? operand) + (error "invalid canonical LR operand" operand))))) + +(def (validate-production-semantics productions) + (unless (list? productions) + (error "invalid canonical LR production list" productions)) + (let loop ((rest productions) (index 0)) + (unless (null? rest) + (let (production (car rest)) + (unless + (match production + ([id (? symbol?) rhs action precedence] + (and (fixnum? id) (fx= id index) (list? rhs) + (or (not precedence) + (match precedence + ([(or 'none 'left 'right 'dynamic) (? integer?)] #t) + (_ #f))) + (cond + ((eq? action 'pass) (and (pair? rhs) (null? (cdr rhs)))) + ((eq? action 'concat) #t) + ((canonical-layout-end-action? action) (null? rhs)) + (else #f)))) + (_ #f)) + (error "invalid canonical LR production" index production)) + (for-each validate-canonical-operand (production-rhs production))) + (loop (cdr rest) (fx+ index 1)))) + ;; Resolve against the complete declaration set: forward references and + ;; recursive unused rules are valid, absent owners are not canonical IR. + (let (owners (production-index-by-lhs productions)) + (for-each + (lambda (production) + (for-each + (lambda (operand) + (match (base-symbol operand) + (['nonterminal name] + (unless (table-ref owners name #f) + (error "invalid canonical LR unresolved reference" + (production-id production) operand))) + (_ (void)))) + (production-rhs production))) + productions)) + productions) + (def (make-production id lhs rhs action precedence) (list id lhs rhs action precedence)) ;; production-id @@ -176,13 +266,15 @@ ;; ;; => augmented production zero followed by ordered lowered productions ;; ``` ;; % -(def (lower-rules rules root (allow-dynamic? #f)) +(def (lower-rules rules root (allow-dynamic? #f) (observe-origin #f)) (let ((next-id 1) (next-synthetic 0) (productions '())) - (def (emit lhs rhs action precedence) + (def (emit-production lhs rhs action precedence owner path expression) (let (production (make-production next-id lhs rhs action precedence)) + (when observe-origin + (observe-origin production owner (reverse path) expression)) (set! next-id (+ next-id 1)) (set! productions (cons production productions)))) (def (fresh owner) @@ -192,27 +284,40 @@ (number->string next-synthetic)))) (set! next-synthetic (+ next-synthetic 1)) name)) - (def (lower-symbol owner expression precedence) + ;; Paths are allocated only for an explicitly requested diagnostic replay. + ;; Canonical productions retain their existing shape and ordering. + (def (child-path path index) (and observe-origin (cons index path))) + (def (map-children procedure children path) + (if observe-origin + (let loop ((rest children) (index 0)) + (if (null? rest) '() + (cons (procedure (car rest) (child-path path index)) + (loop (cdr rest) (+ index 1))))) + (map (lambda (child) (procedure child #f)) children))) + (def (lower-symbol owner expression precedence path) (case (car expression) - ((literal) (list 'terminal 'literal (cadr expression))) + ((literal layout-start layout-next) + (list 'terminal (car expression) (cadr expression))) ((token) (list 'terminal 'token (cadr expression))) ((reference) (list 'nonterminal (cadr expression))) (else (let (name (fresh owner)) - (lower-to name owner expression precedence) + (lower-to name owner expression precedence path) (list 'nonterminal name))))) - (def (lower-operand owner expression precedence) + (def (lower-operand owner expression precedence path) (case (car expression) ((field) (operand-add-action - (lower-operand owner (caddr expression) precedence) + (lower-operand owner (caddr expression) precedence (child-path path 0)) (list 'field (cadr expression)))) ((alias) (operand-add-action - (lower-operand owner (caddr expression) precedence) + (lower-operand owner (caddr expression) precedence (child-path path 0)) (list 'alias (cadr expression)))) - (else (lower-symbol owner expression precedence)))) - (def (lower-to lhs owner expression precedence) + (else (lower-symbol owner expression precedence path)))) + (def (lower-to lhs owner expression precedence path) + (def (emit lhs rhs action precedence) + (emit-production lhs rhs action precedence owner path expression)) (case (car expression) ((precedence) (let ((direction (cadr expression)) @@ -221,46 +326,53 @@ (error "dynamic precedence requires selective GLR admission" owner rank)) (lower-to lhs owner (cadddr expression) - (list direction rank)))) + (list direction rank) (child-path path 0)))) ((choice) - (for-each - (lambda (child) (lower-to lhs owner child precedence)) - (cdr expression))) + (if observe-origin + (let loop ((children (cdr expression)) (index 0)) + (unless (null? children) + (lower-to lhs owner (car children) precedence (child-path path index)) + (loop (cdr children) (+ index 1)))) + (for-each (lambda (child) (lower-to lhs owner child precedence #f)) + (cdr expression)))) ((empty) (emit lhs '() 'concat precedence)) + ((layout-end) + (emit lhs '() (if (null? (cdr expression)) 'layout-end expression) + precedence)) ((sequence) (emit lhs - (map (lambda (child) - (lower-operand owner child precedence)) - (cdr expression)) + (map-children (lambda (child path) + (lower-operand owner child precedence path)) + (cdr expression) path) 'concat precedence)) ((optional) (emit lhs '() 'concat precedence) (emit lhs - (list (lower-operand owner (cadr expression) precedence)) + (list (lower-operand owner (cadr expression) precedence (child-path path 0))) 'pass precedence)) ((repeat) (emit lhs '() 'concat precedence) (emit lhs (list (list 'nonterminal lhs) - (lower-operand owner (cadr expression) precedence)) + (lower-operand owner (cadr expression) precedence (child-path path 0))) 'concat precedence)) ((repeat1) - (let (child (lower-operand owner (cadr expression) precedence)) + (let (child (lower-operand owner (cadr expression) precedence (child-path path 0))) (emit lhs (list child) 'pass precedence) (emit lhs (list (list 'nonterminal lhs) child) 'concat precedence))) ((field) - (emit lhs (list (lower-operand owner expression precedence)) + (emit lhs (list (lower-operand owner expression precedence path)) 'pass precedence)) ((alias) - (emit lhs (list (lower-operand owner expression precedence)) + (emit lhs (list (lower-operand owner expression precedence path)) 'pass precedence)) - ((literal token reference) - (emit lhs (list (lower-symbol owner expression precedence)) + ((literal layout-start layout-next token reference) + (emit lhs (list (lower-symbol owner expression precedence path)) 'pass precedence)) (else (error "unsupported GrammarExpr in LR lowering" owner expression)))) (for-each - (lambda (row) (lower-to (car row) (car row) (cadr row) #f)) + (lambda (row) (lower-to (car row) (car row) (cadr row) #f '())) rules) (cons (make-production 0 '$accept (list (list 'nonterminal root)) 'pass #f) @@ -375,8 +487,112 @@ ;; : (-> List Table Boolean) (def sequence-nullable? rhs-nullable?) +;;; Validate original repetition operands with the lowered grammar's converged +;;; nullable index. Helpers remain private; diagnostics retain the source rule +;;; and operand, including references and wrappers erased by LR lowering. +(def current-grammar-source-map (make-parameter '())) +(defstruct nullable-origin-task (owner expression path references)) + +(def (validate-resolved-repetitions rules nullable) + (def source-map (current-grammar-source-map)) + (def rule-sources (let (entry (assq 'rule source-map)) (if entry (cdr entry) '()))) + (def (occurrence owner path) + (let (source (assq owner rule-sources)) + (list (cons 'rule owner) (cons 'expressionPath (reverse path)) + (cons 'source (if source (cdr source) '()))))) + ;; A diagnostic reports one shortest nullable dependency path, not an expanded + ;; derivation tree. Rule sharing and cycles are visited once. The fixed point + ;; above remains the authority that the complete operand is nullable. + (def (witness owner expression path) + (def rows (make-table test: eq?)) + (def visited (make-table test: eq?)) + (def facts (make-table test: eq?)) + (for-each (lambda (row) (table-set! rows (car row) (cadr row))) rules) + (def (nullable? expr) + (let (cached (table-ref facts expr 'unknown)) + (if (not (eq? cached 'unknown)) cached + (let (value + (case (car expr) + ((empty layout-end repeat optional) #t) + ((reference) (table-ref nullable (cadr expr) #f)) + ((repeat1) (nullable? (cadr expr))) + ((sequence) (every nullable? (cdr expr))) + ((choice) (any nullable? (cdr expr))) + ((field alias) (nullable? (caddr expr))) + ((precedence) (nullable? (cadddr expr))) + (else #f))) + (table-set! facts expr value) value)))) + (def front (list (make-nullable-origin-task owner expression path '()))) + (def back '()) + (def (enqueue owner expr path references) + (set! back (cons (make-nullable-origin-task owner expr path references) back))) + (let loop () + (when (null? front) (set! front (reverse back)) (set! back '())) + (if (null? front) #f + (let* ((task (car front)) + (owner (nullable-origin-task-owner task)) + (expr (nullable-origin-task-expression task)) + (path (nullable-origin-task-path task)) + (references (nullable-origin-task-references task))) + (set! front (cdr front)) + (case (car expr) + ((empty layout-end repeat optional) (reverse references)) + ((reference) + (let ((name (cadr expr)) (row (table-ref rows (cadr expr) #f))) + (when (and row (table-ref nullable name #f) (not (table-ref visited name #f))) + (table-set! visited name #t) + (enqueue name row '() + (cons (append (occurrence name '()) + (list (cons 'referenceOrigin (occurrence owner path)))) references)))) + (loop)) + ((sequence choice) + (for-each + (lambda (child index) + (when (nullable? child) (enqueue owner child (cons index path) references))) + (cdr expr) (iota (length (cdr expr)))) + (loop)) + ((repeat1) (enqueue owner (cadr expr) (cons 0 path) references) (loop)) + ((field alias) (enqueue owner (caddr expr) (cons 0 path) references) (loop)) + ((precedence) (enqueue owner (cadddr expr) (cons 0 path) references) (loop)) + (else (loop))))))) + (def (reject owner expression operand path) + (let (arguments (list owner (car expression) operand)) + (apply error "resolved repetition operand accepts empty input" + (if (null? source-map) arguments + (append arguments + (list (list (cons 'expressionOrigin (occurrence owner path)) + (cons 'nullableReferencePath + (witness owner operand (cons 0 path)))))))))) + ;; One postorder pass validates every child and computes its nullability. + ;; Boolean short-circuiting must never skip a nested repetition obligation. + ;; Paths are only materialized when diagnostic context was supplied. + (def (child-path path index) + (if (null? source-map) '() (cons index path))) + (def (visit owner expression path) + (case (car expression) + ((empty layout-end) #t) + ((reference) (table-ref nullable (cadr expression) #f)) + ((repeat repeat1) + (let (operand (cadr expression)) + (when (visit owner operand (child-path path 0)) + (reject owner expression operand path)) + (eq? (car expression) 'repeat))) + ((sequence choice) + (let loop ((children (cdr expression)) (index 0) + (found (eq? (car expression) 'sequence))) + (if (null? children) found + (let (nullable? (visit owner (car children) (child-path path index))) + (loop (cdr children) (+ index 1) + (if (eq? (car expression) 'sequence) + (and found nullable?) (or found nullable?))))))) + ((optional) (visit owner (cadr expression) (child-path path 0)) #t) + ((field alias) (visit owner (caddr expression) (child-path path 0))) + ((precedence) (visit owner (cadddr expression) (child-path path 0))) + (else #f))) + (for-each (lambda (row) (visit (car row) (cadr row) '())) rules)) + ;;; Computes nullable nonterminals to a monotone fixed point; the returned list -;;; and membership table are materialized from the same completed iteration. +;;; and membership table are materialized after the dependency worklist drains. ;; compute-nullable ;; : (forall (p) (-> [p] [(Pair Symbol Datum)])) ;; compute-nullable @@ -391,23 +607,159 @@ ;; ;; => ordered nullable names and their shared membership index ;; ``` ;; % +;;; Two Boolean domains share one occurrence graph. Bit 1 is nullable, bit 2 +;;; productive; nullable publication also admits productivity. Pending changes +;;; coalesce while queued, so a name's dependencies are visited at most twice. +(defstruct completion-clause (owner nullable-remaining productive-remaining)) + +(def (compute-completion productions) + (let* ((names (nonterminals productions)) + (count (length names)) + (indices (make-table test: eq?)) + (facts (make-vector count 0)) + (pending (make-vector count 0)) + (dependents (make-vector count '())) + (work '())) + (for-each (lambda (name index) (table-set! indices name index)) names (iota count)) + (def (publish! index bits) + (let* ((before (vector-ref facts index)) + (delta (bitwise-and bits (bitwise-not before)))) + (unless (zero? delta) + (vector-set! facts index (bitwise-ior before delta)) + (when (zero? (vector-ref pending index)) (set! work (cons index work))) + (vector-set! pending index (bitwise-ior (vector-ref pending index) delta))))) + (for-each + (lambda (production) + (let ((references '()) (nullable? #t) (remaining 0) + (owner (table-ref indices (production-lhs production)))) + (for-each + (lambda (value) + (let (symbol (base-symbol value)) + (if (nonterminal-symbol? symbol) + (begin + ;; Missing names keep an obligation but have no publisher. + (set! remaining (+ remaining 1)) + (let (index (table-ref indices (nonterminal-name symbol) #f)) + (when index (set! references (cons index references))))) + (set! nullable? #f)))) + (production-rhs production)) + (if (zero? remaining) + (publish! owner (if nullable? 3 2)) + (let (clause (make-completion-clause owner (and nullable? remaining) remaining)) + ;; Register occurrences once, including duplicates, before draining. + (for-each (lambda (index) + (vector-set! dependents index + (cons clause (vector-ref dependents index)))) references))))) + productions) + (let propagate () + (unless (null? work) + (let* ((index (car work)) (delta (vector-ref pending index))) + (set! work (cdr work)) + (vector-set! pending index 0) + (for-each + (lambda (clause) + (when (and (not (zero? (bitwise-and delta 1))) + (completion-clause-nullable-remaining clause)) + (let (remaining (- (completion-clause-nullable-remaining clause) 1)) + (completion-clause-nullable-remaining-set! clause remaining) + (when (zero? remaining) (publish! (completion-clause-owner clause) 3)))) + (when (not (zero? (bitwise-and delta 2))) + (let (remaining (- (completion-clause-productive-remaining clause) 1)) + (completion-clause-productive-remaining-set! clause remaining) + (when (zero? remaining) (publish! (completion-clause-owner clause) 2))))) + (vector-ref dependents index)) + (propagate)))) + ;; Public Boolean tables and first-lhs order are materialized only after both + ;; domains converge; queue order and discovery order never escape the solver. + (let ((nullable (make-table test: eq?)) (productive (make-table test: eq?)) + (nullable-names '()) (productive-names '())) + (for-each + (lambda (name index) + (let (bits (vector-ref facts index)) + (unless (zero? (bitwise-and bits 1)) + (table-set! nullable name #t) (set! nullable-names (cons name nullable-names))) + (unless (zero? (bitwise-and bits 2)) + (table-set! productive name #t) (set! productive-names (cons name productive-names))))) + names (iota count)) + (values (reverse nullable-names) nullable (reverse productive-names) productive)))) + +;;; Standalone observations are projections of the same completed solution. (def (compute-nullable productions) - (let ((names (nonterminals productions)) - (nullable (make-table test: eq?))) - (let loop () - (let (changed? #f) - (for-each - (lambda (production) - (let (lhs (production-lhs production)) - (when (and (not (table-ref nullable lhs #f)) - (rhs-nullable? (production-rhs production) nullable)) - (table-set! nullable lhs #t) - (set! changed? #t)))) - productions) - (if changed? - (loop) - (values (filter (lambda (name) (table-ref nullable name #f)) names) - nullable)))))) + (let-values (((nullable nullable-index _productive _productive-index) + (compute-completion productions))) + (values nullable nullable-index))) + +(def (compute-productive productions) + (let-values (((_nullable _nullable-index productive productive-index) + (compute-completion productions))) + (values productive productive-index))) + +;;; LR parser entries require a finite derivation. Dead alternatives/rules stay +;;; in the CFG; only an unproductive selected start is an admission error. +;;; Explain blockers in selected author expressions, never synthetic helpers. +(def (validate-resolved-start rules root productive) + (unless (table-ref productive root #f) + (let ((rows (make-table test: eq?)) + (seen (make-table test: eq?)) + (facts (make-table test: eq?)) + (sources (let (entry (assq 'rule (current-grammar-source-map))) + (if entry (cdr entry) '())))) + (for-each (lambda (row) (table-set! rows (car row) (cadr row))) rules) + (def (origin owner path) + (let (source (assq owner sources)) + (list (cons 'rule owner) (cons 'expressionPath (reverse path)) + (cons 'source (if source (cdr source) '()))))) + (def (productive? expr) + (let (cached (table-ref facts expr 'unknown)) + (if (not (eq? cached 'unknown)) cached + (let (value + (case (car expr) + ((empty layout-end literal token layout-start layout-next repeat optional) #t) + ((reference) (table-ref productive (cadr expr) #f)) + ((sequence) (every productive? (cdr expr))) + ((choice) (any productive? (cdr expr))) + ((repeat1) (productive? (cadr expr))) + ((field alias) (productive? (caddr expr))) + ((precedence) (productive? (cadddr expr))) + (else #f))) + (table-set! facts expr value) value)))) + (def pending (list root)) + (table-set! seen root #t) + (def blockers '()) + (def (visit owner expr path) + (case (car expr) + ((reference) + (let (name (cadr expr)) + (unless (table-ref seen name #f) + (table-set! seen name #t) + (set! pending (cons name pending))) + (list (list (cons 'rule name) (cons 'referenceOrigin (origin owner path)))))) + ((choice) + ;; Every alternative is blocked. A single cyclic path is not a proof. + (apply append (map (lambda (child index) (visit owner child (cons index path))) + (cdr expr) (iota (length (cdr expr)))))) + ((sequence) + ;; One failed operand suffices to block this complete sequence. + (let loop ((children (cdr expr)) (index 0)) + (cond ((null? children) '()) + ((productive? (car children)) (loop (cdr children) (+ index 1))) + (else (visit owner (car children) (cons index path)))))) + ((repeat1) (visit owner (cadr expr) (cons 0 path))) + ((field alias) (visit owner (caddr expr) (cons 0 path))) + ((precedence) (visit owner (cadddr expr) (cons 0 path))) + (else '()))) + (let loop () + (unless (null? pending) + (let* ((owner (car pending)) (expr (table-ref rows owner #f))) + (set! pending (cdr pending)) + (set! blockers + (cons (list (cons 'ruleOrigin (origin owner '())) + (cons 'defined? (and expr #t)) + (cons 'blockedReferences (if expr (visit owner expr '()) '()))) blockers)) + (loop)))) + (error "LR start rule has no finite terminal derivation" root + (list (cons 'startOrigin (origin root '())) + (cons 'unproductiveRuleBlockers (reverse blockers))))))) (def (symbol-first symbol first) (let (symbol (base-symbol symbol)) @@ -447,8 +799,8 @@ (def (sequence-first symbols first nullable (tail-lookahead #f)) (sequence-first-from symbols first nullable tail-lookahead '())) -;;; Computes FIRST terminal sets after nullable convergence; no partially -;;; updated set escapes an iteration, preserving deterministic table identity. +;;; Computes FIRST terminal sets after nullable convergence. Delta propagation +;;; completes before canonical terminal rows and their lookup table are published. ;; compute-first ;; : (forall (p) (-> [p] Table [(Pair Symbol Datum)])) ;; compute-first @@ -466,53 +818,71 @@ (def (compute-first productions nullable) (let-values (((terminal-values terminal-index) (production-terminal-catalog productions))) - (let ((names (nonterminals productions)) - (first-masks (make-table test: eq?))) - (for-each (lambda (name) (table-set! first-masks name 0)) names) - (def (symbol-first-mask symbol) - (let (symbol (base-symbol symbol)) - (if (terminal-symbol? symbol) - (compiler-index-set-singleton - (table-ref terminal-index symbol)) - (table-ref first-masks (nonterminal-name symbol) 0)))) - (def (sequence-first-mask symbols) - (let loop ((rest symbols) (mask 0)) - (if (null? rest) - mask - (let* ((symbol (car rest)) - (next - (compiler-index-set-union - mask (symbol-first-mask symbol)))) - (if (symbol-nullable? symbol nullable) - (loop (cdr rest) next) - next))))) - (let fixed-point () - (let (changed? #f) - (for-each - (lambda (production) - (let* ((lhs (production-lhs production)) - (before (table-ref first-masks lhs 0)) - (after - (compiler-index-set-union - before - (sequence-first-mask (production-rhs production))))) - (unless (= before after) - (table-set! first-masks lhs after) - (set! changed? #t)))) - productions) - (if changed? - (fixed-point) - (let (first (make-table test: eq?)) - (for-each - (lambda (name) - (table-set! first name - (compiler-index-set->ordered-values - (table-ref first-masks name 0) - terminal-values))) - names) - (values - (map (lambda (name) (cons name (table-ref first name '()))) names) - first)))))))) + (let* ((names (nonterminals productions)) + (count (length names)) + (name-index (make-table test: eq?)) + (first-masks (make-vector count 0)) + (pending (make-vector count 0)) + (dependents (make-vector count '())) + (edges (make-table test: eqv?)) + (work '())) + (for-each (lambda (name index) (table-set! name-index name index)) names (iota count)) + (def (merge! index mask) + (let* ((before (vector-ref first-masks index)) + (after (compiler-index-set-union before mask))) + (unless (= before after) + (let (waiting (vector-ref pending index)) + (vector-set! first-masks index after) + (vector-set! pending index + (compiler-index-set-union waiting + (compiler-index-set-difference after before))) + (when (zero? waiting) (set! work (cons index work))))))) + (def (depend! name target) + (let (source (table-ref name-index name #f)) + ;; An undefined source has no FIRST facts. Retain the existing empty + ;; lookup semantics; reference admission remains a separate owner. + (when source + (let (edge (+ (* source count) target)) + (unless (table-ref edges edge #f) + (table-set! edges edge #t) + (vector-set! dependents source + (cons target (vector-ref dependents source)))))))) + ;; FIRST(lhs) depends on each nonterminal in the completed nullable + ;; prefix, including its first nonnullable operand. A terminal seeds lhs + ;; and blocks the remainder; marked operands retain the same base symbol. + (for-each + (lambda (production) + (let ((lhs (table-ref name-index (production-lhs production)))) + (let prefix ((rhs (production-rhs production))) + (unless (null? rhs) + (let (symbol (base-symbol (car rhs))) + (if (terminal-symbol? symbol) + (merge! lhs (compiler-index-set-singleton + (table-ref terminal-index symbol))) + (begin + (depend! (nonterminal-name symbol) lhs) + (when (symbol-nullable? symbol nullable) + (prefix (cdr rhs)))))))))) + productions) + ;; Coalesce pending facts per dense nonterminal index and propagate only + ;; newly admitted bits. All edges exist before propagation begins. + (let propagate () + (unless (null? work) + (let* ((index (car work)) (delta (vector-ref pending index))) + (set! work (cdr work)) + (vector-set! pending index 0) + (for-each (lambda (target) (merge! target delta)) + (vector-ref dependents index)) + (propagate)))) + (let (first (make-table test: eq?)) + (for-each + (lambda (name index) + (table-set! first name + (compiler-index-set->ordered-values + (vector-ref first-masks index) terminal-values))) + names (iota count)) + (values (map (lambda (name) (cons name (table-ref first name))) names) + first))))) ;;; Reads a field from the canonical LR spec association list without deriving ;;; alternate state; absent fields remain false and therefore fail closed. diff --git a/src/compiler/machine.ss b/src/compiler/machine.ss index 6fcbc40d..5869774e 100644 --- a/src/compiler/machine.ss +++ b/src/compiler/machine.ss @@ -1,33 +1,45 @@ ;;; -*- Gerbil -*- -;;; Hygienic LexicalExpr expansion and deterministic LALR(1) machine binding. +;;; Nominal parser machine, prepared execution and checked backend binding. -(import (only-in :std/vector/vector vector-map/index) +(import (only-in ../runtime/contextual-ir contextual-ir-ref contextual-ir-copy) + (only-in ../runtime/lexical-source + prepare-lexical-source-plan call-with-lexical-source) (only-in ../runtime/lr-parser - lr-lexical-mode-id lr-lexical-mode-terminals - lr-prepare lr-parse/prepared - lr-runtime-lexical-mode-catalog) - (only-in ../runtime/scan - scan-block-comment scan-decimal-digits scan-heredoc - scan-horizontal-whitespace scan-identifier scan-line scan-line-comment - scan-until-delimiters - make-literal-end-scanner scan-longest-literal - scan-nested-block-comment scan-newline - scan-number-literal scan-number-literal/profile - scan-escaped-quoted-strings scan-quoted-strings scan-whitespace - scan-emit) - (only-in ../runtime/token token-end token-kind)) -(export defgeneral-parser-machine + lr-prepare lr-parse/prepared lr-runtime-for-current-semantic-backend lr-runtime-with-action-selector + lr-runtime-lexical-mode-catalog lr-runtime-direct-step lr-runtime-event-step + lr-runtime-event-program? install-lr-runtime-direct-step! + install-lr-runtime-event-step!) + (only-in ../runtime/token + token-kind)) +(import (only-in ./lexical-expression lexical-end lexical-choice + lexical-dispatch lexical-dispatch/ranked) + (only-in ./lexical-lexer generated-lexer current-lexical-plan-sharing-enabled?) + (only-in ../runtime/scan make-ranked-lexical-scanner-factory)) +(export call-with-parser-machine-source + parser-machine-for-current-semantic-backend + current-lexical-plan-sharing-enabled? parser-machine-prepare-lexer + parser-machine-with-ranked-scanner parser-machine-with-lr-action-selector + parser-machine-lexical-plans parser-machine-lexical-modes-compatible? + defgeneral-parser-machine lexical-end lexical-choice lexical-dispatch lexical-dispatch/ranked parser-machine? parser-machine-ir + parser-machine-contextual-ir parser-machine-grammar-digest parser-machine-lex parser-machine-trivia parser-machine-runtime - parser-machine-parse) + parser-machine-parse + parser-machine-direct-drive + parser-machine-direct-source + parser-machine-backend-representation + install-parser-machine-backends! + install-parser-machine-direct-drive! + install-parser-machine-direct-source! + install-parser-machine-direct-step! install-parser-machine-event-step!) ;; parser-machine ;; : ParserMachine @@ -41,334 +53,176 @@ ;; ;; => #t for a generated parser machine ;; ``` ;; % -(defstruct parser-machine (ir grammar-digest lex trivia runtime parse) - transparent: #t) +(defstruct parser-machine + (ir grammar-digest lex trivia runtime parse direct-drive direct-source lexical-plans lexical-certificates lexer-factory source-plan owned-program) + transparent: #t final: #t) -;;; Expands one closed lexical algebra case into its ordinary scanner call. -;;; The templates preserve source/offset bindings; runtime behavior stays in scan.ss. -;; lexical-end -;; : (-> Syntax Syntax) -;; | doc m% -;; `lexical-end` expands a declared lexical expression. -;; -;; # Examples -;; -;; ```scheme -;; (lexical-end source offset (identifier)) -;; ;; => (scan-identifier source offset) -;; ``` -;; % -(defrules lexical-end - (whitespace+ horizontal-whitespace+ newline+ line decimal-digit+ number identifier - heredoc number-literal - quoted-string escaped-quoted-string until-delimiters - line-comment block-comment nested-block-comment - choice literals fallback precedence external) - ((_ source offset (whitespace+)) - (scan-whitespace source offset)) - ((_ source offset (horizontal-whitespace+)) - (scan-horizontal-whitespace source offset)) - ((_ source offset (newline+)) - (scan-newline source offset)) - ((_ source offset (line)) - (scan-line source offset)) - ((_ source offset (decimal-digit+)) - (scan-decimal-digits source offset)) - ((_ source offset (number)) - (scan-number-literal source offset)) - ((_ source offset - (number-literal (prefix ...) separator (suffix ...) - leading-period? trailing-period?)) - (scan-number-literal/profile - source offset (list prefix ...) separator (list suffix ...) - leading-period? trailing-period?)) - ((_ source offset (identifier)) - (scan-identifier source offset)) - ((_ source offset (quoted-string delimiter ...)) - (scan-quoted-strings source offset (list delimiter ...))) - ((_ source offset (escaped-quoted-string delimiter ...)) - (scan-escaped-quoted-strings source offset (list delimiter ...))) - ((_ source offset (until-delimiters characters)) - (scan-until-delimiters source offset characters)) - ((_ source offset (heredoc)) - (scan-heredoc source offset)) - ((_ source offset (line-comment start ...)) - (scan-line-comment source offset (list start ...))) - ((_ source offset (block-comment opening closing)) - (scan-block-comment source offset opening closing)) - ((_ source offset (nested-block-comment opening closing)) - (scan-nested-block-comment source offset opening closing)) - ((_ source offset (choice expression ...)) - (lexical-choice source offset (expression ...))) - ((_ source offset (precedence _rank expression)) - (lexical-end source offset expression)) - ((_ source offset (external _version scanner)) - (scanner source offset)) - ((_ source offset (literals value ...)) - (let (matched (scan-longest-literal source offset (list value ...))) - (and matched (+ offset (string-length matched))))) - ((_ source offset (fallback)) - (+ offset 1))) +;;; The machine caches the exact instruction object in its canonical owner. +;;; Both fields reference the same object; parser hot paths retain a native accessor. +(def (parser-machine-contextual-ir machine) + (contextual-ir-copy (parser-machine-owned-program machine))) -;; : (-> (OrFalse Fixnum) (OrFalse Fixnum) (OrFalse Fixnum)) -(def (prefer-longest-end current candidate) - (cond - ((not current) candidate) - ((not candidate) current) - ((>= current candidate) current) - (else candidate))) +;;; A captured session owns the selected runtime together with its parser machine. +;;; This preserves the exact runtime identity required by certified fragment reuse. +(def (parser-machine-for-current-semantic-backend machine) + (let (runtime (lr-runtime-for-current-semantic-backend (parser-machine-runtime machine))) + (if (eq? runtime (parser-machine-runtime machine)) machine + (make-parser-machine + (parser-machine-ir machine) (parser-machine-grammar-digest machine) + (parser-machine-lex machine) (parser-machine-trivia machine) runtime + (lambda (tokens . maybe-observability) + (lr-parse/prepared runtime tokens + (if (null? maybe-observability) #f (car maybe-observability)))) + (parser-machine-direct-drive machine) (parser-machine-direct-source machine) + (parser-machine-lexical-plans machine) (parser-machine-lexical-certificates machine) + (parser-machine-lexer-factory machine) (parser-machine-source-plan machine) + (parser-machine-owned-program machine))))) -;;; Chooses the longest lexical alternative; declaration order breaks ties. -;; lexical-choice -;; : (-> Syntax Syntax) -;; | doc m% -;; `lexical-choice` expands an ordered lexical alternative. -;; -;; # Examples -;; -;; ```scheme -;; (lexical-choice source offset ((identifier) (fallback))) -;; ;; => first matching end offset -;; ``` -;; % -(defrules lexical-choice - () - ((_ source offset ()) #f) - ((_ source offset (expression rest ...)) - (prefer-longest-end - (lexical-end source offset expression) - (lexical-choice source offset (rest ...))))) +(def (call-with-parser-machine-source machine source thunk) + (call-with-lexical-source (parser-machine-source-plan machine) source thunk)) -;;; Projects optional lexical precedence without inspecting scanner results. -;; lexical-expression-rank -;; : (-> Syntax Syntax) -;; | doc m% -;; Expands the precedence rank carried by one lexical expression. -;; -;; # Examples -;; -;; ```scheme -;; (lexical-expression-rank (precedence 10 (identifier))) -;; ;; => 10 -;; ``` -;; % -(defrules lexical-expression-rank (precedence) - ((_ (precedence rank _expression)) rank) - ((_ _expression) 0)) +(def (parser-machine-prepare-lexer machine) + ((parser-machine-lexer-factory machine))) -;;; Materializes one scanner per declared lexical rule. Static literal -;;; catalogs compile to a trie here, while the parser machine is initialized, -;;; instead of linearly probing every literal for every source token. -(defrules lexical-scanner (literals precedence) - ((_ (literals value ...)) - (make-literal-end-scanner '(value ...))) - ((_ (precedence _rank expression)) - (lexical-scanner expression)) - ((_ expression) - (lambda (source offset) - (lexical-end source offset expression)))) +;;; Rebuild the same checked lexical declarations and mode catalog with an +;;; alternate ranked lexical core. Keep IR/runtime/source ownership together; +;;; generated source backends must not bypass the substituted lexical plan. +(def (parser-machine-with-ranked-scanner machine scanner-factory) + (unless (and (parser-machine? machine) (procedure? scanner-factory)) + (error "ranked scanner substitution requires a machine and factory")) + (let (factory (parser-machine-lexer-factory machine)) + (let-values (((lexer plans certificates) (factory scanner-factory))) + (make-parser-machine + (parser-machine-ir machine) (parser-machine-grammar-digest machine) + lexer (parser-machine-trivia machine) (parser-machine-runtime machine) + (parser-machine-parse machine) #f #f plans certificates + (lambda ((next-scanner-factory scanner-factory)) (factory next-scanner-factory)) + (parser-machine-source-plan machine) + (parser-machine-owned-program machine))))) -;; prefer-ranked-match -;; : (-> (OrFalse List) (OrFalse List) (OrFalse List)) -;; | doc m% -;; Chooses one scanner match by length, precedence, then declaration. -;; -;; # Examples -;; -;; ```scheme -;; (prefer-ranked-match '(left 4 1) '(right 4 2)) -;; ;; => (right 4 2) -;; ``` -;; % -(def (prefer-ranked-match current candidate) - (cond - ((not current) candidate) - ((not candidate) current) - ((> (cadr current) (cadr candidate)) current) - ((< (cadr current) (cadr candidate)) candidate) - ((>= (caddr current) (caddr candidate)) current) - (else candidate))) +(def (parser-machine-with-lr-action-selector machine factory) + (unless (parser-machine? machine) (error "LR selector substitution requires a parser machine")) + (let (runtime (lr-runtime-with-action-selector (parser-machine-runtime machine) factory)) + (make-parser-machine + (parser-machine-ir machine) (parser-machine-grammar-digest machine) + (parser-machine-lex machine) (parser-machine-trivia machine) runtime + (lambda (tokens . observability) + (lr-parse/prepared runtime tokens (if (pair? observability) (car observability) #f))) + #f #f (parser-machine-lexical-plans machine) (parser-machine-lexical-certificates machine) + (parser-machine-lexer-factory machine) (parser-machine-source-plan machine) + (parser-machine-owned-program machine)))) -;;; Keeps large literal catalogs as immutable data instead of expanding one C -;;; branch per literal. Capability checks run only while preparing interned LR -;;; lexical modes, so SRFI-1's maintained list search avoids code-size growth -;;; without entering the source-scanning hot path. -(def (lexical-literal-admitted? literal values case-insensitive?) - (member literal values (if case-insensitive? string-ci=? string=?))) +(def (parser-machine-lexical-modes-compatible? machine old-mode new-mode (first-character #f)) + (or (= old-mode new-mode) + (let (plans (parser-machine-lexical-plans machine)) + (eq? (vector-ref plans old-mode) (vector-ref plans new-mode))) + (and first-character (< (char->integer first-character) 128) + (let* ((certificates (force (parser-machine-lexical-certificates machine))) + (class (vector-ref (vector-ref certificates 0) (char->integer first-character))) + (modes (vector-ref certificates 1))) + (eq? (vector-ref (vector-ref modes old-mode) class) + (vector-ref (vector-ref modes new-mode) class)))))) -;;; AOT capability predicate for literal-producing lexical algebra. Runtime -;;; mode selection never probes scanners with synthetic input: closed literal -;;; and choice forms lower to ordinary comparisons, while opaque external -;;; scanners are conservatively admitted. -(defrules lexical-expression-admits-literal? - (choice literals precedence external) - ((_ literal case-insensitive? (literals value ...)) - (lexical-literal-admitted? literal '(value ...) case-insensitive?)) - ((_ literal case-insensitive? (choice expression ...)) - (or (lexical-expression-admits-literal? - literal case-insensitive? expression) ...)) - ((_ literal case-insensitive? (precedence _rank expression)) - (lexical-expression-admits-literal? - literal case-insensitive? expression)) - ((_ _literal _case-insensitive? (external _version _scanner)) #t) - ((_ _literal _case-insensitive? _expression) #f)) +;;; A generated driver produces recognition values, independently of the +;;; optional prepared event executor. It is admitted only for the Parser IR whose digest +;;; was embedded in its source. Install during language-module initialization, +;;; before the machine is shared with parser requests. +(def (install-parser-machine-direct-drive! machine digest drive) + (unless (and (parser-machine? machine) + (string? digest) + (equal? digest (parser-machine-grammar-digest machine)) + (procedure? drive) + (not (parser-machine-direct-drive machine))) + (error "generated LR driver does not match parser machine" digest)) + (parser-machine-direct-drive-set! machine drive)) -;;; A lexical mode admits complete rules, not already-produced lexemes. This -;;; preserves each admitted rule's ordinary longest-match behavior on source -;;; (for example a rule containing both "*" and "**"). -(defrules lexical-rule-admitted? - () - ((_ terminals name expression extras case-insensitive?) - (or (not terminals) - (memq 'name extras) - (any - (lambda (terminal) - (and (pair? terminal) - (eq? (car terminal) 'terminal) - (case (cadr terminal) - ((token) (eq? (caddr terminal) 'name)) - ((literal) - (lexical-expression-admits-literal? - (caddr terminal) case-insensitive? expression)) - (else #f)))) - terminals)))) +;;; Grammar-derived source parsers produce a ParseArtifact or decline with #f. +;;; They may admit a complete artifact directly for +;;; fresh unobserved requests. Returning #f leaves the ordinary LR path to +;;; own rejection, diagnostics, checkpoints, and unsupported source shapes. +(def (install-parser-machine-direct-source! machine digest parse) + (unless (and (parser-machine? machine) + (string? digest) + (equal? digest (parser-machine-grammar-digest machine)) + (procedure? parse) + (not (parser-machine-direct-source machine))) + (error "generated source parser does not match parser machine" digest)) + (parser-machine-direct-source-set! machine parse)) -;;; One AOT-generated capability/scanner pair. Capability checks run once per -;;; interned LR mode; the scanner remains the ordinary generated lexical rule. -(defrules generated-lexical-rule - () - ((_ (name expression) extras case-insensitive?) - (let (scanner (lexical-scanner expression)) - (cons - (lambda (terminals) - (lexical-rule-admitted? - terminals name expression extras case-insensitive?)) - (lambda (source offset) - (let (end (scanner source offset)) - (and end - (list 'name end (lexical-expression-rank expression))))))))) +(def (install-parser-machine-direct-step! machine digest step) + (unless (and (parser-machine? machine) + (string? digest) + (equal? digest (parser-machine-grammar-digest machine)) + (procedure? step) + (not (lr-runtime-event-program? (parser-machine-runtime machine)))) + (error "generated LR step does not match parser machine" digest)) + (install-lr-runtime-direct-step! + (parser-machine-runtime machine) step)) -;;; Returns name, end offset, and precedence for generated-lexer. Longest -;;; consumption wins globally; lexical precedence breaks equal-length ties. -;; lexical-dispatch/ranked -;; : (-> Syntax Syntax) -;; | doc m% -;; Expands global longest-match dispatch with lexical tie precedence. -;; -;; # Examples -;; -;; ```scheme -;; (lexical-dispatch/ranked source offset rows) -;; ;; => (token-name end-offset precedence) or #f -;; ``` -;; % -(defrules lexical-dispatch/ranked - () - ((_ source offset ()) #f) - ((_ source offset ((name expression) row ...)) - (let ((end (lexical-end source offset expression)) - (rest (lexical-dispatch/ranked source offset (row ...)))) - (prefer-ranked-match - (and end (list 'name end (lexical-expression-rank expression))) - rest)))) +(def (install-parser-machine-event-step! machine digest step) + (unless (and (parser-machine? machine) (string? digest) + (equal? digest (parser-machine-grammar-digest machine)) + (procedure? step)) + (error "generated LR event step does not match parser machine" digest)) + (install-lr-runtime-event-step! (parser-machine-runtime machine) step)) -;;; Binds the matched declaration name to its end offset without runtime macro state. -;; lexical-dispatch -;; : (-> Syntax Syntax) -;; | doc m% -;; `lexical-dispatch` expands the ordered token-rule dispatch. -;; -;; # Examples -;; -;; ```scheme -;; (lexical-dispatch source offset ((Identifier (identifier)))) -;; ;; => (Identifier . end-offset) -;; ``` -;; % -(defrules lexical-dispatch - () - ((_ source offset ()) #f) - ((_ source offset rows) - (let (match (lexical-dispatch/ranked source offset rows)) - (and match (cons (car match) (cadr match)))))) +;;; Installed kinds bind representation obligations without an author annotation. +(def (backend-semantic-representation kind) + (case kind + ((drive step) 'recognition) + ((event-step) 'event-program) + ((source) 'parse-artifact) + (else #f))) -;;; Generates only the source traversal shell; token construction remains scan-owned. -;;; Declaration order is observable only after length and precedence tie. -;; generated-lexer -;; : (-> Syntax Syntax) -;; | doc m% -;; `generated-lexer` expands lexical rows into a source-to-token procedure. -;; -;; # Examples -;; -;; ```scheme -;; (generated-lexer (lexical-rules (Identifier (identifier)))) -;; ;; => source-to-token procedure -;; ``` -;; % -(defrules generated-lexer - (lexical-rules extras) - ((_ (lexical-rules row ...) (extras extra-name ...) case-insensitive? - mode-catalog) - (let* ((rules - (list (generated-lexical-rule - row '(extra-name ...) case-insensitive?) ...)) - (all-scanners (map cdr rules)) - (mode-scanners - (vector-map/index - (lambda (_index mode) - (filter-map - (lambda (rule) - (and ((car rule) (lr-lexical-mode-terminals mode)) - (cdr rule))) - rules)) - mode-catalog))) - (letrec - ((scan-scanners - (lambda (scanners source offset) - (fold - (lambda (scanner selected) - (prefer-ranked-match selected (scanner source offset))) - #f scanners))) - (scan-one - (lambda (source offset byte-offset mode) - (let (match - (or (if mode - (scan-scanners - (vector-ref mode-scanners - (lr-lexical-mode-id mode)) - source offset) - (scan-scanners all-scanners source offset)) - ;; A mode miss must still materialize the offending token - ;; for the LR failure frontier and lossless diagnostics. - ;; Successful directed scans never enter this cold path. - (and mode - (scan-scanners all-scanners source offset)))) - (unless match - (error "no lexical rule matched parser-directed source" - offset mode)) - (let (output-token - (scan-emit source (car match) offset (cadr match) - byte-offset)) - (values output-token (cadr match)))))) - (scan-from - (lambda (source initial-offset initial-byte-offset) - (let (length (string-length source)) - (let loop ((offset initial-offset) - (byte-offset initial-byte-offset) - (tokens '())) - (if (= offset length) - (reverse tokens) - (let-values (((output-token end) - (scan-one source offset byte-offset #f))) - (loop end (token-end output-token) - (cons output-token tokens))))))))) - (case-lambda - ((source) (scan-from source 0 0)) - ((source offset byte-offset) - (scan-from source offset byte-offset)) - ((source offset byte-offset mode) - (scan-one source offset byte-offset mode))))))) +;;; Describe an installed entry, not a requested preference. Generated drive +;;; and source entries retain their own product contracts on a selected machine. +;;; This reads captured runtime identity; it does not select or prepare a backend. +(def (parser-machine-backend-representation machine kind) + (unless (and (parser-machine? machine) + (or (eq? kind 'prepared) (backend-semantic-representation kind))) + (error "unknown parser backend representation" kind)) + (let (runtime (parser-machine-runtime machine)) + (case kind + ((prepared) (if (lr-runtime-event-program? runtime) 'event-program 'recognition)) + ((drive) (and (parser-machine-direct-drive machine) 'recognition)) + ((source) (and (parser-machine-direct-source machine) 'parse-artifact)) + ((step) (and (lr-runtime-direct-step runtime) + (if (lr-runtime-event-program? runtime) 'event-program 'recognition))) + ((event-step) (and (lr-runtime-event-step runtime) 'event-program))))) + +;;; Check the entire declaration before mutating any machine/runtime slot. +;;; Each row is (kind exact-parser-ir-digest generated-procedure). +(def (install-parser-machine-backends! machine rows) + (unless (and (parser-machine? machine) (list? rows)) + (error "invalid parser backend declaration" rows)) + (let ((seen '()) (runtime (parser-machine-runtime machine))) + (for-each + (lambda (row) + (unless (and (list? row) (= (length row) 3) + (backend-semantic-representation (car row)) + (not (memq (car row) seen)) + (string? (cadr row)) + (equal? (cadr row) (parser-machine-grammar-digest machine)) + (procedure? (caddr row))) + (error "invalid or stale parser backend declaration" row)) + (when (and (eq? (car row) 'step) (lr-runtime-event-program? runtime)) + (error "recognition step requires a recognition runtime" row)) + (when (case (car row) + ((drive) (parser-machine-direct-drive machine)) + ((source) (parser-machine-direct-source machine)) + ((step) (lr-runtime-direct-step runtime)) + ((event-step) (or (lr-runtime-event-step runtime) + (lr-runtime-event-program? runtime)))) + (error "parser backend slot is already installed" (car row))) + (set! seen (cons (car row) seen))) rows) + (for-each + (lambda (row) + ((case (car row) + ((drive) install-parser-machine-direct-drive!) + ((source) install-parser-machine-direct-source!) + ((step) install-parser-machine-direct-step!) + ((event-step) install-parser-machine-event-step!)) + machine (cadr row) (caddr row))) rows))) ;;; Connects immutable parser IR to generated lexer and LR runtime entrypoints once. ;;; Runtime calls receive the compiled machine and never re-enter grammar expansion. @@ -380,36 +234,36 @@ ;; # Examples ;; ;; ```scheme -;; (defgeneral-parser-machine parser parser-ir ...) +;; (defgeneral-parser-machine parser contextual-program ...) ;; ;; => immutable parser-machine binding ;; ``` ;; % (defrules defgeneral-parser-machine - (grammar-digest lexical-rules rules extras parser-entrypoints) - ((_ binding parser-ir - (grammar-digest parser-artifact-digest) + (lexical-rules rules extras parser-entrypoints) + ((_ binding program-expression (lexical-rules lexical-row ...) (rules (rule-name rule-expression) ...) (extras extra-name ...) (parser-entrypoints (root-rule root-action root-effect) entry-row ...)) (def binding - (let (runtime - (lr-prepare (cdr (assq 'lr-spec parser-ir)))) - (make-parser-machine - parser-ir - parser-artifact-digest - (generated-lexer - (lexical-rules lexical-row ...) - (extras extra-name ...) - (cdr (assq 'case-insensitive? parser-ir)) - (lr-runtime-lexical-mode-catalog runtime)) - (lambda (input-token) - (memq (token-kind input-token) '(extra-name ...))) - runtime - (lambda (tokens . maybe-observability) - (lr-parse/prepared - runtime tokens - (if (null? maybe-observability) - #f - (car maybe-observability))))))))) + (let* ((program program-expression) + (grammar-digest (contextual-ir-ref program 'base-grammar-digest)) + (owned-ir (contextual-ir-ref (contextual-ir-ref program 'recognition) 'program)) + (runtime (lr-prepare (cdr (assq 'lr-spec owned-ir)))) + (factory (lambda ((ranked-scanner-factory make-ranked-lexical-scanner-factory)) + (generated-lexer + (lexical-rules lexical-row ...) + (extras extra-name ...) + (cdr (assq 'case-insensitive? owned-ir)) + (lr-runtime-lexical-mode-catalog runtime) ranked-scanner-factory)))) + (let-values (((lexer plans certificates) (factory))) + (make-parser-machine + owned-ir grammar-digest lexer + (lambda (input-token) (memq (token-kind input-token) '(extra-name ...))) + runtime + (lambda (tokens . maybe-observability) + (lr-parse/prepared runtime tokens + (if (null? maybe-observability) #f (car maybe-observability)))) + #f #f plans certificates factory + (prepare-lexical-source-plan (cdr (assq 'lexical-rules owned-ir))) program)))))) diff --git a/src/compiler/normalize.ss b/src/compiler/normalize.ss index 87c46835..f532b7be 100644 --- a/src/compiler/normalize.ss +++ b/src/compiler/normalize.ss @@ -4,9 +4,12 @@ grammar-composition grammar-name grammar-parents grammar-role-name grammar-role-ref grammar-roles) (only-in ../runtime/identity sha256-text) + (only-in :std/list/list-builder with-list-builder) (only-in ../utilities/lists append-unique merge-keyed-row)) (export compile-grammar compile-grammar/receipt + compile-grammar/context normalized-grammar-ir normalized-grammar-receipt + normalized-grammar-source-map grammar-composition-receipt grammar-ir-ref grammar-ir-canonical) @@ -19,15 +22,15 @@ ;; : (-> Grammar (List Grammar) (List GrammarRole)) (def (collect-steps grammar (active '())) - (when (memq grammar active) - (error "grammar inheritance cycle" (grammar-name grammar))) - (let (next (cons grammar active)) - (append - (apply append - (map (lambda (parent) (collect-steps parent next)) - (grammar-parents grammar))) - (map (lambda (role) (cons 'merge role)) (grammar-roles grammar)) - (grammar-composition grammar)))) + (with-list-builder (emit) + (def (visit grammar active) + (when (memq grammar active) + (error "grammar inheritance cycle" (grammar-name grammar))) + (let (next (cons grammar active)) + (for-each (lambda (parent) (visit parent next)) (grammar-parents grammar)) + (for-each (lambda (role) (emit (cons 'merge role))) (grammar-roles grammar)) + (for-each emit (grammar-composition grammar)))) + (visit grammar active))) (def (replace-keyed-row rows row section) (let ((key ((section-row-key section) row)) @@ -69,15 +72,96 @@ ((remove) (remove-keyed-row rows row section)) (else (error "unknown grammar composition operation" operation)))) -(def (normalize-keyed-table steps section) - (foldl (lambda (step rows) - (let ((operation (car step)) (role (cdr step))) - (foldl (lambda (row merged) - (apply-keyed-operation merged row section operation)) - rows - (grammar-role-ref role section)))) - '() - steps)) +;;; Context is captured by the same fold that admits effective grammar rows. +;;; Occurrence indices distinguish equal role names and repeated role objects. +(defstruct effective-grammar-row (row step operation role source history)) +(defstruct normalized-grammar (ir receipt selections)) + +(def (normalize-keyed-table steps section source-at capture?) + (def rows '()) + (def selections '()) + (for-each + (lambda (step index) + (let ((operation (car step)) (role (cdr step))) + (for-each + (lambda (row) + ;; This remains the sole authority for conflicts and row ordering. + ;; Source callbacks cannot admit an invalid composition operation. + (set! rows (apply-keyed-operation rows row section operation)) + (when capture? + (let* ((key ((section-row-key section) row)) + (previous (assoc key selections)) + (history-entry (list index operation (grammar-role-name role)))) + (case operation + ((remove) + (set! selections (filter (lambda (entry) (not (equal? (car entry) key))) selections))) + (else + (let* ((old (and previous (cdr previous))) + (history (append (if old (effective-grammar-row-history old) '()) + (list history-entry))) + (selected + (if (and old (eq? operation 'merge)) + ;; An admitted equal duplicate retains the first owner. + (make-effective-grammar-row + (effective-grammar-row-row old) (effective-grammar-row-step old) + (effective-grammar-row-operation old) (effective-grammar-row-role old) + (effective-grammar-row-source old) history) + (let (source (and source-at (source-at role index section row))) + (unless (or (not source) + (and (list? source) + (every (lambda (entry) + (and (pair? entry) (symbol? (car entry)))) source))) + (error "invalid grammar occurrence source" index section key source)) + (make-effective-grammar-row row index operation + (grammar-role-name role) source history))))) + (set! selections + (if previous + (map (lambda (entry) (if (equal? (car entry) key) (cons key selected) entry)) + selections) + (append selections (list (cons key selected))))))))))) + (grammar-role-ref role section)))) + steps (iota (length steps))) + (values rows selections)) + +;;; Project only effective bindings. Removed declarations remain in the receipt, +;;; while a remove/reappend starts a new effective declaration history. +(def (normalized-grammar-source-map normalized origin) + (def selections (normalized-grammar-selections normalized)) + (def (entries section) + (let (entry (assq section selections)) (if entry (cdr entry) '()))) + (def (source selected) + (let* ((provided (or (effective-grammar-row-source selected) '())) + (facts + (list (cons 'componentOwner (effective-grammar-row-role selected)) + (cons 'compositionStep (effective-grammar-row-step selected)) + (cons 'compositionOperation (effective-grammar-row-operation selected)) + (cons 'contributionHistory (effective-grammar-row-history selected)))) + (coordinates + (filter-map (lambda (key) (and (not (assq key provided)) (cons key origin))) + '(path location)))) + ;; The selected role owns composition facts; supplied syntax owns reader + ;; coordinates and generated status. Unknown coordinates stay at origin. + (append coordinates + (filter (lambda (entry) (not (assq (car entry) facts))) provided) + facts))) + (append + (map (lambda (mapping) + (cons (cdr mapping) + (map (lambda (entry) (cons (car entry) (source (cdr entry)))) + (entries (car mapping))))) + '((syntax-kinds . syntax-kind) (terminals . terminal) + (lexical-rules . lexical-rule) (rules . rule))) + (list + (cons 'field + (with-list-builder (emit) + (for-each + (lambda (entry) + (let ((row (effective-grammar-row-row (cdr entry))) + (selected-source (source (cdr entry)))) + (for-each + (lambda (name) (emit (cons (list (car row) name) selected-source))) + (caddr row)))) + (entries 'syntax-kinds))))))) ;; : (-> List List List) (def (append-flow merged edge) @@ -143,16 +227,19 @@ (cons 'compositionDigest identity) (cons 'grammarIrDigest (sha256-text (canonical ir)))))) -(def (compile-grammar/receipt grammar) +(def (normalize-grammar/context grammar source-at capture?) (if (and (list? grammar) (let (row (assq 'schema grammar)) (and row (equal? (cdr row) "gerbil-parser.grammar-ir.v1")))) - (values grammar #f) + (make-normalized-grammar grammar #f '()) (let* ((steps (collect-steps grammar)) - (tables + (normalized-tables (map (lambda (section) - (cons section (normalize-keyed-table steps section))) + (let-values (((rows selections) + (normalize-keyed-table steps section source-at capture?))) + (list section rows selections))) table-slots)) + (tables (map (lambda (entry) (cons (car entry) (cadr entry))) normalized-tables)) (body (append (list (cons 'schema "gerbil-parser.grammar-ir.v1") @@ -165,7 +252,21 @@ (list (cons 'compositionDigest (cdr (assq 'compositionDigest receipt)))) (drop body 2)))) - (values ir receipt)))) + (make-normalized-grammar + ir receipt (if capture? + (map (lambda (entry) (cons (car entry) (caddr entry))) normalized-tables) + '()))))) + +;;; Existing callers do not request occurrence/source capture. +(def (compile-grammar/receipt grammar) + (let (normalized (normalize-grammar/context grammar #f #f)) + (values (normalized-grammar-ir normalized) (normalized-grammar-receipt normalized)))) + +;;; The callback supplies source metadata for the actual accepted occurrence; +;;; it is optional because some POO values have no reader syntax association. +(def (compile-grammar/context grammar (source-at #f)) + (normalize-grammar/context grammar source-at #t)) + (def (compile-grammar grammar) (let-values (((ir _receipt) (compile-grammar/receipt grammar))) ir)) diff --git a/src/compiler/parser-ir.ss b/src/compiler/parser-ir.ss index 69b3c4c0..6573fdd0 100644 --- a/src/compiler/parser-ir.ss +++ b/src/compiler/parser-ir.ss @@ -1,6 +1,7 @@ ;;; Grammar IR to immutable parser-machine IR compilation. -(import (only-in ./lr-compiler compile-lr-spec) +(import (prefix-in ../runtime/parser-ir-data irdata-) + (only-in ./lr-compiler compile-lr-spec) (only-in ../grammar/algebra grammar-expression? grammar-expression-references grammar-expression-terminals) @@ -9,13 +10,17 @@ parser-ir-ref parser-ir-canonical) -;; : (-> (List SyntaxKindRow) Symbol) -;; require-root -;; : (-> List Symbol) -(def (require-root syntax-kinds) - (if (null? syntax-kinds) - (error "parser grammar requires a root syntax kind") - (caar syntax-kinds))) +;; Derive aliased roots from the entry rule; raw contracts retain catalog roots. +;; : (-> List List Symbol Symbol) +(def (require-root syntax-kinds rules root-rule) + (when (null? syntax-kinds) (error "parser grammar requires a root syntax kind")) + (let* ((expression (cadr (assq root-rule rules))) + (kind (if (and (pair? expression) (eq? (car expression) 'alias)) + (cadr expression) (caar syntax-kinds))) + (row (assq kind syntax-kinds))) + (unless (and row (eq? (cadr row) 'node)) + (error "parser entrypoint must publish a declared node root" root-rule kind)) + kind)) (def (row-names rows) (map car rows)) @@ -48,7 +53,7 @@ (for-each (lambda (row) (unless (lexical-expression? (cadr row)) - (error "invalid normalized lexical expression" (car row)))) + (error "invalid normalized lexical expression" (car row) (cadr row)))) lexical-rules))) (def (validate-rules rules terminals) @@ -99,7 +104,7 @@ ;; compile-parser ;; : (-> Alist Alist) -(def (compile-parser grammar) +(def (compile-parser grammar (construction 'lalr)) (unless (and (list? grammar) (equal? (let (row (assq 'schema grammar)) (and row (cdr row))) @@ -128,11 +133,11 @@ (cons 'grammar (parser-ir-ref grammar-ir 'grammar)) (cons 'compositionDigest (parser-ir-ref grammar-ir 'compositionDigest)) - (cons 'root-kind (require-root syntax-kinds)) + (cons 'root-kind (require-root syntax-kinds rules (require-entrypoint entrypoints rules))) (cons 'root-rule (require-entrypoint entrypoints rules)) (cons 'lr-spec (compile-lr-spec rules (require-entrypoint entrypoints rules) - conflict-policy case-insensitive?)) + conflict-policy case-insensitive? construction)) (cons 'conflict-policy conflict-policy) (cons 'case-insensitive? case-insensitive?) (cons 'syntax-kinds syntax-kinds) @@ -147,9 +152,7 @@ ;; parser-ir-ref ;; : (-> Alist Symbol Datum) -(def (parser-ir-ref ir key) - (alet (entry (assq key ir)) - (cdr entry))) +(def parser-ir-ref irdata-parser-ir-ref) (def (parser-ir-canonical ir) (call-with-output-string diff --git a/src/compiler/recursive-source.ss b/src/compiler/recursive-source.ss new file mode 100644 index 00000000..c0f4b873 --- /dev/null +++ b/src/compiler/recursive-source.ss @@ -0,0 +1,683 @@ +;;; Generate rollback-capable HCL event actions from the Scheme Grammar IR. +(import (only-in :clan/poo/object .o .ref) + (only-in ./build-strategy BuildStrategy. declare-build-strategy-provider make-bound-build-strategy) + (only-in :std/misc/ports read-all-as-string) + (only-in :gerbil-parser/src/language/descriptor language-grammar-ir language-grammar-machine) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-grammar-digest)) +(def (rule-procedure name) + (string->symbol (string-append "events-" (symbol->string name)))) + +;;; This closed ASCII subset is safe only while the HCL lexical declarations +;;; retain these exact meanings and precedence relationships. A new lexical +;;; rule must be reviewed before the generated fast path is regenerated. +(def +fast-lexical-contract+ + '((horizontal-whitespace (horizontal-whitespace+)) + (newline (newline+)) + (comment (line-comment "#" "//")) + (block-comment-token (block-comment "/*" "*/")) + (heredoc (heredoc)) + (string (quoted-string "\"")) + (number (number)) + (identifier (identifier)) + (punctuation + (literals "==" "!=" "<=" ">=" "&&" "||" + "{" "}" "[" "]" "(" ")" "=" "," "." ":" + "+" "-" "*" "/" "%" "!" "<" ">" "?")) + (unknown (fallback)))) + +;;; The closed attribute/block event corridor is derived from this Grammar IR. +;;; A grammar edit must regenerate and review its field/node event sequence. +(def +simple-attribute-grammar-digest+ + "sha256:7fa44a0c1fccad8bfc8c10096afd4b11fd7d945dacb88ebc79f494afee851648") + +;;; An emitted expression returns the next significant-token index or #f. +;;; Zero is a valid position, so even an empty match is distinct from failure. +(def (make-expression-compiler) + (let (counter 0) + (def (fresh prefix) + (let (name (string->symbol + (string-append prefix (number->string counter)))) + (set! counter (fx+ counter 1)) + name)) + (def (compile-sequence-steps remaining pos mark) + (if (null? remaining) + pos + (let (next (fresh "next")) + `(let (,next ,(compile-expression (car remaining) pos)) + (if ,next + ,(compile-sequence-steps (cdr remaining) next mark) + (begin (rollback! ,mark) #f)))))) + (def (compile-choice-steps choices pos mark) + (if (null? choices) + #f + (let (next (fresh "next")) + `(let (,next ,(compile-expression (car choices) pos)) + (if ,next + ,next + (begin + (rollback! ,mark) + ,(compile-choice-steps (cdr choices) pos mark))))))) + (def (compile-terminal expected literal? pos) + (let (test (if literal? + `(equal? (token-lexeme input) ,expected) + `(eq? (token-kind input) ',expected))) + `(if (< ,pos limit) + (let (input (vector-ref significant ,pos)) + (if ,test + (begin + (emit-event! 'token input 0) + (fx+ ,pos 1)) + #f)) + #f))) + (def (compile-repeat child required? pos) + (let ((loop-name (fresh "repeat-loop")) + (next (fresh "next")) (count (fresh "count")) + (mark (fresh "mark")) (after (fresh "after"))) + `(let ,loop-name ((,next ,pos) (,count 0)) + (let (,mark event-count) + (let (,after ,(compile-expression child next)) + (if ,after + (begin + (when (= ,after ,next) + (error "zero-width repeated grammar expression")) + (,loop-name ,after (fx+ ,count 1))) + (begin + (rollback! ,mark) + ,(if required? + `(if (> ,count 0) ,next #f) + next)))))))) + (def (compile-wrapper name child field? pos) + (let ((mark (fresh "mark")) (open (fresh "open")) + (next (fresh "next"))) + `(let (,mark event-count) + (emit-event! ',(if field? 'open-field 'open-node) + ',name (offset ,pos)) + (let (,open event-count) + (let (,next ,(compile-expression child pos)) + (if ,next + (begin + ,(if field? + `(if (= event-count ,open) + (rollback! ,mark) + (emit-event! 'close-field ',name + (range-end ,pos ,next))) + `(emit-event! 'close-node ',name + (range-end ,pos ,next))) + ,next) + (begin (rollback! ,mark) #f))))))) + (def (compile-expression expr pos) + (match expr + (['empty] pos) + (['literal expected] (compile-terminal expected #t pos)) + (['token expected] (compile-terminal expected #f pos)) + (['reference name] `(,(rule-procedure name) ,pos)) + (['sequence . children] + (let (mark (fresh "mark")) + `(let (,mark event-count) + ,(compile-sequence-steps children pos mark)))) + (['choice . choices] + (let (mark (fresh "mark")) + `(let (,mark event-count) + ,(compile-choice-steps choices pos mark)))) + (['optional child] + (let ((mark (fresh "mark")) (next (fresh "next"))) + `(let (,mark event-count) + (let (,next ,(compile-expression child pos)) + (if ,next ,next + (begin (rollback! ,mark) ,pos)))))) + (['repeat child] (compile-repeat child #f pos)) + (['repeat1 child] (compile-repeat child #t pos)) + (['field name child] (compile-wrapper name child #t pos)) + (['alias name child] (compile-wrapper name child #f pos)) + (['precedence _ _ child] (compile-expression child pos)) + (else (error "unsupported grammar expression" expr)))) + compile-expression)) +(def (module-expression descriptor) + (let ((hcl-parser-ir (language-grammar-ir descriptor)) + (hcl-parser (language-grammar-machine descriptor))) + (let ((compile-expression (make-expression-compiler)) + (rules (cdr (assq 'rules hcl-parser-ir)))) + (unless (equal? (cdr (assq 'lexical-rules hcl-parser-ir)) + +fast-lexical-contract+) + (error "HCL fast lexer requires lexical contract review")) + (unless (equal? (parser-machine-grammar-digest hcl-parser) + +simple-attribute-grammar-digest+) + (error "HCL closed event path requires grammar review")) + `(begin + (import (only-in :gerbil-parser/src/compiler/machine + parser-machine-trivia) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/scan + scan-quoted-string scan-line-comment scan-block-comment + scan-number-literal) + (only-in :gerbil-parser/src/runtime/significant + parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/token + make-token token-kind token-lexeme token-start token-end) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact/raw-event-tape + make-success-parse-artifact/canonical-events)) + (export direct-parse-hcl direct-lex-hcl direct-hcl-grammar-digest) + (def direct-hcl-grammar-digest + ,(parser-machine-grammar-digest hcl-parser)) + (def (ascii-alpha? ch) + (let (code (char->integer ch)) + (or (<= 65 code 90) (<= 97 code 122) (= code 95)))) + (def (ascii-digit? ch) + (let (code (char->integer ch)) (<= 48 code 57))) + (def (ascii-ident-rest? ch) + (or (ascii-alpha? ch) (ascii-digit? ch) (char=? ch #\-))) + (def (direct-lex-hcl source) + (let (length (string-length source)) + (def (ascii-range-end start end) + (and end + (let check ((index start)) + (if (= index end) + end + (and (< (char->integer (string-ref source index)) 128) + (check (fx+ index 1))))))) + (let loop ((start 0) (tokens '())) + (if (= start length) + (reverse tokens) + (let* ((ch (string-ref source start)) + (kind + (cond + ((ascii-alpha? ch) 'identifier) + ((ascii-digit? ch) 'number) + ((or (char=? ch #\space) (char=? ch #\tab)) + 'horizontal-whitespace) + ((or (char=? ch #\newline) (char=? ch #\return)) + 'newline) + ((or (char=? ch #\=) + (char=? ch #\{) + (char=? ch #\})) + 'punctuation) + ((char=? ch #\") 'string) + ((char=? ch #\#) 'comment) + ((and (char=? ch #\/) + (< (fx+ start 1) length) + (char=? (string-ref source (fx+ start 1)) #\/)) + 'comment) + ((and (char=? ch #\/) + (< (fx+ start 1) length) + (char=? (string-ref source (fx+ start 1)) #\*)) + 'block-comment-token) + (else #f)))) + (if (not kind) + #f + (let (first-end + (case kind + ((string) + (ascii-range-end + start (scan-quoted-string source start "\""))) + ((comment) + (ascii-range-end + start (scan-line-comment source start '("#" "//")))) + ((block-comment-token) + (ascii-range-end + start (scan-block-comment source start "/*" "*/"))) + (else (fx+ start 1)))) + (and first-end + (let scan ((end first-end)) + (if (and (< end length) + (let (next (string-ref source end)) + (case kind + ((identifier) (ascii-ident-rest? next)) + ((number) (ascii-digit? next)) + ((horizontal-whitespace) + (or (char=? next #\space) + (char=? next #\tab))) + ((newline) + (or (char=? next #\newline) + (char=? next #\return))) + (else #f)))) + (scan (fx+ end 1)) + (let (token-end + (if (and (eq? kind 'number) (< end length) + (let (next (string-ref source end)) + (or (char=? next #\.) + (char=? next #\e) + (char=? next #\E)))) + (ascii-range-end + start (scan-number-literal source start)) + end)) + (and token-end + (if (and (eq? kind 'punctuation) + (< token-end length) + (char=? (string-ref source token-end) + #\=)) + #f + (loop token-end + (cons (make-token + kind + (substring source start + token-end) + start token-end) + tokens))))))))))))))) + (def (direct-parse-hcl machine source (use-simple? #t) + (use-fast-lex? #t) (use-canonical? #t)) + (let* ((source-bytes (string->utf8 source)) + (byte-length (u8vector-length source-bytes)) + (tokens (or (and use-fast-lex? + (= byte-length (string-length source)) + (direct-lex-hcl source)) + (lex-source machine source))) + (significant + (list->vector (parser-significant-tokens machine tokens))) + (limit (vector-length significant)) + (events #f) + (event-count 0)) + (def (offset pos) + (if (< pos limit) + (token-start (vector-ref significant pos)) + byte-length)) + (def (range-end start next) + (if (= start next) (offset start) + (token-end (vector-ref significant (fx- next 1))))) + (def (emit-event! operation name byte-offset) + (unless events + (set! events + (make-vector (* 3 (max 64 (* 6 (length tokens)))) #f))) + (when (= (* 3 event-count) (vector-length events)) + (let* ((old events) + (grown (make-vector (* 2 (vector-length old)) #f))) + (let loop ((i 0)) + (when (< i (* 3 event-count)) + (vector-set! grown i (vector-ref old i)) + (loop (fx+ i 1)))) + (set! events grown))) + (let (base (* 3 event-count)) + (vector-set! events base operation) + (vector-set! events (fx+ base 1) name) + (vector-set! events (fx+ base 2) byte-offset)) + (set! event-count (fx+ event-count 1))) + (def (rollback! mark) + (set! event-count mark)) + ;;; The admission pass completes before this fixed event template runs. + ;;; Rejected sources enter the ordinary generated recursive parser. + (def (emit-simple-attribute! name equals value) + (let* ((start (token-start name)) + (name-end (token-end name)) + (value-start (token-start value)) + (value-end (token-end value)) + (value-node + (case (token-kind value) + ((number) 'NumberExpression) + ((string) 'StringExpression) + ((identifier) 'LiteralExpression)))) + (emit-event! 'open-field 'item start) + (emit-event! 'open-node 'Attribute start) + (emit-event! 'open-field 'name start) + (emit-event! 'token name 0) + (emit-event! 'close-field 'name name-end) + (emit-event! 'token equals 0) + (emit-event! 'open-field 'value value-start) + (emit-event! 'open-node 'ConditionalExpression value-start) + (emit-event! 'open-field 'condition value-start) + (emit-event! 'open-node 'BinaryExpression value-start) + (emit-event! 'open-field 'left value-start) + (emit-event! 'open-node 'UnaryExpression value-start) + (emit-event! 'open-field 'operand value-start) + (emit-event! 'open-node 'TraversalExpression value-start) + (emit-event! 'open-field 'root value-start) + (emit-event! 'open-node value-node value-start) + (emit-event! 'open-field 'value value-start) + (emit-event! 'token value 0) + (emit-event! 'close-field 'value value-end) + (emit-event! 'close-node value-node value-end) + (emit-event! 'close-field 'root value-end) + (emit-event! 'close-node 'TraversalExpression value-end) + (emit-event! 'close-field 'operand value-end) + (emit-event! 'close-node 'UnaryExpression value-end) + (emit-event! 'close-field 'left value-end) + (emit-event! 'close-node 'BinaryExpression value-end) + (emit-event! 'close-field 'condition value-end) + (emit-event! 'close-node 'ConditionalExpression value-end) + (emit-event! 'close-field 'value value-end) + (emit-event! 'close-node 'Attribute value-end) + (emit-event! 'close-field 'item value-end))) + (def (simple-attributes?) + (let loop ((pos 0)) + (if (= pos limit) + #t + (let (input (vector-ref significant pos)) + (cond + ((eq? (token-kind input) 'newline) + (loop (fx+ pos 1))) + ((and (< (fx+ pos 2) limit) + (eq? (token-kind input) 'identifier) + (equal? (token-lexeme + (vector-ref significant (fx+ pos 1))) "=") + (memq (token-kind + (vector-ref significant (fx+ pos 2))) + '(number string identifier))) + (loop (fx+ pos 3))) + (else #f)))))) + (def (simple-value-at? pos) + (and (< (fx+ pos 2) limit) + (eq? (token-kind (vector-ref significant pos)) 'identifier) + (equal? (token-lexeme + (vector-ref significant (fx+ pos 1))) "=") + (memq (token-kind + (vector-ref significant (fx+ pos 2))) + '(number string identifier)))) + (def (brace-at? pos brace) + (and (< pos limit) + (equal? (token-lexeme (vector-ref significant pos)) brace))) + (def (block-opening-at pos) + (and (< pos limit) + (eq? (token-kind (vector-ref significant pos)) 'identifier) + (let labels ((cursor (fx+ pos 1))) + (cond + ((= cursor limit) #f) + ((brace-at? cursor "{") cursor) + ((memq (token-kind (vector-ref significant cursor)) + '(string identifier)) + (labels (fx+ cursor 1))) + (else #f))))) + (def (closed-blocks?) + (def (scan-block pos) + (let (opening-pos (block-opening-at pos)) + (and opening-pos + (let body ((cursor (fx+ opening-pos 1))) + (cond + ((= cursor limit) #f) + ((brace-at? cursor "}") (fx+ cursor 1)) + ((eq? (token-kind (vector-ref significant cursor)) + 'newline) + (body (fx+ cursor 1))) + ((simple-value-at? cursor) + (body (fx+ cursor 3))) + (else + (let (next (scan-block cursor)) + (and next (body next))))))))) + (let root ((pos 0) (saw-block? #f)) + (cond + ((= pos limit) saw-block?) + ((eq? (token-kind (vector-ref significant pos)) 'newline) + (root (fx+ pos 1) saw-block?)) + (else + (let (next (scan-block pos)) + (and next (root next #t))))))) + (def (emit-simple-attributes!) + (emit-event! 'open-node 'HclFile 0) + (let loop ((pos 0)) + (if (= pos limit) + (begin + (emit-event! 'close-node 'HclFile byte-length) + (void)) + (let (input (vector-ref significant pos)) + (if (eq? (token-kind input) 'newline) + (begin + (emit-event! 'token input 0) + (loop (fx+ pos 1))) + (begin + (emit-simple-attribute! + input + (vector-ref significant (fx+ pos 1)) + (vector-ref significant (fx+ pos 2))) + (loop (fx+ pos 3)))))))) + (def (make-closed-canonical-artifact shape) + (let ((remaining tokens) + (trivia? (parser-machine-trivia machine)) + (canonical-events (cons #f '())) + (tail #f) + (next-token-id 0) + (next-node-id 0)) + (set! tail canonical-events) + (def (emit! event) + (let (cell (cons event '())) + (set-cdr! tail cell) + (set! tail cell))) + (def (emit-token! input) + (unless (and (pair? remaining) (eq? input (car remaining))) + (error "simple HCL token order changed" input)) + (emit! (vector 'token next-token-id + (token-kind input) (token-lexeme input) + (token-start input) (token-end input))) + (set! next-token-id (fx+ next-token-id 1)) + (set! remaining (cdr remaining))) + (def (emit-trivia-until! boundary) + (let loop () + (when (and (pair? remaining) + (<= (token-end (car remaining)) boundary)) + (unless (trivia? (car remaining)) + (error "unclaimed simple HCL token" (car remaining))) + (emit-token! (car remaining)) + (loop)))) + (def (open-node! name start) + (emit-trivia-until! start) + (let (id next-node-id) + (emit! (vector 'start-node id name start)) + (set! next-node-id (fx+ next-node-id 1)) + id)) + (def (close-node! id name end) + (emit-trivia-until! end) + (emit! (vector 'finish-node id name end))) + (def (open-field! name start) + (emit-trivia-until! start) + (emit! (vector 'start-field name start))) + (def (close-field! name end) + (emit! (vector 'finish-field name end))) + ;;; The admitted attribute shape has a fixed canonical event order. + ;;; Trivia can only precede its three significant source tokens. + (def (emit-attribute! name equals value) + (let* ((start (token-start name)) + (name-end (token-end name)) + (value-start (token-start value)) + (value-end (token-end value)) + (value-node + (case (token-kind value) + ((number) 'NumberExpression) + ((string) 'StringExpression) + ((identifier) 'LiteralExpression))) + (base next-node-id)) + (set! next-node-id (fx+ base 6)) + (emit-trivia-until! start) + (emit! (vector 'start-field 'item start)) + (emit! (vector 'start-node base 'Attribute start)) + (emit! (vector 'start-field 'name start)) + (emit-token! name) + (emit! (vector 'finish-field 'name name-end)) + (emit-trivia-until! (token-start equals)) + (emit-token! equals) + (emit-trivia-until! value-start) + (emit! (vector 'start-field 'value value-start)) + (emit! (vector 'start-node (fx+ base 1) + 'ConditionalExpression value-start)) + (emit! (vector 'start-field 'condition value-start)) + (emit! (vector 'start-node (fx+ base 2) + 'BinaryExpression value-start)) + (emit! (vector 'start-field 'left value-start)) + (emit! (vector 'start-node (fx+ base 3) + 'UnaryExpression value-start)) + (emit! (vector 'start-field 'operand value-start)) + (emit! (vector 'start-node (fx+ base 4) + 'TraversalExpression value-start)) + (emit! (vector 'start-field 'root value-start)) + (emit! (vector 'start-node (fx+ base 5) + value-node value-start)) + (emit! (vector 'start-field 'value value-start)) + (emit-token! value) + (emit! (vector 'finish-field 'value value-end)) + (emit! (vector 'finish-node (fx+ base 5) + value-node value-end)) + (emit! (vector 'finish-field 'root value-end)) + (emit! (vector 'finish-node (fx+ base 4) + 'TraversalExpression value-end)) + (emit! (vector 'finish-field 'operand value-end)) + (emit! (vector 'finish-node (fx+ base 3) + 'UnaryExpression value-end)) + (emit! (vector 'finish-field 'left value-end)) + (emit! (vector 'finish-node (fx+ base 2) + 'BinaryExpression value-end)) + (emit! (vector 'finish-field 'condition value-end)) + (emit! (vector 'finish-node (fx+ base 1) + 'ConditionalExpression value-end)) + (emit! (vector 'finish-field 'value value-end)) + (emit! (vector 'finish-node base 'Attribute value-end)) + (emit! (vector 'finish-field 'item value-end)))) + (def (emit-closed-block! pos) + (let* ((type (vector-ref significant pos)) + (opening-pos (block-opening-at pos)) + (opening (vector-ref significant opening-pos)) + (start (token-start type)) + (type-end (token-end type)) + (body-start (token-start + (vector-ref significant + (fx+ opening-pos 1))))) + (open-field! 'item start) + (let (block-id (open-node! 'Block start)) + (open-field! 'type start) + (emit-trivia-until! start) + (emit-token! type) + (close-field! 'type type-end) + (let labels ((cursor (fx+ pos 1))) + (when (< cursor opening-pos) + (let* ((label (vector-ref significant cursor)) + (label-start (token-start label))) + (emit-trivia-until! label-start) + (open-field! 'label label-start) + (emit-token! label) + (close-field! 'label (token-end label)) + (labels (fx+ cursor 1))))) + (emit-trivia-until! (token-start opening)) + (emit-token! opening) + (open-field! 'body body-start) + (let (body-id (open-node! 'Body body-start)) + (let body ((cursor (fx+ opening-pos 1)) + (body-end body-start)) + (let (input (vector-ref significant cursor)) + (cond + ((equal? (token-lexeme input) "}") + (close-node! body-id 'Body body-end) + (close-field! 'body body-end) + (emit-trivia-until! (token-start input)) + (emit-token! input) + (close-node! block-id 'Block (token-end input)) + (close-field! 'item (token-end input)) + (fx+ cursor 1)) + ((eq? (token-kind input) 'newline) + (emit-trivia-until! (token-start input)) + (emit-token! input) + (body (fx+ cursor 1) (token-end input))) + ((simple-value-at? cursor) + (let (value + (vector-ref significant (fx+ cursor 2))) + (emit-attribute! + input + (vector-ref significant (fx+ cursor 1)) + value) + (body (fx+ cursor 3) + (token-end value)))) + (else + (let (next (emit-closed-block! cursor)) + (body next + (token-end + (vector-ref significant + (fx- next 1))))))))))))) + (let (root-id (open-node! 'HclFile 0)) + (let loop ((pos 0)) + (when (< pos limit) + (let (input (vector-ref significant pos)) + (if (eq? (token-kind input) 'newline) + (begin + (emit-trivia-until! (token-start input)) + (emit-token! input) + (loop (fx+ pos 1))) + (if (eq? shape 'flat-blocks) + (loop (emit-closed-block! pos)) + (begin + (emit-attribute! + input + (vector-ref significant (fx+ pos 1)) + (vector-ref significant (fx+ pos 2))) + (loop (fx+ pos 3)))))))) + (close-node! root-id 'HclFile byte-length)) + (unless (null? remaining) + (error "simple HCL source tokens remain" remaining)) + (make-success-parse-artifact/canonical-events + direct-hcl-grammar-digest source (cdr canonical-events) + source-bytes))) + (let* ((simple? (and use-simple? (simple-attributes?))) + (blocks? (and use-simple? (not simple?) (closed-blocks?)))) + (if (and use-canonical? (or simple? blocks?)) + (make-closed-canonical-artifact + (if simple? 'attributes 'flat-blocks)) + (letrec + ,(map (lambda (row) + `(,(rule-procedure (car row)) + (lambda (pos) + ,(compile-expression (cadr row) 'pos)))) + rules) + (let (end (if simple? + (begin (emit-simple-attributes!) limit) + (events-config-file 0))) + (if (and end (= end limit)) + (make-success-parse-artifact/raw-event-tape + direct-hcl-grammar-digest source tokens events event-count + (parser-machine-trivia machine) source-bytes) + #f)))))))))) +) +;;; Write the generated syntax as source, with one readable form per line of +;;; structure. `write` still owns escaping of symbols, strings, and literals. +;;; Keep the expanded module below the native source-policy limit of 1000 lines. +(def +generated-line-width+ 260) +(def (fits-on-line? form indent) + (<= (+ indent + (string-length + (call-with-output-string (lambda (port) (write form port))))) + +generated-line-width+)) + +(def (write-generated-form form port (indent 0)) + (cond + ((or (not (pair? form)) (fits-on-line? form indent)) + (write form port)) + (else + (display "(" port) + (write-generated-form (car form) port (fx+ indent 2)) + (let loop ((rest (cdr form)) (first? #t)) + (cond + ((null? rest) (display ")" port)) + ((pair? rest) + (if (and first? + (symbol? (car form)) + (memq (car form) '(def lambda let let* letrec if))) + (display " " port) + (if (pair? (car rest)) + (begin + (newline port) + (display (make-string (fx+ indent 2) #\space) port)) + (display " " port))) + (write-generated-form (car rest) port (fx+ indent 2)) + (loop (cdr rest) #f)) + (else + (display " . " port) + (write-generated-form rest port (fx+ indent 2)) + (display ")" port))))))) + +(def (emit-recursive-source descriptor port) + (display ";;; -*- Gerbil -*-\n" port) + (display ";;; Generated by src/compiler/recursive-source.ss from Grammar IR.\n" port) + (display ";;; Edit the generator, then regenerate this module.\n\n" port) + (let loop ((forms (cdr (module-expression descriptor)))) + (unless (null? forms) + (write-generated-form (car forms) port) + (newline port) + (when (pair? (cdr forms)) (newline port)) + (loop (cdr forms))))) +;;; This provider deliberately admits only the reviewed event-action corridor. +;;; The common BuildStrategy owns descriptor identity, inherited metadata and dispatch. +(def +recursive-source-provider+ + (declare-build-strategy-provider 'recursive-source 'scheme-module + (lambda (candidate) + (let (descriptor (.ref candidate 'descriptor)) + (and (equal? (parser-machine-grammar-digest (language-grammar-machine descriptor)) +simple-attribute-grammar-digest+) + (equal? (cdr (assq 'lexical-rules (language-grammar-ir descriptor))) +fast-lexical-contract+)))) + (lambda (candidate port) + (display (call-with-output-string (lambda (output) (emit-recursive-source (.ref candidate 'descriptor) output))) port)))) +(def RecursiveSourceStrategy. (.o (:: self BuildStrategy.) provider: +recursive-source-provider+)) +(def (make-recursive-source-strategy descriptor) + (make-bound-build-strategy descriptor RecursiveSourceStrategy.)) +(export RecursiveSourceStrategy. make-recursive-source-strategy) diff --git a/src/compiler/result-projection.ss b/src/compiler/result-projection.ss new file mode 100644 index 00000000..6ced3b1a --- /dev/null +++ b/src/compiler/result-projection.ss @@ -0,0 +1,82 @@ +;;; Closed result recipes lower to Scheme code with direct capture operations. +(import (only-in ../language/result-profile result-plan? result-plan-recipe)) +(export result-profile-scheme-source) + +(def (projection-operation-source operation index) + (let ((field (cadr operation)) (capture `(vector-ref captures ,index))) + (case (car operation) + ((token) + `(let (value ,capture) + ,@(if (eq? (list-ref operation 4) 'required) + `((unless value (error "missing required projection span" ',field))) '()) + (when value + (unless (and (pair? value) (exact-integer? (car value)) (exact-integer? (cdr value)) + (<= start (car value) (cdr value) end)) + (error "invalid projection span" ',field value)) + (let (token (recognition-source-token source ',(caddr operation) (car value) (cdr value))) + (set! children (cons (make-recognition-child ',field token) children)) + (set! tokens (recognition-sequence-append tokens (list token))))))) + ((one) + `(let (value ,capture) + ,@(if (eq? (list-ref operation 3) 'required) + `((unless value (error "missing required projection node" ',field))) '()) + (when value + (unless (and (pair? value) (recognition-node? (car value))) + (error "invalid projection node capture" ',field)) + (set! children (cons (make-recognition-child ',field (car value)) children)) + (set! tokens (recognition-sequence-append tokens (cdr value)))))) + ((many) + `(let (value ,capture) + (unless (and (pair? value) (list? (car value)) (andmap recognition-node? (car value))) + (error "invalid projection parts capture" ',field)) + (for-each (lambda (part) + (set! children (cons (make-recognition-child ',field part) children))) (car value)) + (set! tokens (recognition-sequence-append tokens (cdr value))))) + (else (error "unadmitted result projection opcode" operation))))) + +(def (projection-executor-source row) + (let* ((operations (cdddr row)) + (single (and (pair? operations) (null? (cdr operations)) (car operations)))) + `(cons ',(car row) + (lambda (source start end captures) + ,(cond + ((and single (eq? (car single) 'token) (eq? (list-ref single 4) 'required)) + `(let (span (vector-ref captures 0)) + (unless (and (pair? span) (exact-integer? (car span)) (exact-integer? (cdr span)) + (<= start (car span) (cdr span) end)) + (error "invalid required projection span" ',(cadr single) span)) + (let (token (recognition-source-token source ',(caddr single) (car span) (cdr span))) + (values (make-recognition-node ',(cadr row) + (recognition-source-offset source start) (recognition-source-offset source end) + (list (make-recognition-child ',(cadr single) token))) (list token))))) + ((and single (eq? (car single) 'many)) + `(let (parts (vector-ref captures 0)) + (unless (and (pair? parts) (list? (car parts)) (andmap recognition-node? (car parts))) + (error "invalid projection parts capture" ',(cadr single))) + (values (result-plan-node plan ',(cadr row) + (recognition-source-offset source start) (recognition-source-offset source end) + (map (lambda (value) (make-recognition-child ',(cadr single) value)) (car parts))) + (cdr parts)))) + (else + ;; Generated emitters own the accumulator spine; captured lists + ;; and shared recognition values never enter this reversal. + `(let ((children '()) (tokens '())) + ,@(map projection-operation-source operations (iota (length operations))) + (values (result-plan-node plan ',(cadr row) + (recognition-source-offset source start) (recognition-source-offset source end) + (reverse! children)) tokens)))))))) + +(def (result-profile-scheme-source plan) + (unless (result-plan? plan) (error "Scheme projection generation requires an admitted result plan")) + (let (recipe (result-plan-recipe plan)) + (call-with-output-string (lambda (port) + (display ";;; Generated by the shared result projection compiler.\n" port) + (for-each (lambda (form) (write form port) (newline port)) + `((import (only-in :gerbil-parser/src/language/result-profile result-plan-node result-plan-bind-projections) + (only-in :gerbil-parser/src/runtime/recognition recognition-node? make-recognition-node make-recognition-child + recognition-source-offset recognition-source-token) + (only-in :gerbil-parser/src/runtime/funcs recognition-sequence-append)) + (export bind-generated-result-projections) + (def (bind-generated-result-projections plan) + (result-plan-bind-projections plan ',recipe + (list ,@(map projection-executor-source (caddr recipe))))))))))) diff --git a/src/compiler/runtime-strategy.ss b/src/compiler/runtime-strategy.ss new file mode 100644 index 00000000..1ee4a2bf --- /dev/null +++ b/src/compiler/runtime-strategy.ss @@ -0,0 +1,35 @@ +;;; -*- Gerbil -*- +;;; Existing canonical Rust producer through the common POO build protocol. +(import (only-in :clan/poo/object .o .ref) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ./build-strategy BuildStrategy. build-strategy-common-shape? + declare-build-strategy-provider make-bound-build-strategy) + (only-in ../language/descriptor language-grammar-parser-policy) + (only-in ./rust-runtime language-rust-runtime-module-source)) +(export RustRuntimeStrategy. RustRuntimeStrategyContract make-rust-runtime-strategy) + +(def (runtime-strategy-shape? candidate) + (and (build-strategy-common-shape? candidate) + (eq? (.ref candidate 'provider) +rust-runtime-provider+) + (not (language-grammar-parser-policy (.ref candidate 'descriptor))))) +(define-type (RustRuntimeStrategyContract @ PooFlowContract.) + identity: 'gerbil-parser/rust-runtime-strategy + .classify: (lambda (candidate context) + (let (accepted? (runtime-strategy-shape? candidate)) + (poo-flow-classification-evidence + 'gerbil-parser/rust-runtime-strategy candidate accepted? + (if accepted? '() '((expected gerbil-parser/rust-runtime-strategy))) context)))) + +(def (emit-runtime-strategy strategy port) + ;; Existing producer retains portable Scanner IR and parser policy gates. + (let (source (language-rust-runtime-module-source (.ref strategy 'descriptor))) + (write-string source port))) +(def +rust-runtime-provider+ + (declare-build-strategy-provider 'rust-runtime 'rust runtime-strategy-shape? emit-runtime-strategy)) +(def RustRuntimeStrategy. + (.o (:: self BuildStrategy.) provider: +rust-runtime-provider+)) +(def (make-rust-runtime-strategy descriptor (prototype RustRuntimeStrategy.)) + (let (strategy (make-bound-build-strategy descriptor prototype)) + (validate RustRuntimeStrategyContract strategy) + strategy)) diff --git a/src/compiler/rust-pure-aot.ss b/src/compiler/rust-pure-aot.ss index 560ffbf0..c3ae2d22 100644 --- a/src/compiler/rust-pure-aot.ss +++ b/src/compiler/rust-pure-aot.ss @@ -9,11 +9,26 @@ rust-string rust-identifier rust-before rust-first-word rust-tuple-index rust-after rust-words rust-any rust-fold rust-number rust-empty - rust-string-in rust-if rust-binary)) + rust-string-in rust-single-ascii-uppercase + rust-unsigned-at-most rust-if rust-binary)) (export define-rust-pure scheme-pure->rust string-before ascii-ci=? string-after string-first-word string-rest-after-first-word - string-last-word string-before-last-word - string-prefix? string-suffix? string-words string-in?) + string-last-word string-before-last-word string-trim-start + string-replace string-lowercase + string-prefix? string-suffix? string-words string-in? + string-single-ascii-uppercase? string-unsigned-at-most?) + +(def (string-single-ascii-uppercase? value) + (and (= (string-length value) 1) + (char<=? #\A (string-ref value 0) #\Z))) + +(def (string-unsigned-at-most? value maximum) + (let (number (string->number value 10)) + (and (> (string-length value) 0) + (andmap (lambda (char) (char<=? #\0 char #\9)) + (string->list value)) + (integer? number) (exact? number) + (<= 0 number maximum)))) (def (string-before value delimiter) (let (index (string-index-from value delimiter)) @@ -26,6 +41,29 @@ (string-length value)) ""))) +(def (string-replace value needle replacement) + (when (equal? needle "") + (error "pure string replacement needs a nonempty needle")) + (let loop ((from 0) (parts '())) + (let (at (string-index-from value needle from)) + (if (fixnum? at) + (loop (+ at (string-length needle)) + (cons replacement (cons (substring value from at) parts))) + (apply string-append + (reverse (cons (substring value from (string-length value)) + parts))))))) + +(def (string-lowercase value) + (list->string (map char-downcase (string->list value)))) + +(def (string-trim-start value) + (let (size (string-length value)) + (let loop ((index 0)) + (if (or (= index size) + (not (char-whitespace? (string-ref value index)))) + (substring value index size) + (loop (+ index 1)))))) + (def (ascii-fold char) (let (code (char->integer char)) (if (and (<= 65 code) (<= code 90)) @@ -134,6 +172,15 @@ (map (lambda (argument type) (compile-pure-expression argument variables type)) arguments argument-types)))) + ((and (pair? expression) (eq? (car expression) 'string-trim-start) + (= (length expression) 2)) + (let (trimmed + (rust-method + (compile-pure-expression (cadr expression) variables "&str") + 'trim_start '())) + (if (equal? result-type "&str") + trimmed + (rust-method trimmed 'to_owned '())))) ((and (pair? expression) (eq? (car expression) 'string-trim) (= (length expression) 2)) (let (trimmed @@ -154,6 +201,40 @@ (rust-after (compile-pure-expression (cadr expression) variables "&str") (compile-pure-expression (caddr expression) variables "&str"))) + ((and (pair? expression) (eq? (car expression) 'string-replace) + (= (length expression) 4)) + (rust-method + (compile-pure-expression (cadr expression) variables "&str") + 'replace + (list (compile-pure-expression (caddr expression) variables "&str") + (compile-pure-expression (cadddr expression) variables "&str")))) + ((and (pair? expression) (eq? (car expression) 'string-lowercase) + (= (length expression) 2)) + (rust-method + (compile-pure-expression (cadr expression) variables "&str") + 'to_lowercase '())) + ((and (pair? expression) (eq? (car expression) 'string-join) + (= (length expression) 3) + (pair? (cadr expression)) + (eq? (caadr expression) 'string-words) + (= (length (cadr expression)) 2) + (string? (caddr expression))) + (rust-method + (rust-call + (rust-identifier 'Vec::from_iter) + (list (compile-pure-expression (cadr expression) variables "iterator"))) + 'join (list (rust-string (caddr expression))))) + ((and (pair? expression) (eq? (car expression) 'string-join) + (= (length expression) 3) + (pair? (cadr expression)) + (eq? (caadr expression) 'list) + (= (length (cadr expression)) 3) + (equal? (caddr expression) "")) + (rust-binary "+" + (rust-method + (compile-pure-expression (cadadr expression) variables "&str") + 'to_owned '()) + (compile-pure-expression (caddr (cadr expression)) variables "&str"))) ((and (pair? expression) (eq? (car expression) 'ascii-ci=?) (= (length expression) 3)) (rust-method @@ -251,6 +332,25 @@ (cons (cons (caadr abstraction) "&str") variables) "bool") slice?))) + ((and (pair? expression) (eq? (car expression) 'map) + (= (length expression) 3)) + (let* ((function (cadr expression)) + (collection (caddr expression)) + (signature (and (symbol? function) + (assq function (current-pure-calls))))) + (unless (and (equal? result-type "Vec") + signature + (equal? (cdr signature) '("&str")) + (pair? collection) + (eq? (car collection) 'string-words) + (= (length collection) 2)) + (error "pure AOT map requires a named &str function over words" + expression)) + (rust-method + (rust-method + (compile-pure-expression collection variables "iterator") + 'map (list (rust-identifier (rust-name-text function)))) + 'collect '()))) ((and (pair? expression) (eq? (car expression) 'foldl) (= (length expression) 4)) (let* ((abstraction (cadr expression)) @@ -290,6 +390,17 @@ (rust-string-in (compile-pure-expression (cadr expression) variables "&str") (compile-pure-expression (caddr expression) variables "&[&str]"))) + ((and (pair? expression) (eq? (car expression) 'string-single-ascii-uppercase?) + (= (length expression) 2)) + (rust-single-ascii-uppercase + (compile-pure-expression (cadr expression) variables "&str"))) + ((and (pair? expression) (eq? (car expression) 'string-unsigned-at-most?) + (= (length expression) 3) + (integer? (caddr expression)) (exact? (caddr expression)) + (<= 0 (caddr expression))) + (rust-unsigned-at-most + (compile-pure-expression (cadr expression) variables "&str") + (caddr expression))) ((and (pair? expression) (eq? (car expression) 'if) (= (length expression) 4)) (rust-if diff --git a/src/compiler/rust-rowan.ss b/src/compiler/rust-rowan.ss deleted file mode 100644 index 099a83b6..00000000 --- a/src/compiler/rust-rowan.ss +++ /dev/null @@ -1,352 +0,0 @@ -;;; -*- Gerbil -*- -;;; Canonical Parser IR v1 to immutable Rust/Rowan table generation. - -(import (only-in ../language/descriptor - language-grammar-contract - language-grammar-ir - language-grammar-language - language-grammar-machine - language-grammar-version) - (only-in ./machine parser-machine-grammar-digest)) -(export generate-language-rust-rowan-module - generate-rust-rowan-module - rust-rowan-module-source) - -(def (alist-ref rows key (default #f)) - (let (entry (assq key rows)) - (if entry (cdr entry) default))) - -(def (rust-string value) - (call-with-output-string (lambda (port) (write value port)))) - -(def (symbol-text value) - (if (symbol? value) (symbol->string value) value)) - -(def (kind-index kinds name) - (let loop ((rest kinds) (offset 0)) - (cond - ((null? rest) (error "unknown Rust/Rowan syntax kind" name)) - ((eq? (caar rest) name) offset) - (else (loop (cdr rest) (+ offset 1)))))) - -(def (emit-terminal port terminal) - (case (cadr terminal) - ((eof) (display "Terminal::Eof" port)) - ((literal) - (display "Terminal::Literal(" port) - (display (rust-string (caddr terminal)) port) - (display ")" port)) - ((token) - (display "Terminal::Token(" port) - (display (rust-string (symbol-text (caddr terminal))) port) - (display ")" port)) - (else (error "unsupported Rust/Rowan terminal" terminal)))) - -(def (emit-parser-action port action) - (case (car action) - ((shift) - (display "ParserAction::Shift(" port) - (display (cadr action) port) - (display ")" port)) - ((reduce) - (display "ParserAction::Reduce(" port) - (display (cadr action) port) - (display ")" port)) - ((accept) (display "ParserAction::Accept" port)) - ((reject-nonassoc) (display "ParserAction::RejectNonAssoc" port)) - ((fork) - (display "ParserAction::Fork(&[" port) - (let loop ((rest (cdr action))) - (unless (null? rest) - (emit-parser-action port (car rest)) - (unless (null? (cdr rest)) (display ", " port)) - (loop (cdr rest)))) - (display "])" port)) - (else (error "unsupported Rust/Rowan parser action" action)))) - -(def (emit-action-rows port rows) - (let loop ((state 0)) - (when (< state (vector-length rows)) - (display "static ACTION_" port) - (display state port) - (display ": &[ActionEntry] = &[\n" port) - (for-each - (lambda (entry) - (display " ActionEntry { terminal: " port) - (emit-terminal port (car entry)) - (display ", action: " port) - (emit-parser-action port (cdr entry)) - (display " },\n" port)) - (vector-ref rows state)) - (display "];\n" port) - (loop (+ state 1)))) - (display "static ACTIONS: &[&[ActionEntry]] = &[\n" port) - (let loop ((state 0)) - (when (< state (vector-length rows)) - (display " ACTION_" port) - (display state port) - (display ",\n" port) - (loop (+ state 1)))) - (display "];\n\n" port)) - -(def (emit-goto-rows port rows) - (let loop ((state 0)) - (when (< state (vector-length rows)) - (display "static GOTO_" port) - (display state port) - (display ": &[GotoEntry] = &[\n" port) - (for-each - (lambda (entry) - (display " GotoEntry { nonterminal: " port) - (display (rust-string (symbol-text (car entry))) port) - (display ", state: " port) - (display (cdr entry) port) - (display " },\n" port)) - (vector-ref rows state)) - (display "];\n" port) - (loop (+ state 1)))) - (display "static GOTOS: &[&[GotoEntry]] = &[\n" port) - (let loop ((state 0)) - (when (< state (vector-length rows)) - (display " GOTO_" port) - (display state port) - (display ",\n" port) - (loop (+ state 1)))) - (display "];\n\n" port)) - -(def (base-operand operand) - (if (and (pair? operand) (eq? (car operand) 'marked)) - (cadr operand) - operand)) - -(def (operand-actions operand) - (if (and (pair? operand) (eq? (car operand) 'marked)) - (caddr operand) - '())) - -(def (emit-symbol port symbol) - (case (car symbol) - ((terminal) - (display "Symbol::Terminal(" port) - (emit-terminal port symbol) - (display ")" port)) - ((nonterminal) - (display "Symbol::Nonterminal(" port) - (display (rust-string (symbol-text (cadr symbol))) port) - (display ")" port)) - (else (error "unsupported Rust/Rowan production symbol" symbol)))) - -(def (emit-operand-action port action kinds) - (case (car action) - ((field) - (display "OperandAction::Field(" port) - (display (rust-string (symbol-text (cadr action))) port) - (display ")" port)) - ((alias) - (display "OperandAction::Alias(" port) - (display (kind-index kinds (cadr action)) port) - (display ")" port)) - (else (error "unsupported Rust/Rowan operand action" action)))) - -(def (emit-operand port operand kinds) - (display "Operand { symbol: " port) - (emit-symbol port (base-operand operand)) - (display ", actions: &[" port) - (let loop ((rest (operand-actions operand))) - (unless (null? rest) - (emit-operand-action port (car rest) kinds) - (unless (null? (cdr rest)) (display ", " port)) - (loop (cdr rest)))) - (display "] }" port)) - -(def (emit-production port production kinds) - (display " Production { lhs: " port) - (display (rust-string (symbol-text (cadr production))) port) - (display ", rhs: &[" port) - (let loop ((rest (caddr production))) - (unless (null? rest) - (emit-operand port (car rest) kinds) - (unless (null? (cdr rest)) (display ", " port)) - (loop (cdr rest)))) - (display "], reduction: " port) - (case (cadddr production) - ((pass) (display "Reduction::Pass" port)) - ((concat) (display "Reduction::Concat" port)) - (else (error "unsupported Rust/Rowan reduction" - (cadddr production)))) - (display ", dynamic_precedence: " port) - (let (precedence (car (cddddr production))) - (display (if (and precedence (eq? (car precedence) 'dynamic)) - (cadr precedence) - 0) - port)) - (display " },\n" port)) - -(def (emit-rust-string-slice port values) - (display "&[" port) - (let loop ((rest values)) - (unless (null? rest) - (display (rust-string (car rest)) port) - (unless (null? (cdr rest)) (display ", " port)) - (loop (cdr rest)))) - (display "]" port)) - -(def (emit-lexical-expression port expression) - (case (car expression) - ((whitespace+) (display "LexicalExpr::Whitespace" port)) - ((horizontal-whitespace+) - (display "LexicalExpr::HorizontalWhitespace" port)) - ((newline+) (display "LexicalExpr::Newline" port)) - ((line) (display "LexicalExpr::Line" port)) - ((decimal-digit+) (display "LexicalExpr::DecimalDigits" port)) - ((number) (display "LexicalExpr::Number" port)) - ((number-literal) - (display "LexicalExpr::NumberLiteral { prefixes: " port) - (emit-rust-string-slice port (cadr expression)) - (display ", separator: " port) - (display (rust-string (caddr expression)) port) - (display ", suffixes: " port) - (emit-rust-string-slice port (cadddr expression)) - (display ", leading_period: " port) - (display (if (car (cddddr expression)) "true" "false") port) - (display ", trailing_period: " port) - (display (if (cadr (cddddr expression)) "true" "false") port) - (display " }" port)) - ((identifier) (display "LexicalExpr::Identifier" port)) - ((until-delimiters) - (display "LexicalExpr::UntilDelimiters(" port) - (display (rust-string (cadr expression)) port) - (display ")" port)) - ((quoted-string) - (display "LexicalExpr::QuotedString(" port) - (emit-rust-string-slice port (cdr expression)) - (display ")" port)) - ((escaped-quoted-string) - (display "LexicalExpr::EscapedQuotedString(" port) - (emit-rust-string-slice port (cdr expression)) - (display ")" port)) - ((heredoc) (display "LexicalExpr::Heredoc" port)) - ((line-comment) - (display "LexicalExpr::LineComment(" port) - (emit-rust-string-slice port (cdr expression)) - (display ")" port)) - ((block-comment nested-block-comment) - (display (if (eq? (car expression) 'block-comment) - "LexicalExpr::BlockComment { opening: " - "LexicalExpr::NestedBlockComment { opening: ") port) - (display (rust-string (cadr expression)) port) - (display ", closing: " port) - (display (rust-string (caddr expression)) port) - (display " }" port)) - ((choice) - (display "LexicalExpr::Choice(&[" port) - (let loop ((rest (cdr expression))) - (unless (null? rest) - (emit-lexical-expression port (car rest)) - (unless (null? (cdr rest)) (display ", " port)) - (loop (cdr rest)))) - (display "])" port)) - ((fallback) (display "LexicalExpr::Fallback" port)) - ((literals) - (display "LexicalExpr::Literals(" port) - (emit-rust-string-slice port (cdr expression)) - (display ")" port)) - ((precedence) - (emit-lexical-expression port (caddr expression))) - (else - (error "lexical expression is not yet supported by Rust/Rowan v1" - expression)))) - -(def (lexical-precedence expression) - (if (eq? (car expression) 'precedence) (cadr expression) 0)) - -(def (rust-rowan-module-source language version contract grammar-digest ir) - (unless (equal? (alist-ref ir 'schema) "gerbil-parser.parser-ir.v1") - (error "Rust/Rowan generation requires Parser IR v1" ir)) - (let* ((kinds (alist-ref ir 'syntax-kinds)) - (terminals (alist-ref ir 'terminals)) - (lexical-rules (alist-ref ir 'lexical-rules)) - (extras (map car (alist-ref ir 'extras))) - (lr (alist-ref ir 'lr-spec)) - (productions (alist-ref lr 'productions)) - (actions (alist-ref lr 'actions)) - (gotos (alist-ref lr 'gotos))) - (call-with-output-string - (lambda (port) - (display "// @generated by gerbil-parser/src/compiler/rust-rowan.ss\n" port) - (display "// Source authority: canonical Grammar IR v1; do not edit.\n\n" port) - (display "use gerbil_parser_rowan::{ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec};\n\n" port) - (display "static KINDS: &[KindSpec] = &[\n" port) - (for-each - (lambda (row) - (display " KindSpec { name: " port) - (display (rust-string (symbol-text (car row))) port) - (display ", category: KindCategory::" port) - (display (case (cadr row) ((node) "Node") ((token) "Token") - (else (error "unknown syntax-kind category" row))) - port) - (display " },\n" port)) - kinds) - (display "];\n\nstatic TERMINALS: &[TerminalSpec] = &[\n" port) - (for-each - (lambda (row) - (display " TerminalSpec { name: " port) - (display (rust-string (symbol-text (car row))) port) - (display ", syntax_kind: " port) - (display (kind-index kinds (cadr row)) port) - (display " },\n" port)) - terminals) - (display "];\n\nstatic LEXICAL_RULES: &[LexicalRule] = &[\n" port) - (for-each - (lambda (row) - (display " LexicalRule { terminal: " port) - (display (rust-string (symbol-text (car row))) port) - (display ", expression: " port) - (emit-lexical-expression port (cadr row)) - (display ", precedence: " port) - (display (lexical-precedence (cadr row)) port) - (display ", extra: " port) - (display (if (memq (car row) extras) "true" "false") port) - (display " },\n" port)) - lexical-rules) - (display "];\n\n" port) - (emit-action-rows port actions) - (emit-goto-rows port gotos) - (display "static PRODUCTIONS: &[Production] = &[\n" port) - (for-each (lambda (production) - (emit-production port production kinds)) - productions) - (display "];\n\npub static LANGUAGE: LanguageSpec = LanguageSpec {\n" port) - (display " language: " port) (display (rust-string language) port) - (display ",\n version: " port) (display (rust-string version) port) - (display ",\n contract: " port) (display (rust-string contract) port) - (display ",\n grammar_digest: " port) - (display (rust-string grammar-digest) port) - (display ",\n case_insensitive: " port) - (display (if (alist-ref ir 'case-insensitive?) "true" "false") port) - (display ",\n root_kind: " port) - (display (kind-index kinds (alist-ref ir 'root-kind)) port) - (display ",\n kinds: KINDS,\n terminals: TERMINALS,\n lexical_rules: LEXICAL_RULES,\n actions: ACTIONS,\n gotos: GOTOS,\n productions: PRODUCTIONS,\n};\n" port))))) - -(def (generate-rust-rowan-module output-path language version contract - grammar-digest ir) - (let (source - (rust-rowan-module-source - language version contract grammar-digest ir)) - (create-directory* (path-directory output-path)) - (call-with-output-file output-path - (lambda (port) (display source port))) - output-path)) - -;;; A language author supplies only deflanguage/grammar.ss. The generated -;;; descriptor already owns identity, Parser IR, machine, and grammar digest; -;;; repository/release tooling supplies only the Rust artifact destination. -(def (generate-language-rust-rowan-module output-path language-grammar) - (generate-rust-rowan-module - output-path - (language-grammar-language language-grammar) - (language-grammar-version language-grammar) - (language-grammar-contract language-grammar) - (parser-machine-grammar-digest - (language-grammar-machine language-grammar)) - (language-grammar-ir language-grammar))) diff --git a/src/compiler/rust-runtime.ss b/src/compiler/rust-runtime.ss new file mode 100644 index 00000000..54ab320e --- /dev/null +++ b/src/compiler/rust-runtime.ss @@ -0,0 +1,505 @@ +;;; -*- Gerbil -*- +;;; Canonical Parser IR v1 to immutable Rust table generation. + +(import (only-in ./lr-table validate-lr-tables layout-productions?) + (only-in ../language/result-profile result-plan? result-plan-recipe compile-result-profile) + (only-in ../grammar/lexical-algebra text-profile?) + (only-in ../language/descriptor + language-grammar-contract + language-grammar-ir + language-grammar-language + language-grammar-machine + language-grammar-version require-portable-language-policy!) + (only-in ./machine parser-machine-grammar-digest) + (only-in ./lr operand-actions validate-production-semantics production-precedence)) +(export result-profile-rust-source rust-text-profile-expression-source + generate-language-rust-runtime-module + generate-rust-runtime-module + language-rust-runtime-module-source rust-runtime-module-source) + +(def (alist-ref rows key (default #f)) + (let (entry (assq key rows)) + (if entry (cdr entry) default))) + +(def (rust-string value) + (call-with-output-string (lambda (port) (write value port)))) + +(def (symbol-text value) + (if (symbol? value) (symbol->string value) value)) + +(def (kind-index kinds name) + (let loop ((rest kinds) (offset 0)) + (cond + ((null? rest) (error "unknown Rust syntax kind" name)) + ((eq? (caar rest) name) offset) + (else (loop (cdr rest) (+ offset 1)))))) + +(def (emit-terminal port terminal) + (case (cadr terminal) + ((eof) (display "Terminal::Eof" port)) + ((literal) + (display "Terminal::Literal(" port) + (display (rust-string (caddr terminal)) port) + (display ")" port)) + ((token) + (display "Terminal::Token(" port) + (display (rust-string (symbol-text (caddr terminal))) port) + (display ")" port)) + (else (error "unsupported Rust terminal" terminal)))) + +(def (emit-parser-action port action) + (case (car action) + ((shift) + (display "ParserAction::Shift(" port) + (display (cadr action) port) + (display ")" port)) + ((reduce) + (display "ParserAction::Reduce(" port) + (display (cadr action) port) + (display ")" port)) + ((accept) (display "ParserAction::Accept" port)) + ((reject-nonassoc) (display "ParserAction::RejectNonAssoc" port)) + ((fork) + (display "ParserAction::Fork(&[" port) + (let loop ((rest (cdr action))) + (unless (null? rest) + (emit-parser-action port (car rest)) + (unless (null? (cdr rest)) (display ", " port)) + (loop (cdr rest)))) + (display "])" port)) + (else (error "unsupported Rust parser action" action)))) + +(def (emit-action-rows port rows) + (let loop ((state 0)) + (when (< state (vector-length rows)) + (display "static ACTION_" port) + (display state port) + (display ": &[ActionEntry] = &[\n" port) + (for-each + (lambda (entry) + (display " ActionEntry { terminal: " port) + (emit-terminal port (car entry)) + (display ", action: " port) + (emit-parser-action port (cdr entry)) + (display " },\n" port)) + (vector-ref rows state)) + (display "];\n" port) + (loop (+ state 1)))) + (display "static ACTIONS: &[&[ActionEntry]] = &[\n" port) + (let loop ((state 0)) + (when (< state (vector-length rows)) + (display " ACTION_" port) + (display state port) + (display ",\n" port) + (loop (+ state 1)))) + (display "];\n\n" port)) + +(def (emit-goto-rows port rows) + (let loop ((state 0)) + (when (< state (vector-length rows)) + (display "static GOTO_" port) + (display state port) + (display ": &[GotoEntry] = &[\n" port) + (for-each + (lambda (entry) + (display " GotoEntry { nonterminal: " port) + (display (rust-string (symbol-text (car entry))) port) + (display ", state: " port) + (display (cdr entry) port) + (display " },\n" port)) + (vector-ref rows state)) + (display "];\n" port) + (loop (+ state 1)))) + (display "static GOTOS: &[&[GotoEntry]] = &[\n" port) + (let loop ((state 0)) + (when (< state (vector-length rows)) + (display " GOTO_" port) + (display state port) + (display ",\n" port) + (loop (+ state 1)))) + (display "];\n\n" port)) + +(def (base-operand operand) + (if (and (pair? operand) (eq? (car operand) 'marked)) + (cadr operand) + operand)) + +(def (emit-symbol port symbol) + (case (car symbol) + ((terminal) + (display "Symbol::Terminal(" port) + (emit-terminal port symbol) + (display ")" port)) + ((nonterminal) + (display "Symbol::Nonterminal(" port) + (display (rust-string (symbol-text (cadr symbol))) port) + (display ")" port)) + (else (error "unsupported Rust production symbol" symbol)))) + +(def (emit-operand-action port action kinds) + (case (car action) + ((field) + (display "OperandAction::Field(" port) + (display (rust-string (symbol-text (cadr action))) port) + (display ")" port)) + ((alias) + (display "OperandAction::Alias(" port) + (display (kind-index kinds (cadr action)) port) + (display ")" port)) + (else (error "unsupported Rust operand action" action)))) + +(def (emit-operand port operand kinds) + (display "Operand { symbol: " port) + (emit-symbol port (base-operand operand)) + (display ", actions: &[" port) + (let loop ((rest (operand-actions operand))) + (unless (null? rest) + (emit-operand-action port (car rest) kinds) + (unless (null? (cdr rest)) (display ", " port)) + (loop (cdr rest)))) + (display "] }" port)) + +(def (emit-production port production kinds) + (display " Production { lhs: " port) + (display (rust-string (symbol-text (cadr production))) port) + (display ", rhs: &[" port) + (let loop ((rest (caddr production))) + (unless (null? rest) + (emit-operand port (car rest) kinds) + (unless (null? (cdr rest)) (display ", " port)) + (loop (cdr rest)))) + (display "], reduction: " port) + (case (cadddr production) + ((pass) (display "Reduction::Pass" port)) + ((concat) (display "Reduction::Concat" port)) + (else (error "unsupported Rust reduction" + (cadddr production)))) + (display ", dynamic_precedence: " port) + (let (precedence (car (cddddr production))) + (display (if (and precedence (eq? (car precedence) 'dynamic)) + (cadr precedence) + 0) + port)) + (display " },\n" port)) + +(def (emit-rust-string-slice port values) + (display "&[" port) + (let loop ((rest values)) + (unless (null? rest) + (display (rust-string (car rest)) port) + (unless (null? (cdr rest)) (display ", " port)) + (loop (cdr rest)))) + (display "]" port)) + +(def (emit-text-class port expression) + (case (car expression) + ((alphabetic) (display "gerbil_parser_runtime::TextClass::Alphabetic" port)) + ((numeric) (display "gerbil_parser_runtime::TextClass::Numeric" port)) + ((ascii-letter) (display "gerbil_parser_runtime::TextClass::AsciiLetter" port)) + ((characters) (display "gerbil_parser_runtime::TextClass::Characters(" port) + (display (rust-string (cadr expression)) port) (display ")" port)) + ((union) (display "gerbil_parser_runtime::TextClass::Union(&[" port) + (for-each (lambda (child) (emit-text-class port child) (display "," port)) (cdr expression)) + (display "])" port)))) + +(def (rust-text-profile-expression-source expression) + (unless (text-profile? expression) (error "invalid or nullable Rust text profile" expression)) + (let (port (open-output-string)) + (emit-text-profile port expression) (get-output-string port))) +(def (emit-text-profile port expression) + (case (car expression) + ((literal) (display "gerbil_parser_runtime::TextProfile::Literal(" port) + (display (rust-string (cadr expression)) port) (display ")" port)) + ((run) (display "gerbil_parser_runtime::TextProfile::Run { class: " port) (emit-text-class port (cadr expression)) + (display ", minimum: " port) (display (caddr expression) port) + (display ", maximum: " port) + (if (cadddr expression) + (begin (display "Some(" port) (display (cadddr expression) port) (display ")" port)) + (display "None" port)) (display " }" port)) + ((run-containing) + (display "gerbil_parser_runtime::TextProfile::RunContaining { class: " port) (emit-text-class port (cadr expression)) + (display ", required: " port) (emit-text-class port (caddr expression)) + (display ", minimum: " port) (display (cadddr expression) port) (display ", maximum: " port) + (if (car (cddddr expression)) (begin (display "Some(" port) (display (car (cddddr expression)) port) (display ")" port)) (display "None" port)) + (display " }" port)) + ((ends-in ends-not-in) + (display "gerbil_parser_runtime::TextProfile::EndsIn { class: " port) (emit-text-class port (cadr expression)) + (display ", body: &" port) (emit-text-profile port (caddr expression)) + (display (if (eq? (car expression) 'ends-in) ", positive: true }" ", positive: false }") port)) + ((seq) (display "gerbil_parser_runtime::TextProfile::Sequence(&[" port) + (for-each (lambda (child) (emit-text-profile port child) (display "," port)) (cdr expression)) + (display "])" port)) + ((optional) (display "gerbil_parser_runtime::TextProfile::Optional(&" port) (emit-text-profile port (cadr expression)) (display ")" port)) + ((if-next) (display "gerbil_parser_runtime::TextProfile::IfNext { class: " port) (emit-text-class port (cadr expression)) + (display ", body: &" port) (emit-text-profile port (caddr expression)) + (display ", otherwise: " port) + (if (= (length expression) 4) + (begin (display "Some(&" port) (emit-text-profile port (cadddr expression)) (display ")" port)) + (display "None" port)) (display " }" port)) + ((not-next) (display "gerbil_parser_runtime::TextProfile::NotNext(" port) (emit-text-class port (cadr expression)) (display ")" port)))) + +(def (emit-lexical-expression port expression) + (case (car expression) + ((text-profile) (display "LexicalExpr::TextProfile(&" port) + (emit-text-profile port (cadr expression)) (display ")" port)) + ((whitespace+) (display "LexicalExpr::Whitespace" port)) + ((horizontal-whitespace+) + (display "LexicalExpr::HorizontalWhitespace" port)) + ((newline+) (display "LexicalExpr::Newline" port)) + ((line) (display "LexicalExpr::Line" port)) + ((decimal-digit+) (display "LexicalExpr::DecimalDigits" port)) + ((number) (display "LexicalExpr::Number" port)) + ((number-literal) + (display "LexicalExpr::NumberLiteral { prefixes: " port) + (emit-rust-string-slice port (cadr expression)) + (display ", separator: " port) + (display (rust-string (caddr expression)) port) + (display ", suffixes: " port) + (emit-rust-string-slice port (cadddr expression)) + (display ", leading_period: " port) + (display (if (car (cddddr expression)) "true" "false") port) + (display ", trailing_period: " port) + (display (if (cadr (cddddr expression)) "true" "false") port) + (display " }" port)) + ((identifier) (display "LexicalExpr::Identifier" port)) + ((quoted-string-profile) + (display "LexicalExpr::QuotedStringProfile { delimiter: " port) + (display (rust-string (cadr expression)) port) + (display ", escapes: " port) + (display (rust-string (caddr expression)) port) + (display ", unicode_width: " port) + (display (cadddr expression) port) + (display " }" port)) + ((module-text) + (display "LexicalExpr::ModuleText(&gerbil_parser_runtime::ModuleTextProfile { " port) + (for-each (lambda (name value) (display name port) (display ": " port) + (display (rust-string value) port) (display ", " port)) + '("header_border" "header_word" "end_border" "block_open" "block_close" "line_comment" "name_extra") + (cdr expression)) + (display "})" port)) + ((header-delimiter header-data) + (display (if (eq? (car expression) 'header-delimiter) + "LexicalExpr::HeaderDelimiter { prefix: " "LexicalExpr::HeaderData { prefix: ") port) + (display (rust-string (cadr expression)) port) + (display ", count: " port) (display (caddr expression) port) + (display (if (eq? (car expression) 'header-delimiter) ", index: " ", stops: ") port) + (display (if (eq? (car expression) 'header-delimiter) + (cadddr expression) (rust-string (cadddr expression))) port) + (display " }" port)) + ((until-delimiters) + (display "LexicalExpr::UntilDelimiters(" port) + (display (rust-string (cadr expression)) port) + (display ")" port)) + ((character-run) + (display "LexicalExpr::CharacterRun { character: " port) + (display (rust-string (cadr expression)) port) + (display ", minimum: " port) + (display (caddr expression) port) + (display " }" port)) + ((quoted-string) + (display "LexicalExpr::QuotedString(" port) + (emit-rust-string-slice port (cdr expression)) + (display ")" port)) + ((escaped-quoted-string) + (display "LexicalExpr::EscapedQuotedString(" port) + (emit-rust-string-slice port (cdr expression)) + (display ")" port)) + ((heredoc) (display "LexicalExpr::Heredoc" port)) + ((line-comment) + (display "LexicalExpr::LineComment(" port) + (emit-rust-string-slice port (cdr expression)) + (display ")" port)) + ((block-comment nested-block-comment) + (display (if (eq? (car expression) 'block-comment) + "LexicalExpr::BlockComment { opening: " + "LexicalExpr::NestedBlockComment { opening: ") port) + (display (rust-string (cadr expression)) port) + (display ", closing: " port) + (display (rust-string (caddr expression)) port) + (display " }" port)) + ((choice) + (display "LexicalExpr::Choice(&[" port) + (let loop ((rest (cdr expression))) + (unless (null? rest) + (emit-lexical-expression port (car rest)) + (unless (null? (cdr rest)) (display ", " port)) + (loop (cdr rest)))) + (display "])" port)) + ((fallback) (display "LexicalExpr::Fallback" port)) + ((literals) + (display "LexicalExpr::Literals(" port) + (emit-rust-string-slice port (cdr expression)) + (display ")" port)) + ((precedence) + (emit-lexical-expression port (caddr expression))) + (else + (error "lexical expression is not yet supported by Rust v1" + expression)))) + +(def (lexical-precedence expression) + (if (eq? (car expression) 'precedence) (cadr expression) 0)) + +;;; Canonical Scheme legality and target representability are separate gates. +;;; Check immutable model capacities before allocating the emitted source. +(def (validate-rust-lr-capability kinds productions actions) + (when (> (length kinds) 65536) + (error "Rust LR kind catalog exceeds u16 capacity")) + (when (or (> (length productions) 4294967296) + (> (vector-length actions) 4294967296)) + (error "Rust LR reference domain exceeds u32 capacity")) + (when (layout-productions? productions) + (error "Rust LR runtime does not support layout productions")) + (for-each + (lambda (production) + (let (precedence (production-precedence production)) + (when (and precedence (eq? (car precedence) 'dynamic) + (not (<= -2147483648 (cadr precedence) 2147483647))) + (error "Rust LR dynamic precedence exceeds i32 capacity" production)))) + productions)) + +(def (rust-runtime-module-source language version contract grammar-digest ir) + (unless (equal? (alist-ref ir 'schema) "gerbil-parser.parser-ir.v1") + (error "Rust generation requires Parser IR v1" ir)) + (let* ((kinds (alist-ref ir 'syntax-kinds)) + (terminals (alist-ref ir 'terminals)) + (lexical-rules (alist-ref ir 'lexical-rules)) + (extras (map car (alist-ref ir 'extras))) + (lr (alist-ref ir 'lr-spec)) + (productions (validate-production-semantics (alist-ref lr 'productions))) + (actions (alist-ref lr 'actions)) + (gotos (alist-ref lr 'gotos))) + (validate-lr-tables productions actions gotos) + (validate-rust-lr-capability kinds productions actions) + (call-with-output-string + (lambda (port) + (display "// @generated by gerbil-parser/src/compiler/rust-runtime.ss\n" port) + (display "// Source authority: canonical Grammar IR v1; do not edit.\n\n" port) + (display "use gerbil_parser_runtime::{ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec};\n\n" port) + (display "static KINDS: &[KindSpec] = &[\n" port) + (for-each + (lambda (row) + (display " KindSpec { name: " port) + (display (rust-string (symbol-text (car row))) port) + (display ", category: KindCategory::" port) + (display (case (cadr row) ((node) "Node") ((token) "Token") + (else (error "unknown syntax-kind category" row))) + port) + (display " },\n" port)) + kinds) + (display "];\n\nstatic TERMINALS: &[TerminalSpec] = &[\n" port) + (for-each + (lambda (row) + (display " TerminalSpec { name: " port) + (display (rust-string (symbol-text (car row))) port) + (display ", syntax_kind: " port) + (display (kind-index kinds (cadr row)) port) + (display " },\n" port)) + terminals) + (display "];\n\nstatic LEXICAL_RULES: &[LexicalRule] = &[\n" port) + (for-each + (lambda (row) + (display " LexicalRule { terminal: " port) + (display (rust-string (symbol-text (car row))) port) + (display ", expression: " port) + (emit-lexical-expression port (cadr row)) + (display ", precedence: " port) + (display (lexical-precedence (cadr row)) port) + (display ", extra: " port) + (display (if (memq (car row) extras) "true" "false") port) + (display " },\n" port)) + lexical-rules) + (display "];\n\n" port) + (emit-action-rows port actions) + (emit-goto-rows port gotos) + (display "static PRODUCTIONS: &[Production] = &[\n" port) + (for-each (lambda (production) + (emit-production port production kinds)) + productions) + (display "];\n\npub static LANGUAGE: LanguageSpec = LanguageSpec {\n" port) + (display " language: " port) (display (rust-string language) port) + (display ",\n version: " port) (display (rust-string version) port) + (display ",\n contract: " port) (display (rust-string contract) port) + (display ",\n grammar_digest: " port) + (display (rust-string grammar-digest) port) + (display ",\n case_insensitive: " port) + (display (if (alist-ref ir 'case-insensitive?) "true" "false") port) + (display ",\n root_kind: " port) + (display (kind-index kinds (alist-ref ir 'root-kind)) port) + (display ",\n kinds: KINDS,\n terminals: TERMINALS,\n lexical_rules: LEXICAL_RULES,\n actions: ACTIONS,\n gotos: GOTOS,\n productions: PRODUCTIONS,\n};\n" port))))) + +(def (write-rust-runtime-module! output-path source) + (create-directory* (path-directory output-path)) + (call-with-output-file output-path (lambda (port) (display source port))) + output-path) + +;;; Raw IR describes recognition only. Complete descriptor products must use +;;; the admission boundary below, including contextual scanner products. +(def (generate-rust-runtime-module output-path language version contract + grammar-digest ir) + (write-rust-runtime-module! + output-path (rust-runtime-module-source language version contract grammar-digest ir))) + +;;; The sole descriptor-to-Rust source boundary. All product frontends delegate +;;; identity, capability admission and recognition emission here. Policy failure +;;; precedes product preparation, directory creation and output publication. +(def (language-rust-runtime-module-source descriptor) + (require-portable-language-policy! descriptor) + (rust-runtime-module-source + (language-grammar-language descriptor) + (language-grammar-version descriptor) + (language-grammar-contract descriptor) + (parser-machine-grammar-digest (language-grammar-machine descriptor)) + (language-grammar-ir descriptor))) + +(def (generate-language-rust-runtime-module output-path descriptor) + (write-rust-runtime-module! output-path (language-rust-runtime-module-source descriptor))) + +;;; Source results use the same admitted constructors as the Scheme executor. +;;; This producer emits data only; languages supply no Rust tree-building code. +(def (result-profile-rust-source profile) + (let* ((plan (if (result-plan? profile) profile (compile-result-profile profile))) + (recipe (result-plan-recipe plan)) (nodes (car recipe)) (tokens (cadr recipe))) + (when (> (+ (length nodes) (length tokens)) 65536) + (error "result catalog exceeds the Rust kind index capacity")) + (def (node-index kind) (kind-index nodes kind)) + (def (token-index kind) (+ (length nodes) (kind-index (map list tokens) kind))) + (call-with-output-string (lambda (port) + (display "// @generated from the admitted Scheme ResultProfile IR.\n" port) + (display "use gerbil_parser_runtime::{CaptureKind, CaptureSpec, KindSpec, KindCategory, ProjectionOpcode, ProjectionInstruction, ResultProjection, ResultNodeSpec, ResultProfileSpec};\n" port) + (display "pub static RESULT_PROFILE: ResultProfileSpec = ResultProfileSpec { kinds: &[\n" port) + (for-each (lambda (row) + (display "KindSpec {name: " port) (write (symbol->string (car row)) port) + (display ", category: KindCategory::Node},\n" port)) nodes) + (for-each (lambda (kind) + (display "KindSpec {name: " port) (write (symbol->string kind) port) + (display ", category: KindCategory::Token},\n" port)) tokens) + (display "], nodes: &[\n" port) + (for-each (lambda (row) + (display "ResultNodeSpec {kind: " port) (display (node-index (car row)) port) + (display ", fields: &[" port) + (for-each (lambda (field) (write (symbol->string field) port) (display "," port)) (cadr row)) + (display "]},\n" port)) nodes) + (display "], projections: &[\n" port) + (for-each (lambda (row) + (display "ResultProjection {id: " port) (write (symbol->string (car row)) port) + (display ", kind: " port) (display (node-index (cadr row)) port) + (display ", captures: &[" port) + (for-each (lambda (capture) + (display "CaptureSpec {name: " port) (write (symbol->string (car capture)) port) + (display ", kind: CaptureKind::" port) + (display (case (cadr capture) ((span) "Span") ((node) "Node") ((parts) "Parts")) port) + (display "}," port)) (caddr row)) + (display "], instructions: &[" port) + (for-each (lambda (operation) + (display "ProjectionInstruction {field: " port) (write (symbol->string (cadr operation)) port) + (display ", opcode: ProjectionOpcode::" port) + (case (car operation) + ((token) (display "Token {kind: " port) (display (token-index (caddr operation)) port) + (display ", required: " port) (display (if (eq? (list-ref operation 4) 'required) "true" "false") port) + (display "}" port)) + ((one) (display "One {required: " port) (display (if (eq? (list-ref operation 3) 'required) "true" "false") port) + (display "}" port)) + ((many) (display "Many" port))) + (display "}," port)) (cdddr row)) + (display "]},\n" port)) (caddr recipe)) + (display "]};\n" port))))) diff --git a/src/compiler/rust-scanner.ss b/src/compiler/rust-scanner.ss new file mode 100644 index 00000000..49f2b9d0 --- /dev/null +++ b/src/compiler/rust-scanner.ss @@ -0,0 +1,358 @@ +;;; -*- Gerbil -*- +;;; Shared contextual Scanner IR to immutable Rust declarations. +(import (only-in :clan/poo/object .ref) + (only-in ../language/source-strategy bind-source-strategy source-engine-program) + (only-in ./contextual-program contextual-program-ir contextual-program-regions + contextual-program-scanner contextual-program-results contextual-program-parts contextual-program-commands) + (only-in ../language/scanner-profile compile-scanner-profile) + (only-in ../language/command-profile compile-command-profile command-plan? command-plan-recipe admit-command-plan) + (only-in ../runtime/region-scanner region-plan-specification prepare-scoped-region-source) + (only-in ../runtime/contextual-scanner prepare-contextual-scanner-plan) + (only-in ./contextual-scanner-ir contextual-scanner-ir-ref) + (only-in ./rust-runtime language-rust-runtime-module-source rust-text-profile-expression-source result-profile-rust-source) + (only-in ../language/part-profile part-plan? compile-part-profile part-plan-recipe part-plan-binding binding-plan-region-plan binding-plan-region-scopes admit-part-plan) + (only-in ../language/result-profile result-plan? compile-result-profile) + (only-in ../language/descriptor language-grammar-ir language-grammar-machine) + (only-in ./machine parser-machine-grammar-digest) + (only-in ../runtime/parser-ir-data parser-ir-ref) + (only-in ../runtime/parser prepare-contextual-parser)) +(export command-source-rust-module-source generate-command-source-rust-module command-profile-rust-source part-profile-rust-source region-plan-rust-source rust-scanner-module-source generate-rust-scanner-module generate-contextual-language-rust-runtime-module) +(def (rust-text port value) + (display "\"" port) + (for-each + (lambda (ch) + (case ch + ((#\\) (display "\\\\" port)) + ((#\") (display "\\\"" port)) + ((#\newline) (display "\\n" port)) + ((#\return) (display "\\r" port)) + ((#\tab) (display "\\t" port)) + (else + (let (code (char->integer ch)) + (if (or (< code 32) (= code 127)) + (begin (display "\\u{" port) (display (number->string code 16) port) (display "}" port)) + (write-char ch port)))))) (string->list value)) + (display "\"" port)) +(def (rust-name port name) (rust-text port (symbol->string name))) +(def (rust-strings port values) + (display "&[" port) + (for-each (lambda (value) (rust-text port value) (display "," port)) values) + (display "]" port)) +(def (rust-symbols port values) (rust-strings port (map symbol->string values))) +(def (rust-char port ch) + (display "'\\u{" port) (display (number->string (char->integer ch) 16) port) + (display "}'" port)) +(def (emit-region-fields port stops quotes pairs initial-stop) + (display "stops: " port) (rust-strings port stops) + (display ", quotes: &[" port) + (for-each + (lambda (row) + (display "gerbil_parser_runtime::scanner::RegionQuote { delimiter: " port) + (rust-char port (car row)) + (display (if (cadr row) ", escaped: true, pairs: " ", escaped: false, pairs: ") port) + (rust-strings port (caddr row)) (display " }," port)) quotes) + (display "], pairs: &[" port) + (for-each + (lambda (row) + (display "gerbil_parser_runtime::scanner::RegionPair { prefix: " port) + (rust-text port (car row)) (display ", opening: " port) (rust-char port (cadr row)) + (display ", closing: " port) (rust-char port (caddr row)) + (display ", depth: " port) (display (cadddr row) port) (display " }," port)) pairs) + (display (if initial-stop "], consume_initial_stop: true" "], consume_initial_stop: false") port)) +(def (region-plan-rust-source plan (scopes '())) + (prepare-scoped-region-source plan "" scopes) + (let ((data (region-plan-specification plan)) (port (open-output-string))) + (display "pub static REGION: gerbil_parser_runtime::scanner::RegionSpec = gerbil_parser_runtime::scanner::RegionSpec { " port) + (apply emit-region-fields port data) + (display " };\npub static SCOPES: &[gerbil_parser_runtime::scanner::RegionScope] = &[" port) + (for-each (lambda (scope) + (display "gerbil_parser_runtime::scanner::RegionScope { prefix: " port) (rust-text port (car scope)) + (display ", pairs: " port) (rust-strings port (cdr scope)) (display " }," port)) scopes) + (display "];\n" port) (get-output-string port))) +(def (emit-matcher port matcher) + (display "ScannerMatcher::" port) + (match matcher + (['literal text] (display "Literal(" port) (rust-text port text) (display ")" port)) + (['literals values] (display "Literals(" port) (rust-strings port values) (display ")" port)) + (['quoted-string values] (display "QuotedString(" port) (rust-strings port values) (display ")" port)) + (['unless-prefix prefixes exceptions child] + (display "UnlessPrefix { prefixes: " port) (rust-strings port prefixes) + (display ", exceptions: " port) (rust-strings port exceptions) + (display ", child: &" port) (emit-matcher port child) (display " }" port)) + (['line-prefix prefix separator] + (display "LinePrefix { prefix: " port) (rust-text port prefix) + (display ", separator: " port) (rust-char port (string-ref separator 0)) (display " }" port)) + (['marker-line-at separator] + (display "MarkerLineAt(" port) (rust-char port (string-ref separator 0)) (display ")" port)) + (['body-line-at separator] + (display "BodyLineAt(" port) (rust-char port (string-ref separator 0)) (display ")" port)) + (['region-word [stops quotes pairs initial-stop]] + (display "RegionWord { " port) + (emit-region-fields port stops quotes pairs initial-stop) + (display " }" port)) + (['balanced-word stops quotes pairs] + (display "BalancedWord { stops: " port) (rust-strings port stops) + (display ", quotes: " port) (rust-strings port quotes) + (display ", pairs: &[" port) + (for-each + (lambda (pair) + (display "BalancedPair { prefix: " port) (rust-text port (car pair)) + (display ", opening: " port) (rust-char port (cadr pair)) + (display ", closing: " port) (rust-char port (caddr pair)) + (display " }," port)) pairs) + (display "] }" port)) + ([opcode] + (display (case opcode + ((horizontal-whitespace+) "HorizontalWhitespace") + ((newline) "Newline") ((newline-one) "NewlineOne") + ((identifier) "Identifier") ((marker-line) "MarkerLine") + ((body-line) "BodyLine") + (else (error "unsupported Rust scanner opcode" opcode))) port)) + (else (error "unsupported Rust scanner matcher" matcher)))) +(def (emit-action port action) + (display "ScannerAction::" port) + (match action + ('keep (display "Keep" port)) + (['expect-marker strip?] + (display (if strip? "ExpectMarker(true)" "ExpectMarker(false)") port)) + (['expect-marker-in strip? mode] + (display (if strip? "ExpectMarkerIn { strip_tabs: true, mode: " "ExpectMarkerIn { strip_tabs: false, mode: ") port) + (rust-name port mode) (display " }" port)) + (['enqueue-marker-in policy mode] + (display "EnqueueMarkerIn { policy: MarkerPolicy::" port) + (display (case policy ((raw) "Raw") ((shell-quote-removal) "ShellQuoteRemoval") + (else (error "unsupported Rust delimiter policy" policy))) port) + (display ", mode: " port) (rust-name port mode) (display " }" port)) + (['enqueue-if-expecting policy] + (display "EnqueueIfExpecting(MarkerPolicy::" port) + (display (case policy ((raw) "Raw") ((shell-quote-removal) "ShellQuoteRemoval") + (else (error "unsupported Rust delimiter policy" policy))) port) + (display ")" port)) + (['activate-next mode] + (display "ActivateNext(" port) (rust-name port mode) (display ")" port)) + (['finish-marker base body] + (display "FinishMarker { base: " port) (rust-name port base) + (display ", body: " port) (rust-name port body) (display " }" port)) + (else (error "unsupported Rust scanner action" action)))) +(def (rust-scanner-module-source ir) + ;; Runtime admission verifies the Scheme-owned digest/opcode contract before + ;; any generated output. Languages provide no Rust source or callbacks. + (prepare-contextual-scanner-plan ir) + (call-with-output-string + (lambda (port) + (display "// @generated from Scheme contextual Scanner IR; do not edit.\n" port) + (display "#[allow(unused_imports)]\nuse gerbil_parser_runtime::scanner::{BalancedPair, MarkerPolicy, ScannerAction, ScannerCell, ScannerMatcher, ScannerRule, ScannerSpec};\n" port) + (display "pub static SCANNER: ScannerSpec = ScannerSpec {\n opcode_contract: " port) + (rust-text port (contextual-scanner-ir-ref ir 'opcode-contract)) + (display ",\n digest: " port) (rust-text port (contextual-scanner-ir-ref ir 'digest)) + (display ",\n base_grammar_digest: " port) + (let (digest (contextual-scanner-ir-ref ir 'grammar-digest)) + (if digest (begin (display "Some(" port) (rust-text port digest) (display ")" port)) + (display "None" port))) + (display ",\n initial_mode: " port) (rust-name port (contextual-scanner-ir-ref ir 'initial-mode)) + (display ",\n modes: " port) (rust-symbols port (contextual-scanner-ir-ref ir 'modes)) + (display ",\n positions: " port) (rust-symbols port (contextual-scanner-ir-ref ir 'positions)) + (display ",\n rules: &[\n" port) + (for-each + (lambda (row) + (display "ScannerRule { name: " port) (rust-name port (list-ref row 0)) + (display ", mode: " port) (rust-name port (list-ref row 1)) + (display ", form: " port) (rust-name port (list-ref row 2)) + (display ", matcher: " port) (emit-matcher port (list-ref row 3)) + (unless (<= -9223372036854775808 (list-ref row 4) 9223372036854775807) + (error "scanner rank exceeds Rust i64" (list-ref row 4))) + (display ", rank: " port) (display (list-ref row 4) port) + (display ", action: " port) (emit-action port (list-ref row 5)) + (display " },\n" port)) (contextual-scanner-ir-ref ir 'rules)) + (display "],\n cells: &[\n" port) + (for-each + (lambda (row) + (display "ScannerCell { mode: " port) (rust-name port (list-ref row 0)) + (display ", position: " port) (rust-name port (list-ref row 1)) + (display ", form: " port) (rust-name port (list-ref row 2)) + (display ", terminal: " port) (rust-name port (car (list-ref row 3))) + (display " },\n" port)) (filter (lambda (row) (list-ref row 3)) (contextual-scanner-ir-ref ir 'cells))) + (display "]\n};\n" port)))) +(def (generate-rust-scanner-module path ir) + (let (source (rust-scanner-module-source ir)) + (create-directory* (path-directory path)) + (call-with-output-file path (lambda (port) (display source port))) + path)) + +;;; One downstream descriptor and its compiled contextual declaration produce +;;; the complete Rust language/parser/scanner data product. +(def (generate-contextual-language-rust-runtime-module path descriptor product) + (let (language-source (language-rust-runtime-module-source descriptor)) + (prepare-contextual-parser (language-grammar-machine descriptor) product) + (let (actions (cdr (assq 'actions (parser-ir-ref (language-grammar-ir descriptor) 'lr-spec)))) + (when (any (lambda (row) (any (lambda (entry) (eq? (cadr entry) 'fork)) row)) + (vector->list actions)) + (error "Rust contextual AOT does not admit GLR scanner branch state"))) + (let (scanner-source (rust-scanner-module-source (cdr (assq 'scanner product)))) + (create-directory* (path-directory path)) + (call-with-output-file path + (lambda (port) + (display language-source port) + (display "pub mod contextual_scanner {\n" port) (display scanner-source port) + (display "}\npub static CONTEXTUAL: gerbil_parser_runtime::ContextualParserSpec = gerbil_parser_runtime::ContextualParserSpec { language: &LANGUAGE, scanner: &contextual_scanner::SCANNER, parser_digest: " port) + (rust-text port (cdr (assq 'digest product))) + (display ", state_positions: " port) + (rust-symbols port (map cdr (cdr (assq 'state-positions (cdr (assq 'context product)))))) + (display " };\n" port))) + path))) + + +;;; Lower admitted POO recognition slots; downstream languages supply no Rust. +(def (part-profile-rust-source profile results regions) + (let* ((plan (if (part-plan? profile) profile (compile-part-profile profile))) + (result-plan (if (result-plan? results) results (compile-result-profile results))) + (_ (admit-part-plan plan result-plan regions)) + (recipe (part-plan-recipe plan)) (bindings (list-ref recipe 3)) + (port (open-output-string))) + (def (binding-index role) + (let loop ((rows bindings) (index 0)) + (cond ((null? rows) (error "unknown part binding" role)) + ((eq? (caar rows) role) index) (else (loop (cdr rows) (+ index 1)))))) + (display "use gerbil_parser_runtime::{BindingSpec, SubscriptSpec, PartGuard, PartOpcode, PartRule, PartProfileSpec};\n" port) + (display "pub mod regions {\n" port) (display (region-plan-rust-source regions) port) (display "}\n" port) + (for-each (lambda (row index) + (let* ((binding (part-plan-binding plan (car row))) (derived (binding-plan-region-plan binding regions))) + (when derived + (display "mod subscript_" port) (display index port) (display " {\n" port) + (display (region-plan-rust-source derived (binding-plan-region-scopes binding regions)) port) + (display "}\n" port)))) bindings (iota (length bindings))) + (display "pub static PART_PROFILE: PartProfileSpec = PartProfileSpec { contexts: " port) + (rust-symbols port (car recipe)) (display ", literal: " port) (rust-name port (caddr recipe)) + (display ", assignment: " port) + (if (assq 'assignment bindings) + (begin (display "Some(" port) (display (binding-index 'assignment) port) (display ")" port)) + (display "None" port)) + (display ", bindings: &[" port) + (for-each (lambda (row index) + (let* ((binding (cadr row)) (subscript (cadddr binding))) + (display "BindingSpec { name: &" port) (display (rust-text-profile-expression-source (car binding)) port) + (display ", prefixes: " port) (rust-strings port (cadr binding)) + (display ", operators: " port) (rust-strings port (caddr binding)) + (display ", subscript: " port) + (if subscript + (begin (display "Some(SubscriptSpec { prefix: " port) (rust-text port (car subscript)) + (display ", closing: " port) (rust-char port (cadr subscript)) + (display ", regions: &subscript_" port) (display index port) (display "::REGION, scopes: subscript_" port) + (display index port) (display "::SCOPES })" port)) + (display "None" port)) + (display " }," port))) bindings (iota (length bindings))) + (display "], rules: &[" port) + (for-each (lambda (row) + (let* ((prefix (cadadr row)) (guard (caddr (cadr row))) (form (caddr row)) + (operation (car form)) (kind (if (eq? operation 'parameter) 'ParameterExpansion (cadr form))) + (opening (case operation ((parameter) 2) ((name escape) 1) (else (caddr form)))) + (closing (case operation ((parameter) 1) ((name escape) 0) (else (cadddr form))))) + (display "PartRule { contexts: " port) (rust-symbols port (car row)) + (display ", prefix: " port) (rust-text port prefix) + (display ", guard: PartGuard::" port) + (case guard ((any) (display "Any" port)) ((next) (display "Next" port)) + (else (display "Characters(" port) (rust-text port guard) (display ")" port))) + (display ", projection: " port) (rust-name port kind) + (display ", opening: " port) (display opening port) (display ", closing: " port) (display closing port) + (display ", opcode: PartOpcode::" port) + (case operation + ((parameter name) + (display (if (eq? operation 'parameter) "Parameter" "Name") port) + (display " { binding: " port) (display (binding-index (if (eq? operation 'parameter) 'parameter (caddr form))) port) + (display " }" port)) + ((quote quoted-body) + (display (if (eq? operation 'quote) "Quote" "QuotedBody") port) + (display " { delimiter: " port) (rust-char port (list-ref form 4)) (display " }" port)) + ((pair) (display "Pair" port)) ((escape) (display "Escape" port))) + (display " }," port))) (cadr recipe)) + (display "] };\n" port) (get-output-string port))) + +;;; Command control programs lower from the admitted POO slots, never hand-written Rust. +(def (emit-command-trigger port trigger) + (display "CommandTrigger::" port) + (case (car trigger) + ((none) (display "None" port)) + ((manual) (if (null? (cdr trigger)) (display "Manual(None)" port) + (begin (display "Manual(Some(&" port) (emit-command-trigger port (cadr trigger)) (display "))" port)))) + ((role) (display "Role(" port) (rust-name port (cadr trigger)) (display ")" port)) + ((or lookahead adjacent) + (display (case (car trigger) ((or) "Or(") ((lookahead) "Lookahead(") (else "Adjacent(")) port) + (display "&[" port) (for-each (lambda (row) (emit-command-trigger port row) (display "," port)) (cdr trigger)) (display "])" port)) + (else (display "Token { kind: " port) (rust-name port (car trigger)) + (display ", literals: " port) (rust-strings port (cdr trigger)) (display " }" port)))) +(def (emit-command-program port program) + (display "&[" port) + (for-each (lambda (step) + (display "CommandInstruction::" port) + (case (car step) + ((as raw word command) + (display (case (car step) ((as) "As(") ((raw) "Raw(") ((word) "Word(") (else "Command(")) port) + (rust-name port (cadr step)) (display ")" port)) + ((take) (display "Take { field: " port) (rust-name port (cadr step)) (display ", trigger: " port) + (emit-command-trigger port (caddr step)) (display " }" port)) + ((list) (display "List { field: " port) (rust-name port (cadr step)) (display ", until: " port) + (emit-command-trigger port (caddr step)) (display (if (cadddr step) ", allow_empty: true }" ", allow_empty: false }") port)) + ((call) (display "Call { field: " port) (rust-name port (cadr step)) (display ", form: " port) (rust-name port (caddr step)) (display " }" port)) + ((node) (display "Node { field: " port) (rust-name port (cadr step)) (display ", kind: " port) (rust-name port (caddr step)) + (display ", program: " port) (emit-command-program port (cdddr step)) (display " }" port)) + ((optional many until) + (display (case (car step) ((optional) "Optional") ((many) "Many") (else "Until")) port) + (display " { trigger: " port) (emit-command-trigger port (cadr step)) + (display ", program: " port) (emit-command-program port (cddr step)) (display " }" port)) + ((branch) (display "Branch { trigger: " port) (emit-command-trigger port (cadr step)) + (display ", yes: " port) (emit-command-program port (caddr step)) (display ", no: " port) (emit-command-program port (cadddr step)) (display " }" port)) + ((choose) (display "Choose(&[" port) (for-each (lambda (row) + (display "CommandChoice { trigger: " port) (emit-command-trigger port (car row)) + (display ", program: " port) (emit-command-program port (cdr row)) (display " }," port)) (cdr step)) (display "])" port)) + ((balance) (display "Balance { open: " port) (emit-command-trigger port (cadr step)) + (display ", close: " port) (emit-command-trigger port (caddr step)) (display ", depth: " port) (display (list-ref step 3) port) + (display ", open_field: " port) (rust-name port (list-ref step 4)) (display ", close_field: " port) (rust-name port (list-ref step 5)) + (display ", program: " port) (emit-command-program port (list-tail step 6)) (display " }" port)) + (else (error "unregistered command Rust instruction" step))) + (display "," port)) program) + (display "]" port)) +(def (command-profile-rust-source profile results scanner) + (unless (or scanner (command-plan? profile)) + (error "Rust command lowering requires an admitted plan or scanner declaration")) + (let* ((plan (if (command-plan? profile) profile (compile-command-profile profile))) + (results (if (result-plan? results) results (compile-result-profile results))) + (_ (when scanner (admit-command-plan plan results scanner))) + (recipe (command-plan-recipe plan)) (port (open-output-string))) + (display "use gerbil_parser_runtime::{CommandProgramSpec, CommandForm, CommandInstruction, CommandChoice, CommandTrigger, CommandRole, CommandTokenClass};\n" port) + (display "pub static COMMAND_PROFILE: CommandProgramSpec = CommandProgramSpec { assignment_tail: " port) + (rust-name port (cadr (assq 'assignment-tail (list-ref recipe 4)))) + (display ", pipeline_head: " port) + (rust-name port (cadr (assq 'pipeline-head (list-ref recipe 4)))) + (display ", nodes: &[" port) + (for-each (lambda (row) (display "(" port) (rust-name port (car row)) (display "," port) (rust-name port (cadr row)) (display ")," port)) (caddr recipe)) + (display "], roles: &[" port) + (for-each (lambda (row) + (display "CommandRole { name: " port) (rust-name port (car row)) (display ", classes: &[" port) + (for-each (lambda (clause) (display "CommandTokenClass { kind: " port) (rust-name port (car clause)) + (display ", literals: " port) (rust-strings port (cdr clause)) (display " }," port)) (cdr row)) + (display "] }," port)) (car recipe)) + (display "], descriptor: &" port) (display (rust-text-profile-expression-source (cadar (cadr recipe))) port) + (display ", forms: &[" port) + (for-each (lambda (row) + (display "CommandForm { id: " port) (rust-name port (car row)) (display ", trigger: " port) (emit-command-trigger port (cadr row)) + (display ", priority: " port) (display (caddr row) port) (display ", kind: " port) (rust-name port (cadddr row)) + (display ", program: " port) (emit-command-program port (list-tail row 4)) (display " }," port)) (list-ref recipe 3)) + (display "] };\n" port) (get-output-string port))) + +(def (command-source-rust-module-source strategy) + (let-values (((recipe engine) (bind-source-strategy strategy))) + (unless (eq? (car recipe) 'shell) (error "Command Source Rust AOT requires the closed shell strategy")) + (let* ((program (source-engine-program engine)) + (results (contextual-program-results program)) (parts (contextual-program-parts program)) + (commands (contextual-program-commands program)) (scanner (contextual-program-scanner program)) + (regions (contextual-program-regions program)) (port (open-output-string))) + (display (result-profile-rust-source results) port) + (display (part-profile-rust-source parts results regions) port) + (display (command-profile-rust-source commands results #f) port) + (display "pub mod scanner {\n" port) (display (rust-scanner-module-source scanner) port) (display "}\n" port) + (display "pub static SOURCE: gerbil_parser_runtime::CommandSourceSpec = gerbil_parser_runtime::CommandSourceSpec { scanner: &scanner::SCANNER, parts: &PART_PROFILE, regions: ®ions::REGION, results: &RESULT_PROFILE, commands: &COMMAND_PROFILE };\n" port) + (get-output-string port)))) +(def (generate-command-source-rust-module strategy path) + (let (source (command-source-rust-module-source strategy)) + (create-directory* (path-directory path)) + (call-with-output-file path (lambda (port) (display source port))) path)) diff --git a/src/compiler/rust-syntax.ss b/src/compiler/rust-syntax.ss index 082ffc22..2fc43fec 100644 --- a/src/compiler/rust-syntax.ss +++ b/src/compiler/rust-syntax.ss @@ -10,7 +10,8 @@ rust-function rust-function-value rust-block rust-let rust-call rust-method rust-tuple-index rust-before rust-after rust-first-word rust-words rust-any rust-fold - rust-empty rust-string-in rust-if + rust-empty rust-string-in rust-single-ascii-uppercase + rust-unsigned-at-most rust-if rust-binary rust-line-event-function rust-event-node rust-event-token rust-event-if rust-line-event-function-form? rust-line-event-function-form-name @@ -37,7 +38,7 @@ rust-identifier-form? rust-identifier-form-value rust-some-form? rust-some-form-value rust-number-form? rust-number-form-value - rust-module-form? rust-module-form-item + rust-module-form? rust-module-form-item rust-module-form-origin rust-static-form? rust-static-form-value) (defstruct rust-field (name value) transparent: #t) @@ -64,6 +65,8 @@ (defstruct rust-fold-form (iterator accumulator item initial step) transparent: #t) (defstruct rust-empty-form (value) transparent: #t) (defstruct rust-string-in-form (value collection) transparent: #t) +(defstruct rust-single-ascii-uppercase-form (value) transparent: #t) +(defstruct rust-unsigned-at-most-form (value maximum) transparent: #t) (defstruct rust-if-form (condition consequent alternate) transparent: #t) (defstruct rust-binary-form (operator left right) transparent: #t) (defstruct rust-line-event-function-form (name root body digest) transparent: #t) @@ -88,7 +91,9 @@ (def (rust-number value) (make-rust-number-form value)) (def (rust-string value) (make-rust-string-form value)) (def (rust-identifier value) (make-rust-identifier-form value)) -(def (rust-module imports item (origin "line-structure-rowan.ss")) +;; Required arity keeps compiled cross-module callers on the same symbol; +;; an optional origin compiled one caller to an undefined rust-module__%. +(def (rust-module imports item origin) (unless (and (string? origin) (not (string-contains origin "\n"))) (error "invalid Rust module origin" origin)) (make-rust-module-form imports item origin)) @@ -148,6 +153,12 @@ (def (rust-empty value) (make-rust-empty-form value)) (def (rust-string-in value collection) (make-rust-string-in-form value collection)) +(def (rust-single-ascii-uppercase value) + (make-rust-single-ascii-uppercase-form value)) +(def (rust-unsigned-at-most value maximum) + (unless (and (integer? maximum) (exact? maximum) (<= 0 maximum)) + (error "Rust unsigned bound must be nonnegative" maximum)) + (make-rust-unsigned-at-most-form value maximum)) (def (rust-if condition consequent alternate) (make-rust-if-form condition consequent alternate)) (def (rust-binary operator left right) @@ -269,7 +280,7 @@ (display "// @generated by gerbil-parser/src/compiler/" port) (display (rust-module-form-origin node) port) (newline port) - (display "use gerbil_parser_rowan::{" port) + (display "use gerbil_parser_runtime::{" port) (let loop ((imports (rust-module-form-imports node)) (first? #t)) (when (pair? imports) (unless first? (display ", " port)) @@ -387,6 +398,16 @@ (display ".contains(&" port) (render-node port (rust-string-in-form-value node)) (display ")" port)) + ((rust-single-ascii-uppercase-form? node) + (write-string "{ let candidate = " port) + (render-node port (rust-single-ascii-uppercase-form-value node)) + (write-string "; candidate.len() == 1 && candidate.as_bytes()[0].is_ascii_uppercase() }" port)) + ((rust-unsigned-at-most-form? node) + (write-string "{ let candidate = " port) + (render-node port (rust-unsigned-at-most-form-value node)) + (write-string "; !candidate.is_empty() && candidate.bytes().all(|byte| byte.is_ascii_digit()) && candidate.parse::().is_ok_and(|number| number <= " port) + (write (rust-unsigned-at-most-form-maximum node) port) + (write-string ") }" port)) ((rust-if-form? node) (display "if " port) (render-node port (rust-if-form-condition node)) @@ -485,6 +506,15 @@ (value (rust-expression-ir (rust-string-in-form-value node))) (collection (rust-expression-ir (rust-string-in-form-collection node))))) + ((rust-single-ascii-uppercase-form? node) + (hash (kind "single_ascii_uppercase") + (value (rust-expression-ir + (rust-single-ascii-uppercase-form-value node))))) + ((rust-unsigned-at-most-form? node) + (hash (kind "unsigned_at_most") + (value (rust-expression-ir + (rust-unsigned-at-most-form-value node))) + (maximum (rust-unsigned-at-most-form-maximum node)))) ((rust-if-form? node) (hash (kind "if") (condition (rust-expression-ir (rust-if-form-condition node))) diff --git a/src/compiler/source-language.ss b/src/compiler/source-language.ss new file mode 100644 index 00000000..e66a6852 --- /dev/null +++ b/src/compiler/source-language.ss @@ -0,0 +1,46 @@ +;;; Native syntax vocabulary elaboration for the common shell source library. +(import (only-in :gerbil/expander core-expand1) + (only-in ../../language-support/command-grammar lower-command-grammar) + (only-in :gerbil/core/expander syntax-case syntax with-syntax datum->syntax syntax->datum stx-map raise-syntax-error)) +(export expand-shell-grammar-syntax) +(def (expand-shell-grammar-syntax stx compile-entry) + (def origins (make-table test: eq?)) + (def constructors '(at node field seq choice optional repeat empty-allowed balanced reference word operator role newline adjacent)) + (def (expand expression (invocation #f)) + (syntax-case expression () + ((head argument ...) + (if (memq (syntax->datum #'head) constructors) + (let (result (cons (syntax->datum #'head) + (stx-map (lambda (child) (expand child invocation)) #'(argument ...)))) + (table-set! origins result (or invocation expression)) + result) + (let (expanded (core-expand1 expression)) + (when (eq? expression expanded) + (raise-syntax-error #f "unknown source grammar constructor" expression stx)) + (expand expanded (or invocation expression))))) + (_ (syntax->datum expression)))) + (syntax-case stx (operators regions bindings words commands roles descriptor rules) + ((_ (operators operator-value ...) (regions region ...) (bindings binding ...) + (words word-value ...) + (commands (roles role-value ...) (descriptor descriptor-value)) (rules (name expression) ...)) + (let* ((rule-data (map (lambda (name expression) (list (syntax->datum name) (expand expression))) + (stx-map values #'(name ...)) (stx-map values #'(expression ...)))) + (declaration (list (cons 'operators (syntax->datum #'(operator-value ...))) + (cons 'regions (syntax->datum #'(region ...))) + (cons 'bindings (syntax->datum #'(binding ...))) + (cons 'words (syntax->datum #'(word-value ...))) + (list 'commands (cons 'roles (syntax->datum #'(role-value ...))) + (list 'descriptor (syntax->datum #'descriptor-value)) + (cons 'rules rule-data))))) + (with-catch + (lambda (condition) + (if (syntax-error? condition) (raise condition) + (raise-syntax-error #f (error-message condition) #'(rules (name expression) ...) stx))) + (lambda () + (lower-command-grammar rule-data + (lambda (message value) + (raise-syntax-error #f message + (table-ref origins value #'(rules (name expression) ...)) stx))))) + (with-syntax ((compile compile-entry) (data (datum->syntax compile-entry declaration))) + #'(compile 'data)))) + (_ (raise-syntax-error #f "shell grammar requires ordered operators, regions, bindings, words and commands sections" stx)))) diff --git a/src/ffi/datum-codec.ss b/src/ffi/datum-codec.ss new file mode 100644 index 00000000..af7876ce --- /dev/null +++ b/src/ffi/datum-codec.ss @@ -0,0 +1,31 @@ +;;; Inert ABI metadata projection; ABI/schema stay v1. +(import (only-in :gerbil/core call-with-output-string)) +(export abi-datum-write) +(def (abi-datum-write value) + (call-with-output-string + (lambda (port) + (def (emit value depth) + (when (> depth 64) (error "ABI datum depth bound")) + (cond + ((hash-table? value) + (display "(object" port) + (for-each + (lambda (entry) + (display " (" port) + (write (car entry) port) + (display " " port) (emit (cdr entry) (+ depth 1)) + (display ")" port)) + (list-sort (lambda (a b) (stringstring (car entry)) (car entry)) + (cdr entry))) (hash->list value)))) + (display ")" port)) + ((or (list? value) (vector? value)) + (display "(list" port) + (for-each (lambda (item) (display " " port) (emit item (+ depth 1))) + (if (vector? value) (vector->list value) value)) + (display ")" port)) + ((or (string? value) (boolean? value) (number? value)) (write value port)) + (else (error "unsupported ABI datum")))) + (emit value 0)))) diff --git a/src/ffi/event-binary.ss b/src/ffi/event-binary.ss new file mode 100644 index 00000000..6090f547 --- /dev/null +++ b/src/ffi/event-binary.ss @@ -0,0 +1,96 @@ +;;; ParseArtifact binary layout and exact-buffer event publication. +(import (only-in ../runtime/parse-cost with-parser-cost-stage) + (only-in ./schema +event-abi-version+) + (only-in :std/vector/u8vector little u8vector-u32-set!) + (only-in :std/encoding/hex hex-decode) + (only-in ../runtime/token token-kind token-start token-end) + (only-in ./language-abi-context language-abi-context-syntax-kind-index + language-abi-context-terminal-index language-abi-context-field-index language-abi-context-event-row-writer)) +(export +binary-header-size+ +binary-event-size+ copy-digest! write-event! abi-event-payload) + +(def +binary-header-size+ 80) +(def +binary-event-size+ 24) + +(def +event-tags+ + (hash (start-node 1) + (finish-node 2) + (start-field 3) + (finish-field 4) + (token 5))) + +(def (required-index table symbol domain) + (or (hash-get table symbol) + (error "ParseArtifact ABI symbol is outside descriptor" domain symbol))) + +(def (copy-digest! payload offset digest) + (let (bytes (hex-decode digest 7)) + (unless (= (u8vector-length bytes) 32) + (error "invalid ParseArtifact digest" digest)) + (subu8vector-move! bytes 0 32 payload offset))) + +(def (write-event! language payload row event) + (let* ((offset (+ +binary-header-size+ (* row +binary-event-size+))) + (tag (vector-ref event 0)) + (number (required-index +event-tags+ tag 'event-tag)) + (write-row! (language-abi-context-event-row-writer language))) + (case tag + ((start-node finish-node) + (let (position (vector-ref event 3)) + (write-row! payload offset number + (required-index (language-abi-context-syntax-kind-index language) + (vector-ref event 2) 'syntax-kind) + (vector-ref event 1) + (if (eq? tag 'start-node) position 0) + (if (eq? tag 'finish-node) position 0)))) + ((start-field finish-field) + (let (position (vector-ref event 2)) + (write-row! payload offset number + (required-index (language-abi-context-field-index language) + (vector-ref event 1) 'field) + 0 (if (eq? tag 'start-field) position 0) + (if (eq? tag 'finish-field) position 0)))) + ((token) + (write-row! payload offset number + (required-index (language-abi-context-terminal-index language) + (vector-ref event 2) 'token-kind) + (vector-ref event 1) (vector-ref event 4) (vector-ref event 5)))))) + +;;; Validate/count without event objects, then publish one exact-sized buffer. +;;; Both passes consume the same immutable recognition/program representation. +;;; The thunk keeps binary allocation/catalog failures outside the parse catch. +(def (abi-event-payload language grammar-digest source-digest walk) + (let (count 0) + (def (count-node _tag _id _kind _position) (set! count (+ count 1))) + (def (count-field _tag _field _position) (set! count (+ count 1))) + (def (count-token _id _token) (set! count (+ count 1))) + (with-parser-cost-stage 'abi-event-validation (walk count-node count-field count-token)) + (lambda () + (let ((payload (make-u8vector (+ +binary-header-size+ (* count +binary-event-size+)) 0)) + (row 0) (write-row! (language-abi-context-event-row-writer language))) + (def (emit-row! tag index id start end) + (let (offset (+ +binary-header-size+ (* row +binary-event-size+))) + (write-row! payload offset tag index id start end) + (set! row (+ row 1)))) + (def (emit-node! tag id kind position) + (emit-row! (if (eq? tag 'start-node) 1 2) + (required-index (language-abi-context-syntax-kind-index language) kind 'syntax-kind) + id (if (eq? tag 'start-node) position 0) + (if (eq? tag 'finish-node) position 0))) + (def (emit-field! tag field position) + (emit-row! (if (eq? tag 'start-field) 3 4) + (required-index (language-abi-context-field-index language) field 'field) + 0 (if (eq? tag 'start-field) position 0) + (if (eq? tag 'finish-field) position 0))) + (def (emit-token! id token) + (emit-row! 5 (required-index (language-abi-context-terminal-index language) + (token-kind token) 'token-kind) + id (token-start token) (token-end token))) + (with-parser-cost-stage 'abi-event-publication (walk emit-node! emit-field! emit-token!)) + (unless (= row count) (error "ABI event count changed during publication")) + (subu8vector-move! #u8(71 80 65 49) 0 4 payload 0) + (u8vector-u32-set! payload 4 +event-abi-version+ little) + (u8vector-u32-set! payload 12 count little) + (copy-digest! payload 16 grammar-digest) + (subu8vector-move! source-digest 0 32 payload 48) + payload)))) + diff --git a/src/ffi/language-abi-context.ss b/src/ffi/language-abi-context.ss new file mode 100644 index 00000000..f2dd251a --- /dev/null +++ b/src/ffi/language-abi-context.ss @@ -0,0 +1,177 @@ +;;; Language ABI context and descriptor catalogs shared by ABI publication. +;;; Contains bounded foreign byte leaves; package builds must use the +;;; explicit FFI module policy, including Darwin loadable-bundle linker flags. +(import (only-in :std/list/list delete-duplicates/hash) + (only-in :std/string/misc string-every) + (only-in ../grammar/algebra grammar-expression-fields) + (only-in ../language/descriptor language-grammar-grammar) + (only-in ../language/source source-language? source-language-result-catalog)) +(export #t) + +(defstruct language-abi-context + (id descriptor parser syntax-kind-index terminal-index field-symbols field-index plan source-admission event-row-writer) + transparent: #t) + +(def (descriptor-section descriptor name) + (if (and (source-language? descriptor) (eq? name 'rules)) '() + (let (row (assq name (if (source-language? descriptor) + (source-language-result-catalog descriptor) + (language-grammar-grammar descriptor)))) + (unless row (error "ABI descriptor lacks a required result section" name)) + (cdr row)))) + +;; Opaque rejected source uses the engine token without changing recognition. +(def (abi-terminal-rows descriptor) + (let (rows (descriptor-section descriptor 'terminals)) + (if (assq 'unknown rows) rows + (append rows '((unknown token)))))) + +(def (indexed-symbols symbols) + (let (table (make-hash-table-eq size: (length symbols))) + (for-each (lambda (symbol index) (hash-put! table symbol index)) + symbols + (iota (length symbols))) + table)) + +;; A shared field has one stable id. The grammar expression algebra is the +;; authority for fields emitted by the parser; declared syntax fields are +;; retained first for the descriptor's complete public surface. +(def (descriptor-field-symbols grammar) + (delete-duplicates/hash + (append + (apply append (map caddr (descriptor-section grammar 'syntax-kinds))) + (apply append + (map (lambda (row) (grammar-expression-fields (cadr row))) + (descriptor-section grammar 'rules)))) + from-end?: #t)) + +(def (prepare-language-abi-context id grammar parser) + (let (field-symbols (descriptor-field-symbols grammar)) + (make-language-abi-context + id grammar parser + (indexed-symbols (map car (descriptor-section grammar 'syntax-kinds))) + (indexed-symbols (map car (abi-terminal-rows grammar))) + field-symbols + (indexed-symbols field-symbols) #f abi-source-text write-abi-row!))) + + +;;; Pack a complete GPA1 row in one synchronous bounded leaf. Conversion of +;;; every integer completes before C touches the buffer; no allocation or +;;; callback can move the borrowed Scheme body during the write. +(extern copy-ascii-source-range! ascii-source-range? pack-abi-row!) +(def (write-abi-row! payload offset tag index id start end) + (declare (not interrupts-enabled)) + ;; Match the upstream encoder's inline checked leaf: ordinary row scalars + ;; remain tagged fixnums instead of entering seven generic FFI converters. + ;; The general integer leaf retains the full unsigned 64-bit ABI domain. + (unless (or (##c-code #<string bytes)) + ;; Gambit characters may extend beyond Unicode scalars. Re-encoding alone + ;; cannot reject surrogate encodings accepted by a permissive decoder. + (unless (and (string-every unicode-scalar? source) + (equal? (string->utf8 source) bytes)) + (error "native source is not canonical UTF-8")) + source))) +(def (unicode-scalar? character) + (let (scalar (char->integer character)) + (and (<= 0 scalar #x10ffff) + (not (<= #xd800 scalar #xdfff))))) +(begin-foreign + (namespace ("gerbil-parser/src/ffi/language-abi-context#" + copy-ascii-source-range! ascii-source-range? pack-abi-row!)) + (c-declare #< +static void gp_store_le(uint8_t *p, uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) { p[i] = (uint8_t)value; value >>= 8; } +} +/* All type, width and full-span checks precede the first borrowed write. */ +static int gp_pack_fixnum_row(___SCMOBJ bytes, ___SCMOBJ position, + ___SCMOBJ tag, ___SCMOBJ index, ___SCMOBJ id, + ___SCMOBJ start, ___SCMOBJ end) { + ___SCMOBJ ___temp; + if (!___U8VECTORP(bytes) || !___FIXNUMP(position) || !___FIXNUMP(tag) || + !___FIXNUMP(index) || !___FIXNUMP(id) || !___FIXNUMP(start) || !___FIXNUMP(end)) + return 0; + if (___INT(position) < 0 || ___INT(tag) < 0 || ___INT(index) < 0 || + ___INT(id) < 0 || ___INT(start) < 0 || ___INT(end) < 0) + return 0; + uint64_t offset = (uint64_t)___INT(position); + uint64_t number = (uint64_t)___INT(tag), symbol = (uint64_t)___INT(index); + uint64_t first = (uint64_t)___INT(start), last = (uint64_t)___INT(end); + size_t length = ___HD_BYTES(___HEADER(bytes)); + if (number > UINT32_MAX || symbol > UINT32_MAX || first > UINT32_MAX || + last > UINT32_MAX || offset > length || length - (size_t)offset < 24) + return 0; + uint8_t *p = (uint8_t*)___BODY_AS(bytes, ___tSUBTYPED) + (size_t)offset; + gp_store_le(p, number, 4); gp_store_le(p+4, symbol, 4); + gp_store_le(p+8, (uint64_t)___INT(id), 8); + gp_store_le(p+16, first, 4); gp_store_le(p+20, last, 4); + return 1; +} +END-C + ) + (define copy-ascii-source-range! + (c-lambda (scheme-object scheme-object unsigned-int64 unsigned-int64) bool + "___SCMOBJ ___temp; + if (!___U8VECTORP(___arg1) || !___STRINGP(___arg2)) { ___return(0); } + size_t length = ___HD_BYTES(___HEADER(___arg1)); + if ((size_t)___STRINGSIZE(___arg2) != length || ___arg3 > ___arg4 || ___arg4 > length || ___arg4 - ___arg3 > 4096) { ___return(0); } + const uint8_t *p = (const uint8_t*)___BODY_AS(___arg1, ___tSUBTYPED); + for (size_t i = (size_t)___arg3; i < (size_t)___arg4; ++i) { + uint8_t scalar = p[i]; if (scalar & 0x80) { ___return(0); } + ___STRINGSET(___arg2, ___FIX(i), ___CHR(scalar)); + } + ___return(1);")) + (define ascii-source-range? + (c-lambda (scheme-object unsigned-int64 unsigned-int64) bool + "___SCMOBJ ___temp; + if (!___U8VECTORP(___arg1)) { ___return(0); } + size_t length = ___HD_BYTES(___HEADER(___arg1)); + if (___arg2 > ___arg3 || ___arg3 > length || ___arg3 - ___arg2 > 4096) { ___return(0); } + const uint8_t *p = (const uint8_t*)___BODY_AS(___arg1, ___tSUBTYPED); + for (size_t i = (size_t)___arg2; i < (size_t)___arg3; ++i) { if (p[i] & 0x80) { ___return(0); } } + ___return(1);")) + (define pack-abi-row! + (c-lambda (scheme-object unsigned-int64 unsigned-int32 unsigned-int32 unsigned-int64 unsigned-int32 unsigned-int32) bool + "___SCMOBJ ___temp; + if (!___U8VECTORP(___arg1)) { ___return(0); } + size_t length = ___HD_BYTES(___HEADER(___arg1)); + if (___arg2 > length || length - (size_t)___arg2 < 24) { ___return(0); } + uint8_t *p = (uint8_t*)___BODY_AS(___arg1, ___tSUBTYPED) + (size_t)___arg2; + gp_store_le(p, ___arg3, 4); gp_store_le(p+4, ___arg4, 4); + gp_store_le(p+8, ___arg5, 8); gp_store_le(p+16, ___arg6, 4); + gp_store_le(p+20, ___arg7, 4); ___return(1);")) +) diff --git a/src/ffi/language-abi.ss b/src/ffi/language-abi.ss new file mode 100644 index 00000000..12fe1eca --- /dev/null +++ b/src/ffi/language-abi.ss @@ -0,0 +1,132 @@ +;;; -*- Gerbil -*- +;;; C ABI for explicit language handles, with no builtin language dependencies. +(import (only-in ../runtime/parse-cost with-parser-cost-stage) + (only-in ./schema declare-language-abi-version) + (only-in ./language-handles language-handle-descriptor + language-handle-parse/publish release-language! + language-handle-owner! language-handle-count +language-source-byte-limit+) + (only-in ./language-artifact-codec abi-error-payload)) +(extern initialize-owner! copy-input! replace-result! result-status) +(def (copy-abi-input! input bytes) + (with-parser-cost-stage 'ffi-input-copy (copy-input! input bytes))) +(def (publish-abi-payload! result payload) + (with-parser-cost-stage 'ffi-result-copy (replace-result! result 0 payload))) +(def (call-with-abi-result result thunk) + (with-exception-catcher + (lambda (exception) + (replace-result! result -1 (string->utf8 (abi-error-payload exception))) + -1) + thunk)) +(def (publish-result result thunk) + (call-with-abi-result result + (lambda () + (replace-result! result 0 (thunk)) + (result-status result)))) +(begin-foreign + (namespace ("gerbil-parser/src/ffi/language-abi#" + initialize-owner! copy-input! replace-result! result-status descriptor-callback + parse-callback release-callback count-callback)) + (c-declare #< +#include +#include +#include +#include +#define GP_DATA(obj) ___CAST(___U8*, ___BODY_AS((obj), ___tSUBTYPED)) +#define GP_LEN(obj) ___HD_BYTES(___HEADER(obj)) +static pthread_t gp_owner; +static _Atomic int gp_owner_ready = 0; +static _Atomic size_t gp_payload_count = 0; +int32_t gerbil_parser_language_is_owner_thread(void) { + return gp_owner_ready && pthread_equal(gp_owner, pthread_self()); +} +void gerbil_parser_result_init(gerbil_parser_result *r) { + if (r) { r->status = 0; r->payload = NULL; r->length = 0; } +} +void gerbil_parser_result_release(gerbil_parser_result *r) { + if (r) { if (r->payload) { free(r->payload); atomic_fetch_sub(&gp_payload_count,1); } gerbil_parser_result_init(r); } +} +/* Match Gambit's generated c-define types at this private boundary. On + LP64 Linux uint64_t may be unsigned long while ___U64 is unsigned long long. + Public declarations retain their fixed-width C ABI types. */ +___S32 gerbil_parser_language_descriptor_impl(___U64, gerbil_parser_result *); +___S32 gerbil_parser_language_parse_impl(___U64, ___U8 *, ___U64, gerbil_parser_result *); +___S32 gerbil_parser_language_release_impl(___U64); +___S32 gerbil_parser_language_handle_count_impl(void); +int32_t gerbil_parser_language_runtime_detach(void) { + if (!gerbil_parser_language_is_owner_thread()) return -1; + if (atomic_load(&gp_payload_count)) return -2; + ___S32 count = gerbil_parser_language_handle_count_impl(); + if (count < 0) return -1; + if (count) return -2; + atomic_store(&gp_owner_ready,0); + return 0; +} +int32_t gerbil_parser_language_descriptor(uint64_t language, gerbil_parser_result *r) { + if (!r) return -1; + if (!gerbil_parser_language_is_owner_thread()) { gerbil_parser_result_release(r); r->status=-1; return -1; } + return gerbil_parser_language_descriptor_impl((___U64)language, r); +} +int32_t gerbil_parser_language_parse(uint64_t language, const uint8_t *source, size_t length, gerbil_parser_result *r) { + if (!r) return -1; + if (!gerbil_parser_language_is_owner_thread() || (!source && length) || length > 67108864) { + gerbil_parser_result_release(r); r->status = -1; return -1; + } + return gerbil_parser_language_parse_impl((___U64)language, (___U8 *)source, (___U64)length, r); +} +int32_t gerbil_parser_language_release(uint64_t language) { + if (!gerbil_parser_language_is_owner_thread()) return -1; + return gerbil_parser_language_release_impl((___U64)language); +} +END-C + ) + (c-define-type abi-result (type "gerbil_parser_result" (gerbil_parser_result))) + (c-define-type abi-result-ptr (pointer abi-result (gerbil_parser_result*))) + (define initialize-owner! (c-lambda () void "gp_owner=pthread_self();gp_owner_ready=1;___return;")) + (define result-status (c-lambda (abi-result-ptr) int32 "___return(___arg1->status);")) + (define copy-input! (c-lambda ((pointer unsigned-int8) scheme-object) void + "if (GP_LEN(___arg2)) memcpy(GP_DATA(___arg2), ___arg1, GP_LEN(___arg2)); ___return;")) + (define replace-result! + (c-lambda (abi-result-ptr int32 scheme-object) void #<status = -1; ___return; } +if (n) { memcpy(p, GP_DATA(___arg3), n); atomic_fetch_add(&gp_payload_count,1); } +___arg1->status = ___arg2; ___arg1->payload = p; ___arg1->length = n; +___return; +END-C + )) + (c-define (descriptor-callback language result) + (unsigned-int64 abi-result-ptr) int32 "gerbil_parser_language_descriptor_impl" "extern" + (gerbil-parser/src/ffi/language-abi#publish-result result (lambda () (string->utf8 (gerbil-parser/src/ffi/language-handles#language-handle-descriptor language))))) + (c-define (parse-callback language input length result) + (unsigned-int64 (pointer unsigned-int8) unsigned-int64 abi-result-ptr) int32 + "gerbil_parser_language_parse_impl" "extern" + (gerbil-parser/src/ffi/language-abi#call-with-abi-result result + (lambda () + (gerbil-parser/src/ffi/language-handles#language-handle-owner!) + (when (> length gerbil-parser/src/ffi/language-handles#+language-source-byte-limit+) (error "native source byte limit")) + (let ((bytes (make-u8vector length))) + (gerbil-parser/src/ffi/language-abi#copy-abi-input! input bytes) + (gerbil-parser/src/ffi/language-handles#language-handle-parse/publish + language bytes + (lambda (payload) + (gerbil-parser/src/ffi/language-abi#publish-abi-payload! result payload) + (zero? (result-status result)))) + (result-status result))))) + (c-define (count-callback) () int32 "gerbil_parser_language_handle_count_impl" "extern" + (with-exception-catcher (lambda (_) -1) + (lambda () + ;; Only emptiness matters; avoid narrowing an unbounded Scheme count. + (if (zero? (gerbil-parser/src/ffi/language-handles#language-handle-count)) 0 1)))) + (c-define (release-callback language) + (unsigned-int64) int32 "gerbil_parser_language_release_impl" "extern" + (with-exception-catcher (lambda (_) -1) + (lambda () (gerbil-parser/src/ffi/language-handles#release-language! language) 0)))) + +;; Pin the C callback boundary during module initialization, before publishing +;; any language handle. Foreign pthreads fail before entering the Gambit VM. +(initialize-owner!) + +(declare-language-abi-version) diff --git a/src/ffi/language-artifact-codec.ss b/src/ffi/language-artifact-codec.ss new file mode 100644 index 00000000..bb6a0c92 --- /dev/null +++ b/src/ffi/language-artifact-codec.ss @@ -0,0 +1,176 @@ +;;; -*- Gerbil -*- +;;; Parser-owned C ABI for language-selected ParseArtifact v1 surfaces. + +(import (only-in ../runtime/parse-cost with-parser-cost-stage) + (only-in ./schema +event-abi-version+ +artifact-abi-version+ +abi-descriptor-schema+ +abi-error-schema+) + (only-in ./language-abi-context prepare-language-abi-context language-abi-context? + descriptor-section abi-terminal-rows language-abi-context-id + language-abi-context-descriptor language-abi-context-parser language-abi-context-field-symbols + language-abi-context-plan language-abi-context-plan-set! + language-abi-context-source-admission) + (only-in ./event-binary +binary-header-size+ +binary-event-size+ + copy-digest! write-event! abi-event-payload) + (only-in :std/crypto/digest sha256) + (only-in :std/vector/u8vector little u8vector-u32-set!) + (only-in ./datum-codec abi-datum-write) + (only-in ../language/descriptor language-grammar-ir language-grammar? language-grammar-grammar language-grammar-language language-grammar-machine + language-grammar-parser-policy language-parser-policy-identity + language-parser-policy-branch-budget) + (only-in ../language/source source-language-digest source-language-root-kind source-language? source-language-language + parse-source-language/session source-language-session-artifact source-language-session-history) + (only-in ../runtime/artifact parse-artifact-events parse-artifact-ref with-parse-event-walk) + (only-in ../language/entry parse-language-source call-with-language-parser-policy) + (only-in ../compiler/machine parser-machine-grammar-digest) + (only-in ../runtime/parser contextual-parser-grammar-digest prepare-contextual-parser parse-source/contextual/prepared + parse-source/contextual/prepared/deferred)) +(export prepare-language-abi-context + bind-language-abi + language-abi-context? + artifact-abi-version + abi-descriptor-payload + abi-parse-binary-payload + abi-parse-binary-payload/bytes + abi-parse-binary-payload/bytes/session + abi-error-payload) + + +(def (abi-error-payload exception) + (abi-datum-write + (hash (schema +abi-error-schema+) + (message + (call-with-output-string + (lambda (port) (display-exception exception port))))))) + +(def (syntax-kind->datum row) + (vector (symbol->string (car row)) + (symbol->string (cadr row)) + (list->vector (map symbol->string (caddr row))))) + +(def (terminal->datum row) + (vector (symbol->string (car row)) + (symbol->string (cadr row)))) + +(def (artifact-abi-version) + +artifact-abi-version+) + +;; One bounded buffer crosses the C ABI. Token lexemes remain zero-copy source +;; slices, represented by their byte ranges instead of duplicated strings. +(def (abi-parse-binary-payload language source) + (abi-artifact-binary-payload language ((language-abi-context-parser language) source))) +(def (abi-artifact-binary-payload language artifact) + (let* ((events (parse-artifact-events artifact)) + (event-count (length events)) + (payload (make-u8vector + (+ +binary-header-size+ + (* event-count +binary-event-size+)) + 0))) + (subu8vector-move! #u8(71 80 65 49) 0 4 payload 0) ; GPA1 + (u8vector-u32-set! payload 4 +event-abi-version+ little) + (u8vector-u32-set! payload 8 + (if (eq? (parse-artifact-ref artifact 'status) + 'accepted) + 0 1) + little) + (u8vector-u32-set! payload 12 event-count little) + (copy-digest! payload 16 (parse-artifact-ref artifact 'grammarDigest)) + (copy-digest! payload 48 (parse-artifact-ref artifact 'sourceDigest)) + ;; Row numbering is traversal state, not another allocated event-sized list. + (let loop ((remaining events) (row 0)) + (unless (null? remaining) + (write-event! language payload row (car remaining)) + (loop (cdr remaining) (+ row 1)))) + payload)) + +;;; Publish before returning history: any transport/admission exception leaves +;;; the handle's previous history untouched. Syntax rejection remains GPA1. +(def (abi-parse-binary-payload/bytes/session language bytes previous) + (let (descriptor (language-abi-context-descriptor language)) + (if (source-language? descriptor) + (let* ((session (parse-source-language/session descriptor (with-parser-cost-stage 'utf8-decode ((language-abi-context-source-admission language) bytes)) previous)) + (payload (with-parser-cost-stage 'payload-encoding + (abi-artifact-binary-payload language (source-language-session-artifact session))))) + (values payload (source-language-session-history session))) + (values (abi-parse-binary-payload/bytes language bytes) #f)))) + +(def (abi-parse-binary-payload/bytes language bytes) + (let (source ((language-abi-context-source-admission language) bytes)) + (let (plan (language-abi-context-plan language)) + (if (or (not plan) + (language-grammar-parser-policy (language-abi-context-descriptor language))) + (abi-parse-binary-payload language source) + (let ((source-byte-length (u8vector-length bytes)) (source-digest (sha256 bytes))) + (parse-source/contextual/prepared/deferred + plan source source-byte-length + (lambda (digest _source tokens root trivia?) + (abi-event-payload + language digest source-digest + (lambda (node-emitter field-emitter token-emitter) + (with-parse-event-walk tokens root trivia? source-byte-length + node-emitter field-emitter token-emitter)))) + (lambda (_machine digest _source tokens _condition) + (let (publish + (abi-event-payload + language digest source-digest + (lambda (_node-emitter _field-emitter token-emitter) + (let loop ((remaining tokens) (id 0)) + (when (pair? remaining) + (token-emitter id (car remaining)) + (loop (cdr remaining) (+ id 1))))))) + (lambda () + (let (payload (publish)) + (u8vector-u32-set! payload 8 1 little) + payload)))))))))) + +(def (abi-descriptor-payload language) + (let (payload + (hash (schema +abi-descriptor-schema+) + (language (language-abi-context-id language)) + (rootKind (symbol->string + (if (source-language? (language-abi-context-descriptor language)) + (source-language-root-kind (language-abi-context-descriptor language)) + (cdr (assq 'root-kind (language-grammar-ir (language-abi-context-descriptor language))))))) + (grammarDigest + (let ((descriptor (language-abi-context-descriptor language)) + (plan (language-abi-context-plan language))) + (cond (plan (contextual-parser-grammar-digest plan)) + ((source-language? descriptor) (source-language-digest descriptor)) + (else (parser-machine-grammar-digest (language-grammar-machine descriptor)))))) + (fields (list->vector + (map symbol->string + (language-abi-context-field-symbols language)))) + (syntaxKinds + (list->vector (map syntax-kind->datum + (descriptor-section + (language-abi-context-descriptor language) + 'syntax-kinds)))) + (terminals + (list->vector (map terminal->datum + (abi-terminal-rows (language-abi-context-descriptor language))))))) + (let* ((descriptor (language-abi-context-descriptor language)) + (policy (and (language-grammar? descriptor) (language-grammar-parser-policy descriptor)))) + (when policy + (hash-put! payload 'parserPolicy + (hash (identity (language-parser-policy-identity policy)) + (branchBudget (language-parser-policy-branch-budget policy)) + (schemeCstValidation #t) (portable #f))))) + (abi-datum-write payload))) + +;;; A language pack supplies its descriptor once; no builtin language imports. +(def (bind-language-abi descriptor (contextual-product #f)) + (unless (or (language-grammar? descriptor) (source-language? descriptor)) + (error "native language requires an admitted descriptor")) + (when (and contextual-product (source-language? descriptor)) + (error "source language does not admit a contextual parser product")) + (let* ((plan (and contextual-product + (prepare-contextual-parser (language-grammar-machine descriptor) contextual-product))) + (language + (prepare-language-abi-context + (if (source-language? descriptor) (source-language-language descriptor) + (language-grammar-language descriptor)) descriptor + (lambda (source) + (if plan + (call-with-language-parser-policy + descriptor source (lambda () (parse-source/contextual/prepared plan source))) + (parse-language-source descriptor source)))))) + (language-abi-context-plan-set! language plan) + language)) diff --git a/src/ffi/language-handles.ss b/src/ffi/language-handles.ss new file mode 100644 index 00000000..423ba8a6 --- /dev/null +++ b/src/ffi/language-handles.ss @@ -0,0 +1,73 @@ +;;; -*- Gerbil -*- +;;; Language-independent ABI admission. All calls belong to one runtime thread. +(import (only-in ./language-abi-context language-abi-context? + language-abi-context-source-admission language-abi-context-event-row-writer) + (only-in ./language-artifact-codec bind-language-abi + abi-parse-binary-payload/bytes/session abi-descriptor-payload) + (only-in ../language/source source-language-session-scanned-token-count + source-language-session-reused-token-count)) +(export register-language! register-language-context! release-language! + language-handle-descriptor language-handle-parse + language-handle-parse/publish + language-handle-scan-counts + language-handle-owner! language-handle-count +language-source-byte-limit+) +(def +language-source-byte-limit+ 67108864) +(def +owner-thread+ #f) +(def +next-handle+ 0) +(def +languages+ (make-hash-table-eqv)) +(defstruct language-handle-entry (language history generation)) +(def (language-handle-owner!) + (unless +owner-thread+ (set! +owner-thread+ (current-thread))) + (unless (eq? +owner-thread+ (current-thread)) + (error "language ABI handle requires its runtime owner thread"))) +(def (language-handle-count) + (language-handle-owner!) + (hash-length +languages+)) +(def (register-language! descriptor (contextual-product #f)) + (register-language-context! (bind-language-abi descriptor contextual-product))) +;;; Prepared contexts retain their decoder and publication catalogs per handle. +(def (register-language-context! language) + (language-handle-owner!) + (unless (and (language-abi-context? language) + (procedure? (language-abi-context-source-admission language)) + (procedure? (language-abi-context-event-row-writer language))) + (error "expected prepared language ABI context")) + (when (>= +next-handle+ 18446744073709551615) + (error "language ABI handle space exhausted")) + (set! +next-handle+ (+ +next-handle+ 1)) + (hash-put! +languages+ +next-handle+ (make-language-handle-entry language #f 0)) + +next-handle+) +(def (release-language! handle) + (language-handle-owner!) + (hash-remove! +languages+ handle)) +(def (admit-handle handle) + (language-handle-owner!) + (or (hash-get +languages+ handle) + (error "unknown or released language ABI handle" handle))) +(def (language-handle-descriptor handle) + (abi-descriptor-payload (language-handle-entry-language (admit-handle handle)))) +(def (language-handle-scan-counts handle) + (let (history (language-handle-entry-history (admit-handle handle))) + (values (and history (source-language-session-scanned-token-count history)) + (if history (source-language-session-reused-token-count history) 0)))) +(def (language-handle-parse handle bytes) + (language-handle-parse/publish handle bytes (lambda (_) #t))) +;;; The C owner commits history only after allocating and copying its result. +(def (language-handle-parse/publish handle bytes publish) + (let (entry (admit-handle handle)) + (unless (and (u8vector? bytes) + (<= (u8vector-length bytes) +language-source-byte-limit+)) + (error "invalid or oversized ABI source bytes")) + ;; Advance before recognition: even a failed newer attempt supersedes the + ;; older callback. Exact integers do not wrap back to an earlier ticket. + (let (generation (+ 1 (language-handle-entry-generation entry))) + (language-handle-entry-generation-set! entry generation) + (let-values (((payload history) + (abi-parse-binary-payload/bytes/session + (language-handle-entry-language entry) bytes + (language-handle-entry-history entry)))) + (when (and (publish payload) + (= generation (language-handle-entry-generation entry)) + (eq? entry (hash-get +languages+ handle))) + (language-handle-entry-history-set! entry history)) + payload)))) diff --git a/src/ffi/parse-artifact-v1-native.ss b/src/ffi/parse-artifact-v1-native.ss deleted file mode 100644 index 4a3e6789..00000000 --- a/src/ffi/parse-artifact-v1-native.ss +++ /dev/null @@ -1,125 +0,0 @@ -;;; -*- Gerbil -*- -;;; C ABI projection for the interpreter-safe ParseArtifact v1 value module. - -(import (only-in ./parse-artifact-v1 - native-abi-version - native-descriptor-payload - native-error-payload - native-parse-binary-payload)) - -(begin-foreign - (namespace - ("gerbil-parser/src/ffi/parse-artifact-v1-native#" - gerbil-parser-native-abi-version - gerbil-parser-native-descriptor - gerbil-parser-native-parse - gerbil-parser-result-v1-set-bytes! - gerbil_parser_result_v1-status - gerbil_parser_result_v1-status-set!)) - - (c-declare #< -#include -#include - -#define GERBIL_PARSER_U8_DATA(obj) \ - ___CAST(___U8*, ___BODY_AS((obj), ___tSUBTYPED)) -#define GERBIL_PARSER_U8_LEN(obj) ___HD_BYTES(___HEADER(obj)) - -typedef struct { - int32_t status; - uint8_t *payload; - size_t length; -} gerbil_parser_result_v1; - -void gerbil_parser_result_v1_init(gerbil_parser_result_v1 *result) { - result->status = 0; - result->payload = NULL; - result->length = 0; -} - -void gerbil_parser_result_v1_release(gerbil_parser_result_v1 *result) { - if (result->payload != NULL) { - free(result->payload); - } - result->status = 0; - result->payload = NULL; - result->length = 0; -} -END-C - ) - - (c-define-type gerbil_parser_result_v1 - (type "gerbil_parser_result_v1" (gerbil_parser_result_v1))) - (c-define-type gerbil_parser_result_v1-borrowed-ptr* - (pointer gerbil_parser_result_v1 (gerbil_parser_result_v1*))) - - (define gerbil_parser_result_v1-status - (c-lambda (gerbil_parser_result_v1-borrowed-ptr*) int32 - "___return(___arg1->status);")) - - (define gerbil_parser_result_v1-status-set! - (c-lambda (gerbil_parser_result_v1-borrowed-ptr* int32) void - "___arg1->status = ___arg2; ___return;")) - - (define gerbil-parser-result-v1-set-bytes! - (c-lambda (gerbil_parser_result_v1-borrowed-ptr* scheme-object) void - #<length = GERBIL_PARSER_U8_LEN(___arg2); -___arg1->payload = (uint8_t *)malloc(___arg1->length); -if (___arg1->payload == NULL && ___arg1->length > 0) { - ___arg1->length = 0; - ___arg1->status = -1; - ___return; -} -if (___arg1->length > 0) { - memcpy(___arg1->payload, GERBIL_PARSER_U8_DATA(___arg2), ___arg1->length); -} -___return; -END-C - )) - - (c-define (gerbil-parser-native-abi-version) - () unsigned-int32 "gerbil_parser_native_abi_version" "extern" - (gerbil-parser/src/ffi/parse-artifact-v1#native-abi-version)) - - (c-define (gerbil-parser-native-descriptor language result) - (UTF-8-string gerbil_parser_result_v1-borrowed-ptr*) int32 - "gerbil_parser_native_descriptor" "extern" - (with-exception-catcher - (lambda (exception) - (gerbil_parser_result_v1-status-set! result -1) - (gerbil-parser/src/ffi/parse-artifact-v1-native#gerbil-parser-result-v1-set-bytes! - result - (string->utf8 - (gerbil-parser/src/ffi/parse-artifact-v1#native-error-payload - exception))) - -1) - (lambda () - (gerbil_parser_result_v1-status-set! result 0) - (gerbil-parser/src/ffi/parse-artifact-v1-native#gerbil-parser-result-v1-set-bytes! - result - (string->utf8 - (gerbil-parser/src/ffi/parse-artifact-v1#native-descriptor-payload - language))) - (gerbil_parser_result_v1-status result)))) - - (c-define (gerbil-parser-native-parse language source result) - (UTF-8-string UTF-8-string gerbil_parser_result_v1-borrowed-ptr*) int32 - "gerbil_parser_native_parse" "extern" - (with-exception-catcher - (lambda (exception) - (gerbil_parser_result_v1-status-set! result -1) - (gerbil-parser/src/ffi/parse-artifact-v1-native#gerbil-parser-result-v1-set-bytes! - result - (string->utf8 - (gerbil-parser/src/ffi/parse-artifact-v1#native-error-payload - exception))) - -1) - (lambda () - (gerbil_parser_result_v1-status-set! result 0) - (gerbil-parser/src/ffi/parse-artifact-v1-native#gerbil-parser-result-v1-set-bytes! - result - (gerbil-parser/src/ffi/parse-artifact-v1#native-parse-binary-payload - language source)) - (gerbil_parser_result_v1-status result))))) diff --git a/src/ffi/parse-artifact-v1.ss b/src/ffi/parse-artifact-v1.ss deleted file mode 100644 index 8a3ad4b1..00000000 --- a/src/ffi/parse-artifact-v1.ss +++ /dev/null @@ -1,204 +0,0 @@ -;;; -*- Gerbil -*- -;;; Parser-owned C ABI for language-selected ParseArtifact v1 surfaces. - -(import (only-in :std/vector/u8vector little u8vector-u32-set! u8vector-u64-set!) - (only-in :std/list/list delete-duplicates/hash) - :std/encoding/json - (only-in :std/encoding/hex hex-decode) - (only-in ../grammar/algebra grammar-expression-fields) - (only-in ../language/descriptor language-grammar-grammar) - (only-in ../runtime/artifact parse-artifact-events parse-artifact-ref) - (only-in ../../languages/gql/iso-39075-2024/grammar - gql-iso-language-grammar) - (only-in ../../languages/gql/iso-39075-2024/parser - parse-gql-iso-39075-2024) - (only-in ../../languages/cypher/opencypher-2024-1/grammar - opencypher-2024-1-language-grammar) - (only-in ../../languages/cypher/opencypher-2024-1/parser - parse-opencypher-2024-1)) -(export native-abi-version - native-descriptor-payload - native-parse-binary-payload - native-error-payload) - -(def +gerbil-parser-native-abi-version+ 1) -(def +gerbil-parser-native-descriptor-schema+ - "gerbil-parser.native-descriptor.v1") -(def +gerbil-parser-native-error-schema+ - "gerbil-parser.native-error.v1") - -(def (native-error-payload exception) - (json->string - (hash (schema +gerbil-parser-native-error-schema+) - (message - (call-with-output-string - (lambda (port) (display-exception exception port))))))) - -(defstruct native-language - (id grammar parser syntax-kind-index terminal-index field-symbols field-index) - transparent: #t) - -(def (grammar-section grammar name) - (cdr (assq name (language-grammar-grammar grammar)))) - -(def (syntax-kind->json row) - (vector (symbol->string (car row)) - (symbol->string (cadr row)) - (list->vector (map symbol->string (caddr row))))) - -(def (terminal->json row) - (vector (symbol->string (car row)) - (symbol->string (cadr row)))) - -(def (native-abi-version) - +gerbil-parser-native-abi-version+) - -(def +binary-header-size+ 80) -(def +binary-event-size+ 24) - -(def +event-tags+ - (hash (start-node 1) - (finish-node 2) - (start-field 3) - (finish-field 4) - (token 5))) - -(def (indexed-symbols symbols) - (let (table (make-hash-table-eq size: (length symbols))) - (for-each (lambda (symbol index) (hash-put! table symbol index)) - symbols - (iota (length symbols))) - table)) - -;; A shared field has one stable id. The grammar expression algebra is the -;; authority for fields emitted by the parser; declared syntax fields are -;; retained first for the descriptor's complete public surface. -(def (grammar-field-symbols grammar) - (delete-duplicates/hash - (append - (apply append (map caddr (grammar-section grammar 'syntax-kinds))) - (apply append - (map (lambda (row) (grammar-expression-fields (cadr row))) - (grammar-section grammar 'rules)))) - from-end?: #t)) - -(def (make-native-language-context id grammar parser) - (let (field-symbols (grammar-field-symbols grammar)) - (make-native-language - id grammar parser - (indexed-symbols (map car (grammar-section grammar 'syntax-kinds))) - (indexed-symbols (map car (grammar-section grammar 'terminals))) - field-symbols - (indexed-symbols field-symbols)))) - -(def +gql-native-language+ - (delay - (make-native-language-context "gql" gql-iso-language-grammar - parse-gql-iso-39075-2024))) - -(def +cypher-native-language+ - (delay - (make-native-language-context "cypher" opencypher-2024-1-language-grammar - parse-opencypher-2024-1))) - -(def (resolve-native-language language) - (cond - ((string=? language "gql") (force +gql-native-language+)) - ((string=? language "cypher") (force +cypher-native-language+)) - (else (error "unsupported native parser language" language)))) - -(def (required-index table symbol domain) - (or (hash-get table symbol) - (error "native ParseArtifact symbol is outside descriptor" domain symbol))) - -(def (copy-digest! payload offset digest) - (let (bytes (hex-decode digest 7)) - (unless (= (u8vector-length bytes) 32) - (error "invalid ParseArtifact digest" digest)) - (subu8vector-move! bytes 0 32 payload offset))) - -(def (write-event! language payload row event) - (let* ((offset (+ +binary-header-size+ (* row +binary-event-size+))) - (tag (vector-ref event 0))) - (u8vector-u32-set! payload offset - (required-index +event-tags+ tag 'event-tag) little) - (case tag - ((start-node finish-node) - (u8vector-u32-set! - payload (+ offset 4) - (required-index (native-language-syntax-kind-index language) - (vector-ref event 2) 'syntax-kind) - little) - (u8vector-u64-set! payload (+ offset 8) (vector-ref event 1) little) - (let (position (vector-ref event 3)) - (u8vector-u32-set! payload (+ offset 16) - (if (eq? tag 'start-node) position 0) little) - (u8vector-u32-set! payload (+ offset 20) - (if (eq? tag 'finish-node) position 0) little))) - ((start-field finish-field) - (u8vector-u32-set! - payload (+ offset 4) - (required-index (native-language-field-index language) - (vector-ref event 1) 'field) - little) - (let (position (vector-ref event 2)) - (u8vector-u32-set! payload (+ offset 16) - (if (eq? tag 'start-field) position 0) little) - (u8vector-u32-set! payload (+ offset 20) - (if (eq? tag 'finish-field) position 0) little))) - ((token) - (u8vector-u32-set! - payload (+ offset 4) - (required-index (native-language-terminal-index language) - (vector-ref event 2) 'token-kind) - little) - (u8vector-u64-set! payload (+ offset 8) (vector-ref event 1) little) - (u8vector-u32-set! payload (+ offset 16) (vector-ref event 4) little) - (u8vector-u32-set! payload (+ offset 20) (vector-ref event 5) little))))) - -;; One bounded buffer crosses the C ABI. Token lexemes remain zero-copy source -;; slices, represented by their byte ranges instead of duplicated strings. -(def (native-parse-binary-payload language-id source) - (let* ((language (resolve-native-language language-id)) - (artifact ((native-language-parser language) source)) - (events (parse-artifact-events artifact)) - (payload (make-u8vector - (+ +binary-header-size+ - (* (length events) +binary-event-size+)) - 0))) - (subu8vector-move! #u8(71 80 65 49) 0 4 payload 0) ; GPA1 - (u8vector-u32-set! payload 4 1 little) - (u8vector-u32-set! payload 8 - (if (eq? (parse-artifact-ref artifact 'status) - 'accepted) - 0 1) - little) - (u8vector-u32-set! payload 12 (length events) little) - (copy-digest! payload 16 (parse-artifact-ref artifact 'grammarDigest)) - (copy-digest! payload 48 (parse-artifact-ref artifact 'sourceDigest)) - (for-each (lambda (event row) (write-event! language payload row event)) - events - (iota (length events))) - payload)) - -(def (native-descriptor-payload language-id) - (let (language (resolve-native-language language-id)) - (json->string - (hash (schema +gerbil-parser-native-descriptor-schema+) - (language (native-language-id language)) - (grammarDigest - (parse-artifact-ref ((native-language-parser language) "") - 'grammarDigest)) - (fields (list->vector - (map symbol->string - (native-language-field-symbols language)))) - (syntaxKinds - (list->vector (map syntax-kind->json - (grammar-section - (native-language-grammar language) - 'syntax-kinds)))) - (terminals - (list->vector (map terminal->json - (grammar-section - (native-language-grammar language) - 'terminals)))))))) diff --git a/src/ffi/rust-aot-abi.ss b/src/ffi/rust-aot-abi.ss new file mode 100644 index 00000000..a4b3360d --- /dev/null +++ b/src/ffi/rust-aot-abi.ss @@ -0,0 +1,121 @@ +;;; -*- Gerbil -*- +;;; C ABI projection for the interpreter-safe Rust generator. + +(import (only-in ./rust-runtime-aot + rust-aot-abi-version + rust-aot-error-payload + rust-runtime-source)) + +(begin-foreign + (namespace + ("gerbil-parser/src/ffi/rust-aot-abi#" + gerbil-parser-runtime-aot-abi-version + gerbil-parser-runtime-compile + gerbil-parser-runtime-result-set-bytes! + gerbil_parser_runtime_result-status + gerbil_parser_runtime_result-status-set!)) + + (c-declare #< +#include +#include + +#define GERBIL_PARSER_U8_DATA(obj) \ + ___CAST(___U8*, ___BODY_AS((obj), ___tSUBTYPED)) +#define GERBIL_PARSER_U8_LEN(obj) ___HD_BYTES(___HEADER(obj)) + +#include + +void gerbil_parser_runtime_result_init( + gerbil_parser_runtime_result *result) { + if (result == NULL) return; + result->status = 0; + result->payload = NULL; + result->length = 0; +} + +void gerbil_parser_runtime_result_release( + gerbil_parser_runtime_result *result) { + if (result == NULL) return; + if (result->payload != NULL) { + free(result->payload); + } + result->status = 0; + result->payload = NULL; + result->length = 0; +} + +/* Gambit's UTF-8-string callback accepts char*. The public API reads + immutable strings; conversion does not modify the caller's buffer. */ +int32_t gerbil_parser_runtime_compile_impl(char *grammar_path, gerbil_parser_runtime_result *result); +int32_t gerbil_parser_runtime_compile(const char *grammar_path, gerbil_parser_runtime_result *result) { + if (result == NULL) return -1; + if (grammar_path == NULL) { + gerbil_parser_runtime_result_release(result); + result->status = -1; + return -1; + } + return gerbil_parser_runtime_compile_impl((char *)grammar_path, result); +} +END-C + ) + + (c-define-type gerbil_parser_runtime_result + (type "gerbil_parser_runtime_result" (gerbil_parser_runtime_result))) + (c-define-type gerbil_parser_runtime_result-borrowed-ptr* + (pointer gerbil_parser_runtime_result + (gerbil_parser_runtime_result*))) + + (define gerbil_parser_runtime_result-status + (c-lambda (gerbil_parser_runtime_result-borrowed-ptr*) int32 + "___return(___arg1->status);")) + + (define gerbil_parser_runtime_result-status-set! + (c-lambda (gerbil_parser_runtime_result-borrowed-ptr* int32) void + "___arg1->status = ___arg2; ___return;")) + + (define gerbil-parser-runtime-result-set-bytes! + (c-lambda (gerbil_parser_runtime_result-borrowed-ptr* scheme-object) void + #< 0) { + gerbil_parser_runtime_result_release(___arg1); + ___arg1->status = -1; + ___return; +} +if (length > 0) { + memcpy(payload, GERBIL_PARSER_U8_DATA(___arg2), length); +} +/* Results must be initialized once; every call replaces their owned buffer. */ +free(___arg1->payload); +___arg1->payload = payload; +___arg1->length = length; +___return; +END-C + )) + + (c-define (gerbil-parser-runtime-aot-abi-version) + () unsigned-int32 "gerbil_parser_runtime_aot_abi_version" "extern" + (gerbil-parser/src/ffi/rust-runtime-aot#rust-aot-abi-version)) + + (c-define (gerbil-parser-runtime-compile grammar-path result) + (UTF-8-string gerbil_parser_runtime_result-borrowed-ptr*) int32 + "gerbil_parser_runtime_compile_impl" "extern" + (with-exception-catcher + (lambda (exception) + (gerbil_parser_runtime_result-status-set! result -1) + (gerbil-parser/src/ffi/rust-aot-abi#gerbil-parser-runtime-result-set-bytes! + result + (string->utf8 + (gerbil-parser/src/ffi/rust-runtime-aot#rust-aot-error-payload + exception))) + -1) + (lambda () + (gerbil_parser_runtime_result-status-set! result 0) + (gerbil-parser/src/ffi/rust-aot-abi#gerbil-parser-runtime-result-set-bytes! + result + (string->utf8 + (gerbil-parser/src/ffi/rust-runtime-aot#rust-runtime-source + grammar-path))) + (gerbil_parser_runtime_result-status result))))) diff --git a/src/ffi/rust-rowan-aot-v1-native.ss b/src/ffi/rust-rowan-aot-v1-native.ss deleted file mode 100644 index 4612b8ab..00000000 --- a/src/ffi/rust-rowan-aot-v1-native.ss +++ /dev/null @@ -1,106 +0,0 @@ -;;; -*- Gerbil -*- -;;; C ABI projection for the interpreter-safe Rust/Rowan generator. - -(import (only-in ./rust-rowan-aot-v1 - native-rowan-aot-abi-version - native-rowan-aot-error-payload - native-rust-rowan-source)) - -(begin-foreign - (namespace - ("gerbil-parser/src/ffi/rust-rowan-aot-v1-native#" - gerbil-parser-rowan-aot-abi-version - gerbil-parser-rowan-compile - gerbil-parser-rowan-result-v1-set-bytes! - gerbil_parser_rowan_result_v1-status - gerbil_parser_rowan_result_v1-status-set!)) - - (c-declare #< -#include -#include - -#define GERBIL_PARSER_U8_DATA(obj) \ - ___CAST(___U8*, ___BODY_AS((obj), ___tSUBTYPED)) -#define GERBIL_PARSER_U8_LEN(obj) ___HD_BYTES(___HEADER(obj)) - -typedef struct { - int32_t status; - uint8_t *payload; - size_t length; -} gerbil_parser_rowan_result_v1; - -void gerbil_parser_rowan_result_v1_init( - gerbil_parser_rowan_result_v1 *result) { - result->status = 0; - result->payload = NULL; - result->length = 0; -} - -void gerbil_parser_rowan_result_v1_release( - gerbil_parser_rowan_result_v1 *result) { - if (result->payload != NULL) { - free(result->payload); - } - result->status = 0; - result->payload = NULL; - result->length = 0; -} -END-C - ) - - (c-define-type gerbil_parser_rowan_result_v1 - (type "gerbil_parser_rowan_result_v1" (gerbil_parser_rowan_result_v1))) - (c-define-type gerbil_parser_rowan_result_v1-borrowed-ptr* - (pointer gerbil_parser_rowan_result_v1 - (gerbil_parser_rowan_result_v1*))) - - (define gerbil_parser_rowan_result_v1-status - (c-lambda (gerbil_parser_rowan_result_v1-borrowed-ptr*) int32 - "___return(___arg1->status);")) - - (define gerbil_parser_rowan_result_v1-status-set! - (c-lambda (gerbil_parser_rowan_result_v1-borrowed-ptr* int32) void - "___arg1->status = ___arg2; ___return;")) - - (define gerbil-parser-rowan-result-v1-set-bytes! - (c-lambda (gerbil_parser_rowan_result_v1-borrowed-ptr* scheme-object) void - #<length = GERBIL_PARSER_U8_LEN(___arg2); -___arg1->payload = (uint8_t *)malloc(___arg1->length); -if (___arg1->payload == NULL && ___arg1->length > 0) { - ___arg1->length = 0; - ___arg1->status = -1; - ___return; -} -if (___arg1->length > 0) { - memcpy(___arg1->payload, GERBIL_PARSER_U8_DATA(___arg2), ___arg1->length); -} -___return; -END-C - )) - - (c-define (gerbil-parser-rowan-aot-abi-version) - () unsigned-int32 "gerbil_parser_rowan_aot_abi_version" "extern" - (gerbil-parser/src/ffi/rust-rowan-aot-v1#native-rowan-aot-abi-version)) - - (c-define (gerbil-parser-rowan-compile grammar-path result) - (UTF-8-string gerbil_parser_rowan_result_v1-borrowed-ptr*) int32 - "gerbil_parser_rowan_compile" "extern" - (with-exception-catcher - (lambda (exception) - (gerbil_parser_rowan_result_v1-status-set! result -1) - (gerbil-parser/src/ffi/rust-rowan-aot-v1-native#gerbil-parser-rowan-result-v1-set-bytes! - result - (string->utf8 - (gerbil-parser/src/ffi/rust-rowan-aot-v1#native-rowan-aot-error-payload - exception))) - -1) - (lambda () - (gerbil_parser_rowan_result_v1-status-set! result 0) - (gerbil-parser/src/ffi/rust-rowan-aot-v1-native#gerbil-parser-rowan-result-v1-set-bytes! - result - (string->utf8 - (gerbil-parser/src/ffi/rust-rowan-aot-v1#native-rust-rowan-source - grammar-path))) - (gerbil_parser_rowan_result_v1-status result))))) diff --git a/src/ffi/rust-rowan-aot-v1.ss b/src/ffi/rust-rowan-aot-v1.ss deleted file mode 100644 index 2c07e5dc..00000000 --- a/src/ffi/rust-rowan-aot-v1.ss +++ /dev/null @@ -1,62 +0,0 @@ -;;; -*- Gerbil -*- -;;; Build-only C ABI for grammar.ss to immutable Rust/Rowan source. - -(import :gerbil/expander - :std/encoding/json - (only-in ../compiler/machine parser-machine-grammar-digest) - (only-in ../compiler/rust-rowan rust-rowan-module-source) - (only-in ../language/descriptor - language-grammar? - language-grammar-contract - language-grammar-ir - language-grammar-language - language-grammar-machine - language-grammar-version)) -(export native-rowan-aot-abi-version - native-rust-rowan-source - native-rowan-aot-error-payload) - -(def +native-rowan-aot-abi-version+ 1) -(def +native-rowan-aot-error-schema+ - "gerbil-parser.rust-rowan-aot-error.v1") - -(def (native-rowan-aot-abi-version) - +native-rowan-aot-abi-version+) - -(def (native-rowan-aot-error-payload exception) - (json->string - (hash (schema +native-rowan-aot-error-schema+) - (message - (call-with-output-string - (lambda (port) (display-exception exception port))))))) - -(def (eval-runtime-export exported) - (let (binding (core-resolve-module-export exported)) - (eval (binding-id binding)))) - -;;; The precompiled Gerbil expander remains the only grammar syntax owner. -;;; Exactly one exported descriptor prevents accidental multi-language output. -(def (grammar-module-language grammar-path) - (import-module ':gerbil-parser/rust-rowan-grammar-support #t #t) - (let* ((context (import-module grammar-path #t #t)) - (languages - (filter-map - (lambda (exported) - (and (= (module-export-phi exported) 0) - (let (value (eval-runtime-export exported)) - (and (language-grammar? value) value)))) - (module-context-export context)))) - (unless (= (length languages) 1) - (error "grammar module must export exactly one language descriptor" - grammar-path (length languages))) - (car languages))) - -(def (native-rust-rowan-source grammar-path) - (let (language (grammar-module-language grammar-path)) - (rust-rowan-module-source - (language-grammar-language language) - (language-grammar-version language) - (language-grammar-contract language) - (parser-machine-grammar-digest - (language-grammar-machine language)) - (language-grammar-ir language)))) diff --git a/src/ffi/rust-rowan-aot-main.ss b/src/ffi/rust-runtime-aot-main.ss similarity index 56% rename from src/ffi/rust-rowan-aot-main.ss rename to src/ffi/rust-runtime-aot-main.ss index 54971294..2f359af9 100644 --- a/src/ffi/rust-rowan-aot-main.ss +++ b/src/ffi/rust-runtime-aot-main.ss @@ -1,11 +1,11 @@ ;;; -*- Gerbil -*- -;;; Standalone build-time frontend for the Scheme-owned Rust/Rowan generator. +;;; Standalone build-time frontend for the Scheme-owned Rust generator. (import (only-in :gerbil/runtime gerbil-load-expander!) (only-in :gerbil/runtime/loader set-load-path!) - (only-in :gerbil-parser/rust-rowan-grammar-support deflanguage) - (for-syntax :gerbil-parser/rust-rowan-grammar-support) - (only-in ./rust-rowan-aot-v1 native-rust-rowan-source)) + (only-in :gerbil-parser/rust-runtime-grammar-support deflanguage) + (for-syntax :gerbil-parser/rust-runtime-grammar-support) + (only-in ./rust-runtime-aot rust-runtime-source)) (export main) (def (write-generated-source output-path source) @@ -14,14 +14,14 @@ (def (main . arguments) (unless (= (length arguments) 2) - (display "usage: gerbil-parser-rowan-aot GRAMMAR.ss OUTPUT.rs" + (display "usage: gerbil-parser-runtime-aot GRAMMAR.ss OUTPUT.rs" (current-error-port)) (newline (current-error-port)) (exit 64)) (cond - ((getenv "GERBIL_PARSER_ROWAN_AOT_LIB" #f) + ((getenv "GERBIL_PARSER_RUNTIME_AOT_LIB" #f) => (lambda (paths) (set-load-path! (string-split paths #\:))))) (gerbil-load-expander!) (write-generated-source (cadr arguments) - (native-rust-rowan-source (car arguments)))) + (rust-runtime-source (car arguments)))) diff --git a/src/ffi/rust-runtime-aot.ss b/src/ffi/rust-runtime-aot.ss new file mode 100644 index 00000000..a7c9c062 --- /dev/null +++ b/src/ffi/rust-runtime-aot.ss @@ -0,0 +1,26 @@ +;;; -*- Gerbil -*- +;;; Build-only C ABI for grammar.ss to immutable Rust source. + +(import (only-in ./schema +rust-aot-abi-version+ +rust-aot-error-schema+) + :gerbil/expander + :std/encoding/json + (only-in ../compiler/rust-runtime language-rust-runtime-module-source) + (only-in ../language/module-input language-module-descriptor)) +(export rust-aot-abi-version + rust-runtime-source + rust-aot-error-payload) + + +(def (rust-aot-abi-version) + +rust-aot-abi-version+) + +(def (rust-aot-error-payload exception) + (json->string + (hash (schema +rust-aot-error-schema+) + (message + (call-with-output-string + (lambda (port) (display-exception exception port))))))) + +(def (rust-runtime-source grammar-path reload?: (reload? #f)) + (import-module ':gerbil-parser/rust-runtime-grammar-support #f #t) + (language-rust-runtime-module-source (language-module-descriptor grammar-path reload?: reload?))) diff --git a/src/ffi/schema.ss b/src/ffi/schema.ss new file mode 100644 index 00000000..b660fd93 --- /dev/null +++ b/src/ffi/schema.ss @@ -0,0 +1,31 @@ +;;; Versioned FFI contracts. File paths and namespaces carry no version. +(begin-syntax + (def contracts + '((+language-abi-version+ 2) + (+artifact-abi-version+ 1) + (+event-abi-version+ 1) + (+rust-aot-abi-version+ 1) + (+abi-descriptor-schema+ "gerbil-parser.native-descriptor.v1") + (+abi-error-schema+ "gerbil-parser.native-error.v1") + (+rust-aot-error-schema+ "gerbil-parser.rust-runtime-aot-error.v1")))) +(defsyntax (abi-schema-value stx) + (syntax-case stx () + ((_ key) + (datum->syntax #'key (cadr (assq (syntax->datum #'key) contracts)))))) +(def +language-abi-version+ (abi-schema-value +language-abi-version+)) +(def +artifact-abi-version+ (abi-schema-value +artifact-abi-version+)) +(def +event-abi-version+ (abi-schema-value +event-abi-version+)) +(def +rust-aot-abi-version+ (abi-schema-value +rust-aot-abi-version+)) +(def +abi-descriptor-schema+ (abi-schema-value +abi-descriptor-schema+)) +(def +abi-error-schema+ (abi-schema-value +abi-error-schema+)) +(def +rust-aot-error-schema+ (abi-schema-value +rust-aot-error-schema+)) +(export +event-abi-version+ +language-abi-version+ +artifact-abi-version+ +rust-aot-abi-version+ + +abi-descriptor-schema+ +abi-error-schema+ +rust-aot-error-schema+) +;;; Derive the pure C query from the same table; it never enters the Scheme VM. +(defsyntax (declare-language-abi-version stx) + (datum->syntax #'declare-language-abi-version + (list 'begin-foreign + (list 'c-declare + (string-append "#include \nuint32_t gerbil_parser_language_abi_version(void) { return " + (number->string (cadr (assq '+language-abi-version+ contracts))) "; }"))))) +(export declare-language-abi-version) diff --git a/src/grammar/algebra.ss b/src/grammar/algebra.ss index bbadc600..8f8b6069 100644 --- a/src/grammar/algebra.ss +++ b/src/grammar/algebra.ss @@ -1,46 +1,56 @@ ;;; -*- Gerbil -*- ;;; Canonical GrammarExpr v1 constructors and structural analysis. +(import (only-in :gerbil/runtime/syntax __AST-list?)) + (export grammar-expression grammar-expression? + grammar-expression-header? grammar-expression-kind grammar-expression-nullable? grammar-expression-fields grammar-expression-references grammar-expression-terminals) -;; grammar-expression? +;; Shared constructor arity and scalar metadata checks. Children are checked by +;; grammar-expression? or by the source-aware compiler traversal, not here. ;; : (-> Datum Boolean) -(def (grammar-expression? value) - (and (list? value) +(def (grammar-expression-header? value) + (and (__AST-list? value) (pair? value) (case (car value) - ((empty) - (null? (cdr value))) - ((literal) + ((empty) (null? (cdr value))) + ((layout-end) + (every (lambda (boundary) + (and (string? boundary) (positive? (string-length boundary)))) + (cdr value))) + ((literal layout-start layout-next) (and (= (length value) 2) (string? (cadr value)) (positive? (string-length (cadr value))))) ((token reference) - (and (= (length value) 2) - (symbol? (cadr value)))) - ((sequence choice) - (and (pair? (cdr value)) - (grammar-expressions? (cdr value)))) - ((optional repeat repeat1) - (and (= (length value) 2) - (grammar-expression? (cadr value)))) + (and (= (length value) 2) (symbol? (cadr value)))) + ((sequence choice) (pair? (cdr value))) + ((optional repeat repeat1) (= (length value) 2)) ((field alias) - (and (= (length value) 3) - (symbol? (cadr value)) - (grammar-expression? (caddr value)))) + (and (= (length value) 3) (symbol? (cadr value)))) ((precedence) (and (= (length value) 4) (memq (cadr value) '(none left right dynamic)) - (integer? (caddr value)) - (grammar-expression? (cadddr value)))) + (integer? (caddr value)))) (else #f)))) +;; grammar-expression? +;; : (-> Datum Boolean) +(def (grammar-expression? value) + (and (grammar-expression-header? value) + (case (car value) + ((sequence choice) (grammar-expressions? (cdr value))) + ((optional repeat repeat1) (grammar-expression? (cadr value))) + ((field alias) (grammar-expression? (caddr value))) + ((precedence) (grammar-expression? (cadddr value))) + (else #t)))) + ;; grammar-expressions? ;; : (-> List Boolean) (def (grammar-expressions? expressions) @@ -68,6 +78,16 @@ (error "grammar literal must be a non-empty string" value)) (list 'literal value)) +(def (grammar-layout-marker role value) + (unless (and (memq role '(layout-start layout-next)) + (string? value) (positive? (string-length value))) + (error "layout marker must be a non-empty literal" role value)) + (list role value)) + +(def (grammar-layout-end (boundaries '())) + (let (expression (cons 'layout-end boundaries)) + (require-expression expression 'layout-end))) + ;; grammar-token ;; : (-> Symbol List) (def (grammar-token name) @@ -156,12 +176,19 @@ ;; ``` ;; % (defrules grammar-expression - (empty literal token reference seq choice optional repeat repeat1 + (empty literal layout-start layout-next layout-end + token reference seq choice optional repeat repeat1 field alias prec none left right dynamic) ((_ (empty)) (grammar-empty)) ((_ (literal value)) (grammar-literal value)) + ((_ (layout-start value)) + (grammar-layout-marker 'layout-start value)) + ((_ (layout-next value)) + (grammar-layout-marker 'layout-next value)) + ((_ (layout-end boundary ...)) + (grammar-layout-end (list boundary ...))) ((_ (token name)) (grammar-token 'name)) ((_ (reference name)) @@ -209,7 +236,7 @@ (def (grammar-expression-nullable? expression) (require-expression expression 'nullable) (case (car expression) - ((empty optional repeat) #t) + ((empty layout-end optional repeat) #t) ((sequence) (let loop ((rest (cdr expression))) (or (null? rest) diff --git a/src/grammar/lexical-algebra.ss b/src/grammar/lexical-algebra.ss index 006a653b..d1236cb6 100644 --- a/src/grammar/lexical-algebra.ss +++ b/src/grammar/lexical-algebra.ss @@ -1,8 +1,44 @@ ;;; -*- Gerbil -*- ;;; Canonical LexicalExpr v1 constructors. +(import (for-syntax (only-in :gerbil/expander core-expand1))) (export lexical-expression - lexical-expression?) + lexical-expression? text-profile? text-profile-minimum-width + text-profile-data deftext-profile + (for-syntax expand-text-lexical-rows)) + +;;; Profile expansion is a compiler helper, separate from grammar inference. +(begin-syntax + (def (expand-text-lexical-rows rows owner) + (stx-map + (lambda (row) + (syntax-case row () + ((name kind expression) + (list (syntax->datum #'name) (syntax->datum #'kind) + (expand-text-profile-syntax #'expression owner))) + (_ (raise-syntax-error #f "invalid concise lexical declaration" owner)))) + rows)) + (def (expand-text-profile-syntax expression owner (profile? #f) (depth 0)) + (when (>= depth 64) (raise-syntax-error #f "text profile declaration exceeds depth limit" owner)) + (syntax-case expression () + ((head argument ...) + (let (name (syntax->datum #'head)) + (if (and profile? (eq? name 'ref)) + (syntax-case #'(argument ...) () + ((fragment) + (identifier? #'fragment) + (let* ((use #'(fragment)) (expanded (core-expand1 use))) + (when (eq? expanded use) + (raise-syntax-error #f "unknown text profile fragment" owner)) + (expand-text-profile-syntax expanded owner #t (+ depth 1)))) + (_ (raise-syntax-error #f "text profile ref requires one declared fragment" owner))) + (cons name + (stx-map + (lambda (child) + (expand-text-profile-syntax child owner (or profile? (eq? name 'text-profile)) (+ depth 1))) + #'(argument ...)))))) + (atom (syntax->datum #'atom))))) + (def (strings? values) (let loop ((rest values)) @@ -17,12 +53,83 @@ (and (lexical-expression? (car rest)) (loop (cdr rest)))))) +;;; Closed text profiles have no callbacks or recursive references. Nullable +;;; children support guards, but each admitted scanner must make progress. +(def (text-class? value (depth 0)) + (and (< depth 64) (pair? value) (list? value) + (case (car value) + ((numeric alphabetic ascii-letter) (= (length value) 1)) + ((characters) (and (= (length value) 2) (string? (cadr value)) + (positive? (string-length (cadr value))))) + ((union) (and (pair? (cdr value)) + (andmap (lambda (child) (text-class? child (+ depth 1))) (cdr value)))) + (else #f)))) + +(def (text-profile-minimum-width value (depth 0)) + (and (< depth 64) (pair? value) (list? value) + (case (car value) + ((literal) (and (= (length value) 2) (string? (cadr value)) + (positive? (string-length (cadr value))) (string-length (cadr value)))) + ((run) (and (= (length value) 4) (text-class? (cadr value)) + (exact-integer? (caddr value)) (<= 0 (caddr value) 4294967295) + (or (not (cadddr value)) + (and (exact-integer? (cadddr value)) (<= (caddr value) (cadddr value) 4294967295))) + (caddr value))) + ((run-containing) + (and (= (length value) 5) (text-class? (cadr value)) (text-class? (caddr value)) + (let (width (text-profile-minimum-width (list 'run (cadr value) (cadddr value) (car (cddddr value))) depth)) + (and width (max 1 width))))) + ((ends-in ends-not-in) + (and (= (length value) 3) (text-class? (cadr value)) + (let (width (text-profile-minimum-width (caddr value) (+ depth 1))) + (and width (positive? width) width)))) + ((seq) (and (pair? (cdr value)) + (let loop ((rest (cdr value)) (width 0)) + (if (null? rest) width + (let (next (text-profile-minimum-width (car rest) (+ depth 1))) + (and next (<= (+ width next) 4294967295) + (loop (cdr rest) (+ width next)))))))) + ((optional) (and (= (length value) 2) + (text-profile-minimum-width (cadr value) (+ depth 1)) 0)) + ((if-next) (and (memv (length value) '(3 4)) (text-class? (cadr value)) + (let ((yes (text-profile-minimum-width (caddr value) (+ depth 1))) + (no (if (= (length value) 4) + (text-profile-minimum-width (cadddr value) (+ depth 1)) 0))) + (and yes no (min yes no))))) + ((not-next) (and (= (length value) 2) (text-class? (cadr value)) 0)) + (else #f)))) + + +;;; Named fragments expand only during declaration lowering. Runtime IR is +;;; closed data and has no name lookup or callback. +(defrules text-profile-data (literal run run-containing ends-in ends-not-in seq optional if-next not-next) + ((_ (literal value)) (list 'literal value)) + ((_ (run class minimum maximum)) (list 'run 'class minimum maximum)) + ((_ (run-containing class required minimum maximum)) + (list 'run-containing 'class 'required minimum maximum)) + ((_ (ends-in class body)) (list 'ends-in 'class (text-profile-data body))) + ((_ (ends-not-in class body)) (list 'ends-not-in 'class (text-profile-data body))) + ((_ (seq expression ...)) (cons 'seq (list (text-profile-data expression) ...))) + ((_ (optional expression)) (list 'optional (text-profile-data expression))) + ((_ (if-next class yes)) (list 'if-next 'class (text-profile-data yes))) + ((_ (if-next class yes no)) + (list 'if-next 'class (text-profile-data yes) (text-profile-data no))) + ((_ (not-next class)) (list 'not-next 'class))) + +(defrules deftext-profile () + ((_ binding expression) + (defrules binding () ((_ ) expression)))) + +(def (text-profile? value) + (let (width (text-profile-minimum-width value)) (and width (positive? width)))) + (def (lexical-expression? value) (and (list? value) (case (car value) ((whitespace+ horizontal-whitespace+ newline+ line decimal-digit+ number identifier heredoc fallback) (null? (cdr value))) + ((text-profile) (and (= (length value) 2) (text-profile? (cadr value)))) ((number-literal) (and (= (length value) 6) (pair? (cadr value)) @@ -34,10 +141,36 @@ (boolean? (cadr (cddddr value))))) ((quoted-string escaped-quoted-string) (and (pair? (cdr value)) (strings? (cdr value)))) + ((quoted-string-profile) + (and (= (length value) 4) + (string? (cadr value)) (= (string-length (cadr value)) 1) + (string? (caddr value)) + (exact-integer? (cadddr value)) (<= 0 (cadddr value) 8))) + ((module-text) + (and (= (length value) 8) (strings? (cdr value)) + (andmap (lambda (text) (<= (string-length text) 128)) (cdr value)) + (= (string-length (list-ref value 7)) 1) + (andmap (lambda (border) + (andmap (cut char=? <> (string-ref border 0)) (string->list border))) + (list (cadr value) (list-ref value 3))))) + ((header-delimiter header-data) + (and (= (length value) 4) + (string? (cadr value)) (positive? (string-length (cadr value))) + (exact-integer? (caddr value)) (<= 1 (caddr value) 32) + (if (eq? (car value) 'header-delimiter) + (and (exact-integer? (cadddr value)) (<= 0 (cadddr value)) + (< (cadddr value) (caddr value))) + (string? (cadddr value))))) ((until-delimiters) (and (= (length value) 2) (string? (cadr value)) (positive? (string-length (cadr value))))) + ((character-run) + (and (= (length value) 3) + (string? (cadr value)) + (= (string-length (cadr value)) 1) + (exact-integer? (caddr value)) + (positive? (caddr value)))) ((line-comment) (and (pair? (cdr value)) (strings? (cdr value)))) ((block-comment nested-block-comment) @@ -99,8 +232,8 @@ ;; % (defrules lexical-expression (whitespace+ horizontal-whitespace+ newline+ line decimal-digit+ number identifier - heredoc number-literal - quoted-string escaped-quoted-string until-delimiters + heredoc number-literal text-profile + quoted-string escaped-quoted-string quoted-string-profile until-delimiters character-run header-delimiter header-data module-text line-comment block-comment nested-block-comment choice literals fallback precedence external) ((_ (whitespace+)) @@ -119,14 +252,26 @@ leading-period? trailing-period?)) (list 'number-literal (list prefix ...) separator (list suffix ...) leading-period? trailing-period?)) + ((_ (text-profile expression)) + (list 'text-profile (text-profile-data expression))) ((_ (identifier)) (lexical-primitive 'identifier)) ((_ (quoted-string delimiter ...)) (cons 'quoted-string (list delimiter ...))) ((_ (escaped-quoted-string delimiter ...)) (cons 'escaped-quoted-string (list delimiter ...))) + ((_ (quoted-string-profile delimiter escapes unicode-width)) + (list 'quoted-string-profile delimiter escapes unicode-width)) + ((_ (module-text border word end-border open close comment extra)) + (list 'module-text border word end-border open close comment extra)) + ((_ (header-delimiter prefix count index)) + (list 'header-delimiter prefix count index)) + ((_ (header-data prefix count stops)) + (list 'header-data prefix count stops)) ((_ (until-delimiters characters)) (list 'until-delimiters characters)) + ((_ (character-run character minimum)) + (list 'character-run character minimum)) ((_ (heredoc)) (lexical-primitive 'heredoc)) ((_ (line-comment start ...)) diff --git a/src/language-support/antlr4-language.ss b/src/language-support/antlr4-language.ss deleted file mode 100644 index fb4330c6..00000000 --- a/src/language-support/antlr4-language.ss +++ /dev/null @@ -1,110 +0,0 @@ -;;; -*- Gerbil -*- -;;; Expansion-time ANTLR4 language-pack compiler. - -(import (for-syntax :std/misc/ports - (only-in ./antlr4-source - antlr4-source-declaration-sources - antlr4-source-parser-grammar-rules - antlr4-source-parser-literals - antlr4-source-parser-syntax-kinds - parse-antlr4-source/expected)) - :gerbil-parser/src/language/grammar) -(export deflanguage-antlr4-grammar) - -;;; A language pack declares identity and pinned native grammar once; expansion -;;; materializes its catalog, Grammar IR, LALR table, and generated machine. -;; deflanguage-antlr4-grammar -;; : (-> Syntax Syntax) -;; | doc m% -;; Lowers a pinned ANTLR4 grammar into the stable language v1 contract. -;; -;; # Examples -;; -;; ```scheme -;; (deflanguage-antlr4-grammar language -;; (identity "lang" "v1" "contract.v1") -;; (reference "upstream-v1" "commit") -;; (digest "sha256:...") (source "grammar.g4") -;; (entrypoint program) (conflicts reject) (case-insensitive #f)) -;; ;; => grammar, Parser IR, machine, and descriptor bindings -;; ``` -;; Result: runtime receives immutable v1 data and never provider source. -;; % -(defsyntax (deflanguage-antlr4-grammar stx) - (syntax-case stx - (identity reference digest source entrypoint conflicts case-insensitive) - ((_ prefix - (identity language version contract) - (reference source-version source-commit) - (digest expected-digest) - (source path) - (entrypoint entry-name) - (conflicts conflict-policy) - (case-insensitive case-insensitive-value)) - (and (identifier? #'prefix) - (stx-string? #'language) - (stx-string? #'version) - (stx-string? #'contract) - (stx-string? #'source-version) - (stx-string? #'source-commit) - (stx-string? #'expected-digest) - (stx-string? #'path) - (identifier? #'entry-name)) - (let* ((resolved (gx#core-resolve-path #'path (stx-source stx))) - (content (call-with-input-file resolved read-all-as-string)) - (catalog - (parse-antlr4-source/expected - (stx-e #'language) (stx-e #'source-version) - (stx-e #'source-commit) (stx-e #'expected-digest) content)) - (syntax-rows - (append - (antlr4-source-parser-syntax-kinds catalog) - '((LexicalIdentifier token (text)) - (NumericLiteralToken token (text)) - (StringLiteralToken token (text)) - (WhitespaceTrivia token (text)) - (CommentTrivia token (text)) - (PunctuationToken token (text)) - (UnknownToken token (text))))) - (rule-rows (antlr4-source-parser-grammar-rules catalog)) - (literal-values (antlr4-source-parser-literals catalog)) - (source-map - (antlr4-source-declaration-sources catalog (stx-e #'path)))) - (with-syntax ((((syntax-row ...)) (list syntax-rows)) - (((rule-row ...)) (list rule-rows)) - (((literal-value ...)) (list literal-values)) - (source-map-value source-map)) - #'(deflanguage-grammar prefix - (identity language version contract) - (syntax-kinds syntax-row ...) - (terminals - (identifier LexicalIdentifier) - (number NumericLiteralToken) - (string StringLiteralToken) - (whitespace WhitespaceTrivia) - (comment CommentTrivia) - (punctuation PunctuationToken) - (unknown UnknownToken)) - (lexical-rules - (whitespace (whitespace+)) - (comment (choice (line-comment "//") - (block-comment "/*" "*/"))) - (string (quoted-string "\"" "'" "`")) - (number - (number-literal ("0x" "0o" "0b") "_" - ("M" "m" "F" "f" "D" "d") #t #t)) - (identifier (identifier)) - (punctuation (literals literal-value ...)) - (unknown (fallback))) - (rules rule-row ...) - (extras whitespace comment) - (keywords) - (parser-entrypoints (entry-name parse pure)) - (recoveries - (entry-name "GERBIL-PARSER-ANTLR4-SOURCE" preserve-source)) - (conflicts conflict-policy) - (case-insensitive case-insensitive-value) - (source-ownership source-map-value) - (lineage deflanguage-antlr4-grammar source-version source-commit) - (flow (source lexical) (lexical cst)))))) - (_ (raise-syntax-error #f "invalid ANTLR4 language declaration" stx)))) diff --git a/src/language/assembly.ss b/src/language/assembly.ss new file mode 100644 index 00000000..df5f4226 --- /dev/null +++ b/src/language/assembly.ss @@ -0,0 +1,52 @@ +;;; -*- Gerbil -*- +;;; Internal assembly of already admitted immutable compiler products. + +(import (only-in ../compiler/machine defgeneral-parser-machine parser-machine-ir) + (only-in ../runtime/language-artifact load-compiled-language-artifact/embedded)) +(export assemble-language-parser) + +;; Internal runtime emission shared by native and imported declarations. +;; Expansion publishes grammar and backend instructions once. Native binding +;; loads the admitted common product and publishes the machine-owned instruction view. +;; assemble-language-parser +;; : (-> Syntax Syntax) +;; | doc m% +;; `assemble-language-parser` expands validated grammar data into bindings. +;; +;; # Examples +;; +;; ```scheme +;; (assemble-language-parser grammar ir parser grammar-data ir-data ...) +;; ;; => immutable grammar, IR, and parser bindings +;; ``` +;; % +(defrules assemble-language-parser + (syntax-kinds terminals lexical-rules rules extras keywords + parser-entrypoints recoveries flow) + ((_ grammar-binding bound-binding ir-binding parser-binding + grammar-encoded grammar-payload + bound-encoded bound-payload + program-encoded program-payload + (syntax-kinds (kind-name kind-category (field-name ...)) ...) + (lexical-rules (lexical-name lexical-expression-value) ...) + (rules (rule-name rule-expression) ...) + (extras extra-name ...) + (parser-entrypoints (entry-name entry-action entry-effect) ...) + remainder ...) + (begin + (def grammar-binding + (load-compiled-language-artifact/embedded + "gerbil-parser.grammar-ir.v1" 'grammar-encoded 'grammar-payload)) + (def bound-binding + (load-compiled-language-artifact/embedded + "gerbil-parser.bound-grammar-ir.v1" 'bound-encoded 'bound-payload)) + (defgeneral-parser-machine parser-binding + (load-compiled-language-artifact/embedded + "gerbil-parser.contextual-parser-ir.v2" 'program-encoded 'program-payload) + (lexical-rules (lexical-name lexical-expression-value) ...) + (rules (rule-name rule-expression) ...) + (extras extra-name ...) + (parser-entrypoints + (entry-name entry-action entry-effect) ...)) + (def ir-binding (parser-machine-ir parser-binding))))) + diff --git a/src/language/command-profile.ss b/src/language/command-profile.ss new file mode 100644 index 00000000..ca76c221 --- /dev/null +++ b/src/language/command-profile.ss @@ -0,0 +1,292 @@ +;;; Closed command token roles and node projections prepare once per source engine. +(import (only-in :clan/poo/object .o .ref object?) + (only-in ../grammar/lexical-algebra text-profile-data) + (only-in ../runtime/scan make-text-profile-scanner) + (only-in ../runtime/token token-kind token-lexeme token-start token-end) + (only-in ./result-profile result-plan-catalog)) +(export command-node-schemas CommandProfile. defcommand-profile compile-command-profile admit-command-plan command-plan? + command-plan-recipe command-plan-role-matcher command-plan-token? command-plan-text? command-plan-kind command-plan-forms command-plan-match-form command-plan-trigger? command-plan-entry) +(def +command-shapes+ '( + (Redirection descriptor operator target) + (ArrayAssignment assignment open close separator element) + (SimpleCommand assignment name argument redirect) + (CommandList command separator here-document) + (IfCommand keyword condition body else-body) + (WhileCommand keyword condition body) + (UntilCommand keyword condition body) + (ForCommand keyword header variable item separator body) + (SelectCommand keyword header variable item separator body) + (ArithmeticForCommand keyword header variable item separator body) + (CaseCommand keyword subject separator clause) + (CaseClause open pattern alternate close body terminator) + (FunctionDefinition keyword name open close body) + (ConditionalCommand open close operator operand) + (ArithmeticCommand open close expression operator) + (RedirectedCommand command redirect) + (BraceGroup open body close) + (Subshell open body close) + (Pipeline keyword option negate command operator) + (AndOrList command operator) + (HereDocument delimiter content) + (BashFile command separator here-document) +)) +(def (command-node-schemas) + (map (lambda (row) (list (car row) (map values (cdr row)))) +command-shapes+)) +(def CommandProfile. (.o ports: '() roles: '() texts: '() forms: '() + projections: (map (lambda (shape) (list (car shape) (car shape))) +command-shapes+))) +(def +command-roles+ '(trivia separator redirect here-redirect strip-redirect reserved conditional-operator pipeline and-or case-end)) +(defstruct command-plan (recipe-field roles texts projections dispatch fallback)) +(def (copy-data value) + (cond ((pair? value) (cons (copy-data (car value)) (copy-data (cdr value)))) + ((string? value) (string-copy value)) (else value))) +(def (unique? values) + (let loop ((remaining values) (seen '())) + (or (null? remaining) + (and (not (member (car remaining) seen)) (loop (cdr remaining) (cons (car remaining) seen)))))) +(def (same-keys? rows keys) + (and (list? rows) (andmap (lambda (row) (and (list? row) (pair? row) (symbol? (car row)))) rows) + (unique? (map car rows)) (= (length rows) (length keys)) + (andmap (lambda (key) (assq key rows)) keys))) +(def (compile-command-profile profile) + (unless (object? profile) (error "command profile requires a POO value")) + (let* ((roles (copy-data (.ref profile 'roles))) (texts (copy-data (.ref profile 'texts))) + (projections (copy-data (.ref profile 'projections))) + (forms (copy-data (.ref profile 'forms))) + (ports (copy-data (.ref profile 'ports))) + (role-index (make-hash-table-eq)) (text-index (make-hash-table-eq)) (projection-index (make-hash-table-eq))) + (unless (and (same-keys? roles +command-roles+) (same-keys? texts '(descriptor)) + (same-keys? projections (map car +command-shapes+))) + (error "command profile requires every closed role and projection exactly once")) + (for-each (lambda (row) + (let (kinds (make-hash-table-eq)) + (unless (and (pair? (cdr row)) (andmap (lambda (clause) + (and (list? clause) (pair? clause) (symbol? (car clause)) + (andmap (lambda (literal) (and (string? literal) (> (string-length literal) 0))) (cdr clause)) + (unique? (cdr clause)))) (cdr row)) (unique? (map car (cdr row)))) + (error "invalid command token role" (car row))) + (for-each (lambda (clause) + (let (index (make-table test: equal?)) + (for-each (lambda (literal) (hash-put! index literal #t)) (cdr clause)) + (hash-put! kinds (car clause) (if (null? (cdr clause)) #t index)))) (cdr row)) + (hash-put! role-index (car row) kinds))) roles) + (for-each (lambda (row) + (unless (= (length row) 2) (error "invalid command text profile" row)) + (hash-put! text-index (car row) (make-text-profile-scanner (cadr row)))) texts) + (for-each (lambda (row) + (unless (and (= (length row) 2) (symbol? (cadr row))) (error "invalid command projection" row)) + (hash-put! projection-index (car row) (cadr row))) projections) + (let-values (((dispatch fallback) (prepare-command-forms forms ports))) + (make-command-plan (list roles texts projections forms ports) role-index text-index projection-index dispatch fallback)))) +(def (command-plan-recipe plan) (copy-data (command-plan-recipe-field plan))) +(def (command-plan-kind plan role) + (or (hash-get (command-plan-projections plan) role) (error "undeclared command projection" role))) +(def (command-plan-role-matcher plan role) + (let (kinds (or (hash-get (command-plan-roles plan) role) (error "undeclared command role" role))) + (lambda (token) + (and token (let (accepted (hash-get kinds (token-kind token))) + (and accepted (or (eq? accepted #t) (hash-get accepted (token-lexeme token))))))))) +(def (command-plan-token? plan role token) ((command-plan-role-matcher plan role) token)) +(def (command-plan-text? plan role text) + (let* ((scanner (or (hash-get (command-plan-texts plan) role) (error "undeclared command text role" role))) + (end (scanner text 0))) + (and end (= end (string-length text))))) +(def (admit-command-plan plan results scanner) + (let* ((catalog (result-plan-catalog results)) (nodes (cdr (assq 'syntax-kinds catalog))) + (tokens (map car (cdr (assq 'terminals catalog)))) (scanner-tokens (.ref scanner 'tokens))) + (for-each (lambda (shape) + (let* ((kind (command-plan-kind plan (car shape))) (entry (assq kind nodes))) + (unless (and entry (andmap (lambda (field) (memq field (caddr entry))) (cdr shape))) + (error "command projection disagrees with result fields" (car shape) kind)))) +command-shapes+) + (for-each (lambda (row) + (for-each (lambda (clause) + (unless (and (memq (car clause) tokens) (memq (car clause) scanner-tokens)) + (error "command role uses an undeclared scanner terminal" (car row) (car clause)))) (cdr row))) + (car (command-plan-recipe-field plan))) + plan)) + +;;; Programs are closed data: no procedures, foreign references or nullable loops. +(def (valid-command-trigger? trigger (depth 0)) + (and (< depth 32) (list? trigger) (pair? trigger) + (case (car trigger) + ((none) (null? (cdr trigger))) + ((manual) (or (null? (cdr trigger)) + (and (= (length trigger) 2) (valid-command-trigger? (cadr trigger) (+ depth 1)) + (memq (car (cadr trigger)) '(word operator))))) + ((role) (and (= (length trigger) 2) (memq (cadr trigger) +command-roles+))) + ((or lookahead adjacent) + (and (pair? (cdr trigger)) + (andmap (lambda (item) (valid-command-trigger? item (+ depth 1))) (cdr trigger)) + (or (eq? (car trigger) 'or) + (and (<= (length (cdr trigger)) 32) + (andmap (lambda (item) (memq (car item) '(word operator))) (cdr trigger)))))) + ((word operator newline) + (andmap (lambda (literal) (and (string? literal) (> (string-length literal) 0))) (cdr trigger))) + (else #f)))) +(def (command-fields kind) + (or (assq kind +command-shapes+) (error "unknown command node role" kind))) +(def (validate-command-program program kind forms (depth 0)) + (unless (and (< depth 128) (list? program)) (error "invalid command program depth or body" kind)) + (def (field! field) + (unless (memq field (cdr (command-fields kind))) (error "command program uses a foreign field" kind field))) + (def (trigger! value) + (unless (valid-command-trigger? value) (error "invalid command trigger" value))) + (def (body steps (owner kind)) (validate-command-program steps owner forms (+ depth 1))) + (let loop ((remaining program) (minimum 0)) + (if (null? remaining) minimum + (let* ((step (car remaining)) (width + (begin + (unless (and (list? step) (pair? step)) (error "invalid command operation" step)) + (case (car step) + ((raw word command) + (unless (= (length step) 2) (error "invalid command leaf operation" step)) (field! (cadr step)) 1) + ((take) + (unless (= (length step) 3) (error "invalid command take operation" step)) + (field! (cadr step)) (trigger! (caddr step)) 1) + ((list) + (unless (and (= (length step) 4) (boolean? (cadddr step))) (error "invalid command list operation" step)) + (field! (cadr step)) (trigger! (caddr step)) (if (cadddr step) 0 1)) + ((call) + (unless (and (= (length step) 3) (symbol? (caddr step)) (assq (caddr step) forms)) + (error "undeclared command form reference" step)) (field! (cadr step)) 1) + ((node) + (unless (and (>= (length step) 3) (symbol? (caddr step))) (error "invalid command node operation" step)) + (field! (cadr step)) (command-fields (caddr step)) (body (cdddr step) (caddr step))) + ((as) + (unless (and (= (length step) 2) (symbol? (cadr step))) (error "invalid command projection branch" step)) + (unless (andmap (lambda (field) (memq field (cdr (command-fields (cadr step))))) + (cdr (command-fields kind))) (error "branch projection loses fields" step)) 0) + ((optional many until) + (unless (>= (length step) 2) (error "invalid command repetition" step)) + (trigger! (cadr step)) + (let (consumed (body (cddr step))) + (when (and (memq (car step) '(many until)) (zero? consumed)) + (error "command repetition requires a consuming body" step))) 0) + ((branch) + (unless (= (length step) 4) (error "invalid command branch" step)) (trigger! (cadr step)) + (min (body (caddr step)) (body (cadddr step)))) + ((choose) + (unless (pair? (cdr step)) (error "empty command choice" step)) + (apply min (map (lambda (row) + (unless (and (list? row) (pair? row)) (error "invalid command choice row" row)) + (trigger! (car row)) (body (cdr row))) (cdr step)))) + ((balance) + (unless (and (>= (length step) 6) (exact-integer? (list-ref step 3)) + (> (list-ref step 3) 0)) (error "invalid balanced command body" step)) + (trigger! (cadr step)) (trigger! (caddr step)) + (field! (list-ref step 4)) (field! (list-ref step 5)) + (when (zero? (body (list-tail step 6))) (error "balanced body must consume input" step)) 1) + (else (error "unregistered command operation" (car step))))))) + (loop (cdr remaining) (+ minimum width)))))) +(def (prepare-command-forms forms ports) + (unless (and (same-keys? ports '(assignment-tail pipeline-head)) + (andmap (lambda (row) (and (= (length row) 2) (symbol? (cadr row)))) ports)) + (error "command profile requires distinct assignment-tail and pipeline-head entry positions")) + (unless (and (list? forms) (pair? forms) + (andmap (lambda (row) (and (list? row) (>= (length row) 4) (symbol? (car row)))) forms) + (unique? (map car forms))) (error "invalid or duplicate command forms")) + (let ((dispatch (make-hash-table-eq)) (fallback '())) + (for-each (lambda (row) + (let ((trigger (cadr row)) (priority (caddr row)) (kind (cadddr row))) + (unless (and (valid-command-trigger? trigger) (exact-integer? priority) (<= 0 priority 1000)) + (error "invalid command form selector" (car row))) + (command-fields kind) + (when (and (not (eq? (car trigger) 'manual)) + (zero? (validate-command-program (list-tail row 4) kind forms))) + (error "command form must consume input" (car row))) + (case (car trigger) + ((manual) #t) + ((word operator) + (unless (pair? (cdr trigger)) (error "entry selectors require literal tokens" (car row))) + (let (index (or (hash-get dispatch (car trigger)) + (let (table (make-table test: equal?)) (hash-put! dispatch (car trigger) table) table))) + (for-each (lambda (literal) + (when (hash-get index literal) (error "ambiguous command form literal" literal)) + (hash-put! index literal row)) (cdr trigger)))) + ((lookahead adjacent) + (when (ormap (lambda (candidate) (equal? (cadr candidate) trigger)) fallback) + (error "duplicate command lookahead selector" trigger)) + (set! fallback (cons row fallback))) + (else (error "unregistered command entry selector" trigger))))) forms) + ;; Manual calls cannot form a cycle: they run before an input-dependent command dispatch. + (def colors (make-hash-table-eq)) + (def (visit id) + (case (hash-get colors id) + ((visiting) (error "cyclic command form call" id)) + ((done) #t) + (else + (hash-put! colors id 'visiting) + (let walk ((data (list-tail (assq id forms) 4))) + (when (pair? data) + (when (eq? (car data) 'call) (visit (caddr data))) + (walk (car data)) (walk (cdr data)))) + (hash-put! colors id 'done)))) + (for-each (lambda (row) (visit (car row))) forms) + (for-each (lambda (entry) + (let (row (assq (cadr (assq (car entry) ports)) forms)) + (unless (and row (eq? (caadr row) 'manual) (eq? (cadddr row) (cadr entry))) + (error "missing or invalid manual command entrypoint" (car entry))))) + '((assignment-tail ArrayAssignment) (pipeline-head Pipeline))) + (let* ((header (assq (cadr (assq 'pipeline-head ports)) forms)) + (array (assq (cadr (assq 'assignment-tail ports)) forms))) + ;; A nullable header is a guarded sequence. Its FIRST set can skip the VM + ;; for ordinary commands without changing publication or projection. + (unless (andmap (lambda (step) + (and (memq (car step) '(optional many)) + (> (validate-command-program (cddr step) 'Pipeline forms) 0))) + (list-tail header 4)) + (error "pipeline header requires guarded consuming operations")) + (when (zero? (validate-command-program (list-tail array 4) 'ArrayAssignment forms)) + (error "array command body must consume input"))) + (let walk ((data forms)) + (when (pair? data) + (when (eq? (car data) 'call) + (let (target (assq (caddr data) forms)) + (when (zero? (validate-command-program (list-tail target 4) (cadddr target) forms)) + (error "command call requires a consuming target" (caddr data))))) + (walk (car data)) (walk (cdr data)))) + (values dispatch fallback))) +(def (command-plan-entry plan position) + (let (row (assq position (list-ref (command-plan-recipe-field plan) 4))) + (or (and row (cadr row)) (error "unknown command entry position" position)))) +(def (command-plan-forms plan) (copy-data (list-ref (command-plan-recipe-field plan) 3))) +(def (command-plan-trigger? plan trigger tokens) + (let (token (and (pair? tokens) (car tokens))) + (case (car trigger) + ((none) #f) + ((manual) (and (pair? (cdr trigger)) (command-plan-trigger? plan (cadr trigger) tokens))) + ((role) (command-plan-token? plan (cadr trigger) token)) + ((or) (ormap (lambda (choice) (command-plan-trigger? plan choice tokens)) (cdr trigger))) + ((lookahead adjacent) + (let loop ((matchers (cdr trigger)) (remaining tokens) (previous #f)) + (if (null? matchers) #t + (let skip ((rest remaining)) + (if (and (eq? (car trigger) 'lookahead) (pair? rest) + (command-plan-token? plan 'trivia (car rest))) (skip (cdr rest)) + (and (pair? rest) + (or (not previous) (not (eq? (car trigger) 'adjacent)) + (= (token-end previous) (token-start (car rest)))) + (command-plan-trigger? plan (car matchers) rest) + (loop (cdr matchers) (cdr rest) (car rest)))))))) + (else (and token (eq? (token-kind token) (car trigger)) + (or (null? (cdr trigger)) (and (member (token-lexeme token) (cdr trigger)) #t))))))) +(def (command-plan-match-form plan tokens) + (let* ((token (and (pair? tokens) (car tokens))) + (index (and token (hash-get (command-plan-dispatch plan) (token-kind token)))) + (literal (and index (hash-get index (token-lexeme token))))) + (let loop ((remaining (command-plan-fallback plan)) (selected literal)) + (if (null? remaining) (and selected (car selected)) + (let (candidate (car remaining)) + (loop (cdr remaining) + (if (and (or (not selected) (> (caddr candidate) (caddr selected))) + (command-plan-trigger? plan (cadr candidate) tokens)) candidate selected))))))) +(defsyntax (defcommand-profile stx) + (syntax-case stx (roles texts forms projections) + ((_ (name marker self base) (roles role ...) (texts (key expression) ...) (forms form ...)) + (and (identifier? #'name) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) roles: '(role ...) forms: '(form ...) + texts: (list (list 'key (text-profile-data expression)) ...)))) + ((_ (name marker self base) (roles role ...) (texts (key expression) ...) (forms form ...) (projections projection ...)) + (and (identifier? #'name) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) roles: '(role ...) forms: '(form ...) + texts: (list (list 'key (text-profile-data expression)) ...) + projections: '(projection ...)))))) diff --git a/src/language/descriptor.ss b/src/language/descriptor.ss index fe55a590..1edebf49 100644 --- a/src/language/descriptor.ss +++ b/src/language/descriptor.ss @@ -3,7 +3,7 @@ (export +language-grammar-schema+ language-grammar? - make-language-grammar + make-language-grammar language-grammar-with-identity language-grammar-language language-grammar-version language-grammar-contract @@ -11,7 +11,10 @@ language-grammar-ir language-grammar-machine language-grammar-observability - language-grammar-with-observability) + language-grammar-with-observability + language-grammar-with-parser-policy language-grammar-parser-policy + language-parser-policy-identity language-parser-policy-branch-budget + language-parser-policy-cst-validator require-portable-language-policy!) (def +language-grammar-schema+ "gerbil-parser.language-grammar.v1") @@ -19,15 +22,56 @@ (schema language version contract grammar ir machine observability) transparent: #t) -;;; Configuration is immutable and grammar-local. The supplied value is a POO -;;; Flow observability policy; its owner validates it when a phase is invoked. +(defstruct language-parser-policy (identity branch-budget cst-validator) + transparent: #t) + +;;; Preserve the ordinary descriptor layout and its eight-argument constructor. +;;; A policy is admitted once and cannot be supplied as an unchecked loader slot. +(defstruct (policy-language-grammar language-grammar) (policy) transparent: #t) + +(def (language-grammar-parser-policy grammar) + (and (policy-language-grammar? grammar) (policy-language-grammar-policy grammar))) + +(def (copy-language-grammar grammar observability policy) + (let (fields (list (language-grammar-schema grammar) + (language-grammar-language grammar) + (language-grammar-version grammar) + (language-grammar-contract grammar) + (language-grammar-grammar grammar) + (language-grammar-ir grammar) + (language-grammar-machine grammar) observability)) + (if policy + (apply make-policy-language-grammar (append fields (list policy))) + (apply make-language-grammar fields)))) + +(def (language-grammar-with-parser-policy grammar identity branch-budget cst-validator) + (unless (and (language-grammar? grammar) + (string? identity) (positive? (string-length identity)) + (fixnum? branch-budget) (positive? branch-budget) + (procedure? cst-validator)) + (error "invalid language parser policy" identity branch-budget)) + (copy-language-grammar + grammar (language-grammar-observability grammar) + (make-language-parser-policy identity branch-budget cst-validator))) + +;;; Scheme CST callbacks have no portable lowering yet. Reject before emission. +(def (require-portable-language-policy! grammar) + (let (policy (language-grammar-parser-policy grammar)) + (when policy + (error "language parser policy is unsupported by standalone Rust AOT" + (language-parser-policy-identity policy)))) + grammar) + +;;; Observability changes preserve the recognition and validation policy. (def (language-grammar-with-observability grammar observability) - (make-language-grammar - (language-grammar-schema grammar) - (language-grammar-language grammar) - (language-grammar-version grammar) - (language-grammar-contract grammar) - (language-grammar-grammar grammar) - (language-grammar-ir grammar) - (language-grammar-machine grammar) - observability)) + (copy-language-grammar grammar observability (language-grammar-parser-policy grammar))) + +;;; Bind release identity after syntax compilation; preserve machine and policy. +(def (language-grammar-with-identity definition language version contract) + (let ((policy (language-grammar-parser-policy definition)) + (fields (list (language-grammar-schema definition) + (string-copy language) (string-copy version) (string-copy contract) + (language-grammar-grammar definition) (language-grammar-ir definition) + (language-grammar-machine definition) (language-grammar-observability definition)))) + (if policy (apply make-policy-language-grammar (append fields (list policy))) + (apply make-language-grammar fields)))) diff --git a/src/language/entry.ss b/src/language/entry.ss index 5b2ef54f..52acdea7 100644 --- a/src/language/entry.ss +++ b/src/language/entry.ss @@ -1,59 +1,56 @@ ;;; -*- Gerbil -*- -;;; Declarative public entry boundary for a versioned language parser. +;;; Engine descriptor dispatch and lossless policy execution. (import (only-in ../runtime/parser parse-source) + (only-in ../runtime/lr-parser current-lr-branch-budget) + (only-in ../runtime/cst parse-artifact->cst) + (only-in ../runtime/token make-token) + (only-in ../runtime/artifact + parse-artifact-success? parse-artifact-valid? parse-artifact-ref + parse-artifact-roundtrip parse-artifact-status event-kind + parse-artifact-events token-event? token-event-token-kind + token-event-lexeme event-start event-end make-failure-parse-artifact) (only-in ./descriptor - language-grammar-contract language-grammar-language + language-grammar? language-grammar-contract language-grammar-language language-grammar-machine language-grammar-observability - language-grammar-version)) -(export deflanguage-parser - +language-parser-entry-schema+ - language-parser-entry-ref - parse-language-source) + language-grammar-version language-grammar-parser-policy + language-parser-policy-branch-budget language-parser-policy-cst-validator) + (only-in ./source + parse-source-language source-language? + source-language-contract source-language-language + source-language-version source-language-digest source-language-scanner-factory)) +(export parse-language-source call-with-language-parser-policy) -(def +language-parser-entry-schema+ - "gerbil-parser.language-entry.v1") +(def (call-with-language-parser-policy descriptor source recognize) + (let (policy (language-grammar-parser-policy descriptor)) + (if (not policy) (recognize) + (parameterize ((current-lr-branch-budget (language-parser-policy-branch-budget policy))) + (let* ((artifact (recognize)) + (diagnostic + (and (parse-artifact-success? artifact) + ((language-parser-policy-cst-validator policy) + (parse-artifact->cst artifact))))) + (if (not diagnostic) artifact + (let (rejected + (make-failure-parse-artifact + (parse-artifact-ref artifact 'grammarDigest) source + (map (lambda (event) + (make-token (token-event-token-kind event) + (token-event-lexeme event) + (event-start event) (event-end event))) + (filter token-event? (parse-artifact-events artifact))) + diagnostic)) + (unless (parse-artifact-valid? rejected) + (error "language parser policy returned an invalid diagnostic" diagnostic)) + rejected))))))) -;; language-parser-entry-ref -;; : (-> LanguageParserEntry Symbol Datum) -(def (language-parser-entry-ref entry key) - (let (row (assq key entry)) - (and row (cdr row)))) - -;;; Parse through one immutable LanguageGrammar configuration. Observability is -;;; therefore request-local by construction and cannot leak into another -;;; language, grammar version, or concurrent parse. -(def (parse-language-source language-grammar source) - (parse-source - (language-grammar-machine language-grammar) - source - (language-grammar-observability language-grammar))) - -;;; This macro is the sole public projection from a versioned language -;;; descriptor to its runtime parse entry, keeping identity and machine bound. -;; deflanguage-parser -;; : (-> Syntax Syntax) -;; | doc m% -;; `deflanguage-parser` expands an immutable language entry and parse binding. -;; -;; # Examples -;; -;; ```scheme -;; (deflanguage-parser parse-v1 (grammar grammar-v1) (parse parse-v1-source)) -;; ;; => version-bound parser entry and procedure -;; ``` -;; % -(defrules deflanguage-parser - (grammar parse) - ((_ binding - (grammar language-grammar-value) - (parse parse-binding)) - (begin - (def binding - (list - (cons 'schema +language-parser-entry-schema+) - (cons 'language (language-grammar-language language-grammar-value)) - (cons 'version (language-grammar-version language-grammar-value)) - (cons 'contract (language-grammar-contract language-grammar-value)))) - (def (parse-binding source) - (parse-language-source language-grammar-value source))))) +(def (parse-language-source descriptor source) + (cond + ((source-language? descriptor) (parse-source-language descriptor source)) + ((language-grammar? descriptor) + (call-with-language-parser-policy + descriptor source + (lambda () + (parse-source (language-grammar-machine descriptor) source + (language-grammar-observability descriptor))))) + (else (error "language loader requires a language descriptor" descriptor)))) diff --git a/src/language/grammar.ss b/src/language/grammar.ss index 95800c2f..99a02b4e 100644 --- a/src/language/grammar.ss +++ b/src/language/grammar.ss @@ -1,19 +1,13 @@ ;;; -*- Gerbil -*- ;;; Single declarative grammar-to-parser generation boundary. -(import (for-syntax (only-in ../compiler/parser-ir compile-parser) +(import (only-in ../grammar/lexical-algebra expand-text-lexical-rows) + (for-syntax (only-in ../compiler/parser-ir compile-parser) (only-in ../compiler/bound-ir bind-grammar-ir) - (only-in ../compiler/language-artifact - compile-language-declaration-artifacts - encode-compiled-language-artifact) - (only-in :gerbil/expander core-expand1) - (only-in :std/list/list delete-duplicates/hash)) - (only-in ../compiler/machine defgeneral-parser-machine) + (only-in ../compiler/concise-language expand-lexical-language-syntax)) (only-in ./descriptor make-language-grammar) - (only-in ../runtime/language-artifact - load-compiled-language-artifact/embedded)) + (only-in ./assembly assemble-language-parser)) (export deflanguage - deflanguage-grammar defgrammar-syntax) ;;; Defines an ordinary hygienic Gerbil macro whose expansion is consumed only @@ -34,700 +28,22 @@ ((_ (name argument ...) template) (defrules name () ((_ argument ...) template)))) -;; This lower-level expander is deliberately private. Language packs have one -;; authoring surface: deflanguage-grammar. -;; Expansion materializes immutable grammar and parser IR once; runtime code -;; only receives the generated machine and never invokes the compiler. -;; assemble-language-parser -;; : (-> Syntax Syntax) -;; | doc m% -;; `assemble-language-parser` expands validated grammar data into bindings. -;; -;; # Examples -;; -;; ```scheme -;; (assemble-language-parser grammar ir parser grammar-data ir-data ...) -;; ;; => immutable grammar, IR, and parser bindings -;; ``` -;; % -(defrules assemble-language-parser - (syntax-kinds terminals lexical-rules rules extras keywords - parser-entrypoints recoveries flow) - ((_ grammar-binding bound-binding ir-binding parser-binding - grammar-encoded grammar-payload - bound-encoded bound-payload - ir-encoded ir-payload - (syntax-kinds (kind-name kind-category (field-name ...)) ...) - (lexical-rules (lexical-name lexical-expression-value) ...) - (rules (rule-name rule-expression) ...) - (extras extra-name ...) - (parser-entrypoints (entry-name entry-action entry-effect) ...) - remainder ...) - (begin - (def grammar-binding - (load-compiled-language-artifact/embedded - "gerbil-parser.grammar-ir.v1" 'grammar-encoded grammar-payload)) - (def bound-binding - (load-compiled-language-artifact/embedded - "gerbil-parser.bound-grammar-ir.v1" 'bound-encoded bound-payload)) - (def ir-binding - (load-compiled-language-artifact/embedded - "gerbil-parser.parser-ir.v1" 'ir-encoded ir-payload)) - (defgeneral-parser-machine parser-binding ir-binding - (grammar-digest (cadr 'ir-encoded)) - (lexical-rules (lexical-name lexical-expression-value) ...) - (rules (rule-name rule-expression) ...) - (extras extra-name ...) - (parser-entrypoints - (entry-name entry-action entry-effect) ...))))) - -;;; Owns validation and expansion of the complete versioned language declaration. -;;; Generated grammar, IR, and machine bindings share one expansion-time identity. -;; deflanguage-grammar -;; : (-> Syntax Syntax) -;; | doc m% -;; `deflanguage-grammar` expands a language declaration into immutable bindings. -;; -;; # Examples -;; -;; ```scheme -;; (deflanguage-grammar example-v1 ...) -;; ;; => grammar, parser IR, and parser machine bindings -;; ``` -;; % -(defsyntax (deflanguage-grammar stx) - (def (grammar-expression-datum expression) - (unless (and (pair? expression) (symbol? (car expression))) - (raise-syntax-error #f "invalid GrammarExpr declaration" stx)) - (case (car expression) - ((empty literal token reference) expression) - ((seq sequence) - (cons 'sequence (map grammar-expression-datum (cdr expression)))) - ((choice) - (cons 'choice (map grammar-expression-datum (cdr expression)))) - ((optional repeat repeat1) - (list (car expression) - (grammar-expression-datum (cadr expression)))) - ((field alias) - (list (car expression) (cadr expression) - (grammar-expression-datum (caddr expression)))) - ((prec precedence) - (list 'precedence (cadr expression) (caddr expression) - (grammar-expression-datum (cadddr expression)))) - (else - (raise-syntax-error #f "unknown GrammarExpr constructor" stx)))) - (def (compile-language-declaration - grammar-name syntax-rows terminal-rows lexical-rows rule-rows - extra-names keyword-rows entry-rows recovery-rows flow-rows - conflict-policy case-insensitive? origin lineage explicit-source-map) - (def (source-location value) - (let (source (stx-source value)) - (list - (cons 'path origin) - (cons 'location - (if source - (call-with-output-string - (lambda (port) (display source port))) - origin)) - (cons 'generated? #f)))) - (def (section-sources namespace rows) - (cons - namespace - (stx-map - (lambda (row) - (syntax-case row () - ((name . _) - (cons (syntax->datum #'name) (source-location #'name))) - (_ (raise-syntax-error #f "invalid bound declaration row" row)))) - rows))) - (def (field-sources rows) - (cons - 'field - (apply append - (stx-map - (lambda (row) - (syntax-case row () - ((kind _ (field ...)) - (map - (lambda (field) - (cons - (list (syntax->datum #'kind) - (syntax->datum field)) - (source-location field))) - (stx-map (lambda (value) value) #'(field ...)))) - (_ (raise-syntax-error #f - "invalid bound syntax-kind row" row)))) - rows)))) - (let* ((source-map - (append - explicit-source-map - (list - (section-sources 'syntax-kind syntax-rows) - (section-sources 'terminal terminal-rows) - (section-sources 'lexical-rule lexical-rows) - (section-sources 'rule rule-rows) - (field-sources syntax-rows)))) - (grammar - (list - (cons 'schema "gerbil-parser.grammar-ir.v1") - (cons 'grammar (syntax->datum grammar-name)) - (cons 'syntax-kinds (syntax->datum syntax-rows)) - (cons 'terminals (syntax->datum terminal-rows)) - (cons 'lexical-rules (syntax->datum lexical-rows)) - (cons 'rules - (map (lambda (row) - (list (car row) - (grammar-expression-datum (cadr row)))) - (syntax->datum rule-rows))) - (cons 'extras - (map list (syntax->datum extra-names))) - (cons 'keywords (syntax->datum keyword-rows)) - (cons 'parser-entrypoints (syntax->datum entry-rows)) - (cons 'recoveries (syntax->datum recovery-rows)) - (cons 'conflict-policy conflict-policy) - (cons 'case-insensitive? case-insensitive?) - (cons 'flow (syntax->datum flow-rows)))) - (declaration-identity - (list origin lineage source-map))) - (let-values (((grammar-locator bound-locator parser-locator _status) - (compile-language-declaration-artifacts - declaration-identity grammar - (lambda () - (bind-grammar-ir grammar origin lineage source-map)) - (lambda () - (compile-parser grammar))))) - (values grammar-locator - (encode-compiled-language-artifact grammar-locator) - bound-locator - (encode-compiled-language-artifact bound-locator) - parser-locator - (encode-compiled-language-artifact parser-locator))))) - -;; Identity, grammar, canonical IR, and generated parser machine are owned by -;; one declaration. parser.ss consumes the descriptor and adds no authority. - (syntax-case stx - (identity syntax-kinds terminals lexical-rules rules extras keywords - parser-entrypoints recoveries conflicts case-insensitive flow - lineage source-ownership) - ((_ prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (conflicts conflict-policy-value) - (case-insensitive case-insensitive-value) - (lineage lineage-item ...) - (flow flow-row ...)) - #'(deflanguage-grammar prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (conflicts conflict-policy-value) - (case-insensitive case-insensitive-value) - (source-ownership ()) - (lineage lineage-item ...) - (flow flow-row ...))) - ((_ prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (lineage lineage-item ...) - (flow flow-row ...)) - #'(deflanguage-grammar prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (source-ownership ()) - (lineage lineage-item ...) - (flow flow-row ...))) - ((_ prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (conflicts conflict-policy-value) - (case-insensitive case-insensitive-value) - (source-ownership source-map-value) - (lineage lineage-item ...) - (flow flow-row ...)) - (identifier? #'prefix) - (let* ((stem (symbol->string (syntax->datum #'prefix))) - (origin - (let (source (stx-source stx)) - (if source - (call-with-output-string - (lambda (port) (display source port))) - ""))) - (binding - (lambda (suffix) - (datum->syntax - #'prefix - (string->symbol (string-append stem suffix)))))) - (with-syntax ((grammar-binding (binding "-grammar")) - (bound-binding (binding "-bound-grammar-ir")) - (ir-binding (binding "-parser-ir")) - (machine-binding (binding "-parser")) - (language-binding (binding "-language-grammar"))) - (let-values (((grammar-encoded grammar-payload - bound-encoded bound-payload - ir-encoded ir-payload) - (compile-language-declaration - #'grammar-binding - #'(syntax-row ...) - #'(terminal-row ...) - #'(lexical-row ...) - #'(rule-row ...) - #'(extra-name ...) - #'(keyword-row ...) - #'(entry-row ...) - #'(recovery-row ...) - #'(flow-row ...) - (syntax->datum #'conflict-policy-value) - (syntax->datum #'case-insensitive-value) - origin (syntax->datum #'(lineage-item ...)) - (syntax->datum #'source-map-value)))) - (with-syntax - ((grammar-encoded grammar-encoded) - (grammar-payload grammar-payload) - (bound-encoded bound-encoded) - (bound-payload bound-payload) - (ir-encoded ir-encoded) - (ir-payload ir-payload)) - #'(begin - (assemble-language-parser - grammar-binding bound-binding ir-binding machine-binding - grammar-encoded grammar-payload - bound-encoded bound-payload - ir-encoded ir-payload - (syntax-kinds syntax-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (parser-entrypoints entry-row ...)) - (def language-binding - (make-language-grammar - "gerbil-parser.language-grammar.v1" - language-value version-value contract-value - grammar-binding ir-binding machine-binding #f)))))))) - ((_ prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (source-ownership source-map-value) - (lineage lineage-item ...) - (flow flow-row ...)) - (identifier? #'prefix) - (let* ((stem (symbol->string (syntax->datum #'prefix))) - (origin - (let (source (stx-source stx)) - (if source - (call-with-output-string - (lambda (port) (display source port))) - ""))) - (binding - (lambda (suffix) - (datum->syntax - #'prefix - (string->symbol (string-append stem suffix)))))) - (with-syntax ((grammar-binding (binding "-grammar")) - (bound-binding (binding "-bound-grammar-ir")) - (ir-binding (binding "-parser-ir")) - (machine-binding (binding "-parser")) - (language-binding (binding "-language-grammar"))) - (let-values (((grammar-encoded grammar-payload - bound-encoded bound-payload - ir-encoded ir-payload) - (compile-language-declaration - #'grammar-binding - #'(syntax-row ...) - #'(terminal-row ...) - #'(lexical-row ...) - #'(rule-row ...) - #'(extra-name ...) - #'(keyword-row ...) - #'(entry-row ...) - #'(recovery-row ...) - #'(flow-row ...) - 'reject #f origin - (syntax->datum #'(lineage-item ...)) - (syntax->datum #'source-map-value)))) - (with-syntax - ((grammar-encoded grammar-encoded) - (grammar-payload grammar-payload) - (bound-encoded bound-encoded) - (bound-payload bound-payload) - (ir-encoded ir-encoded) - (ir-payload ir-payload)) - #'(begin - (assemble-language-parser - grammar-binding bound-binding ir-binding machine-binding - grammar-encoded grammar-payload - bound-encoded bound-payload - ir-encoded ir-payload - (syntax-kinds syntax-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (parser-entrypoints entry-row ...)) - (def language-binding - (make-language-grammar - "gerbil-parser.language-grammar.v1" - language-value version-value contract-value - grammar-binding ir-binding machine-binding #f)))))))) - ((_ prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (conflicts conflict-policy-value) - (case-insensitive case-insensitive-value) - (flow flow-row ...)) - #'(deflanguage-grammar prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (conflicts conflict-policy-value) - (case-insensitive case-insensitive-value) - (source-ownership ()) - (lineage deflanguage-grammar) - (flow flow-row ...))) - ((_ prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (flow flow-row ...)) - #'(deflanguage-grammar prefix - (identity language-value version-value contract-value) - (syntax-kinds syntax-row ...) - (terminals terminal-row ...) - (lexical-rules lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (parser-entrypoints entry-row ...) - (recoveries recovery-row ...) - (source-ownership ()) - (lineage deflanguage-grammar) - (flow flow-row ...))) - (_ (raise-syntax-error #f "invalid language grammar declaration" stx)))) - -;;; Concise v1 authoring projection. It infers terminal rows, syntax-kind -;;; rows, rule/token references, the single parser entry, and the connected -;;; parser flow before delegating to deflanguage-grammar. The latter remains -;;; the sole Grammar IR and parser compilation authority. -;; deflanguage -;; : (-> Syntax Syntax) -;; | doc m% -;; `deflanguage` projects one hygienic language declaration into the -;; stable `deflanguage-grammar` v1 contract. -;; -;; # Examples -;; -;; ```scheme -;; (deflanguage arithmetic -;; (identity "arithmetic" "v1" "arithmetic-expression.v1") -;; (root source-file) -;; (lex (number Number (number))) -;; (rules (source-file (node SourceFile (field value number)))) -;; (extras) (keywords) (recoveries) -;; (conflicts reject) (case-insensitive #f)) -;; ;; => arithmetic grammar, Parser IR, machine, and descriptor bindings -;; ``` -;; Result: every generated binding retains the stable v1 schemas. -;; % +;;; One author contract for all vocabularies. Vocabulary transformers receive +;;; the complete declaration; they cannot silently accept the retired flat form. (defsyntax (deflanguage stx) - (def (syntax-failure message) - (raise-syntax-error #f message stx)) - (def (require condition message) - (unless condition (syntax-failure message))) - (def (unique values) - (delete-duplicates/hash values)) - (def macro-lineage '()) - (def grammar-constructors - '(node alias seq sequence choice optional repeat repeat1 field - prec precedence empty literal token reference)) - (def (expand-surface-syntax expression) - (syntax-case expression () - ((head argument ...) - (let (head-value (syntax->datum #'head)) - (if (memq head-value grammar-constructors) - (cons head-value - (stx-map expand-surface-syntax #'(argument ...))) - (let (expanded (core-expand1 expression)) - (when (eq? expanded expression) - (syntax-failure "unknown concise GrammarExpr constructor")) - (set! macro-lineage - (cons head-value macro-lineage)) - (expand-surface-syntax expanded))))) - (atom (syntax->datum #'atom)))) - (syntax-case stx - (identity root lex rules extras keywords recoveries - conflicts case-insensitive) - ((_ prefix - (identity language-value version-value contract-value) - (root root-name) - (lex lexical-row ...) - (rules rule-row ...) - (extras extra-name ...) - (keywords keyword-row ...) - (recoveries recovery-row ...) - (conflicts conflict-policy) - (case-insensitive case-insensitive-value)) + (when (and (getenv "GERBIL_PARSER_LR_TRACE" #f) (stx-pair? (stx-cdr stx))) + (displayln "AUTHOR-DECLARATION " (syntax->datum (stx-car (stx-cdr stx)))) + (force-output)) + (syntax-case stx (syntax rules lexical) + ((_ prefix (syntax (lexical clause ...)) (rules rule ...)) (identifier? #'prefix) - (let* ((prefix-name (syntax->datum #'prefix)) - (root-value (syntax->datum #'root-name)) - (lexical-rows (syntax->datum #'(lexical-row ...))) - (rule-rows - (stx-map - (lambda (row) - (syntax-case row () - ((name expression ...) - (cons (syntax->datum #'name) - (stx-map expand-surface-syntax - #'(expression ...)))) - (_ (syntax-failure - "invalid concise rule declaration")))) - #'(rule-row ...))) - (token-names - (map (lambda (row) - (require (and (list? row) - (= (length row) 3) - (symbol? (car row)) - (symbol? (cadr row))) - "invalid concise lexical declaration") - (car row)) - lexical-rows)) - (token-kinds (unique (map cadr lexical-rows))) - (rule-names - (map (lambda (row) - (require (and (list? row) - (pair? (cdr row)) - (symbol? (car row))) - "invalid concise rule declaration") - (car row)) - rule-rows)) - (node-rows '())) - (require (memq root-value rule-names) - "concise language root must name a declared rule") - (for-each - (lambda (name) - (when (memq name rule-names) - (syntax-failure - "concise language identity is both a token and a rule"))) - token-names) - (def (record-node! kind fields) - (require (symbol? kind) - "concise node kind must be an identifier") - (let (current (assq kind node-rows)) - (if current - (unless (equal? (cdr current) fields) - (syntax-failure - "concise node kind has conflicting inferred fields")) - (set! node-rows - (append node-rows (list (cons kind fields))))))) - (def (surface-fields expression) - (cond - ((not (pair? expression)) '()) - ((memq (car expression) '(node alias)) '()) - ((eq? (car expression) 'field) - (require (= (length expression) 3) - "invalid concise field expression") - (cons (cadr expression) - (surface-fields (caddr expression)))) - ((memq (car expression) '(prec precedence)) - (surface-fields (cadddr expression))) - ((memq (car expression) '(optional repeat repeat1)) - (surface-fields (cadr expression))) - ((memq (car expression) '(seq sequence choice)) - (apply append (map surface-fields (cdr expression)))) - (else '()))) - (def (surface-body expressions) - (require (pair? expressions) - "concise rule or node requires an expression") - (if (null? (cdr expressions)) - (surface-expression (car expressions)) - (cons 'seq (map surface-expression expressions)))) - (def (surface-expression expression) - (cond - ((string? expression) (list 'literal expression)) - ((symbol? expression) - (cond - ((memq expression token-names) (list 'token expression)) - ((memq expression rule-names) (list 'reference expression)) - (else - (syntax-failure - "unresolved concise grammar identifier")))) - ((not (pair? expression)) - (syntax-failure "invalid concise grammar expression")) - (else - (case (car expression) - ((node alias) - (require (and (pair? (cdr expression)) - (symbol? (cadr expression)) - (pair? (cddr expression))) - "invalid concise node expression") - (let* ((kind (cadr expression)) - (body (surface-body (cddr expression))) - (fields (unique - (apply append - (map surface-fields - (cddr expression)))))) - (record-node! kind fields) - (list 'alias kind body))) - ((seq sequence choice) - (require (pair? (cdr expression)) - "concise sequence or choice cannot be empty") - (cons (if (eq? (car expression) 'sequence) - 'seq - (car expression)) - (map surface-expression (cdr expression)))) - ((optional repeat repeat1) - (require (= (length expression) 2) - "invalid concise unary grammar expression") - (list (car expression) - (surface-expression (cadr expression)))) - ((field) - (require (and (= (length expression) 3) - (symbol? (cadr expression))) - "invalid concise field expression") - (list 'field (cadr expression) - (surface-expression (caddr expression)))) - ((prec precedence) - (require (= (length expression) 4) - "invalid concise precedence expression") - (list 'prec (cadr expression) (caddr expression) - (surface-expression (cadddr expression)))) - ((empty) - (require (null? (cdr expression)) - "invalid concise empty expression") - '(empty)) - ((literal) - (require (and (= (length expression) 2) - (string? (cadr expression))) - "invalid concise literal expression") - expression) - ((token) - (require (and (= (length expression) 2) - (memq (cadr expression) token-names)) - "explicit token names an unknown lexical declaration") - expression) - ((reference) - (require (and (= (length expression) 2) - (memq (cadr expression) rule-names)) - "explicit reference names an unknown rule") - expression) - (else - (syntax-failure - "unknown concise GrammarExpr constructor")))))) - (let* ((normalized-rules - (map (lambda (row) - (list (car row) (surface-body (cdr row)))) - rule-rows)) - (root-rule (assq root-value normalized-rules)) - (root-expression (and root-rule (cadr root-rule))) - (root-kind - (and (pair? root-expression) - (eq? (car root-expression) 'alias) - (cadr root-expression)))) - (require root-kind - "concise language root rule must construct one node") - (for-each - (lambda (kind) - (when (assq kind node-rows) - (syntax-failure - "concise syntax kind is both a node and a token"))) - token-kinds) - (let* ((root-node (assq root-kind node-rows)) - (ordered-nodes - (cons root-node - (filter (lambda (row) - (not (eq? (car row) root-kind))) - node-rows))) - (syntax-kinds - (append - (map (lambda (row) - (list (car row) 'node (cdr row))) - ordered-nodes) - (map (lambda (kind) - (list kind 'token '(text))) - token-kinds))) - (terminals - (map (lambda (row) (list (car row) (cadr row))) - lexical-rows)) - (lexical-rules - (map (lambda (row) (list (car row) (caddr row))) - lexical-rows)) - (arguments - `(,prefix-name - (identity ,(syntax->datum #'language-value) - ,(syntax->datum #'version-value) - ,(syntax->datum #'contract-value)) - (syntax-kinds ,@syntax-kinds) - (terminals ,@terminals) - (lexical-rules ,@lexical-rules) - (rules ,@normalized-rules) - (extras ,@(syntax->datum #'(extra-name ...))) - (keywords ,@(syntax->datum #'(keyword-row ...))) - (parser-entrypoints (,root-value parse pure)) - (recoveries ,@(syntax->datum #'(recovery-row ...))) - (conflicts ,(syntax->datum #'conflict-policy)) - (case-insensitive - ,(syntax->datum #'case-insensitive-value)) - (lineage deflanguage - ,@(reverse (unique macro-lineage))) - (flow (source lexical) (lexical parser) (parser cst))))) - (with-syntax (((argument ...) - (datum->syntax #'prefix arguments))) - #'(deflanguage-grammar argument ...)))))) - (_ (raise-syntax-error #f "invalid concise language declaration" stx)))) + (expand-lexical-language-syntax + stx expand-text-lexical-rows compile-parser bind-grammar-ir + #'assemble-language-parser #'make-language-grammar + #'deflanguage #'begin #'def)) + ((_ binding (syntax (vocabulary clause ...)) (rules rule ...)) + (and (identifier? #'binding) (identifier? #'vocabulary)) + #'(vocabulary binding (syntax clause ...) (rules rule ...))) + (_ (raise-syntax-error #f + "deflanguage requires exactly one syntax vocabulary and one rules block; flat declarations and release identity are not author syntax" + stx)))) diff --git a/src/language/module-input.ss b/src/language/module-input.ss new file mode 100644 index 00000000..b35d5ba2 --- /dev/null +++ b/src/language/module-input.ss @@ -0,0 +1,47 @@ +;;; Build input selects one descriptor; multi-scope packs require an export name. +(import :gerbil/expander + (only-in ./descriptor language-grammar?) + (only-in ../runtime/identity sha256-text)) +(export language-module-descriptor) +;; Track source identity only; contexts and evaluation stay SDK-owned. +(def source-identities (make-hash-table)) +;; File inputs re-admit changed content, including same-size/same-time edits. +;; Library inputs use the SDK snapshot; explicit reload is not recursive. +(def (language-module-descriptor input reload?: (reload? #f)) + (unless (and (string? input) (positive? (string-length input))) + (error "language build input requires a module path" input)) + (let* ((split (let loop ((at 0)) + (and (< at (string-length input)) + (if (char=? (string-ref input at) #\#) at (loop (+ at 1)))))) + (path (if split (substring input 0 split) input)) + (name (and split (substring input (+ split 1) (string-length input))))) + (when (or (zero? (string-length path)) (and name (zero? (string-length name)))) + (error "invalid language descriptor selector" input)) + (let* ((file? (not (char=? (string-ref path 0) #\:))) + (canonical (and file? (path-normalize (path-expand path)))) + (digest (and file? (sha256-text + (call-with-input-file canonical + (lambda (port) (read-line port #f)))))) + (changed? (and file? (not (equal? digest (hash-get source-identities canonical))))) + (context + (begin + ;; Official reload replaces the context before evaluation. A failed + ;; replacement must not leave a historical identity admitting it. + (when (and file? (or reload? changed?)) + (hash-remove! source-identities canonical)) + (import-module + (if (char=? (string-ref path 0) #\:) (string->symbol path) path) + (or reload? changed?) #t))) + (descriptors + (filter-map + (lambda (exported) + (and (= (module-export-phi exported) 0) + (or (not name) (eq? (module-export-name exported) (string->symbol name))) + (let (value (eval (binding-id (core-resolve-module-export exported)))) + (and (language-grammar? value) value)))) + (module-context-export context)))) + (unless (= (length descriptors) 1) + (error "language build input must select exactly one descriptor" input (length descriptors))) + ;; Commit only after evaluation and descriptor selection succeed. + (when file? (hash-put! source-identities canonical digest)) + (car descriptors)))) diff --git a/src/language/part-profile.ss b/src/language/part-profile.ss new file mode 100644 index 00000000..dfc7ac88 --- /dev/null +++ b/src/language/part-profile.ss @@ -0,0 +1,270 @@ +;;; Closed recognition rules compile once to context/first-character dispatch. +(import (only-in :clan/poo/object .o .ref object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ../runtime/region-scanner source-prefix-at? region-plan-specification prepare-region-plan) + (only-in ../runtime/scan make-bounded-text-profile-scanner make-bounded-literal-end-scanner) + (only-in ../grammar/lexical-algebra text-profile-data) + (only-in ./result-profile result-plan-projection)) +(export BindingProfile. defbinding-profile binding-plan-name-end binding-plan-prefix-end + binding-plan-operator-end binding-plan-subscript-at? binding-plan-subscript-width binding-plan-region-plan binding-plan-region-scopes part-plan-binding part-step-binding + PartProfile. PartProfileContract defpart-profile compile-part-profile + part-plan? part-plan-recipe part-plan-literal part-plan-match part-plan-literal-end + part-step-operation part-step-kind part-step-opening part-step-closing part-step-delimiter + part-plan-name-end admit-part-plan + part-plan-context part-context-match part-context-literal-end) +(def PartProfile. (.o contexts: '() rules: '() literal: 'LiteralPart bindings: '())) +(defstruct part-step (operation kind opening closing delimiter binding)) +(defstruct part-rule (prefix guard step)) +(defstruct part-context-index (ascii wide)) +(def (context-rules context character) + (let (code (char->integer character)) + (if (< code 128) (vector-ref (part-context-index-ascii context) code) + (or (hash-get (part-context-index-wide context) character) '())))) +(def (context-rules-put! context character rules) + (let (code (char->integer character)) + (if (< code 128) (vector-set! (part-context-index-ascii context) code rules) + (hash-put! (part-context-index-wide context) character rules)))) +(defstruct part-plan (recipe-field dispatch literal bindings)) +(def (copy-datum value) + (cond ((pair? value) (cons (copy-datum (car value)) (copy-datum (cdr value)))) + ((string? value) (string-copy value)) (else value))) +(def (distinct? values) + (let loop ((remaining values) (seen '())) + (or (null? remaining) + (and (not (member (car remaining) seen)) + (loop (cdr remaining) (cons (car remaining) seen)))))) +(def BindingProfile. + (.o name: '(seq (run (union (alphabetic) (characters "_")) 1 1) + (run (union (alphabetic) (numeric) (characters "_")) 0 #f)) + prefixes: '() operators: '() subscript: #f)) +(defstruct binding-plan (recipe name prefix operator subscript-field)) +(def (compile-binding profile) + (unless (object? profile) (error "binding profile requires a POO value")) + (let* ((name (copy-datum (.ref profile 'name))) (prefixes (copy-datum (.ref profile 'prefixes))) + (operators (copy-datum (.ref profile 'operators))) (subscript (copy-datum (.ref profile 'subscript)))) + (unless (and (list? prefixes) (list? operators) (distinct? prefixes) (distinct? operators) + (andmap (lambda (text) (and (string? text) (> (string-length text) 0))) (append prefixes operators)) + (or (not subscript) + (and (list? subscript) (= (length subscript) 2) + (string? (car subscript)) (> (string-length (car subscript)) 0) (char? (cadr subscript))))) + (error "invalid binding profile slots")) + (make-binding-plan (list name prefixes operators subscript) + (make-bounded-text-profile-scanner name) + (make-bounded-literal-end-scanner prefixes) + (make-bounded-literal-end-scanner operators) subscript))) +(def (binding-plan-name-end plan text start end) + (or ((binding-plan-name plan) text start end) start)) +(def (binding-plan-prefix-end plan text start end) + ;; A prefix requires an actual following body character, as declared syntax. + ((binding-plan-prefix plan) text start (max start (- end 1)))) +(def (binding-plan-operator-end plan text start end) + ((binding-plan-operator plan) text start end)) +(def (binding-plan-subscript-width plan) + (let (spec (binding-plan-subscript-field plan)) (and spec (string-length (car spec))))) +(def (binding-plan-subscript-at? plan text start end) + (let (spec (binding-plan-subscript-field plan)) + (and spec (<= (+ start (string-length (car spec))) end) (source-prefix-at? text start (car spec))))) +(def (binding-plan-region-plan plan regions) + (let (subscript (binding-plan-subscript-field plan)) + (and subscript + (let* ((spec (region-plan-specification regions)) (prefix (car subscript)) + (row (list prefix (string-ref prefix (- (string-length prefix) 1)) (cadr subscript) 1)) + (existing (assoc prefix (caddr spec)))) + (when (and existing (not (equal? existing row))) (error "binding region conflicts with declared pair" prefix)) + (prepare-region-plan (list (car spec) (cadr spec) + (if existing (caddr spec) (append (caddr spec) (list row))) (cadddr spec))))))) +(def (binding-plan-region-scopes plan regions) + (let (subscript (binding-plan-subscript-field plan)) + (and subscript + (let* ((prefix (car subscript)) + (base (filter (lambda (candidate) (not (equal? candidate prefix))) + (map car (caddr (region-plan-specification regions)))))) + (append (map (lambda (parent) (cons parent base)) base) + (list (cons prefix (append base (list prefix))))))))) +(def (part-plan-binding plan key) + (hash-get (part-plan-bindings plan) key)) +(def (binding-ref bindings key) + (or (hash-get bindings key) (error "undeclared binding profile" key))) +(defsyntax (defbinding-profile stx) + (syntax-case stx (name prefixes operators subscript) + ((_ (binding marker self base) (name expression) (prefixes prefix ...) (operators operator ...) (subscript #f)) + (and (identifier? #'binding) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def binding (.o (:: self base) name: (text-profile-data expression) + prefixes: '(prefix ...) operators: '(operator ...) subscript: #f))) + ((_ (binding marker self base) (name expression) (prefixes prefix ...) (operators operator ...) (subscript opener closer)) + (and (identifier? #'binding) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def binding (.o (:: self base) name: (text-profile-data expression) + prefixes: '(prefix ...) operators: '(operator ...) subscript: (list opener closer)))))) +(def (compile-step form bindings) + (unless (and (list? form) (pair? form)) (error "invalid part operation" form)) + (case (car form) + ((parameter) + (unless (= (length form) 1) (error "invalid parameter operation" form)) + (make-part-step 'parameter 'ParameterExpansion 2 1 #f (binding-ref bindings 'parameter))) + ((quote quoted-body) + (unless (and (= (length form) 5) (symbol? (cadr form)) + (exact-integer? (caddr form)) (> (caddr form) 0) + (exact-integer? (cadddr form)) (= (cadddr form) 1) + (char? (list-ref form 4))) (error "invalid quote operation" form)) + (make-part-step (car form) (cadr form) (caddr form) (cadddr form) (list-ref form 4) #f)) + ((pair) + (unless (and (= (length form) 4) (symbol? (cadr form)) + (exact-integer? (caddr form)) (> (caddr form) 0) + (exact-integer? (cadddr form)) (> (cadddr form) 0)) + (error "invalid pair operation" form)) + (make-part-step 'pair (cadr form) (caddr form) (cadddr form) #f #f)) + ((name) + (unless (and (= (length form) 3) (symbol? (cadr form)) (symbol? (caddr form))) + (error "invalid name operation" form)) + (make-part-step 'name (cadr form) 1 0 #f (binding-ref bindings (caddr form)))) + ((escape) + (unless (and (= (length form) 2) (symbol? (cadr form))) (error "invalid escape operation" form)) + (make-part-step 'escape (cadr form) 1 0 #f #f)) + (else (error "unknown part operation" form)))) +(def (profile-plan profile) + (unless (object? profile) (error "part profile requires a POO object")) + (let* ((contexts (.ref profile 'contexts)) (rules (.ref profile 'rules)) + (literal (.ref profile 'literal)) (dispatch (make-hash-table-eq)) + (bindings (make-hash-table-eq)) (declarations (.ref profile 'bindings))) + (unless (and (list? contexts) (pair? contexts) (andmap symbol? contexts) + (distinct? contexts) + (list? rules) (symbol? literal)) (error "invalid part profile slots")) + (unless (and (list? declarations) + (andmap (lambda (row) (and (pair? row) (symbol? (car row)) (object? (cdr row)))) declarations) + (distinct? (map car declarations))) (error "invalid named binding profiles")) + (for-each (lambda (row) (hash-put! bindings (car row) (compile-binding (cdr row)))) declarations) + (for-each (lambda (context) (hash-put! dispatch context (make-part-context-index (make-vector 128 '()) (make-hash-table-eqv)))) contexts) + (for-each + (lambda (row) + (unless (and (list? row) (= (length row) 3) (list? (car row)) (pair? (car row)) + (andmap (lambda (context) (memq context contexts)) (car row)) + (distinct? (car row)) + (list? (cadr row)) (= (length (cadr row)) 3) + (eq? (caadr row) 'prefix) (string? (cadadr row)) + (> (string-length (cadadr row)) 0) + (or (memq (caddr (cadr row)) '(any next)) (string? (caddr (cadr row))))) + (error "invalid part recognition rule" row)) + (let* ((prefix (string-copy (cadadr row))) (guard (let (value (caddr (cadr row))) (if (string? value) (string->list value) value))) + (step (compile-step (caddr row) bindings)) (rule (make-part-rule prefix guard step))) + (unless (and (or (not (memq (part-step-operation step) '(name escape parameter))) + (= (string-length prefix) (part-step-opening step))) + (or (not (memq (part-step-operation step) '(quote pair quoted-body))) + (= (string-length prefix) (part-step-opening step))) + (or (not (memq (part-step-operation step) '(quote quoted-body))) + (char=? (string-ref prefix (- (string-length prefix) 1)) (part-step-delimiter step)))) + (error "part opening does not match recognition prefix" row)) + (for-each + (lambda (context) + (let* ((bucket (hash-ref dispatch context)) (key (string-ref prefix 0)) + (previous (context-rules bucket key))) + (when (ormap (lambda (other) (and (equal? prefix (part-rule-prefix other)) + (equal? guard (part-rule-guard other)))) previous) + (error "duplicate part recognition rule" row)) + ;; Keep declaration precedence, including shorter-prefix overrides. + (context-rules-put! bucket key (append previous (list rule))))) (car row)))) rules) + (make-part-plan (copy-datum (list contexts rules literal + (map (lambda (row) (list (car row) (binding-plan-recipe (hash-ref bindings (car row))))) declarations))) + dispatch literal bindings))) +(def (profile-valid? profile) + (with-catch (lambda (_) #f) (lambda () (profile-plan profile) #t))) +(define-type (PartProfileContract @ PooFlowContract.) + identity: 'gerbil-parser/part-profile + .classify: (lambda (candidate context) + (let (accepted? (profile-valid? candidate)) + (poo-flow-classification-evidence 'gerbil-parser/part-profile candidate accepted? + (if accepted? '() '((expected gerbil-parser/part-profile))) context)))) +(def (compile-part-profile profile) + (validate PartProfileContract profile) (profile-plan profile)) +(def (part-plan-recipe plan) (copy-datum (part-plan-recipe-field plan))) +(def (admit-part-plan plan results regions) + (let* ((recipe (part-plan-recipe plan)) (contexts (car recipe)) + (region-spec (region-plan-specification regions)) (bindings (part-plan-bindings plan))) + (unless (and (memq 'word contexts) (memq 'HereDocument contexts)) + (error "part profile lacks required contexts")) + (def (requires! id captures) (result-plan-projection results id captures)) + (requires! 'Word '((parts parts))) + (requires! 'HereDocumentLine '((parts parts))) + (let (assignment (hash-get bindings 'assignment)) + (when assignment + (requires! 'Assignment '((name span required) (operator span required) (parts parts))) + (unless (and (pair? (caddr (binding-plan-recipe assignment))) + (null? (cadr (binding-plan-recipe assignment))) (not (binding-plan-subscript-field assignment))) + (error "assignment profile requires operators and excludes prefixes/subscripts")))) + (requires! (part-plan-literal plan) '((text span required))) + (for-each + (lambda (row) + (let* ((step (compile-step (caddr row) bindings)) (operation (part-step-operation step)) + (kind (part-step-kind step)) (prefix (cadadr row))) + (case operation + ((name escape) (requires! kind '((text span required)))) + ((quote) (requires! kind '((open span required) (parts parts) (close span required)))) + ((pair quoted-body) (requires! kind '((open span required) (body span optional) (close span required)))) + ((parameter) + (requires! kind '((open span required) (prefix span optional) (name span optional) (subscript node optional) (operator span optional) (parts parts) (close span required))) + (when (binding-plan-subscript-field (part-step-binding step)) + (requires! 'ArraySubscript '((open span required) (parts parts) (close span required))) + (binding-plan-region-plan (part-step-binding step) regions)))) + (when (eq? operation 'quote) + (unless (memq kind contexts) (error "quote result lacks part context" kind))) + (when (memq operation '(quote quoted-body)) + (unless (assv (part-step-delimiter step) (cadr region-spec)) + (error "undeclared part quote delimiter" prefix))) + (when (memq operation '(pair parameter)) + (let (region (assoc prefix (caddr region-spec))) + (unless (and region (= (part-step-closing step) (cadddr region))) + (error "part operation disagrees with region declaration" prefix)))))) + (cadr recipe))) + plan) +(def (part-plan-context plan context) + (or (hash-get (part-plan-dispatch plan) context) (error "undeclared part context" context))) +(def (match-part-rules rules text offset end) + (let loop ((rules rules)) + (and (pair? rules) + (let* ((rule (car rules)) (prefix (part-rule-prefix rule)) + (after (+ offset (string-length prefix))) (guard (part-rule-guard rule))) + (if (and (<= after end) (source-prefix-at? text offset prefix) + (or (eq? guard 'any) + (and (< after end) (or (eq? guard 'next) (memv (string-ref text after) guard))))) + (part-rule-step rule) (loop (cdr rules))))))) +(def (part-context-match bucket text offset end) + (and (< offset end) + (match-part-rules (context-rules bucket (string-ref text offset)) text offset end))) +(def (part-plan-match plan text offset end context) + (part-context-match (part-plan-context plan context) text offset end)) +(def (part-plan-literal-end plan text start end context) + (part-context-literal-end (part-plan-context plan context) text start end)) +(def (part-context-literal-end bucket text start end) + ;; The context is constant throughout a literal run. Probe its prepared + ;; index directly, invoking the shared matcher only for candidate initials. + (unless (and (string? text) (exact-integer? start) (exact-integer? end) + (<= 0 start end (string-length text))) + (error "invalid literal source range" start end)) + (let ((ascii (part-context-index-ascii bucket)) (wide (part-context-index-wide bucket))) + (let loop ((offset (if (= start end) end (+ start 1)))) + (if (= offset end) offset + (let* ((character (string-ref text offset)) (code (char->integer character)) + (rules (if (< code 128) (vector-ref ascii code) (hash-get wide character)))) + (if (and (pair? rules) (match-part-rules rules text offset end)) + offset (loop (+ offset 1)))))))) +(def (part-plan-name-end step text start end) + (binding-plan-name-end (part-step-binding step) text start end)) +(defsyntax (defpart-profile stx) + (syntax-case stx (contexts rules literal bindings) + ((_ (name marker self base) (contexts context ...) (bindings (key value) ...) + (rules rule ...) (literal kind)) + (and (identifier? #'name) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) contexts: '(context ...) + bindings: (list (cons 'key value) ...) rules: '(rule ...) literal: 'kind))) + ((_ (name marker self base) (bindings (key value) ...)) + (and (identifier? #'name) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) bindings: (list (cons 'key value) ...)))) + ((_ (name marker self base) (contexts context ...) (rules rule ...) (literal kind)) + (and (identifier? #'name) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) contexts: '(context ...) rules: '(rule ...) literal: 'kind))) + ((_ (name marker self base) (rules rule ...)) + (and (identifier? #'name) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) rules: '(rule ...)))) + ((_ (name marker self base) (literal kind)) + (and (identifier? #'name) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) literal: 'kind))))) diff --git a/src/language/result-profile.ss b/src/language/result-profile.ss new file mode 100644 index 00000000..151b949f --- /dev/null +++ b/src/language/result-profile.ss @@ -0,0 +1,321 @@ +;;; Declarative result construction is the authority for publication and catalogs. +(import (only-in :clan/poo/object .o .ref object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ./scanner-profile compile-scanner-profile) + (only-in ../runtime/token token? token-kind token-start token-end) + (only-in ../runtime/recognition + make-recognition-node recognition-node? recognition-node-kind recognition-node-start recognition-node-end + recognition-child? recognition-child-field recognition-child-value + make-recognition-child recognition-source-offset recognition-source-token) + (only-in ../runtime/funcs recognition-sequence-append)) +(export ResultProfile. ResultProfileContract defresult-profile compile-result-profile + result-plan? result-plan-recipe result-plan-catalog result-plan-node result-plan-token + result-plan-projection result-projection-build result-projection-parts-field result-projection-build-parts result-plan-bind-projections) +(def ResultProfile. (.o nodes: '() tokens: '() scanner: #f projections: '())) +(defstruct result-plan (owned-recipe node-index token-index projection-index)) +(defstruct result-projection (owner kind captures operations arity executor direct-parts?)) +(def (copy-datum value) + (cond ((pair? value) (cons (copy-datum (car value)) (copy-datum (cdr value)))) + ((string? value) (string-copy value)) (else value))) +(def (unique-symbols? values) + (and (list? values) (andmap symbol? values) + (let (seen (make-hash-table-eq)) + (andmap (lambda (value) + (and (not (hash-get seen value)) + (begin (hash-put! seen value #t) #t))) values)))) +(def (projection-catalog nodes tokens projections) + (unless (and (list? projections) + (andmap (lambda (row) (and (list? row) (>= (length row) 3) + (symbol? (car row)) (symbol? (cadr row)))) projections) + (unique-symbols? (map car projections))) + (error "invalid result projection declarations")) + (let ((derived (make-hash-table-eq)) (order '()) (terminals tokens)) + (for-each (lambda (row) + (let ((kind (cadr row)) (fields '())) + (for-each (lambda (operation) + (unless (and (list? operation) (>= (length operation) 2) (symbol? (cadr operation)) + (case (car operation) + ((token) (and (= (length operation) 5) (symbol? (caddr operation)))) + ((one) (= (length operation) 4)) + ((many) (= (length operation) 3)) (else #f))) + (error "invalid result projection operation" (car row) operation)) + (unless (memq (cadr operation) fields) (set! fields (cons (cadr operation) fields))) + (when (and (eq? (car operation) 'token) (not (memq (caddr operation) terminals))) + (set! terminals (append terminals (list (caddr operation)))))) (cdddr row)) + ;; Explicit node declarations constrain fields; otherwise derive them. + ;; Several projection programs may construct the same derived node. + (unless (assq kind nodes) + (unless (hash-key? derived kind) (set! order (cons kind order))) + (let (prior (or (hash-get derived kind) '())) + (hash-put! derived kind (append prior (filter (lambda (field) (not (memq field prior))) (reverse fields)))))))) projections) + (values (append nodes (map (lambda (kind) (list kind (hash-get derived kind))) (reverse order))) terminals))) +(def (profile-plan profile) + (unless (object? profile) (error "result profile requires a POO object")) + (let ((nodes (.ref profile 'nodes)) (tokens (.ref profile 'tokens)) + (scanner (.ref profile 'scanner)) (projections (copy-datum (.ref profile 'projections)))) + (unless (and (list? nodes) + (andmap (lambda (row) + (and (list? row) (= (length row) 2) (symbol? (car row)) + (unique-symbols? (cadr row)))) nodes) + (unique-symbols? (map car nodes)) (unique-symbols? tokens)) + (error "invalid result construction declaration")) + (when scanner (compile-scanner-profile scanner)) + (let-values (((all-nodes derived-tokens) (projection-catalog nodes tokens projections))) + (unless (pair? all-nodes) (error "result profile requires a node construction")) + (let* ((published-tokens + (if scanner + (append derived-tokens (filter (lambda (kind) (not (memq kind derived-tokens))) (.ref scanner 'tokens))) + derived-tokens)) + (node-index (make-hash-table-eq)) (token-index (make-hash-table-eq)) + (owned (map (lambda (row) (list (car row) (map values (cadr row)))) all-nodes))) + (for-each (lambda (row) + (let (fields (make-hash-table-eq)) + (for-each (lambda (field) (hash-put! fields field #t)) (cadr row)) + (hash-put! node-index (car row) fields))) owned) + (for-each (lambda (kind) (hash-put! token-index kind #t)) published-tokens) + (let (plan (make-result-plan (list owned (map values published-tokens) projections) + node-index token-index (make-hash-table-eq))) + (for-each (lambda (row) (compile-projection! plan row)) projections) + plan))))) + +;;; Each capture is consumed once, in declaration order. Compilation resolves +;;; field and terminal membership and assigns fixed vector indexes; execution +;;; never searches an association list or flattens nested token sequences. +(def (compile-projection! plan row) + (let* ((id (car row)) (kind (cadr row)) (captures (caddr row)) (operations (cdddr row)) + (fields (hash-get (result-plan-node-index plan) kind))) + (unless (and fields (list? captures) + (andmap (lambda (capture) (and (list? capture) (= (length capture) 2) + (symbol? (car capture)) (memq (cadr capture) '(span node parts)))) captures) + (unique-symbols? (map car captures)) (= (length captures) (length operations))) + (error "invalid result projection captures" id)) + (let* ((index -1) + (compiled + (map (lambda (operation capture) + (set! index (+ index 1)) + (unless (and (list? operation) (>= (length operation) 2) + (symbol? (cadr operation)) (hash-get fields (cadr operation)) + (case (car operation) + ((token) (and (= (length operation) 5) (eq? (cadr capture) 'span) + (hash-get (result-plan-token-index plan) (caddr operation)) + (eq? (list-ref operation 3) (car capture)) + (memq (list-ref operation 4) '(required optional)))) + ((one) (and (= (length operation) 4) (eq? (cadr capture) 'node) + (eq? (caddr operation) (car capture)) + (memq (list-ref operation 3) '(required optional)))) + ((many) (and (= (length operation) 3) (eq? (cadr capture) 'parts) + (eq? (caddr operation) (car capture)))) + (else #f))) + (error "invalid result projection operation" id operation)) + (list->vector (cons index operation))) + operations captures))) + (hash-put! (result-plan-projection-index plan) id + (make-result-projection plan kind + (map (lambda (capture operation) + (if (eq? (cadr capture) 'parts) capture + (append capture (list (list-ref operation (- (length operation) 1)))))) captures operations) + compiled (length captures) + (compile-projection-executor plan kind compiled) + (and (pair? compiled) (null? (cdr compiled)) + (eq? (vector-ref (car compiled) 1) 'many))))))) +(def (result-plan-projection plan id expected-captures) + (let (projection (hash-get (result-plan-projection-index plan) id)) + (unless (and projection (equal? expected-captures (result-projection-captures projection))) + (error "missing or incompatible result projection" id expected-captures)) + projection)) +(def (result-plan-bind-projections plan expected bindings) + ;; Generated producers bind against the complete admitted recipe. Clone the + ;; executable plan so another consumer cannot change this owner's execution. + (unless (and (result-plan? plan) (equal? expected (result-plan-recipe plan)) + (list? bindings) + (andmap (lambda (row) (and (pair? row) (symbol? (car row)) (procedure? (cdr row)))) bindings) + (unique-symbols? (map car bindings)) + (equal? (map car bindings) (map car (caddr expected)))) + (error "incompatible generated result projections")) + (let (bound (make-result-plan (result-plan-owned-recipe plan) + (result-plan-node-index plan) (result-plan-token-index plan) (make-hash-table-eq))) + (for-each (lambda (row) + (let (original (hash-get (result-plan-projection-index plan) (car row))) + (hash-put! (result-plan-projection-index bound) (car row) + (make-result-projection bound (result-projection-kind original) + (result-projection-captures original) (result-projection-operations original) + (result-projection-arity original) (cdr row) #f)))) bindings) + bound)) +(def (compile-projection-executor plan kind operations) + ;; Static one-operation programs avoid VM dispatch and redundant catalog + ;; checks. Capture bounds and source offsets are still checked on every call. + (let (operation (and (pair? operations) (null? (cdr operations)) (car operations))) + (cond + ((and operation (eq? (vector-ref operation 1) 'token) (eq? (vector-ref operation 5) 'required)) + (let ((field (vector-ref operation 2)) (terminal (vector-ref operation 3))) + (lambda (source start end captures) + (let (span (vector-ref captures 0)) + (unless (and (pair? span) (exact-integer? (car span)) (exact-integer? (cdr span)) + (<= start (car span) (cdr span) end)) + (error "invalid required projection span" field span)) + (let (token (recognition-source-token source terminal (car span) (cdr span))) + (values (make-recognition-node kind + (recognition-source-offset source start) (recognition-source-offset source end) + (list (make-recognition-child field token))) (list token))))))) + ((and operation (eq? (vector-ref operation 1) 'many)) + (let (field (vector-ref operation 2)) + (lambda (source start end captures) + (let (parts (vector-ref captures 0)) + (unless (and (pair? parts) (list? (car parts)) (andmap recognition-node? (car parts))) + (error "invalid projection parts capture" field)) + (values (result-plan-node plan kind + (recognition-source-offset source start) (recognition-source-offset source end) + (map (lambda (value) (make-recognition-child field value)) (car parts))) (cdr parts)))))) + (else #f)))) +(def (result-projection-build projection source start end captures) + (unless (and (result-projection? projection) (vector? captures) + (= (vector-length captures) (result-projection-arity projection)) + (exact-integer? start) (exact-integer? end) (<= 0 start end)) + (error "invalid result projection frame")) + (let (executor (result-projection-executor projection)) + (if executor (executor source start end captures) + (interpret-result-projection projection source start end captures)))) +;;; The engine can construct final child cells for a default unary many +;;; projection. Bound executors retain node captures and own their publication. +(def (result-projection-parts-field projection) + (unless (result-projection? projection) (error "invalid parts projection")) + (let* ((operations (result-projection-operations projection)) + (operation (and (pair? operations) (null? (cdr operations)) (car operations)))) + (unless (and operation (eq? (vector-ref operation 1) 'many)) + (error "parts construction requires a unary many projection")) + (and (result-projection-direct-parts? projection) (vector-ref operation 2)))) +(def (result-projection-build-parts projection source start end parts tokens) + (unless (and (result-projection? projection) + (exact-integer? start) (exact-integer? end) (<= 0 start end)) + (error "invalid parts projection frame")) + (let (field (result-projection-parts-field projection)) + (if field + (begin + (unless (and (list? parts) + (andmap (lambda (child) + (and (recognition-child? child) + (eq? (recognition-child-field child) field) + (recognition-node? (recognition-child-value child)))) parts)) + (error "invalid direct projection parts capture" field)) + (values (result-plan-node (result-projection-owner projection) + (result-projection-kind projection) + (recognition-source-offset source start) (recognition-source-offset source end) parts) tokens)) + (result-projection-build projection source start end (vector (cons parts tokens)))))) +(def (interpret-result-projection projection source start end captures) + (let ((plan (result-projection-owner projection)) (children '()) (tokens '())) + (def (emit! field value) (set! children (cons (make-recognition-child field value) children))) + (for-each + (lambda (operation) + (let* ((value (vector-ref captures (vector-ref operation 0))) + (opcode (vector-ref operation 1)) (field (vector-ref operation 2))) + (case opcode + ((token) + (when (and (not value) (eq? (vector-ref operation 5) 'required)) + (error "missing required projection span" field)) + (when value + (unless (and (pair? value) (exact-integer? (car value)) (exact-integer? (cdr value)) + (<= start (car value) (cdr value) end)) + (error "invalid projection span" field value)) + (let (token (recognition-source-token source (vector-ref operation 3) (car value) (cdr value))) + (emit! field token) + (set! tokens (recognition-sequence-append tokens (list token)))))) + ((one) + (when (and (not value) (eq? (vector-ref operation 4) 'required)) + (error "missing required projection node" field)) + (when value + (unless (and (pair? value) (recognition-node? (car value))) + (error "invalid projection node capture" field)) + (emit! field (car value)) + (set! tokens (recognition-sequence-append tokens (cdr value))))) + ((many) + (unless (and (pair? value) (list? (car value)) (andmap recognition-node? (car value))) + (error "invalid projection parts capture" field)) + (for-each (lambda (part) (emit! field part)) (car value)) + (set! tokens (recognition-sequence-append tokens (cdr value))))))) + (result-projection-operations projection)) + (values (result-plan-node plan (result-projection-kind projection) + (recognition-source-offset source start) (recognition-source-offset source end) + ;; Only emit! allocated this spine; captures retain their own lists. + (reverse! children)) tokens))) +(def (profile-valid? profile) + (with-catch (lambda (_) #f) (lambda () (profile-plan profile) #t))) +(define-type (ResultProfileContract @ PooFlowContract.) + identity: 'gerbil-parser/result-profile + .classify: (lambda (candidate context) + (let (accepted? (profile-valid? candidate)) + (poo-flow-classification-evidence 'gerbil-parser/result-profile candidate accepted? + (if accepted? '() '((expected gerbil-parser/result-profile))) context)))) +(def (compile-result-profile profile) + (validate ResultProfileContract profile) + (profile-plan profile)) +(def (result-plan-recipe plan) + (let (recipe (result-plan-owned-recipe plan)) + (list (map (lambda (row) (list (car row) (map values (cadr row)))) (car recipe)) + (map values (cadr recipe)) (copy-datum (caddr recipe))))) +(def (result-plan-catalog plan) + ;; Catalog publication copies only the catalog, never the projection program. + (let (recipe (result-plan-owned-recipe plan)) + (list (cons 'syntax-kinds + (map (lambda (row) (list (car row) 'result (map values (cadr row)))) (car recipe))) + (cons 'terminals (map (lambda (kind) (list kind 'token)) (cadr recipe)))))) +(def (result-plan-token plan token) + (unless (and (token? token) (exact-integer? (token-start token)) + (exact-integer? (token-end token)) (<= 0 (token-start token) (token-end token)) + (hash-get (result-plan-token-index plan) (token-kind token))) + (error "undeclared result terminal" token)) + token) +(def (result-plan-node plan kind start end children) + (let (fields (hash-get (result-plan-node-index plan) kind)) + (unless (and fields (exact-integer? start) (exact-integer? end) (<= 0 start end) + (list? children)) + (error "invalid or undeclared result node" kind start end)) + (for-each + (lambda (child) + (unless (and (recognition-child? child) + (hash-get fields (recognition-child-field child))) + (error "undeclared result field" kind child)) + (let (value (recognition-child-value child)) + ;; Dispatch once, retain both bounds, then validate the local operation. + ;; Do not call the generic span dispatcher repeatedly for each child. + (let-values (((child-start child-end) + (cond + ((token? value) + (unless (hash-get (result-plan-token-index plan) (token-kind value)) + (error "undeclared child result terminal" kind)) + (values (token-start value) (token-end value))) + ((recognition-node? value) + (unless (hash-get (result-plan-node-index plan) (recognition-node-kind value)) + (error "undeclared child result node" kind)) + (values (recognition-node-start value) (recognition-node-end value))) + (else (error "invalid result child value" kind))))) + (unless (and (exact-integer? child-start) (exact-integer? child-end) + (<= start child-start child-end end)) + (error "result child outside parent span" kind))))) children) + (make-recognition-node kind start end children))) +(defsyntax (defresult-profile stx) + (syntax-case stx () + ((_ (name marker self base) clause ...) + (and (identifier? #'name) (identifier? #'self) (eq? (syntax->datum #'marker) '::)) + (let (clauses + (map (lambda (clause) + (syntax-case clause (scanner nodes tokens projections) + ((scanner source-scanner) + (cons 'scanner (syntax->list #'(scanner: source-scanner)))) + ((nodes (kind field ...) ...) + (cons 'nodes (syntax->list #'(nodes: '((kind (field ...)) ...))))) + ((tokens token ...) + (cons 'tokens (syntax->list #'(tokens: '(token ...))))) + ((projections projection ...) + (cons 'projections (syntax->list #'(projections: '(projection ...))))) + (_ (raise-syntax-error 'defresult-profile "invalid result profile clause" stx clause)))) + (syntax->list #'(clause ...)))) + ;; Preserve the existing declaration language, including clause order. + ;; Parse each clause once instead of expanding every full combination. + (unless (member (map car clauses) + '((scanner nodes tokens projections) (nodes tokens projections) + (projections) (scanner nodes tokens) (nodes tokens) + (nodes) (tokens) (scanner))) + (raise-syntax-error 'defresult-profile "invalid result profile clause combination" stx)) + (with-syntax (((slot ...) (apply append (map cdr clauses)))) + #'(def name (.o (:: self base) slot ...))))))) diff --git a/src/language/scanner-profile.ss b/src/language/scanner-profile.ss new file mode 100644 index 00000000..2cfe6d40 --- /dev/null +++ b/src/language/scanner-profile.ss @@ -0,0 +1,50 @@ +;;; POO scanner declarations compile to closed shared contextual IR. +(import (only-in :clan/poo/object .o .ref object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ../modules/parser/contextual-objects make-contextual-role make-contextual-method make-contextual-scan-rule) + (only-in ../compiler/contextual-dispatch compile-contextual-dispatch) + (only-in ../compiler/contextual-scanner-ir compile-contextual-scanner) + (only-in ../runtime/region-scanner region-plan-specification)) +(export ScannerProfile. ScannerProfileContract defscanner-profile compile-scanner-profile) +(def ScannerProfile. (.o modes: '() tokens: '() initial-mode: #f rules: '())) +(def (profile-ir profile) + (let* ((tokens (.ref profile 'tokens)) + (role (make-contextual-role 'source + (map (lambda (token) (make-contextual-method token 'any 'any token token)) tokens))) + (dispatch (compile-contextual-dispatch (list role) (.ref profile 'modes) '(source) tokens)) + (rules (map (lambda (row) (apply make-contextual-scan-rule row)) (.ref profile 'rules)))) + (compile-contextual-scanner rules dispatch (.ref profile 'initial-mode)))) +(def (profile-valid? profile) + (with-catch (lambda (_) #f) (lambda () + (and (object? profile) (profile-ir profile) #t)))) +(define-type (ScannerProfileContract @ PooFlowContract.) + identity: 'gerbil-parser/scanner-profile + .classify: (lambda (candidate context) + (let (ok? (profile-valid? candidate)) + (poo-flow-classification-evidence 'gerbil-parser/scanner-profile candidate ok? + (if ok? '() '((expected gerbil-parser/scanner-profile))) context)))) +(def (compile-scanner-profile profile) + (validate ScannerProfileContract profile) + (profile-ir profile)) +(defrules scanner-matcher-value (region-word unless-prefix) + ((_ (region-word plan initial?)) + (list 'region-word (append (take (region-plan-specification plan) 3) (list initial?)))) + ((_ (unless-prefix (prefix ...) (exception ...) child)) + (list 'unless-prefix (list prefix ...) (list exception ...) (scanner-matcher-value child))) + ((_ matcher) 'matcher)) +(defsyntax (defscanner-profile stx) + (syntax-case stx (modes initial-mode tokens rules) + ((_ (name marker self base) + (modes mode ...) (initial-mode initial) (tokens token ...) + (rules (rule mode-name form matcher rank action) ...)) + (and (identifier? #'name) (identifier? #'self) + (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) modes: '(mode ...) initial-mode: 'initial + tokens: '(token ...) rules: (list (list 'rule 'mode-name 'form + (scanner-matcher-value matcher) rank 'action) ...)))) + ((_ (name marker self base) (rules (rule mode-name form matcher rank action) ...)) + (and (identifier? #'name) (identifier? #'self) + (eq? (syntax->datum #'marker) '::)) + #'(def name (.o (:: self base) rules: (list (list 'rule 'mode-name 'form + (scanner-matcher-value matcher) rank 'action) ...)))))) diff --git a/src/language/source-strategy.ss b/src/language/source-strategy.ss new file mode 100644 index 00000000..b3d37633 --- /dev/null +++ b/src/language/source-strategy.ss @@ -0,0 +1,57 @@ +;;; Common admitted POO source engines. Providers own binding and closed recipes. +(import (only-in ../compiler/contextual-program contextual-program? contextual-program-ir contextual-program-results) + (only-in ../runtime/contextual-ir contextual-ir-ref) + (only-in :clan/poo/object .o .ref .slot? object?) + (only-in :clan/poo/mop define-type validate) + (only-in ./result-profile result-plan? result-plan-catalog) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence)) +(export SourceStrategy. SourceStrategyContract bind-source-strategy + declare-source-strategy-provider make-source-engine + source-engine? source-engine-program source-engine-scanner source-engine-factory + source-engine-parse source-engine-receipt source-engine-results source-engine-root-kind) +(defstruct source-strategy-provider (name admit compile)) +(defstruct source-engine (scanner factory parse receipt results root-kind program)) +(def (declare-source-strategy-provider name admit compile) + (unless (and (symbol? name) (procedure? admit) (procedure? compile)) + (error "invalid engine source strategy registration" name)) + (make-source-strategy-provider name admit compile)) +(def (source-strategy? candidate) + (with-catch (lambda (_) #f) + (lambda () + (and (object? candidate) + (andmap (lambda (slot) (.slot? candidate slot)) '(provider kind metadata)) + (source-strategy-provider? (.ref candidate 'provider)) + (eq? (.ref candidate 'kind) (source-strategy-provider-name (.ref candidate 'provider))) + (object? (.ref candidate 'metadata)) + ((source-strategy-provider-admit (.ref candidate 'provider)) candidate))))) +(define-type (SourceStrategyContract @ PooFlowContract.) + identity: 'gerbil-parser/source-strategy + .classify: (lambda (candidate context) + (let (accepted? (source-strategy? candidate)) + (poo-flow-classification-evidence + 'gerbil-parser/source-strategy candidate accepted? + (if accepted? '() '((expected gerbil-parser/source-strategy))) context)))) +(def SourceStrategy. + (.o (:: self) provider: #f metadata: (.o) + (kind (source-strategy-provider-name (.ref self 'provider))))) +(def (bind-source-strategy strategy) + (validate SourceStrategyContract strategy) + (let (provider (.ref strategy 'provider)) + (let-values (((recipe engine) ((source-strategy-provider-compile provider) strategy))) + (unless (and (source-engine? engine) + (procedure? (source-engine-scanner engine)) + (procedure? (source-engine-factory engine)) + (procedure? (source-engine-parse engine)) + (contextual-program? (source-engine-program engine)) + (equal? recipe (contextual-program-ir (source-engine-program engine))) + (eq? (source-engine-root-kind engine) + (contextual-ir-ref recipe 'root-kind)) + (result-plan? (source-engine-results engine)) + (eq? (source-engine-results engine) + (contextual-program-results (source-engine-program engine))) + (symbol? (source-engine-root-kind engine)) + (assq (source-engine-root-kind engine) + (cdr (assq 'syntax-kinds (result-plan-catalog (source-engine-results engine))))) + (or (not (source-engine-receipt engine)) (procedure? (source-engine-receipt engine)))) + (error "source provider returned an invalid engine" (.ref strategy 'kind))) + (values (list (.ref strategy 'kind) recipe) engine)))) diff --git a/src/language/source.ss b/src/language/source.ss new file mode 100644 index 00000000..fc3f942e --- /dev/null +++ b/src/language/source.ss @@ -0,0 +1,141 @@ +;;; -*- Gerbil -*- +;;; Versioned language ownership for stateful source parsers. The scanner and +;;; parser are one immutable declaration and must publish a valid artifact. + +(import (only-in ../runtime/parse-cost with-parser-cost-stage) + (only-in ../compiler/contextual-program contextual-program-ir) + (only-in ../runtime/artifact + parse-artifact-ref parse-artifact-valid-for-source? sha256-text) + (only-in ../runtime/source-scanner source-scanner-contextual? make-source-scan-session source-scan-session-tokens + source-scan-session-scanned-token-count source-scan-session-reused-token-count) + (only-in ./source-strategy bind-source-strategy source-engine-scanner source-engine-factory + source-engine-parse source-engine-receipt source-engine-results source-engine-root-kind source-engine-program) + (only-in ./result-profile result-plan-catalog)) +(export +source-language-schema+ + declare-source-language declare-source-syntax source-language-with-identity source-syntax-strategy + source-language? + source-language-language + source-language-version + source-language-contract + source-language-digest source-language-result-catalog source-language-root-kind source-language-contextual-ir + source-language-scanner-factory parse-source-language parse-source-language/receipt + deflanguage-parser-receipt + parse-source-language/session source-language-session? source-language-session-artifact + source-language-history? source-language-session-history + source-language-session-scanned-token-count source-language-session-reused-token-count) + +(def +source-language-schema+ "gerbil-parser.source-language.v1") + +(defstruct source-language + (schema language version contract digest scanner parse factory receipt results root-kind program) + transparent: #t) + +(def (declare-source-language language version contract strategy) + (unless (and (string? language) (positive? (string-length language)) + (string? version) (positive? (string-length version)) + (string? contract) (positive? (string-length contract))) + (error "invalid source language identity")) + (let ((language (string-copy language)) (version (string-copy version)) + (contract (string-copy contract))) + (let-values (((recipe engine) (bind-source-strategy strategy))) + (make-source-language + +source-language-schema+ language version contract + (sha256-text (call-with-output-string + (lambda (port) (write (list language version contract recipe) port)))) + (source-engine-scanner engine) (source-engine-parse engine) + (source-engine-factory engine) (source-engine-receipt engine) (source-engine-results engine) (source-engine-root-kind engine) (source-engine-program engine))))) +(def (source-language-contextual-ir descriptor) + (contextual-program-ir (source-language-program descriptor))) +(def (source-language-result-catalog descriptor) + (result-plan-catalog (source-language-results descriptor))) +(def (source-language-scanner-factory descriptor) (source-language-factory descriptor)) +(def (validate-source-publication! descriptor source artifact) + (unless (and (equal? (parse-artifact-ref artifact 'grammarDigest) (source-language-digest descriptor)) + (parse-artifact-valid-for-source? artifact source)) + (error "source parser published an invalid artifact")) + artifact) +(def (parse-source-language/receipt descriptor source) + (unless (and (string? source) (source-language? descriptor) (procedure? (source-language-receipt descriptor))) + (error "source engine does not declare receipts for this input")) + (let-values (((artifact receipt) + ((source-language-receipt descriptor) source (source-language-scanner descriptor) + (source-language-digest descriptor)))) + (with-parser-cost-stage 'artifact-validation + (validate-source-publication! descriptor source artifact)) + (values artifact receipt))) + +(defrules deflanguage-parser-receipt () + ((_ binding descriptor) + (def (binding source) (parse-source-language/receipt descriptor source)))) + +(def (parse-source-language descriptor source) + (unless (string? source) + (error "source language requires a string" source)) + (let* ((digest (source-language-digest descriptor)) + (artifact + ((source-language-parse descriptor) + source (source-language-scanner descriptor) digest))) + (with-parser-cost-stage 'artifact-validation + (validate-source-publication! descriptor source artifact)))) + +;;; An internal unbound syntax product; parser metadata supplies release identity. +(defstruct (source-syntax source-language) (recipe strategy) transparent: #t) +(def (declare-source-syntax strategy) + (let-values (((recipe engine) (bind-source-strategy strategy))) + (make-source-syntax +source-language-schema+ #f #f #f #f + (source-engine-scanner engine) (source-engine-parse engine) + (source-engine-factory engine) (source-engine-receipt engine) + (source-engine-results engine) (source-engine-root-kind engine) (source-engine-program engine) recipe strategy))) +(def (source-language-with-identity definition language version contract) + (unless (source-syntax? definition) (error "source identity requires unbound syntax")) + (let ((language-value (string-copy language)) (version-value (string-copy version)) + (contract-value (string-copy contract))) + (make-source-language +source-language-schema+ language-value version-value contract-value + (sha256-text (call-with-output-string + (lambda (port) (write (list language-value version-value contract-value + (source-syntax-recipe definition)) port)))) + (source-language-scanner definition) (source-language-parse definition) + (source-language-factory definition) (source-language-receipt definition) + (source-language-results definition) (source-language-root-kind definition) + (source-language-program definition)))) + +;;; Source sessions reuse only certified lexical suffixes. Recognition and +;;; canonical publication run through the existing engine on the new source. +(defstruct source-language-history (descriptor snapshot)) +(defstruct (source-language-session source-language-history) (artifact)) +;;; Retain only lexical history between foreign calls, not the published tree. +(def (source-language-session-history session) + (make-source-language-history (source-language-history-descriptor session) + (source-language-history-snapshot session))) +(def (source-language-session-scanned-token-count session) + (let (snapshot (source-language-history-snapshot session)) + (and snapshot (source-scan-session-scanned-token-count snapshot)))) +(def (source-language-session-reused-token-count session) + (let (snapshot (source-language-history-snapshot session)) + (if snapshot (source-scan-session-reused-token-count snapshot) 0))) +(def (parse-source-language/session descriptor source (previous #f)) + (unless (and (source-language? descriptor) (string? source) + (or (not previous) + (and (source-language-history? previous) + (eq? descriptor (source-language-history-descriptor previous))))) + (error "source session requires the same language declaration")) + (let* ((owned (with-parser-cost-stage 'source-copy (string-copy source))) + (worker (with-parser-cost-stage 'source-prepare ((source-language-factory descriptor) owned))) + (snapshot #f) + (artifact + ((source-language-parse descriptor) owned + (lambda (requested) + (unless (eq? requested owned) (error "source session scanner received foreign input")) + (if (source-scanner-contextual? worker) + (begin + (set! snapshot (with-parser-cost-stage 'source-scan + (make-source-scan-session worker #f + (and previous (source-language-history-snapshot previous))))) + (source-scan-session-tokens snapshot)) + ((source-language-scanner descriptor) requested))) + (source-language-digest descriptor)))) + ;; A lexical failure is handled by the original parser's diagnostic path; + ;; it produces no cache and cannot authorize reuse in the next edit. + (with-parser-cost-stage 'artifact-validation + (validate-source-publication! descriptor owned artifact)) + (make-source-language-session descriptor snapshot artifact))) diff --git a/languages/tla-plus/v1/qualification.ss b/src/language/tlc.ss similarity index 84% rename from languages/tla-plus/v1/qualification.ss rename to src/language/tlc.ss index 09c52fde..8cfeeab0 100644 --- a/languages/tla-plus/v1/qualification.ss +++ b/src/language/tlc.ss @@ -1,18 +1,20 @@ ;;; -*- Gerbil -*- -;;; Native TLA+ syntax and official TLC model-check qualification boundary. +;;; Descriptor-bound syntax and official TLC model-check qualification. +;;; The engine selects no builtin language; the admitted descriptor owns parsing. (import (only-in :std/misc/ports read-all-as-string) (only-in :std/misc/process run-process) (only-in :std/string/misc string-trim-eol) - (only-in ./parser parse-tla-plus-v1) - (only-in ../../../src/runtime/artifact + (only-in ./entry parse-language-source) + (only-in ./descriptor language-grammar-contract) + (only-in ../runtime/artifact parse-artifact-roundtrip parse-artifact-success? parse-artifact-valid?) - (only-in ../../../src/runtime/identity sha256-text)) + (only-in ../runtime/identity sha256-text)) (export +tla-plus-model-qualification-schema+ - qualify-tla-plus-model + qualify-language-tlc-model tla-plus-model-receipt? tla-plus-model-receipt-admitted tla-plus-model-receipt-output @@ -42,14 +44,19 @@ (def (line-with-prefix lines prefix) (find (lambda (line) (string-prefix? prefix line)) lines)) -(def (line-containing lines fragment) - (find (lambda (line) (string-contains line fragment)) lines)) +(def (final-states-line lines) + ;; TLC emits intermediate Progress(...) lines with the same summary phrase. + ;; Only the final, number-prefixed line describes the completed state space. + (find (lambda (line) + (and (decimal-prefix line) + (string-contains line " states generated, "))) + (reverse lines))) (def (parse-tlc-output output) (let* ((lines (string-split output #\newline)) (version-prefix "TLC2 Version ") (version-line (line-with-prefix lines version-prefix)) - (states-line (line-containing lines " states generated, ")) + (states-line (final-states-line lines)) (depth-line (line-with-prefix lines "The depth of the complete state graph search is ")) @@ -120,11 +127,11 @@ (equal? (summary-ref summary 'states-left) 0) (natural-number? (summary-ref summary 'graph-depth)))) -(def (syntax-rejection-receipt model source config-source +(def (syntax-rejection-receipt grammar model source config-source syntax-accepted roundtrip tlc workers) (make-tla-plus-model-receipt +tla-plus-model-qualification-schema+ - "tla-plus.native-core.v1" + (language-grammar-contract grammar) model (sha256-text source) (sha256-text config-source) @@ -138,9 +145,9 @@ (sha256-text "") "")) -(def (qualify-tla-plus-model spec-path config-path - tlc: (tlc "tlc") - workers: (workers 1)) +(def (qualify-language-tlc-model grammar spec-path config-path + tlc: (tlc "tlc") + workers: (workers 1)) (unless (and (exact-integer? workers) (> workers 0)) (error "TLC workers must be a positive integer" workers)) (let* ((spec (path-normalize spec-path)) @@ -151,7 +158,7 @@ spec config)) (let* ((source (read-text spec)) (config-source (read-text config)) - (artifact (parse-tla-plus-v1 source)) + (artifact (parse-language-source grammar source)) (syntax-accepted (and (parse-artifact-success? artifact) (parse-artifact-valid? artifact))) @@ -161,7 +168,7 @@ (model (path-strip-extension (path-strip-directory spec)))) (if (not roundtrip) (syntax-rejection-receipt - model source config-source syntax-accepted roundtrip tlc workers) + grammar model source config-source syntax-accepted roundtrip tlc workers) (let* ((tool-path (resolve-tool tlc)) (tool-source (read-text tool-path))) (call-with-tlc-temporary-directory @@ -180,7 +187,7 @@ syntax-accepted roundtrip exit-status summary))) (make-tla-plus-model-receipt +tla-plus-model-qualification-schema+ - "tla-plus.native-core.v1" + (language-grammar-contract grammar) model (sha256-text source) (sha256-text config-source) @@ -228,3 +235,10 @@ (cons 'admitted (tla-plus-model-receipt-admitted receipt)) (cons 'output-digest (tla-plus-model-receipt-output-digest receipt)))) + +;;; Public language entries declare the descriptor; process handling stays here. +(defrules deflanguage-model-entry (grammar) + ((_ binding (grammar descriptor)) + (def (binding spec config tlc: (tlc "tlc") workers: (workers 1)) + (qualify-language-tlc-model descriptor spec config tlc: tlc workers: workers)))) +(export deflanguage-model-entry) diff --git a/src/modules/parser/contextual-objects.ss b/src/modules/parser/contextual-objects.ss new file mode 100644 index 00000000..6c4d12f9 --- /dev/null +++ b/src/modules/parser/contextual-objects.ss @@ -0,0 +1,61 @@ +;;; -*- Gerbil -*- +;;; POO authoring objects for contextual method declarations. + +(import (only-in :clan/poo/object .o .ref) + (only-in :clan/poo/mop .defgeneric element? validate) + (only-in ./contextual-types + +contextual-role-kind+ ContextualRoleContract)) +(export ContextualRole. + make-contextual-role contextual-role? contextual-role-ref + make-contextual-method contextual-method? + contextual-method-name contextual-method-mode + contextual-method-position contextual-method-form + contextual-method-result + make-contextual-scan-rule contextual-scan-rule? + contextual-scan-rule-name contextual-scan-rule-mode + contextual-scan-rule-form contextual-scan-rule-matcher + contextual-scan-rule-rank contextual-scan-rule-action) + +;;; Method payloads are closed IR identities. =any= is resolved only by the +;;; compiler, after it checks the full axis catalogs. +(defstruct contextual-method (name mode position form result) transparent: #t) +(defstruct contextual-scan-rule (name mode form matcher rank action) + transparent: #t) + +(def ContextualRoleContract. (.ref ContextualRoleContract 'proto)) + +(.defgeneric (contextual-role-ref role field) slot: .section) + +(def ContextualRole. + (.o (:: self ContextualRoleContract.) + (.section (lambda (field) (.ref self field))))) + +(def (unique-method-names? methods) + (let loop ((rest methods) (seen '())) + (or (null? rest) + (and (not (memq (contextual-method-name (car rest)) seen)) + (loop (cdr rest) + (cons (contextual-method-name (car rest)) seen)))))) + +(def (contextual-method-valid? method) + (and (contextual-method? method) + (symbol? (contextual-method-name method)) + (symbol? (contextual-method-mode method)) + (symbol? (contextual-method-position method)) + (symbol? (contextual-method-form method)) + (symbol? (contextual-method-result method)))) + +(def (make-contextual-role name-value methods-value) + (unless (and (symbol? name-value) (list? methods-value) + (andmap contextual-method-valid? methods-value) + (unique-method-names? methods-value)) + (error "invalid contextual role" name-value methods-value)) + (validate + ContextualRoleContract + (.o (:: @ ContextualRole.) + kind: +contextual-role-kind+ + name: name-value + methods: methods-value))) + +(def (contextual-role? value) + (element? ContextualRoleContract value)) diff --git a/src/modules/parser/contextual-types.ss b/src/modules/parser/contextual-types.ss new file mode 100644 index 00000000..7f79fd60 --- /dev/null +++ b/src/modules/parser/contextual-types.ss @@ -0,0 +1,21 @@ +;;; -*- Gerbil -*- +;;; Contract for compile-time contextual roles. Generated machines contain +;;; only the role's normalized rows, never this object or its prototype. + +(import (only-in :clan/poo/object .o) + (only-in :clan/poo/mop define-type) + (only-in :core/types PooFlowNativeObjectContract.) + (only-in ./types ParserSymbol ParserList)) +(export +contextual-role-kind+ ContextualRoleContract) + +(def +contextual-role-kind+ 'gerbil-parser-contextual-role) + +(def (contextual-empty-prototype) (.o)) + +(define-type (ContextualRoleContract @ PooFlowNativeObjectContract.) + identity: 'gerbil-parser/contextual-role + proto: (contextual-empty-prototype) + responsibilities: + (.o kind: ParserSymbol + name: ParserSymbol + methods: ParserList)) diff --git a/src/modules/parser/funcs.ss b/src/modules/parser/funcs.ss index 9337218c..fbbc63a2 100644 --- a/src/modules/parser/funcs.ss +++ b/src/modules/parser/funcs.ss @@ -2,7 +2,11 @@ ;;; Invariant: alists are emitted only at an explicit projection boundary and ;;; never become the semantic owner of grammar composition or behavior. -(import (only-in ./objects +(import (only-in ./source-pattern-funs + source-priority-forms + source-offset-after source-pattern-end source-pattern-at? + source-ascii-ci-pattern-at?) + (only-in ./objects grammar-role-name grammar-role-ref grammar-schema @@ -11,7 +15,10 @@ grammar-roles grammar-composition)) (export grammar-role->alist - grammar->alist) + grammar->alist + source-priority-forms + source-offset-after source-pattern-end source-pattern-at? + source-ascii-ci-pattern-at?) ;; : (-> GrammarRole Alist) (def (grammar-role->alist role) diff --git a/src/modules/parser/graph-projection-types.ss b/src/modules/parser/graph-projection-types.ss index 9ee7e5c1..d1ea4e97 100644 --- a/src/modules/parser/graph-projection-types.ss +++ b/src/modules/parser/graph-projection-types.ss @@ -55,7 +55,7 @@ .classify: (lambda (candidate context) (graph-classify 'gerbil-parser/graph-field-mode (lambda (value) - (memq value '(append each append-or-empty))) + (memq value '(append each append-or-empty node-text each-node-text))) candidate context))) (define-type (GraphFieldContract @ PooFlowNativeObjectContract.) diff --git a/src/modules/parser/graph-query-funs.ss b/src/modules/parser/graph-query-funs.ss new file mode 100644 index 00000000..bf4cc2e3 --- /dev/null +++ b/src/modules/parser/graph-query-funs.ss @@ -0,0 +1,53 @@ +;;; -*- Gerbil -*- +;;; Pure, language-neutral selection over POO projected graph views. + +(import (only-in :clan/poo/object .ref) + (only-in ./graph-query-objects + graph-query-context-view graph-query-context-index + graph-query-context-count)) +(export graph-query-map graph-query-lineage? graph-query-select) + +(def (graph-query-map context kind predicate) + (let (view (graph-query-context-view context)) + (filter + (lambda (record) + (and (equal? ((.ref view 'kind-of) record) kind) + (predicate record))) + (.ref view 'records)))) + +(def (graph-query-lineage? context ancestor descendant) + (let* ((view (graph-query-context-view context)) + (by-id (graph-query-context-index context)) + (parent-of (.ref view 'parent-of))) + (let loop ((id descendant) (remaining (graph-query-context-count context))) + (and (> remaining 0) + (let (record (hash-get by-id id)) + (and record + (let (parent (parent-of record)) + (and parent + (or (equal? parent ancestor) + (loop parent (- remaining 1))))))))))) + +(def (graph-query-select context kind scope-id relation targets predicate) + (unless (memq relation '(any at child-of descendant-of)) + (error "unknown graph relation" relation)) + (unless (list? targets) + (error "graph query targets must be a list" targets)) + (let* ((view (graph-query-context-view context)) + (id-of (.ref view 'id-of)) + (parent-of (.ref view 'parent-of))) + (graph-query-map + context kind + (lambda (record) + (let (id (id-of record)) + (and (or (equal? id scope-id) + (graph-query-lineage? context scope-id id)) + (case relation + ((any) #t) + ((at) (member id targets)) + ((child-of) (member (parent-of record) targets)) + ((descendant-of) + (ormap (lambda (target) + (graph-query-lineage? context target id)) + targets))) + (predicate record))))))) diff --git a/src/modules/parser/graph-query-objects.ss b/src/modules/parser/graph-query-objects.ss new file mode 100644 index 00000000..9ae68a59 --- /dev/null +++ b/src/modules/parser/graph-query-objects.ss @@ -0,0 +1,53 @@ +;;; -*- Gerbil -*- +;;; POO-native graph query context; language packs supply their own view. + +(import (only-in :clan/poo/object .o .ref) + (only-in :core/types + poo-flow-contract-admit + poo-flow-validation-evidence-accepted?) + (only-in :core/object-family/interface defpoo-object-family) + (only-in ./graph-query-types + +graph-query-context-kind+ + GraphQueryContextContract graph-query-view?)) +(export GraphQueryContext. make-graph-query-context + graph-query-context-view graph-query-context-index + graph-query-context-count) + +(def GraphQueryContext. (.ref GraphQueryContextContract 'proto)) + +(def (make-graph-query-context graph-view) + (unless (graph-query-view? graph-view) + (error "graph query requires a POO graph view" graph-view)) + (let (by-id (make-hash-table)) + (for-each + (lambda (record) + (let (id ((.ref graph-view 'id-of) record)) + (when (hash-get by-id id) + (error "duplicate graph record id" id)) + (hash-put! by-id id record))) + (.ref graph-view 'records)) + (for-each + (lambda (record) + (let (parent ((.ref graph-view 'parent-of) record)) + (when (and parent (not (hash-get by-id parent))) + (error "missing graph parent" parent)))) + (.ref graph-view 'records)) + (let* ((candidate + (.o (:: @ GraphQueryContext.) + kind: +graph-query-context-kind+ + view: graph-view + index: by-id + count: (length (.ref graph-view 'records)))) + (evidence (poo-flow-contract-admit + GraphQueryContextContract candidate #f))) + (unless (poo-flow-validation-evidence-accepted? evidence) + (error "invalid POO graph query context" + (.ref evidence 'diagnostics))) + candidate))) + +(defpoo-object-family + (accessors + (graph-query-context-view view) + (graph-query-context-index index) + (graph-query-context-count count)) + (projections)) diff --git a/src/modules/parser/graph-query-types.ss b/src/modules/parser/graph-query-types.ss new file mode 100644 index 00000000..7cfacfe6 --- /dev/null +++ b/src/modules/parser/graph-query-types.ss @@ -0,0 +1,77 @@ +;;; -*- Gerbil -*- +;;; Language-neutral POO admission for projected graph query contexts. + +(import (only-in :clan/poo/object .o .ref .slot? object?) + (only-in :clan/poo/mop define-type) + (only-in :core/types + PooFlowContract. PooFlowNativeObjectContract. + poo-flow-contract-admit + poo-flow-validation-evidence-accepted? + poo-flow-classification-evidence)) +(export +graph-query-context-kind+ + GraphQueryViewContract GraphQueryContextContract + graph-query-view? graph-query-context?) + +(def +graph-query-context-kind+ 'gerbil-parser-graph-query-context) + +(def (classify identity predicate candidate context) + (let (accepted? (predicate candidate)) + (poo-flow-classification-evidence + identity candidate accepted? + (if accepted? '() (list (list 'expected identity))) context))) + +(def (graph-query-view-shape? value) + (and (object? value) + (.slot? value 'records) + (list? (.ref value 'records)) + (andmap (lambda (slot) + (and (.slot? value slot) + (procedure? (.ref value slot)))) + '(id-of parent-of kind-of)))) + +(define-type (GraphQueryViewContract @ PooFlowContract.) + identity: 'gerbil-parser/graph-query-view + .classify: (lambda (candidate context) + (classify 'gerbil-parser/graph-query-view + graph-query-view-shape? candidate context))) + +(define-type (GraphQueryContextKind @ PooFlowContract.) + identity: 'gerbil-parser/graph-query-context-kind + .classify: (lambda (candidate context) + (classify 'gerbil-parser/graph-query-context-kind + (lambda (value) + (eq? value +graph-query-context-kind+)) + candidate context))) + +(define-type (GraphQueryIndex @ PooFlowContract.) + identity: 'gerbil-parser/graph-query-index + .classify: (lambda (candidate context) + (classify 'gerbil-parser/graph-query-index + hash-table? candidate context))) + +(define-type (GraphQueryCount @ PooFlowContract.) + identity: 'gerbil-parser/graph-query-count + .classify: (lambda (candidate context) + (classify 'gerbil-parser/graph-query-count + (lambda (value) + (and (exact-integer? value) (>= value 0))) + candidate context))) + +(define-type (GraphQueryContextContract @ PooFlowNativeObjectContract.) + identity: 'gerbil-parser/graph-query-context + proto: (.o) + responsibilities: + (.o kind: GraphQueryContextKind + view: GraphQueryViewContract + index: GraphQueryIndex + count: GraphQueryCount)) + +(def (admitted? contract candidate) + (poo-flow-validation-evidence-accepted? + (poo-flow-contract-admit contract candidate #f))) + +(def (graph-query-view? value) + (admitted? GraphQueryViewContract value)) + +(def (graph-query-context? value) + (admitted? GraphQueryContextContract value)) diff --git a/src/modules/parser/graph-syntax.ss b/src/modules/parser/graph-syntax.ss new file mode 100644 index 00000000..b1cb15c4 --- /dev/null +++ b/src/modules/parser/graph-syntax.ss @@ -0,0 +1,161 @@ +;;; -*- Gerbil -*- +;;; Shared POO graph-pattern syntax and constrained MATCH/WHERE/RETURN rendering. + +(import (only-in :clan/poo/object .o .ref .slot? object?) + (only-in :gerbil/core string-join) + (only-in :std/list/list every) + (only-in :std/string/misc string-concatenate string-subst)) + +(export GraphSyntaxNode. + GraphSyntaxStep. + GraphSyntaxPath. + GraphSyntaxProperty. + GraphSyntaxLiteral. + GraphSyntaxEquals. + GraphSyntaxProjection. + GraphSyntaxProgram. + graph-syntax-program? + graph-syntax-program->source) + +(def (graph-has-slots? value slots) + (and (object? value) + (every (lambda (slot) (.slot? value slot)) slots))) + +(def GraphSyntaxNode. + (.o syntax-kind: 'graph.syntax.node binding: #f label: #f)) + +(def GraphSyntaxStep. + (.o syntax-kind: 'graph.syntax.step relation: #f target: #f next: #f)) + +(def GraphSyntaxPath. + (.o syntax-kind: 'graph.syntax.path start: #f next: #f)) + +(def GraphSyntaxProperty. + (.o syntax-kind: 'graph.syntax.property binding: #f property: #f)) + +(def GraphSyntaxLiteral. + (.o syntax-kind: 'graph.syntax.literal literal-kind: #f value: #f)) + +(def GraphSyntaxEquals. + (.o syntax-kind: 'graph.syntax.equals left: #f right: #f)) + +(def GraphSyntaxProjection. + (.o syntax-kind: 'graph.syntax.projection expression: #f next: #f)) + +(def GraphSyntaxProgram. + (.o syntax-kind: 'graph.syntax.program match: #f where: #f project: #f)) + +(def (graph-syntax-node? value) + (and (graph-has-slots? value '(syntax-kind binding label)) + (eq? (.ref value 'syntax-kind) 'graph.syntax.node) + (symbol? (.ref value 'binding)) + (symbol? (.ref value 'label)))) + +(def (graph-syntax-step? value (remaining 256)) + (and (positive? remaining) (graph-has-slots? value '(syntax-kind relation target next)) + (eq? (.ref value 'syntax-kind) 'graph.syntax.step) + (symbol? (.ref value 'relation)) + (graph-syntax-node? (.ref value 'target)) + (or (not (.ref value 'next)) + (graph-syntax-step? (.ref value 'next) (- remaining 1))))) + +(def (graph-syntax-path? value) + (and (graph-has-slots? value '(syntax-kind start next)) + (eq? (.ref value 'syntax-kind) 'graph.syntax.path) + (graph-syntax-node? (.ref value 'start)) + (or (not (.ref value 'next)) + (graph-syntax-step? (.ref value 'next) 256)))) + +(def (graph-syntax-property? value) + (and (graph-has-slots? value '(syntax-kind binding property)) + (eq? (.ref value 'syntax-kind) 'graph.syntax.property) + (symbol? (.ref value 'binding)) + (symbol? (.ref value 'property)))) + +(def (graph-syntax-literal? value) + (and (graph-has-slots? value '(syntax-kind literal-kind value)) + (eq? (.ref value 'syntax-kind) 'graph.syntax.literal) + (case (.ref value 'literal-kind) + ((string) (string? (.ref value 'value))) + ((symbol) (symbol? (.ref value 'value))) + ((integer) (exact-integer? (.ref value 'value))) + ((boolean) (boolean? (.ref value 'value))) + (else #f)))) + +(def (graph-syntax-equals? value) + (and (graph-has-slots? value '(syntax-kind left right)) + (eq? (.ref value 'syntax-kind) 'graph.syntax.equals) + (graph-syntax-property? (.ref value 'left)) + (graph-syntax-literal? (.ref value 'right)))) + +(def (graph-syntax-projection? value (remaining 256)) + (and (positive? remaining) (graph-has-slots? value '(syntax-kind expression next)) + (eq? (.ref value 'syntax-kind) 'graph.syntax.projection) + (graph-syntax-property? (.ref value 'expression)) + (or (not (.ref value 'next)) + (graph-syntax-projection? (.ref value 'next) (- remaining 1))))) + +(def (graph-syntax-program? value) + (and (graph-has-slots? value '(syntax-kind match where project)) + (eq? (.ref value 'syntax-kind) 'graph.syntax.program) + (graph-syntax-path? (.ref value 'match)) + (or (not (.ref value 'where)) + (graph-syntax-equals? (.ref value 'where))) + (graph-syntax-projection? (.ref value 'project)))) + +(def (graph-name value) + (if (symbol? value) (symbol->string value) value)) + +(def (graph-string value) + (string-append "'" (string-subst value "'" "''") "'")) + +(def (graph-node-document node) + (string-append "(" (graph-name (.ref node 'binding)) ":" + (graph-name (.ref node 'label)) ")")) + +(def (graph-path-document path) + (let loop ((step (.ref path 'next)) + (documents (list (graph-node-document (.ref path 'start))))) + (if step + (loop (.ref step 'next) + (cons + (string-append "-[:" (graph-name (.ref step 'relation)) "]->" + (graph-node-document (.ref step 'target))) + documents)) + (string-concatenate (reverse documents))))) + +(def (graph-property-document property) + (string-append (graph-name (.ref property 'binding)) "." + (graph-name (.ref property 'property)))) + +(def (graph-literal-document literal) + (case (.ref literal 'literal-kind) + ((string) (graph-string (.ref literal 'value))) + ((symbol) (graph-string (symbol->string (.ref literal 'value)))) + ((integer) (number->string (.ref literal 'value))) + ((boolean) (if (.ref literal 'value) "TRUE" "FALSE")))) + +(def (graph-equals-document equals) + (string-append (graph-property-document (.ref equals 'left)) " = " + (graph-literal-document (.ref equals 'right)))) + +(def (graph-projection-document projection) + (let loop ((current projection) (documents '())) + (if current + (loop (.ref current 'next) + (cons (graph-property-document (.ref current 'expression)) + documents)) + (string-join (reverse documents) ", ")))) + +(def (graph-syntax-program->source program) + (unless (graph-syntax-program? program) + (error "invalid graph-pattern syntax" program)) + (string-concatenate + (list + (string-append "MATCH " (graph-path-document (.ref program 'match)) "\n") + (if (.ref program 'where) + (string-append "WHERE " + (graph-equals-document (.ref program 'where)) "\n") + "") + (string-append "RETURN " + (graph-projection-document (.ref program 'project)) "\n")))) diff --git a/src/modules/parser/inline-link-event-funs.ss b/src/modules/parser/inline-link-event-funs.ss new file mode 100644 index 00000000..797b29a6 --- /dev/null +++ b/src/modules/parser/inline-link-event-funs.ss @@ -0,0 +1,85 @@ +;;; Native link recognition from admitted POO rules, without language policy. +(import (only-in ./line-structure-objects + text-line? text-line-token text-line-inline-link + inline-link-opening inline-link-separator inline-link-closing + inline-link-node inline-link-target-token + inline-link-description-token inline-link-trivia-token) + (only-in ./source-pattern-funs source-pattern-at? source-pattern-end)) +(import (only-in ./source-event-scope-funs source-event-name)) +(export source-inline-link-initial source-inline-link-scan-forms source-inline-link-open-condition) +(def (admit-link rule) + (unless (and (text-line? rule) (text-line-inline-link rule)) + (error "native link requires a declared inline-link rule" rule)) + (text-line-inline-link rule)) +(def (source-inline-link-initial rule scope) + (admit-link rule) + (map (lambda (binding) (list (source-event-name scope (car binding)) (cadr binding))) + '((inline-open #f) (inline-has-separator #f) (inline-failed #f) + (inline-open-at 0) (inline-target-start 0) (inline-separator-at 0)))) +(def (source-inline-link-open-condition scope) + `(state ,(source-event-name scope 'inline-open))) +(def (source-inline-link-scan-forms rule scope + description-helper: (helper #f) parameters: (parameters '()) + description-node: (node #f) literal-description?: (literal? #f) + invalid-target: (invalid-target 'recover-span) + index: (index-name #f) cursor: (cursor-name #f)) + (source-inline-link-initial rule scope) + (def (name role) (source-event-name scope role)) + (let* ((link (admit-link rule)) + (index `(line-index ,index-name)) (cursor `(state-offset ,cursor-name)) + (opening (inline-link-opening link)) (separator (inline-link-separator link)) + (closing (inline-link-closing link)) (trivia (inline-link-trivia-token link)) + (target (inline-link-target-token link)) + (description (inline-link-description-token link)) + (separator-end (source-pattern-end `(state-offset ,(name 'inline-separator-at)) separator)) + (close-end (source-pattern-end index closing))) + (unless (and (symbol? index-name) (symbol? cursor-name) (boolean? literal?) + (memq invalid-target '(recover-span recover-region)) + (or (not node) (symbol? node)) (or (not helper) (symbol? helper)) + (list? parameters) (andmap symbol? parameters) + (let loop ((rest parameters) (seen '())) + (or (null? rest) + (and (not (memq (car rest) seen)) + (loop (cdr rest) (cons (car rest) seen))))) + (or helper (null? parameters))) + (error "invalid native inline description binding" helper parameters node)) + (def description-forms + (let (content + (if (or literal? (not helper)) + `(token ,description ,separator-end ,index) + `(call-source-helper ,helper ,separator-end ,index + ,@(if (null? parameters) '() + (list (map (lambda (name) `(state ,name)) parameters)))))) + (if (and node (not literal?)) + `((start-node ,node) ,content (finish-node)) (list content)))) + (def events + `((token ,(text-line-token rule) ,cursor (state-offset ,(name 'inline-open-at))) + (start-node ,(inline-link-node link)) + (token ,trivia (state-offset ,(name 'inline-open-at)) (state-offset ,(name 'inline-target-start))) + (if (state ,(name 'inline-has-separator)) + ((token ,target (state-offset ,(name 'inline-target-start)) (state-offset ,(name 'inline-separator-at))) + (token ,trivia (state-offset ,(name 'inline-separator-at)) ,separator-end) + ,@description-forms) + ((token ,target (state-offset ,(name 'inline-target-start)) ,index))) + (token ,trivia ,index ,close-end) (finish-node) + (set-uint ,cursor-name (offset ,close-end)) (set-bool ,(name 'inline-open) (bool #f)))) + (def (commit until) + `((if (offset-less? (state-offset ,(name 'inline-target-start)) ,until) + ,events ,(if (eq? invalid-target 'recover-region) + `((set-bool ,(name 'inline-failed) (bool #t))) + `((set-bool ,(name 'inline-open) (bool #f))))))) + `((if (offset-less? ,index ,cursor) () + ((if (and (not (state ,(name 'inline-failed))) (not (state ,(name 'inline-open)))) + ((if ,(source-pattern-at? index opening) + ((set-bool ,(name 'inline-open) (bool #t)) + (set-bool ,(name 'inline-has-separator) (bool #f)) + (set-uint ,(name 'inline-open-at) (offset ,index)) + (set-uint ,(name 'inline-target-start) (offset ,(source-pattern-end index opening)))) ())) + ((if (offset-less? ,index (state-offset ,(name 'inline-target-start))) () + ((if (and (state ,(name 'inline-open)) ,(source-pattern-at? index closing)) + ((if (state ,(name 'inline-has-separator)) + ,(commit `(state-offset ,(name 'inline-separator-at))) ,(commit index))) + ((if (and (state ,(name 'inline-open)) (not (state ,(name 'inline-has-separator))) + ,(source-pattern-at? index separator)) + ((set-bool ,(name 'inline-has-separator) (bool #t)) + (set-uint ,(name 'inline-separator-at) (offset ,index))) ())))))))))))) diff --git a/src/modules/parser/interface.ss b/src/modules/parser/interface.ss index f2e6fb75..376673ed 100644 --- a/src/modules/parser/interface.ss +++ b/src/modules/parser/interface.ss @@ -6,8 +6,28 @@ (import ./types ./objects ./funcs - ./syntax) + ./syntax + ./line-structure-objects + ./source-event-scope-types + ./source-event-scope-objects + ./source-event-scope-funs + ./line-event-funs + ./list-event-funs + ./inline-link-event-funs + ./source-boundary-types + ./source-boundary-objects + ./source-boundary-funs) (export (import: ./types) (import: ./objects) (import: ./funcs) - (import: ./syntax)) + (import: ./syntax) + (import: ./line-structure-objects) + (import: ./source-event-scope-types) + (import: ./source-event-scope-objects) + (import: ./source-event-scope-funs) + (import: ./line-event-funs) + (import: ./list-event-funs) + (import: ./inline-link-event-funs) + (import: ./source-boundary-types) + (import: ./source-boundary-objects) + (import: ./source-boundary-funs)) diff --git a/src/modules/parser/line-event-funs.ss b/src/modules/parser/line-event-funs.ss new file mode 100644 index 00000000..4c06ecee --- /dev/null +++ b/src/modules/parser/line-event-funs.ss @@ -0,0 +1,142 @@ +;;; -*- Gerbil -*- +;;; Pure native event lowering of admitted contextual line declarations. +;;; Domain nodes and content helpers are supplied by the language, not the engine. +(import (only-in ./line-structure-objects + table-line? text-line? + table-line-delimiter table-line-row-node table-line-rule-row-node + table-line-cell-node table-line-separator-token table-line-cell-token + table-line-trivia-token table-line-rule-token + text-line-node text-line-token text-line-paragraph-node)) +(import (only-in ./source-event-scope-funs source-event-name)) +(export source-table-row-initial source-table-row-forms + source-paragraph-initial source-paragraph-close-form source-paragraph-line-form + source-paragraph-open-condition source-paragraph-span-forms source-paragraph-blank-forms) + +;;; Named helper references are bindings, never arbitrary raw-form callbacks. +(def (content-call helper parameters from until token) + (unless (and (or (not helper) (symbol? helper)) + (list? parameters) (andmap symbol? parameters) + (let loop ((rest parameters) (seen '())) + (or (null? rest) + (and (not (memq (car rest) seen)) + (loop (cdr rest) (cons (car rest) seen)))))) + (error "invalid native content helper binding" helper parameters)) + (when (and (not helper) (pair? parameters)) + (error "content parameters require a helper" parameters)) + (if helper + `(call-source-helper ,helper ,from ,until + ,@(if (null? parameters) '() + (list (map (lambda (name) `(state ,name)) parameters)))) + `(token ,token ,from ,until))) + +(def (source-table-row-initial rule scope) + (unless (table-line? rule) (error "unadmitted native table rule" rule)) + `((,(source-event-name scope 'table-seen-separator) #f) + (,(source-event-name scope 'table-escaped) #f) + (,(source-event-name scope 'table-cell-start) 0))) + +(def (source-table-row-forms rule scope (helper #f) (parameters '()) + rule-bytes: (rule-bytes '()) rule-marker: (rule-marker #f) + escape-byte: (escape-byte #f)) + (source-table-row-initial rule scope) + (def (name role) (source-event-name scope role)) + (def (byte? value) (and (integer? value) (<= 0 value 255))) + (unless (and (list? rule-bytes) (andmap byte? rule-bytes) + (or (not rule-marker) (and (byte? rule-marker) (memv rule-marker rule-bytes))) + (or (not escape-byte) (byte? escape-byte))) + (error "invalid native table lexical policy" rule-bytes rule-marker escape-byte)) + (let* ((byte (char->integer (string-ref (table-line-delimiter rule) 0))) + (indent '(line-skip-horizontal start)) + (until '(line-content-end)) + (index `(line-index ,(name 'table-byte-index))) + (next `(line-step ,index)) + (cell-start `(state-offset ,(name 'table-cell-start))) + (trivia (table-line-trivia-token rule))) + (def (cell end) + `((start-node ,(table-line-cell-node rule)) + ,(content-call helper parameters cell-start end (table-line-cell-token rule)) + (finish-node))) + (let (row + `((start-node ,(table-line-row-node rule)) + (for-line-bytes ,(name 'table-byte-index) ,indent ,until + ((if ,(if escape-byte `(line-byte-equal? ,index ,escape-byte) + '(uint-equal? (uint 0) (uint 1))) + ((set-bool ,(name 'table-escaped) (not (state ,(name 'table-escaped))))) + ((if (and (line-byte-equal? ,index ,byte) (not (state ,(name 'table-escaped)))) + ((if (state ,(name 'table-seen-separator)) + ,(cell index) ((token ,trivia start ,index))) + (token ,(table-line-separator-token rule) ,index ,next) + (set-uint ,(name 'table-cell-start) (offset ,next)) + (set-bool ,(name 'table-seen-separator) (bool #t))) ()) + (set-bool ,(name 'table-escaped) (bool #f)))))) + (if (state ,(name 'table-seen-separator)) + ((if (line-bytes-all-in? ,cell-start ,until (9 32)) + ((token ,trivia ,cell-start ,until)) ,(cell until))) ()) + (token ,trivia ,until end) (finish-node) + (set-bool ,(name 'table-seen-separator) (bool #f)) + (set-bool ,(name 'table-escaped) (bool #f)))) + (if rule-marker + (list `(if (and (line-bytes-all-in? ,indent ,until ,(cons byte rule-bytes)) + (line-bytes-any-in? ,indent ,until (,rule-marker))) + ((start-node ,(table-line-rule-row-node rule)) + (token ,(table-line-rule-token rule) start end) (finish-node)) + ,row)) + row)))) + +(def (source-paragraph-initial rule scope pending-blank?: (pending? #t)) + (unless (and (text-line? rule) (text-line-paragraph-node rule)) + (error "native paragraph requires a declared paragraph node" rule)) + (unless (boolean? pending?) (error "invalid paragraph blank policy" pending?)) + (append + `((,(source-event-name scope 'paragraph-open) #f) + (,(source-event-name scope 'paragraph-start) 0) + (,(source-event-name scope 'paragraph-end) 0)) + (if pending? + `((,(source-event-name scope 'paragraph-blank-end) 0) + (,(source-event-name scope 'paragraph-post-blank) #f)) '()))) + +(def (source-paragraph-close-form rule scope (helper #f) (parameters '()) (reset? #t) + pending-blank?: (pending? #t)) + (source-paragraph-initial rule scope pending-blank?: pending?) + (def (name role) (source-event-name scope role)) + (unless (boolean? reset?) (error "invalid native paragraph reset policy" reset?)) + `(if (state ,(name 'paragraph-open)) + (,(content-call helper parameters `(state-offset ,(name 'paragraph-start)) + `(state-offset ,(name 'paragraph-end)) (text-line-token rule)) + ,@(if pending? + `((if (state ,(name 'paragraph-post-blank)) + ((start-node ,(text-line-node rule)) + (token ,(text-line-token rule) (state-offset ,(name 'paragraph-end)) + (state-offset ,(name 'paragraph-blank-end))) (finish-node)) ())) '()) + (finish-node) + ,@(if reset? + (append `((set-bool ,(name 'paragraph-open) (bool #f))) + (if pending? `((set-bool ,(name 'paragraph-post-blank) (bool #f))) '())) + '())) ())) + +(def (source-paragraph-open-condition scope) + `(state ,(source-event-name scope 'paragraph-open))) + +(def (source-paragraph-span-forms rule scope from until) + (source-paragraph-initial rule scope) + (def (name role) (source-event-name scope role)) + `((if (not ,(source-paragraph-open-condition scope)) + ((start-node ,(text-line-paragraph-node rule)) + (set-bool ,(name 'paragraph-open) (bool #t)) + (set-uint ,(name 'paragraph-start) (offset ,from))) ()) + (set-uint ,(name 'paragraph-end) (offset ,until)))) + +(def (source-paragraph-blank-forms scope until) + `((set-bool ,(source-event-name scope 'paragraph-post-blank) (bool #t)) + (set-uint ,(source-event-name scope 'paragraph-blank-end) (offset ,until)))) + +(def (source-paragraph-line-form rule scope (helper #f) (parameters '())) + (def (name role) (source-event-name scope role)) + (let (close (source-paragraph-close-form rule scope helper parameters)) + `(if (line-blank?) + ((if (state ,(name 'paragraph-open)) + ,(source-paragraph-blank-forms scope 'end) + ((start-node ,(text-line-node rule)) + (token ,(text-line-token rule) start end) (finish-node)))) + ((if (state ,(name 'paragraph-post-blank)) (,close) ()) + ,@(source-paragraph-span-forms rule scope 'start 'end))))) diff --git a/src/modules/parser/line-structure-objects.ss b/src/modules/parser/line-structure-objects.ss index e6e75e58..5417d02b 100644 --- a/src/modules/parser/line-structure-objects.ss +++ b/src/modules/parser/line-structure-objects.ss @@ -21,7 +21,7 @@ make-line-structure make-heading-line make-block-line make-text-line make-table-line make-list-line make-key-line make-key-value-line make-block-header make-inline-link make-heading-fields - line-structure? heading-line? block-line? text-line? key-value-line? + line-structure? heading-line? block-line? text-line? key-value-line? table-line? list-line? line-structure-heading line-structure-blocks line-structure-text line-structure-table line-structure-list line-structure-key-lines heading-line-marker heading-line-separator @@ -312,3 +312,8 @@ (admitted-line? TextLineContract value)) (def (key-value-line? value) (admitted-line? KeyValueLineContract value)) +(def (table-line? value) + (admitted-line? TableLineContract value)) + +(def (list-line? value) + (admitted-line? ListLineContract value)) diff --git a/src/modules/parser/list-event-funs.ss b/src/modules/parser/list-event-funs.ss new file mode 100644 index 00000000..6b33d415 --- /dev/null +++ b/src/modules/parser/list-event-funs.ss @@ -0,0 +1,86 @@ +;;; Native list ownership from admitted POO declarations and explicit scopes. +(import (only-in ./line-structure-objects + list-line? list-line-unordered-markers list-line-ordered + list-line-tab-width list-line-list-node list-line-item-node + list-line-bullet-token list-line-trivia-token) + (only-in ./source-event-scope-funs source-event-name) + (only-in ./line-event-funs source-paragraph-initial + source-paragraph-span-forms source-paragraph-close-form)) +(export source-list-initial source-list-scan-form source-list-present-condition + source-list-ordered-condition source-list-active-condition + source-list-first-blank-condition source-list-continuation-condition + source-list-content-offset source-list-item-open-forms + source-list-paragraph-span-forms source-list-paragraph-close-form + source-list-close-forms source-list-first-blank-forms source-list-reset-blank-form) +(def (admit-list rule) + (unless (list-line? rule) (error "unadmitted native list rule" rule)) + rule) +(def (source-list-initial rule scope text) + (admit-list rule) + (append + (map (lambda (binding) (list (source-event-name scope (car binding)) (cadr binding))) + '((list-frames (uint-stack)) (list-present #f) (list-ordered #f) + (list-column 0) (list-bullet-start 0) (list-bullet-end 0) + (list-content-start 0) (list-blank-count 0))) + (source-paragraph-initial text scope pending-blank?: #f))) +(def (source-list-scan-form rule scope) + (admit-list rule) + `(scan-list-marker ,(list-line-unordered-markers rule) ,(list-line-ordered rule) + ,(list-line-tab-width rule) + ,@(map (lambda (role) (source-event-name scope role)) + '(list-present list-column list-ordered + list-bullet-start list-bullet-end list-content-start)))) +(def (source-list-present-condition scope) + `(state ,(source-event-name scope 'list-present))) +(def (source-list-ordered-condition scope) + `(state ,(source-event-name scope 'list-ordered))) +(def (source-list-active-condition scope) + `(stack-nonempty? ,(source-event-name scope 'list-frames))) +(def (source-list-first-blank-condition scope) + `(uint-equal? (state ,(source-event-name scope 'list-blank-count)) (uint 0))) +(def (source-list-continuation-condition rule scope) + (admit-list rule) + `(uint-greater? (line-indent-column ,(list-line-tab-width rule)) + (uint-divide (stack-top ,(source-event-name scope 'list-frames)) (uint 2)))) +(def (source-list-content-offset scope) + `(state-offset ,(source-event-name scope 'list-content-start))) +(def (source-list-reset-blank-form scope) + `(set-uint ,(source-event-name scope 'list-blank-count) (uint 0))) +(def (source-list-paragraph-span-forms text scope from until) + (source-paragraph-span-forms text scope from until)) +(def (source-list-paragraph-close-form text scope (helper #f) (parameters '()) (reset? #t)) + (source-paragraph-close-form text scope helper parameters reset? pending-blank?: #f)) +(def (source-list-close-forms text scope (helper #f) (parameters '()) (reset? #t)) + `(,(source-list-paragraph-close-form text scope helper parameters reset?) + (close-all-frames ,(source-event-name scope 'list-frames) 2) + ,@(if reset? (list (source-list-reset-blank-form scope)) '()))) +(def (source-list-first-blank-forms rule text scope (helper #f) (parameters '())) + (admit-list rule) + `(,(source-list-paragraph-close-form text scope helper parameters) + (token ,(list-line-trivia-token rule) start end) + (set-uint ,(source-event-name scope 'list-blank-count) (uint 1)))) +(def (source-list-item-open-forms rule scope ordered? bullet-span: (span 'marker)) + (admit-list rule) + (unless (and (boolean? ordered?) (memq span '(marker with-separator))) + (error "invalid native list item policy" ordered? span)) + (let* ((frames (source-event-name scope 'list-frames)) + (column `(state ,(source-event-name scope 'list-column))) + (top `(uint-divide (stack-top ,frames) (uint 2))) + (base `(uint-multiply ,column (uint 2))) + (frame (if ordered? `(uint-add ,base (uint 1)) base)) + (from `(state-offset ,(source-event-name scope 'list-bullet-start))) + (marker-end `(state-offset ,(source-event-name scope 'list-bullet-end))) + (content (source-list-content-offset scope)) + (trivia (list-line-trivia-token rule))) + `((close-frames-while ,frames + (or (uint-greater? ,top ,column) + (and (uint-equal? ,top ,column) + (uint-not-equal? (stack-top ,frames) ,frame))) 2) + (if (and ,(source-list-active-condition scope) (uint-equal? ,top ,column)) + ((finish-node)) + ((start-node ,(list-line-list-node rule)) (push-frame ,frames ,frame))) + (start-node ,(list-line-item-node rule)) + (token ,trivia start ,from) + (token ,(list-line-bullet-token rule) ,from + ,(if (eq? span 'with-separator) content marker-end)) + ,@(if (eq? span 'marker) `((token ,trivia ,marker-end ,content)) '())))) diff --git a/src/modules/parser/source-boundary-funs.ss b/src/modules/parser/source-boundary-funs.ss new file mode 100644 index 00000000..be942774 --- /dev/null +++ b/src/modules/parser/source-boundary-funs.ss @@ -0,0 +1,55 @@ +;;; Compose parent-aware recovery through the existing cached future queries. +(import (only-in :clan/poo/object .ref) + (only-in ./source-boundary-types source-boundary-query? source-boundary-parent?) + (only-in ./source-pattern-funs source-pattern-end) + (only-in ./line-structure-objects + heading-line-marker heading-line-separator block-line-closing + block-line-indent block-line-contents block-line-body-line + block-line-unclosed block-line-heading-bound block-line-case-insensitive + key-value-line-marker)) +(export source-container-boundary-condition) +(def (query-condition query stop parent) + (let* ((heading (.ref query 'heading)) + (marker (heading-line-marker heading)) (separator (heading-line-separator heading))) + (case (.ref query 'mode) + ((fixed) + (when (and parent (.ref parent 'name-start)) + (error "fixed future query requires a literal parent boundary" parent)) + (let* ((block (.ref query 'block)) (body (block-line-body-line block))) + (unless (and (eq? (block-line-unclosed block) 'recover-as-text) + (block-line-heading-bound block) + (block-line-case-insensitive block)) + (error "unsupported native block recovery policy" block)) + `(future-line-marker-before-boundary? ,(block-line-closing block) ,stop + ,marker ,separator ,(block-line-indent block) + ,(eq? (block-line-contents block) 'elements) + ,(if body (key-value-line-marker body) "")))) + ((named) + (let* ((indent (if (.ref query 'indent) '(line-skip-horizontal start) 'start)) + (from (source-pattern-end indent (.ref query 'opening))) + (terminator (.ref query 'terminator)) + (until (if terminator `(line-scan-until ,from ,terminator) `(line-scan-key ,from)))) + (append + `(future-named-line-marker-before-boundary? ,from ,until + ,(.ref query 'closing) ,(.ref query 'suffix) ,stop ,marker ,separator + ,(.ref query 'indent) ,(.ref query 'heading-bound) ,(.ref query 'case-insensitive)) + (if (and parent (.ref parent 'name-start)) + `((state-offset ,(.ref parent 'name-start)) (state-offset ,(.ref parent 'name-end)) + ,(.ref parent 'closing) ,(.ref parent 'suffix) ,(.ref parent 'case-insensitive)) + '()))))))) +(def (source-container-boundary-condition query parents stack) + (unless (and (source-boundary-query? query) (symbol? stack) (list? parents) + (andmap source-boundary-parent? parents) + (let loop ((rest parents) (ids '())) + (or (null? rest) + (and (not (memv (.ref (car rest) 'id) ids)) + (loop (cdr rest) (cons (.ref (car rest) 'id) ids)))))) + (error "invalid native container boundary composition" query parents stack)) + (foldr + (lambda (parent otherwise) + (let (named? (.ref parent 'name-start)) + `(or (and (uint-equal? (stack-top ,stack) (uint ,(.ref parent 'id))) + ,(query-condition query (if named? "" (.ref parent 'closing)) parent)) + ,otherwise))) + `(and (uint-equal? (stack-top ,stack) (uint 0)) ,(query-condition query "" #f)) + parents)) diff --git a/src/modules/parser/source-boundary-objects.ss b/src/modules/parser/source-boundary-objects.ss new file mode 100644 index 00000000..542f14f4 --- /dev/null +++ b/src/modules/parser/source-boundary-objects.ss @@ -0,0 +1,24 @@ +;;; Inert public boundary declarations with native POO admission. +(import (only-in :clan/poo/object .o) + ./source-boundary-types) +(export make-source-block-boundary make-source-named-boundary make-source-boundary-parent) +(def (admit predicate value) + (unless (predicate value) (error "invalid source boundary declaration" value)) + value) +(def (make-source-block-boundary block-value heading-value) + (admit source-boundary-query? + (.o kind: 'source-boundary-query mode: 'fixed block: block-value heading: heading-value + opening: #f closing: #f suffix: "" terminator: #f + indent: #t heading-bound: #t case-insensitive: #t))) +(def (make-source-named-boundary opening-value closing-value suffix-value heading-value + (terminator-value #f) (indent? #t) + (heading-bound? #t) (case-insensitive? #t)) + (admit source-boundary-query? + (.o kind: 'source-boundary-query mode: 'named block: #f heading: heading-value + opening: opening-value closing: closing-value suffix: suffix-value terminator: terminator-value + indent: indent? heading-bound: heading-bound? case-insensitive: case-insensitive?))) +(def (make-source-boundary-parent id-value closing-value (name-start-value #f) (name-end-value #f) + (suffix-value "") (case-insensitive? #t)) + (admit source-boundary-parent? + (.o kind: 'source-boundary-parent id: id-value closing: closing-value name-start: name-start-value + name-end: name-end-value suffix: suffix-value case-insensitive: case-insensitive?))) diff --git a/src/modules/parser/source-boundary-types.ss b/src/modules/parser/source-boundary-types.ss new file mode 100644 index 00000000..6d7ef87f --- /dev/null +++ b/src/modules/parser/source-boundary-types.ss @@ -0,0 +1,39 @@ +;;; Admitted POO boundary queries and parent bindings; no raw form callbacks. +(import (only-in :clan/poo/object .ref .slot? object?) + (only-in :clan/poo/mop define-type Type. element?) + (only-in ./line-structure-objects heading-line? block-line?)) +(export SourceBoundaryQuery SourceBoundaryParent source-boundary-query? source-boundary-parent?) +(def (ascii? text) + (and (string? text) + (andmap (lambda (ch) (<= 1 (char->integer ch) 127)) (string->list text)))) +(def (slots? value names) + (and (object? value) (andmap (lambda (name) (.slot? value name)) names))) +(def (query-shape? value) + (and (slots? value '(kind mode heading block opening closing suffix terminator indent heading-bound case-insensitive)) + (eq? (.ref value 'kind) 'source-boundary-query) + (heading-line? (.ref value 'heading)) + (andmap (lambda (name) (boolean? (.ref value name))) '(indent heading-bound case-insensitive)) + (case (.ref value 'mode) + ((fixed) (block-line? (.ref value 'block))) + ((named) + (and (ascii? (.ref value 'opening)) (> (string-length (.ref value 'opening)) 0) + (ascii? (.ref value 'closing)) (> (string-length (.ref value 'closing)) 0) + (ascii? (.ref value 'suffix)) + (or (not (.ref value 'terminator)) + (and (ascii? (.ref value 'terminator)) + (= (string-length (.ref value 'terminator)) 1))))) + (else #f)))) +(def (parent-shape? value) + (and (slots? value '(kind id closing name-start name-end suffix case-insensitive)) + (eq? (.ref value 'kind) 'source-boundary-parent) + (integer? (.ref value 'id)) (> (.ref value 'id) 0) + (ascii? (.ref value 'closing)) (> (string-length (.ref value 'closing)) 0) + (ascii? (.ref value 'suffix)) (boolean? (.ref value 'case-insensitive)) + (or (and (not (.ref value 'name-start)) (not (.ref value 'name-end)) + (string=? (.ref value 'suffix) "") + (.ref value 'case-insensitive)) + (and (symbol? (.ref value 'name-start)) (symbol? (.ref value 'name-end)))))) +(define-type (SourceBoundaryQuery @ Type.) .element?: query-shape?) +(define-type (SourceBoundaryParent @ Type.) .element?: parent-shape?) +(def (source-boundary-query? value) (element? SourceBoundaryQuery value)) +(def (source-boundary-parent? value) (element? SourceBoundaryParent value)) diff --git a/src/modules/parser/source-event-scope-funs.ss b/src/modules/parser/source-event-scope-funs.ss new file mode 100644 index 00000000..81ff0cd5 --- /dev/null +++ b/src/modules/parser/source-event-scope-funs.ss @@ -0,0 +1,16 @@ +;;; Pure deterministic allocation; no runtime name lookup or tree rewrite. +(import (only-in :clan/poo/object .ref) + (only-in ./source-event-scope-types source-event-scope?)) +(export source-event-name) +(def (identifier-part value) + (apply string-append + (map (lambda (byte) + (let (digits (number->string byte 16)) + (if (< byte 16) (string-append "0" digits) digits))) + (u8vector->list (string->utf8 (symbol->string value)))))) +(def (source-event-name scope role) + (unless (and (source-event-scope? scope) (symbol? role)) + (error "invalid native event scope binding" scope role)) + (string->symbol + (string-append "source_" (identifier-part (.ref scope 'owner)) + "_" (identifier-part role)))) diff --git a/src/modules/parser/source-event-scope-objects.ss b/src/modules/parser/source-event-scope-objects.ss new file mode 100644 index 00000000..b56c5620 --- /dev/null +++ b/src/modules/parser/source-event-scope-objects.ss @@ -0,0 +1,9 @@ +;;; Inert native POO scope with a stable, explicit author-owned identity. +(import (only-in :clan/poo/object .o) + (only-in ./source-event-scope-types source-event-scope?)) +(export make-source-event-scope) +(def (make-source-event-scope owner-value) + (let (scope (.o kind: 'source-event-scope owner: owner-value)) + (unless (source-event-scope? scope) + (error "invalid native event scope owner" owner-value)) + scope)) diff --git a/src/modules/parser/source-event-scope-types.ss b/src/modules/parser/source-event-scope-types.ss new file mode 100644 index 00000000..ed33402f --- /dev/null +++ b/src/modules/parser/source-event-scope-types.ss @@ -0,0 +1,10 @@ +;;; Rule-instance ownership is a POO declaration, not a runtime registry. +(import (only-in :clan/poo/object .ref .slot? object?) + (only-in :clan/poo/mop define-type Type. element?)) +(export SourceEventScope source-event-scope?) +(def (scope-shape? value) + (and (object? value) (.slot? value 'kind) (.slot? value 'owner) + (eq? (.ref value 'kind) 'source-event-scope) + (symbol? (.ref value 'owner)))) +(define-type (SourceEventScope @ Type.) .element?: scope-shape?) +(def (source-event-scope? value) (element? SourceEventScope value)) diff --git a/src/modules/parser/source-fragment-funs.ss b/src/modules/parser/source-fragment-funs.ss new file mode 100644 index 00000000..a9e16907 --- /dev/null +++ b/src/modules/parser/source-fragment-funs.ss @@ -0,0 +1,175 @@ +;;; -*- Gerbil -*- +;;; Lower an admitted POO fragment to the existing Scheme/event-AOT algebra. +;;; One pass over the bounded slice emits source-backed separator and item events. + +(import (only-in ./source-pattern-funs + source-pattern-at?) + (only-in ./source-fragment-types + source-delimited-fragment? source-first-split? + source-reference-scan?) + (only-in ./source-fragment-objects + source-delimited-fragment-delimiter + source-delimited-fragment-separator-token + source-delimited-fragment-item-helper + source-delimited-fragment-trivia-token + source-first-split-delimiter + source-first-split-separator-token + source-first-split-before-helper + source-first-split-after-helper + source-reference-scan-markers + source-reference-scan-continuation + source-reference-scan-call-prefix + source-reference-scan-call-token + source-reference-scan-reference-node + source-reference-scan-text-token)) +(export source-delimited-fragment-initial + source-delimited-fragment-forms + source-first-split-initial source-first-split-forms + source-reference-scan-initial source-reference-scan-forms) + +(def segment-index '(line-index segment-byte-index)) +(def segment-next `(line-step ,segment-index)) +(def segment-start '(state-offset segment-start)) + +(def (segment-events rule from until) + `(if (line-bytes-all-in? ,from ,until (9 32)) + ((token ,(source-delimited-fragment-trivia-token rule) ,from ,until)) + ((call-source-helper ,(source-delimited-fragment-item-helper rule) + ,from ,until)))) + +(def (source-delimited-fragment-initial rule) + (unless (source-delimited-fragment? rule) + (error "unadmitted source fragment" rule)) + '((segment-start 0) (segment-skip-next #f))) + +(def (source-delimited-fragment-forms rule) + (unless (source-delimited-fragment? rule) + (error "unadmitted source fragment" rule)) + (let* ((delimiter (source-delimited-fragment-delimiter rule)) + (first (char->integer (string-ref delimiter 0))) + (double? (= (string-length delimiter) 2)) + (match? (if double? + `(and (line-byte-equal? ,segment-index ,first) + (line-byte-equal? ,segment-next + ,(char->integer + (string-ref delimiter 1)))) + `(line-byte-equal? ,segment-index ,first))) + (separator-end (if double? `(line-step ,segment-next) segment-next)) + (separator-token (source-delimited-fragment-separator-token rule))) + `((set-uint segment-start (offset start)) + (for-line-bytes segment-byte-index start end + ((if (state segment-skip-next) + ((set-bool segment-skip-next (bool #f))) + ((if ,match? + (,(segment-events rule segment-start segment-index) + (token ,separator-token ,segment-index ,separator-end) + (set-uint segment-start (offset ,separator-end)) + ,@(if double? + '((set-bool segment-skip-next (bool #t))) + '())) ()))))) + ,(segment-events rule segment-start 'end)))) + +(def split-index '(line-index split-byte-index)) +(def split-at '(state-offset split-at)) +(def split-after `(line-step ,split-at)) + +(def (source-first-split-initial rule) + (unless (source-first-split? rule) + (error "unadmitted source first-split" rule)) + '((split-at 0) (split-seen #f))) + +(def (source-first-split-forms rule) + (unless (source-first-split? rule) + (error "unadmitted source first-split" rule)) + (let ((delimiter (char->integer + (string-ref (source-first-split-delimiter rule) 0))) + (separator-token (source-first-split-separator-token rule)) + (before-helper (source-first-split-before-helper rule)) + (after-helper (source-first-split-after-helper rule))) + `((for-line-bytes split-byte-index start end + ((if (and (not (state split-seen)) + (line-byte-equal? ,split-index ,delimiter)) + ((set-uint split-at (offset ,split-index)) + (set-bool split-seen (bool #t))) ()))) + (if (state split-seen) + ((call-source-helper ,before-helper start ,split-at) + (token ,separator-token ,split-at ,split-after) + (call-source-helper ,after-helper ,split-after end)) + ((call-source-helper ,before-helper start end)))))) + +(def ref-index '(line-index ref-byte-index)) +(def ref-next `(line-step ,ref-index)) +(def ref-start '(state-offset ref-start)) +(def ref-text-start '(state-offset ref-text-start)) + +(def (ref-node rule token until (continue? #t)) + `((start-node ,(source-reference-scan-reference-node rule)) + (token ,token ,ref-start ,until) + (finish-node) + (set-uint ref-text-start (offset ,until)) + ,@(if continue? '((set-uint ref-kind (uint 0))) '()))) + +(def (ref-begin rule kind) + `((if (offset-less? ,ref-text-start ,ref-index) + ((token ,(source-reference-scan-text-token rule) + ,ref-text-start ,ref-index)) ()) + (set-uint ref-start (offset ,ref-index)) + (set-uint ref-text-start (offset ,ref-index)) + (set-uint ref-kind (uint ,kind)))) + +(def (marker-start-chain rule markers call-kind) + `(if ,(source-pattern-at? ref-index (source-reference-scan-call-prefix rule)) + ,(ref-begin rule call-kind) + ,(let loop ((rest markers) (kind 1)) + (if (null? rest) '() + `((if (line-byte-equal? ,ref-index + ,(char->integer (caar rest))) + ,(ref-begin rule kind) + ,(loop (cdr rest) (+ kind 1)))))))) + +(def (marker-end-chain rule markers until (continue? #t)) + (let loop ((rest markers) (kind 1)) + (if (null? rest) '() + `((if (uint-equal? (state ref-kind) (uint ,kind)) + ,(ref-node rule (cdar rest) until continue?) + ,(loop (cdr rest) (+ kind 1))))))) + +(def (source-reference-scan-initial rule) + (unless (source-reference-scan? rule) + (error "unadmitted source reference scan" rule)) + '((ref-start 0) (ref-text-start 0) (ref-kind 0) + (ref-opens 0) (ref-closes 0))) + +(def (source-reference-scan-forms rule) + (unless (source-reference-scan? rule) + (error "unadmitted source reference scan" rule)) + (let* ((markers (source-reference-scan-markers rule)) + (call-kind (+ (length markers) 1)) + (continuation (map char->integer + (string->list + (source-reference-scan-continuation rule)))) + (call-token (source-reference-scan-call-token rule))) + `((set-uint ref-text-start (offset start)) + (for-line-bytes ref-byte-index start end + ((if (uint-equal? (state ref-kind) (uint ,call-kind)) + ((if (line-byte-equal? ,ref-index 40) + ((set-uint ref-opens + (uint-add (state ref-opens) (uint 1)))) + ((if (and (line-byte-equal? ,ref-index 41) + (uint-greater? (state ref-opens) + (state ref-closes))) + ((set-uint ref-closes + (uint-add (state ref-closes) (uint 1))) + (if (uint-equal? (state ref-opens) + (state ref-closes)) + ,(ref-node rule call-token ref-next) ())) ())))) + ((if (and (uint-positive? (state ref-kind)) + (not (line-bytes-any-in? + ,ref-index ,ref-next ,continuation))) + ,(marker-end-chain rule markers ref-index) ()) + (if (uint-equal? (state ref-kind) (uint 0)) + (,(marker-start-chain rule markers call-kind)) ()))))) + ,@(marker-end-chain rule markers 'end #f) + (if (offset-less? ,ref-text-start end) + ((token ,(source-reference-scan-text-token rule) + ,ref-text-start end)) ())))) diff --git a/src/modules/parser/source-fragment-objects.ss b/src/modules/parser/source-fragment-objects.ss new file mode 100644 index 00000000..6f9522a7 --- /dev/null +++ b/src/modules/parser/source-fragment-objects.ss @@ -0,0 +1,80 @@ +;;; -*- Gerbil -*- +;;; Source fragment declarations are admitted POO values, not raw form tuples. + +(import (only-in :clan/poo/object .o .ref) + (only-in ./source-fragment-types + source-delimited-fragment? source-first-split? + source-reference-scan? + +source-delimited-fragment-kind+ +source-first-split-kind+ + +source-reference-scan-kind+)) +(export make-source-delimited-fragment + source-delimited-fragment-delimiter + source-delimited-fragment-separator-token + source-delimited-fragment-item-helper + source-delimited-fragment-trivia-token + make-source-first-split source-first-split-delimiter + source-first-split-separator-token source-first-split-before-helper + source-first-split-after-helper + make-source-reference-scan source-reference-scan-markers + source-reference-scan-continuation + source-reference-scan-call-prefix source-reference-scan-call-token + source-reference-scan-reference-node source-reference-scan-text-token) + +(def (make-source-delimited-fragment delimiter-value separator-token-value + item-helper-value trivia-token-value) + (let (candidate + (.o kind: +source-delimited-fragment-kind+ + delimiter: delimiter-value + separator-token: separator-token-value + item-helper: item-helper-value + trivia-token: trivia-token-value)) + (unless (source-delimited-fragment? candidate) + (error "invalid source-delimited fragment" delimiter-value + separator-token-value item-helper-value trivia-token-value)) + candidate)) + +(def (source-delimited-fragment-delimiter value) (.ref value 'delimiter)) +(def (source-delimited-fragment-separator-token value) + (.ref value 'separator-token)) +(def (source-delimited-fragment-item-helper value) (.ref value 'item-helper)) +(def (source-delimited-fragment-trivia-token value) (.ref value 'trivia-token)) + +(def (make-source-first-split delimiter-value separator-token-value + before-helper-value after-helper-value) + (let (candidate + (.o kind: +source-first-split-kind+ + delimiter: delimiter-value + separator-token: separator-token-value + before-helper: before-helper-value + after-helper: after-helper-value)) + (unless (source-first-split? candidate) + (error "invalid source first-split" delimiter-value + separator-token-value before-helper-value after-helper-value)) + candidate)) + +(def (source-first-split-delimiter value) (.ref value 'delimiter)) +(def (source-first-split-separator-token value) (.ref value 'separator-token)) +(def (source-first-split-before-helper value) (.ref value 'before-helper)) +(def (source-first-split-after-helper value) (.ref value 'after-helper)) + +(def (make-source-reference-scan markers-value continuation-value + call-prefix-value call-token-value + reference-node-value text-token-value) + (let (candidate + (.o kind: +source-reference-scan-kind+ + markers: markers-value + continuation: continuation-value + call-prefix: call-prefix-value + call-token: call-token-value + reference-node: reference-node-value + text-token: text-token-value)) + (unless (source-reference-scan? candidate) + (error "invalid source reference scan" candidate)) + candidate)) + +(def (source-reference-scan-markers value) (.ref value 'markers)) +(def (source-reference-scan-continuation value) (.ref value 'continuation)) +(def (source-reference-scan-call-prefix value) (.ref value 'call-prefix)) +(def (source-reference-scan-call-token value) (.ref value 'call-token)) +(def (source-reference-scan-reference-node value) (.ref value 'reference-node)) +(def (source-reference-scan-text-token value) (.ref value 'text-token)) diff --git a/src/modules/parser/source-fragment-types.ss b/src/modules/parser/source-fragment-types.ss new file mode 100644 index 00000000..861f18c7 --- /dev/null +++ b/src/modules/parser/source-fragment-types.ss @@ -0,0 +1,98 @@ +;;; -*- Gerbil -*- +;;; POO admission for source-bounded event fragments. + +(import (only-in :clan/poo/object .ref .slot? object?) + (only-in :clan/poo/mop define-type Type. element?)) +(export SourceDelimitedFragment source-delimited-fragment? + SourceFirstSplit source-first-split? + SourceReferenceScan source-reference-scan? + +source-delimited-fragment-kind+ +source-first-split-kind+ + +source-reference-scan-kind+) + +(def +source-delimited-fragment-kind+ 'gerbil-parser-source-delimited-fragment) +(def +source-first-split-kind+ 'gerbil-parser-source-first-split) +(def +source-reference-scan-kind+ 'gerbil-parser-source-reference-scan) + +(def (source-ascii? value) + (and (string? value) + (> (string-length value) 0) + (every (lambda (char) + (and (<= 1 (char->integer char)) + (< (char->integer char) 128))) + (string->list value)))) + +(def (source-delimiter? value) + (and (source-ascii? value) + (<= (string-length value) 2))) + +(def (source-delimited-fragment-shape? value) + (and (object? value) + (.slot? value 'kind) + (.slot? value 'delimiter) + (.slot? value 'separator-token) + (.slot? value 'item-helper) + (.slot? value 'trivia-token) + (eq? (.ref value 'kind) +source-delimited-fragment-kind+) + (source-delimiter? (.ref value 'delimiter)) + (symbol? (.ref value 'separator-token)) + (symbol? (.ref value 'item-helper)) + (symbol? (.ref value 'trivia-token)))) + +(define-type (SourceDelimitedFragment @ Type.) + .element?: source-delimited-fragment-shape?) + +(def (source-delimited-fragment? value) + (element? SourceDelimitedFragment value)) + +(def (source-first-split-shape? value) + (and (object? value) + (.slot? value 'kind) + (.slot? value 'delimiter) + (.slot? value 'separator-token) + (.slot? value 'before-helper) + (.slot? value 'after-helper) + (eq? (.ref value 'kind) +source-first-split-kind+) + (source-delimiter? (.ref value 'delimiter)) + (= (string-length (.ref value 'delimiter)) 1) + (symbol? (.ref value 'separator-token)) + (symbol? (.ref value 'before-helper)) + (symbol? (.ref value 'after-helper)))) + +(define-type (SourceFirstSplit @ Type.) + .element?: source-first-split-shape?) + +(def (source-first-split? value) + (element? SourceFirstSplit value)) + +(def (source-reference-scan-shape? value) + (and (object? value) + (.slot? value 'kind) + (.slot? value 'markers) + (.slot? value 'continuation) + (.slot? value 'call-prefix) + (.slot? value 'call-token) + (.slot? value 'reference-node) + (.slot? value 'text-token) + (eq? (.ref value 'kind) +source-reference-scan-kind+) + (let (markers (.ref value 'markers)) + (and (pair? markers) + (every (lambda (entry) + (and (pair? entry) + (char? (car entry)) + (< (char->integer (car entry)) 128) + (symbol? (cdr entry)))) + markers))) + (source-ascii? (.ref value 'continuation)) + (source-ascii? (.ref value 'call-prefix)) + (char=? (string-ref (.ref value 'call-prefix) + (- (string-length (.ref value 'call-prefix)) 1)) + #\() + (symbol? (.ref value 'call-token)) + (symbol? (.ref value 'reference-node)) + (symbol? (.ref value 'text-token)))) + +(define-type (SourceReferenceScan @ Type.) + .element?: source-reference-scan-shape?) + +(def (source-reference-scan? value) + (element? SourceReferenceScan value)) diff --git a/src/modules/parser/source-pattern-funs.ss b/src/modules/parser/source-pattern-funs.ss new file mode 100644 index 00000000..36058f0d --- /dev/null +++ b/src/modules/parser/source-pattern-funs.ss @@ -0,0 +1,62 @@ +;;; -*- Gerbil -*- +;;; Pure projections of source patterns into the shared event-fold algebra. +;;; Strings are literal grammar inputs, not a second runtime matcher or DSL. + +(import (only-in :std/string/utf8 string-utf8-length)) +(export source-priority-forms source-offset-after source-pattern-end source-pattern-at? + source-ascii-ci-pattern-at?) + +;;; Ordered choice is a pure lowering operation, not another runtime matcher. +;;; Keep every predicate in source order, including branches with no emissions: +;;; a matched empty branch still suppresses the fallback. +(def (source-priority-forms predicates branches (otherwise '())) + (unless (and (list? predicates) (list? branches) + (= (length predicates) (length branches)) + (every list? branches) (list? otherwise)) + (error "source priority requires equally sized predicate/statement lists" + predicates branches otherwise)) + (let loop ((tests predicates) (bodies branches)) + (if (null? tests) otherwise + `((if ,(car tests) ,(car bodies) + ,(loop (cdr tests) (cdr bodies))))))) + +(def (source-offset-after from count) + (unless (and (exact-integer? count) (>= count 0)) + (error "source byte offset requires a nonnegative exact step count" count)) + (let loop ((offset from) (remaining count)) + (if (= remaining 0) offset + (loop `(line-step ,offset) (- remaining 1))))) + +(def (source-pattern-end from pattern) + (unless (string? pattern) + (error "source pattern must be a string" pattern)) + (source-offset-after from (string-utf8-length pattern))) + +(def (source-pattern-at? from pattern) + (unless (and (string? pattern) (> (string-length pattern) 0)) + (error "source byte pattern must not be empty" pattern)) + (let (bytes (string->utf8 pattern)) + (let loop ((offset from) (index 0) (predicate #f)) + (if (= index (u8vector-length bytes)) predicate + (let (check `(line-byte-equal? ,offset ,(u8vector-ref bytes index))) + (loop `(line-step ,offset) (+ index 1) + (if predicate `(and ,predicate ,check) check))))))) + +(def (source-ascii-ci-pattern-at? from pattern) + (unless (and (string? pattern) (> (string-length pattern) 0) + (every (lambda (char) (< (char->integer char) 128)) + (string->list pattern))) + (error "case-insensitive source pattern requires nonempty ASCII" pattern)) + (let loop ((rest (string->list pattern)) (offset from) (predicates [])) + (if (null? rest) + ;; A single predicate must not become an invalid unary `and` form. + (if (null? (cdr predicates)) (car predicates) + (cons 'and (reverse predicates))) + (let* ((byte (char->integer (car rest))) + (upper (if (<= 97 byte 122) (- byte 32) byte)) + (lower (if (<= 65 byte 90) (+ byte 32) byte)) + (check (if (= upper lower) + `(line-byte-equal? ,offset ,upper) + `(line-bytes-any-in? ,offset (line-step ,offset) + (,upper ,lower))))) + (loop (cdr rest) `(line-step ,offset) (cons check predicates)))))) diff --git a/src/modules/parser/types.ss b/src/modules/parser/types.ss index cf1c3e74..37b186cd 100644 --- a/src/modules/parser/types.ss +++ b/src/modules/parser/types.ss @@ -2,12 +2,14 @@ ;;; Invariant: grammar values prove prototype ancestry and every public slot ;;; before object behavior or compiler normalization can observe them. -(import (only-in :clan/poo/object .o .ref) - (only-in :clan/poo/mop define-type element?) +(import (only-in :clan/poo/object .o .ref .slot? object? compute-precedence-list!) + (only-in :clan/poo/mop define-type element? raise-type-error) (only-in :core/types PooFlowContract. PooFlowNativeObjectContract. - poo-flow-classification-evidence)) + poo-flow-classification-evidence + poo-flow-contract-admit + poo-flow-validation-evidence-accepted?)) (export +grammar-role-kind+ +grammar-kind+ @@ -106,7 +108,9 @@ keywords: ParserList parser-entrypoints: ParserList recoveries: ParserList - flow: ParserList)) + flow: ParserList) + .element?: (cut parser-object-element? @ <>) + .validate: (cut parser-object-validate @ <>)) ;; : (-> Object Object [Object]) (def (grammar-object-obligations candidate _context) @@ -139,4 +143,59 @@ parents: ParserList roles: ParserList composition: ParserList) - .obligations: grammar-object-obligations) + .obligations: grammar-object-obligations + .element?: (cut parser-object-element? @ <>) + .validate: (cut parser-object-validate @ <>)) + +;; Capture exact builtin descriptors privately, rather than following any +;; subsequent rebinding of an exported contract to a refinement. +(def +grammar-role-boolean-contract+ GrammarRoleContract) +(def +grammar-boolean-contract+ GrammarContract) + +;;; Boolean membership and successful validation do not need responsibility +;;; evidence objects. Explicit .admit still belongs to the full Core protocol. +;;; Only these exact builtin descriptors use the specialization; refinements +;;; keep their own classifiers, responsibilities and obligations. +(def (parser-slot-matches? candidate slot predicate) + (and (.slot? candidate slot) (predicate (.ref candidate slot)))) + +(def (parser-native-ancestry? descriptor candidate) + (and (object? candidate) + (if (memq (.ref descriptor 'proto) (compute-precedence-list! candidate)) #t #f))) + +(def (parser-role-slots? candidate) + (and (parser-slot-matches? candidate 'kind (lambda (value) (eq? value +grammar-role-kind+))) + (parser-slot-matches? candidate 'name symbol?) + (andmap (lambda (slot) (parser-slot-matches? candidate slot list?)) + '(syntax-kinds terminals lexical-rules rules extras keywords + parser-entrypoints recoveries flow)))) + +(def (parser-grammar-slots? candidate) + (and (parser-slot-matches? candidate 'kind (lambda (value) (eq? value +grammar-kind+))) + (parser-slot-matches? candidate 'schema (lambda (value) (equal? value +grammar-schema+))) + (parser-slot-matches? candidate 'name symbol?) + (andmap (lambda (slot) (parser-slot-matches? candidate slot list?)) + '(parents roles composition)) + (null? (grammar-object-obligations candidate #f)))) + +(def (parser-object-element? descriptor candidate) + (cond + ((eq? descriptor +grammar-role-boolean-contract+) + (and (parser-native-ancestry? descriptor candidate) (parser-role-slots? candidate))) + ((eq? descriptor +grammar-boolean-contract+) + (and (parser-native-ancestry? descriptor candidate) (parser-grammar-slots? candidate))) + (else + (poo-flow-validation-evidence-accepted? + (poo-flow-contract-admit descriptor candidate #f))))) + +(def (parser-object-validate descriptor candidate) + (if (and (or (eq? descriptor +grammar-role-boolean-contract+) + (eq? descriptor +grammar-boolean-contract+)) + (parser-object-element? descriptor candidate)) + candidate + ;; A refined descriptor is admitted exactly once. Its classifier and + ;; obligations belong to the open protocol, including invocation count. + (let (evidence (poo-flow-contract-admit descriptor candidate #f)) + (if (poo-flow-validation-evidence-accepted? evidence) + candidate + (raise-type-error descriptor candidate (.ref evidence 'diagnostics)))))) diff --git a/src/runtime/actor.ss b/src/runtime/actor.ss new file mode 100644 index 00000000..5fa404dc --- /dev/null +++ b/src/runtime/actor.ss @@ -0,0 +1,56 @@ +;;; -*- Gerbil -*- +;;; Optional in-process actor boundary; importing this library starts no threads. +(import :std/actor + (only-in ../compiler/machine parser-machine?) + (only-in ./parser parse-source)) +(export spawn-parser-actor parser-actor-parse parser-actor-stop!) + +(def +parser-protocol+ "/gerbil-parser/parse/v1") +(defstruct !ParseSource (source)) + +;; The application owns the returned LocalActor and the machine's lifetime. +;; Install generated drivers before spawning; never mutate an active machine. +(def (spawn-parser-actor machine) + (unless (parser-machine? machine) (error "expected a prepared parser machine")) + (LocalActor + (spawn + (lambda () + (let loop () + (<- protocol: +parser-protocol+ + ((!ParseSource source) + (let (result + (with-catch + (lambda (exception) + (!Error (call-with-output-string + (lambda (port) (display-exception exception port))))) + (lambda () (!OK (parse-source machine source))))) + (--> result)) + (loop)) + protocol: proto:/actor/control + ((!Ping) (--> (!OK 'OK)) (loop)) + ((!Shutdown) (--> (!OK 'shutdown))))))))) + +(def (check-timeout timeout) + (unless (and (real? timeout) (> timeout 0)) + (error "expected a positive actor request timeout" timeout))) + +;; Gerbil's envelope continuation correlates replies from concurrent callers. +;; Expired queued envelopes are discarded by <-. Timeout does not preempt a +;; parse that already started. Snapshot caller-owned mutable strings on submit. +(def (parser-actor-parse actor source (timeout 5)) + (unless (string? source) (error "parse source must be a string" source)) + (check-timeout timeout) + (with-result + (->> actor (!ParseSource (string-copy source)) +parser-protocol+ + timeout: timeout) + (lambda (message) (error "parser actor request failed" message)))) + +;; Shutdown is ordered after previously queued work; completion joins the child. +(def (parser-actor-stop! actor (timeout 5)) + (check-timeout timeout) + (let (result + (with-result + (->> actor (!Shutdown) proto:/actor/control timeout: timeout) + (lambda (message) (error "parser actor shutdown failed" message)))) + (thread-join! (LocalActor-thread actor) timeout) + result)) diff --git a/src/runtime/artifact.ss b/src/runtime/artifact.ss index 4d2e450b..f9bbf989 100644 --- a/src/runtime/artifact.ss +++ b/src/runtime/artifact.ss @@ -1,29 +1,44 @@ ;;; -*- Gerbil -*- ;;; Canonical backend-neutral ParseArtifact v1 and CST event authority. -(import (only-in :std/func compose every-of) +(import (only-in :std/string/utf8 string-utf8-length) + (only-in :std/list/list-builder with-list-builder) + (only-in ./parse-cost with-parser-cost-stage) + (only-in ./event-program event-program-value? event-program-value-kind + event-program-value-code event-program-walk/inline event-program-relocate) + (only-in ./funcs recognition-sequence-for-each) + (only-in :std/func compose every-of) + (only-in :std/string/misc string-concatenate-reverse) (only-in ../modules/parser/types +diagnostic-schema+ +parse-artifact-schema+) - (only-in ./identity sha256-text) + (only-in ./identity sha256-bytes sha256-text) (only-in ./recognition + recognition-relocation? recognition-relocation-value recognition-relocation-delta recognition-child-field recognition-child-value recognition-fragment-children recognition-fragment-end + recognition-fragment-start recognition-fragment? recognition-node-children recognition-node-end recognition-node-kind recognition-node? - recognition-node-start recognition-value-end - recognition-value-start) + recognition-node-start) (only-in ./token token? token-end token-kind token-lexeme token-start)) (export +parse-artifact-schema+ +diagnostic-schema+ sha256-text make-success-parse-artifact + make-success-parse-artifact/canonical-events + make-raw-parse-event + make-success-parse-artifact/raw-event-tape + make-same-width-token-artifact + make-shifted-token-artifact + make-certified-window-artifact make-failure-parse-artifact parse-artifact-ref parse-artifact-events + with-parse-event-walk parse-artifact-status parse-artifact-success? - parse-artifact-valid? + parse-artifact-valid? parse-artifact-valid-for-source? parse-artifact-roundtrip event-kind token-event? @@ -114,124 +129,511 @@ ;;; Node/token identifiers are allocated once; source gaps become trivia only here. ;; recognition-events ;; : (-> List Recognition Boolean Fixnum List) +(defrule (with-recognition-event-walk tokens root trivia? source-byte-length + node-emitter field-emitter token-emitter) + (let* ((remaining tokens) + (next-node-id 0) (next-token-id 0)) + (def (emit-source-token! source-token) + (token-emitter next-token-id source-token) + (set! next-token-id (+ next-token-id 1)) + (set! remaining (cdr remaining))) + (def (emit-trivia-until! boundary) + (let loop () + (when (and (pair? remaining) (<= (token-end (car remaining)) boundary)) + (unless (trivia? (car remaining)) + (error "unclaimed significant token" (token-kind (car remaining)) + (token-start (car remaining)))) + (emit-source-token! (car remaining)) (loop)))) + (def (emit-token! source-token delta translated?) + (unless (and (pair? remaining) + (if translated? + (let (current (car remaining)) + (and (= (token-start current) (+ (token-start source-token) delta)) + (= (token-end current) (+ (token-end source-token) delta)) + (eq? (token-kind current) (token-kind source-token)) + (equal? (token-lexeme current) (token-lexeme source-token)))) + (eq? source-token (car remaining)))) + (error "recognition token does not match source order" + (token-kind source-token) (+ delta (token-start source-token)))) + ;; The canonical event always uses this request's actual source token. + (emit-source-token! (car remaining))) + (def (emit-children! children end delta translated?) + ;; Most production children already are canonical lists. Retain their + ;; direct consumer rather than adding sequence callbacks and zero deltas. + ;; Classify the outer representation; the checked visitor owns tail admission. + ;; A list? preflight would traverse every child again on each publication. + (if (or (null? children) (pair? children)) + (let visit ((remaining children)) + (unless (null? remaining) + (let (child (car remaining)) + (emit-value! (recognition-child-value child) + (recognition-child-field child) delta translated? #f)) + (visit (cdr remaining)))) + (recognition-sequence-for-each + (lambda (child child-delta child-translated?) + (emit-value! (recognition-child-value child) (recognition-child-field child) + (+ delta child-delta) (or translated? child-translated?) #f)) children)) + (emit-trivia-until! end)) + (def (emit-value! value field delta translated? root?) + (cond + ((recognition-relocation? value) + (emit-value! (recognition-relocation-value value) field + (+ delta (recognition-relocation-delta value)) #t root?)) + ((token? value) + (when root? (error "parse root must be a recognition node" value)) + (let ((start (+ delta (token-start value))) (end (+ delta (token-end value)))) + (emit-trivia-until! start) + (when field (field-emitter 'start-field field start)) + (emit-token! value delta translated?) + (when field (field-emitter 'finish-field field end)))) + ((recognition-node? value) + (let* ((start (if root? 0 (+ delta (recognition-node-start value)))) + (end (if root? source-byte-length (+ delta (recognition-node-end value)))) + (id next-node-id) (kind (recognition-node-kind value))) + (emit-trivia-until! start) + (when field (field-emitter 'start-field field start)) + (set! next-node-id (+ next-node-id 1)) + (node-emitter 'start-node id kind start) + (emit-children! (recognition-node-children value) end delta translated?) + (node-emitter 'finish-node id kind end) + (when field (field-emitter 'finish-field field end)))) + ((recognition-fragment? value) + (when root? (error "parse root must be a recognition node" value)) + (let ((start (+ delta (recognition-fragment-start value))) + (end (+ delta (recognition-fragment-end value)))) + (emit-trivia-until! start) + (when field (field-emitter 'start-field field start)) + (emit-children! (recognition-fragment-children value) end delta translated?) + (when field (field-emitter 'finish-field field end)))) + (else (error "invalid recognition value" value)))) + (emit-value! root #f 0 #f #t) + (unless (null? remaining) (error "source tokens remain outside parse root" remaining)) + (void))) + +(defrule (collect-canonical-events node-emitter field-emitter token-emitter walk) + (with-list-builder (emit!) + (def (node-emitter tag id kind position) (emit! (vector tag id kind position))) + (def (field-emitter tag field position) (emit! (vector tag field position))) + (def (token-emitter id token) (emit! (make-token-event id token))) + walk)) (def (recognition-events tokens root trivia? source-byte-length) - (let ((remaining tokens) - (events '()) - (next-node-id 0) - (next-token-id 0)) - (letrec - ((emit! - (lambda (event) - (set! events (cons event events)))) - (emit-source-token! - (lambda (source-token) - (emit! (make-token-event next-token-id source-token)) - (set! next-token-id (+ next-token-id 1)) - (set! remaining (cdr remaining)))) - (emit-trivia-until! - (lambda (boundary) - (let loop () - (when (and (pair? remaining) - (<= (token-end (car remaining)) boundary)) - (unless (trivia? (car remaining)) - (error "unclaimed significant token" - (token-kind (car remaining)) - (token-start (car remaining)))) - (emit-source-token! (car remaining)) - (loop))))) - (emit-child! - (lambda (child) - (let* ((value (recognition-child-value child)) - (field (recognition-child-field child)) - (start (recognition-value-start value)) - (end (recognition-value-end value))) - (emit-trivia-until! start) - (when field (emit! (vector 'start-field field start))) - (emit-value! value) - (when field (emit! (vector 'finish-field field end)))))) - (emit-children! - (lambda (children end) - (for-each emit-child! children) - (emit-trivia-until! end))) - (emit-node! - (lambda (node root?) - (let* ((id next-node-id) - (kind (recognition-node-kind node)) - (start (if root? 0 (recognition-node-start node))) - (end (if root? - source-byte-length - (recognition-node-end node)))) - (set! next-node-id (+ next-node-id 1)) - (emit! (vector 'start-node id kind start)) - (emit-children! (recognition-node-children node) end) - (emit! (vector 'finish-node id kind end))))) - (emit-fragment! - (lambda (fragment) - (emit-children! (recognition-fragment-children fragment) - (recognition-fragment-end fragment)))) - (emit-token! - (lambda (source-token) - (emit-trivia-until! (token-start source-token)) - (unless (and (pair? remaining) - (eq? source-token (car remaining))) - (error "recognition token does not match source order" - (token-kind source-token) - (token-start source-token))) - (emit-source-token! source-token))) - (emit-value! - (lambda (value) - (cond - ((token? value) (emit-token! value)) - ((recognition-node? value) (emit-node! value #f)) - ((recognition-fragment? value) (emit-fragment! value)) - (else (error "invalid recognition value" value)))))) - (unless (recognition-node? root) - (error "parse root must be a recognition node" root)) - (emit-node! root #t) - (unless (null? remaining) - (error "source tokens remain outside parse root" remaining)) - (let reverse! ((rest events) (found '())) - (if (null? rest) - found - (let (next (cdr rest)) - (set-cdr! rest found) - (reverse! next rest))))))) + (collect-canonical-events node-emitter field-emitter token-emitter + (with-recognition-event-walk tokens root trivia? source-byte-length + node-emitter field-emitter token-emitter))) ;; flat-token-events ;; : (-> List List) (def (flat-token-events tokens) - (map make-token-event (iota (length tokens)) tokens)) + (with-list-builder (emit) + (let loop ((rest tokens) (id 0)) + (unless (null? rest) + (emit (make-token-event id (car rest))) + (loop (cdr rest) (+ id 1)))))) ;; artifact -;; : (-> String String Symbol List List Alist) -(def (artifact grammar-digest source status events diagnostics) - (map cons - '(schema grammarDigest sourceDigest sourceByteLength - status events diagnostics) - (list +parse-artifact-schema+ - grammar-digest - (sha256-text source) - (u8vector-length (string->utf8 source)) - status - events - diagnostics))) +;; : (-> String String Symbol List List (? U8Vector) Alist) +(def (artifact grammar-digest source status events diagnostics + (source-bytes #f)) + (let (bytes (or source-bytes + (with-parser-cost-stage 'artifact-source-encoding (string->utf8 source)))) + (with-parser-cost-stage 'artifact-source-identity + (map cons + '(schema grammarDigest sourceDigest sourceByteLength + status events diagnostics) + (list +parse-artifact-schema+ + grammar-digest + (sha256-bytes bytes) + (u8vector-length bytes) + status + events + diagnostics))))) ;; make-success-parse-artifact ;; : (-> String String List Recognition Boolean Alist) (def (make-success-parse-artifact grammar-digest source tokens root trivia?) - (let* ((source-byte-length (u8vector-length (string->utf8 source))) + (let* ((source-bytes + (with-parser-cost-stage 'artifact-source-encoding (string->utf8 source))) + (source-byte-length (u8vector-length source-bytes)) (value (artifact grammar-digest source 'accepted - (recognition-events tokens root trivia? source-byte-length) - '()))) + (with-parser-cost-stage 'artifact-event-publication + (if (or (event-program-value? root) + (and (recognition-relocation? root) + (event-program-root? root))) + (event-program-events tokens root trivia? source-byte-length) + (recognition-events tokens root trivia? source-byte-length))) + '() source-bytes))) value)) +;;; Interpret committed event bytecode directly into the canonical artifact. +;;; The program retains old source tokens only as proof operands: moved leaves +;;; bind and verify the actual current source token, never a copied substitute. +(def (event-program-root? value) + (if (recognition-relocation? value) + (event-program-root? (recognition-relocation-value value)) + (event-program-value? value))) +(def (event-program-root-code root) + (let loop ((value root) (delta 0) (moved? #f)) + (if (recognition-relocation? value) + (loop (recognition-relocation-value value) + (+ delta (recognition-relocation-delta value)) #t) + (begin + (unless (and (event-program-value? value) (event-program-value-kind value)) + (error "event program root must be a node")) + (event-program-relocate (event-program-value-code value) delta moved?))))) +(defrule (with-event-program-walk tokens root trivia? source-byte-length + node-emitter field-emitter token-emitter) + (let ((remaining tokens) (next-token-id 0) + (next-node-id 0)) + (let () + (def (emit-source-token!) + (let (input (car remaining)) + (token-emitter next-token-id input) + (set! next-token-id (+ next-token-id 1)) + (set! remaining (cdr remaining)))) + (def (emit-trivia-until! boundary) + (let loop () + (when (and (pair? remaining) (<= (token-end (car remaining)) boundary)) + (unless (trivia? (car remaining)) + (error "unclaimed significant program token" (token-kind (car remaining)))) + (emit-source-token!) (loop)))) + (event-program-walk/inline + (lambda (operation name offset delta moved? node-id) + (let (position (+ offset delta)) + (case operation + ((token) + (emit-trivia-until! position) + (unless (and (pair? remaining) + (if moved? + (let (actual (car remaining)) + (and (= (token-start actual) (+ (token-start name) delta)) + (= (token-end actual) (+ (token-end name) delta)) + (eq? (token-kind actual) (token-kind name)) + (equal? (token-lexeme actual) (token-lexeme name)))) + (eq? name (car remaining)))) + (error "event program token does not bind current source")) + (emit-source-token!)) + ((open-node) + (let ((start (if (= next-node-id 0) 0 position)) (id next-node-id)) + (emit-trivia-until! start) + (node-emitter 'start-node id name start) + (set! next-node-id (+ next-node-id 1)) + id)) + ((close-node) + ;; The immutable node instruction supplies its own footer. Its + ;; opening ID is retained by the traversal frame, including the + ;; iterative depth fallback, rather than a second node-ID stack. + (unless (integer? node-id) (error "event node has no opening ID")) + (let (end (if (= node-id 0) source-byte-length position)) + (emit-trivia-until! end) + (node-emitter 'finish-node node-id name end))) + ((open-field) + (emit-trivia-until! position) + (field-emitter 'start-field name position)) + ((close-field) (field-emitter 'finish-field name position)) + ((boundary) (emit-trivia-until! position)) + (else (error "unknown committed event program operation" operation))))) + (event-program-root-code root)) + (unless (null? remaining) + (error "event program is incomplete")) + (void)))) + +(def (event-program-events tokens root trivia? source-byte-length) + (collect-canonical-events node-emitter field-emitter token-emitter + (with-event-program-walk tokens root trivia? source-byte-length + node-emitter field-emitter token-emitter))) + +;;; One source-order authority; sinks choose their own representation. Expansion +;;; keeps canonical emitters local and avoids imposing a generic event object. +(defrule (with-parse-event-walk tokens root trivia? source-byte-length + node-emitter field-emitter token-emitter) + (if (or (event-program-value? root) + (and (recognition-relocation? root) (event-program-root? root))) + (with-event-program-walk tokens root trivia? source-byte-length + node-emitter field-emitter token-emitter) + (with-recognition-event-walk tokens root trivia? source-byte-length + node-emitter field-emitter token-emitter))) + +;;; Assemble already canonical events from a committed generated path. +(def (make-success-parse-artifact/canonical-events + grammar-digest source events source-bytes) + (artifact grammar-digest source 'accepted events '() source-bytes)) + +;;; Named raw descriptors remain available for inspection. The generated HCL +;;; hot path stores their operation, name, and byte offset in a flat tape. +(defstruct raw-parse-event (operation name byte-offset) transparent: #t) + +;;; Resolve trivia and allocate canonical IDs once, after the speculative +;;; parser commits a populated prefix of the request-local tape. +(def (make-success-parse-artifact/raw-event-tape + grammar-digest source tokens tape raw-count trivia? source-bytes) + (let (events + (with-parser-cost-stage 'artifact-event-publication + (raw-event-tape-events tokens tape raw-count trivia? + (u8vector-length source-bytes)))) + (artifact grammar-digest source 'accepted events '() source-bytes))) + +;; Event resolution owns exactly the committed tape and canonical event list. +;; Keep source encoding/hash/header construction outside this boundary. +(def (raw-event-tape-events tokens tape raw-count trivia? source-byte-length) + (let* ((remaining tokens) + (events (cons #f '())) + (tail events) + (next-token-id 0) + (next-node-id 0) + (node-ids '())) + (def (emit! event) + (let (cell (cons event '())) + (set-cdr! tail cell) + (set! tail cell))) + (def (emit-token! input) + (unless (and (pair? remaining) (eq? input (car remaining))) + (error "event token differs from source order" input)) + (emit! (vector 'token next-token-id + (token-kind input) (token-lexeme input) + (token-start input) (token-end input))) + (set! next-token-id (fx+ next-token-id 1)) + (set! remaining (cdr remaining))) + (def (emit-trivia-until! boundary) + (let loop () + (when (and (pair? remaining) + (<= (token-end (car remaining)) boundary)) + (unless (trivia? (car remaining)) + (error "unclaimed significant event token" + (token-kind (car remaining)))) + (emit-token! (car remaining)) + (loop)))) + (let loop ((index 0)) + (when (< index raw-count) + (let* ((base (* 3 index)) + (operation (vector-ref tape base)) + (name (vector-ref tape (fx+ base 1))) + (byte-offset (vector-ref tape (fx+ base 2)))) + (case operation + ((token) + (emit-trivia-until! (token-start name)) + (emit-token! name)) + ((open-node) + (let* ((start (if (= next-node-id 0) 0 byte-offset)) + (id next-node-id)) + (emit-trivia-until! start) + (emit! (vector 'start-node id name start)) + (set! next-node-id (fx+ next-node-id 1)) + (set! node-ids (cons id node-ids)))) + ((close-node) + (let ((end (if (null? (cdr node-ids)) + source-byte-length byte-offset)) + (id (car node-ids))) + (emit-trivia-until! end) + (emit! (vector 'finish-node id name end)) + (set! node-ids (cdr node-ids)))) + ((open-field) + (emit-trivia-until! byte-offset) + (emit! (vector 'start-field name byte-offset))) + ((close-field) + (emit! (vector 'finish-field name byte-offset))) + (else (error "unknown generated event" operation))) + (loop (fx+ index 1))))) + (unless (and (null? remaining) (null? node-ids)) + (error "generated event stream is incomplete")) + (cdr events))) + +;;; A certified same-width edit changes exactly one token event. The unchanged +;;; suffix remains shared, including node and field events. +(def (make-same-width-token-artifact old-artifact source token-id source-token) + (unless (parse-artifact-success? old-artifact) + (error "token event reuse requires an accepted artifact")) + (let (events + (let loop ((remaining (parse-artifact-events old-artifact)) + (prefix '())) + (cond + ((null? remaining) + (error "token event id is absent" token-id)) + ((and (token-event? (car remaining)) + (= (token-event-id (car remaining)) token-id)) + (foldl cons + (cons (make-token-event token-id source-token) + (cdr remaining)) + prefix)) + (else (loop (cdr remaining) + (cons (car remaining) prefix)))))) + (artifact (parse-artifact-ref old-artifact 'grammarDigest) + source 'accepted events '()))) + +;;; Preserve the event topology while rebasing offsets after one token whose +;;; byte width changed. Event boundaries inside a token are not admissible. +(def (make-shifted-token-artifact old-artifact source token-id + token-start token-end source-token delta) + (unless (parse-artifact-success? old-artifact) + (error "shifted token reuse requires an accepted artifact")) + (let ((replaced? #f) (shared 0)) + (def (offset value) + (cond + ((<= value token-start) value) + ((>= value token-end) (+ value delta)) + (else (error "event boundary is inside the edited token" value)))) + (def (keep event) + (set! shared (+ shared 1)) + event) + (let (events + (map + (lambda (event) + (case (event-kind event) + ((token) + (let ((id (token-event-id event)) + (start (event-start event)) + (end (event-end event))) + (cond + ((= id token-id) + (set! replaced? #t) + (make-token-event id source-token)) + ((<= end token-start) (keep event)) + ((>= start token-end) + (vector 'token id (token-event-token-kind event) + (token-event-lexeme event) + (+ start delta) (+ end delta))) + (else (error "token overlaps the edited token" event))))) + ((start-node finish-node) + (let* ((value (vector-ref event 3)) + (shifted (offset value))) + (if (= value shifted) + (keep event) + (vector (event-kind event) (vector-ref event 1) + (vector-ref event 2) shifted)))) + ((start-field finish-field) + (let* ((value (vector-ref event 2)) + (shifted (offset value))) + (if (= value shifted) + (keep event) + (vector (event-kind event) (vector-ref event 1) + shifted)))) + (else (error "unknown recognition event" event)))) + (parse-artifact-events old-artifact))) + (unless replaced? + (error "token event id is absent" token-id)) + (values + (artifact (parse-artifact-ref old-artifact 'grammarDigest) + source 'accepted events '()) + shared)))) + +;;; Each old token boundary in a certified window has one new boundary. LR +;;; reductions can only place node and field boundaries at token boundaries, +;;; so the event topology remains valid when their offsets are remapped. +(def (token-groups-match? groups tokens) + (let groups-loop ((groups groups) (tokens tokens)) + (if (null? groups) (null? tokens) + (let group-loop ((group (car groups)) (tokens tokens)) + (cond + ((null? group) (groups-loop (cdr groups) tokens)) + ((or (null? tokens) (not (equal? (car group) (car tokens)))) #f) + (else (group-loop (cdr group) (cdr tokens)))))))) + +(def (make-certified-window-artifact old-artifact source first-id + old-window new-window delta + (assigned #f) (boundaries #f)) + (unless (and (parse-artifact-success? old-artifact) + (pair? old-window) + (or (not assigned) + (and (= (length assigned) (length old-window)) + (token-groups-match? assigned new-window)))) + (error "invalid certified token window")) + (let* ((boundary-map (make-table test: equal?)) + (start (token-start (car old-window))) + (old-count (length old-window)) + (end (token-end (list-ref old-window (- old-count 1)))) + (limit (+ first-id old-count)) + (id-delta (- (length new-window) old-count)) + (groups + (or assigned + (if (zero? id-delta) + (map list new-window) + (cons new-window + (make-list (- old-count 1) '()))))) + (next-id first-id) + ;; Borrow certified token groups until publication; construct each + ;; replacement event directly in its final request-owned list spine. + (replacement-vector (list->vector groups)) + (shared 0)) + (table-set! boundary-map start start) + (table-set! boundary-map end (+ end delta)) + (cond + (boundaries + (for-each (lambda (entry) + (table-set! boundary-map (car entry) (cdr entry))) + boundaries)) + ((zero? id-delta) + (for-each + (lambda (old-token new-token) + (table-set! boundary-map (token-start old-token) + (token-start new-token)) + (table-set! boundary-map (token-end old-token) + (token-end new-token))) + old-window new-window))) + (def (offset value) + (cond + ((< value start) value) + ((> value end) (+ value delta)) + (else + (let (mapped (table-ref boundary-map value 'missing)) + (if (eq? mapped 'missing) + (error "event boundary is inside a certified token" value) + mapped))))) + (def (keep event) + (set! shared (+ shared 1)) + event) + (let (events + (with-list-builder (emit) + (for-each + (lambda (event) + (case (event-kind event) + ((token) + (let ((id (token-event-id event)) + (old-start (event-start event)) + (old-end (event-end event))) + (cond + ((< id first-id) (emit (keep event))) + ((< id limit) + (for-each + (lambda (source-token) + (emit (make-token-event next-id source-token)) + (set! next-id (+ next-id 1))) + (vector-ref replacement-vector (- id first-id)))) + ((and (zero? delta) (zero? id-delta)) + (emit (keep event))) + (else + (emit + (vector 'token (+ id id-delta) + (token-event-token-kind event) + (token-event-lexeme event) + (offset old-start) (offset old-end))))))) + ((start-node finish-node) + (let* ((value (vector-ref event 3)) + (mapped (offset value))) + (if (= value mapped) + (emit (keep event)) + (emit (vector (event-kind event) (vector-ref event 1) + (vector-ref event 2) mapped))))) + ((start-field finish-field) + (let* ((value (vector-ref event 2)) + (mapped (offset value))) + (if (= value mapped) + (emit (keep event)) + (emit (vector (event-kind event) (vector-ref event 1) + mapped))))) + (else (error "unknown recognition event" event)))) + (parse-artifact-events old-artifact)))) + (values + (artifact (parse-artifact-ref old-artifact 'grammarDigest) + source 'accepted events '()) + shared)))) + ;; make-failure-parse-artifact ;; : (-> String String List Datum Alist) (def (make-failure-parse-artifact grammar-digest source tokens diagnostic) (let (value (artifact grammar-digest source 'rejected - (flat-token-events tokens) + (with-parser-cost-stage 'artifact-event-publication + (flat-token-events tokens)) (list diagnostic))) value)) @@ -250,11 +652,16 @@ ;;; Validates identity, byte ranges, nesting, and terminal balance before publication. ;;; The admitted source string is returned so roundtrip does not traverse and -;;; concatenate the complete event stream a second time. Malformed streams +;;; concatenate the complete event stream a second time. Validated lexemes are +;;; copied into one exactly sized string; no general text port participates in +;;; reconstruction and no advertised byte length drives an eager allocation. +;;; Publication can instead bind the actual input: compare admitted lexemes +;;; directly, prove complete character coverage and hash that input once. +;;; Malformed streams ;;; still fail closed and never become observable parse artifacts. ;; validate-parse-artifact! ;; : (-> Alist String) -(def (validate-parse-artifact! artifact) +(def (validate-parse-artifact! artifact (expected-source #f)) (unless (and (list? artifact) (equal? (parse-artifact-ref artifact 'schema) +parse-artifact-schema+) @@ -271,7 +678,8 @@ (expected-token-id 0) (expected-node-id 0) (root-count 0) - (source-port (open-output-string))) + (source-chunks '()) + (character-coverage 0)) (unless (and (integer? source-byte-length) (>= source-byte-length 0) (memq status '(accepted rejected)) @@ -337,16 +745,23 @@ (= start coverage) (> end start) (= (- end start) - (u8vector-length (string->utf8 lexeme)))) + (string-utf8-length lexeme))) (error "invalid token event coverage" event coverage)) - (display lexeme source-port) + (if expected-source + (let (width (string-length lexeme)) + (unless (and (<= (+ character-coverage width) (string-length expected-source)) + (source-lexeme-at? expected-source character-coverage lexeme)) + (error "ParseArtifact lexeme differs from input source" event)) + (set! character-coverage (+ character-coverage width))) + (set! source-chunks (cons lexeme source-chunks))) (set! coverage end) (set! expected-token-id (+ expected-token-id 1)))) (else (error "unknown CST event" event)))) events) - (let (source (get-output-string source-port)) + (let (source (or expected-source (string-concatenate-reverse source-chunks))) (unless (and (null? stack) (= coverage source-byte-length) + (or (not expected-source) (= character-coverage (string-length expected-source))) (equal? (sha256-text source) (parse-artifact-ref artifact 'sourceDigest))) (error "ParseArtifact source coverage mismatch" artifact)) @@ -359,6 +774,20 @@ (error "rejected ParseArtifact exposes partial structure" artifact)))) source))) +(def (source-lexeme-at? source offset lexeme) + ;; Callers prove containment first. Compare characters directly, without a + ;; temporary substring or encoding; byte coverage is checked independently. + (let (width (string-length lexeme)) + (let loop ((index 0)) + (or (= index width) + (and (char=? (string-ref source (+ offset index)) (string-ref lexeme index)) + (loop (+ index 1))))))) + +(def (parse-artifact-valid-for-source? artifact source) + (and (string? source) + (with-catch (lambda (_) #f) + (lambda () (validate-parse-artifact! artifact source) #t)))) + ;; parse-artifact-valid? ;; : (forall (a) (-> [(Pair Symbol a)] Boolean)) ;; : (-> Alist Boolean) diff --git a/src/runtime/byte-spans.ss b/src/runtime/byte-spans.ss new file mode 100644 index 00000000..2182571b --- /dev/null +++ b/src/runtime/byte-spans.ss @@ -0,0 +1,15 @@ +;;; -*- Gerbil -*- +;;; Pure borrowed-buffer traversal. Grammar supplies predicates, not cursors. +(export byte-span-skip byte-span-all?) + +;; Callers own valid request bounds. Checked vector reads remain in place; +;; the result is an absolute offset, including until for an empty/all-hit span. +(def (byte-span-skip (bytes : :u8vector) (from : :fixnum) + (until : :fixnum) accept?) + (let loop ((cursor from)) + (if (and (fx< cursor until) (accept? (u8vector-ref bytes cursor))) + (loop (fx+ cursor 1)) + cursor))) + +(def (byte-span-all? bytes from until accept?) + (= (byte-span-skip bytes from until accept?) until)) diff --git a/src/runtime/contextual-ir.ss b/src/runtime/contextual-ir.ss new file mode 100644 index 00000000..ba9ba95c --- /dev/null +++ b/src/runtime/contextual-ir.ss @@ -0,0 +1,65 @@ +;;; Common closed recognition/context/construction identity, without compiler imports. +(import (only-in ./identity sha256-text)) +(export +contextual-parser-ir-schema+ contextual-ir-ref contextual-ir-copy + make-contextual-parser-ir contextual-parser-ir-valid?) +(def +contextual-parser-ir-schema+ "gerbil-parser.contextual-parser-ir.v2") +(def (rows? datum) + (and (list? datum) (andmap (lambda (row) (and (pair? row) (symbol? (car row)))) datum))) +(def (contextual-ir-ref ir key) + (let (row (and (rows? ir) (assq key ir))) (and row (cdr row)))) +(def (fields? datum required optional) + (and (rows? datum) + (andmap (lambda (key) (assq key datum)) required) + (let loop ((rest datum) (seen '())) + (or (null? rest) + (let (key (caar rest)) + (and (memq key (append required optional)) (not (memq key seen)) + (loop (cdr rest) (cons key seen)))))))) +(def (contextual-ir-copy value) + (cond ((pair? value) (cons (contextual-ir-copy (car value)) (contextual-ir-copy (cdr value)))) + ((vector? value) (list->vector (map contextual-ir-copy (vector->list value)))) + ((string? value) (string-copy value)) (else value))) +(def (closed? value) + (cond ((pair? value) (and (closed? (car value)) (closed? (cdr value)))) + ((vector? value) (andmap closed? (vector->list value))) + (else (or (null? value) (boolean? value) (symbol? value) (string? value) + (char? value) (exact-integer? value))))) +(def (canonical value) (call-with-output-string (lambda (port) (write value port)))) +(def (shape? body) + (and (fields? body '(schema recognition scanner context construction root-kind) + '(digest base-grammar-digest parser-ir-digest)) + (closed? body) + (let* ((recognition (contextual-ir-ref body 'recognition)) + (construction (contextual-ir-ref body 'construction)) + (root (contextual-ir-ref body 'root-kind)) + (kinds (contextual-ir-ref construction 'syntax-kinds)) + (dialect (contextual-ir-ref recognition 'dialect))) + (and (equal? (contextual-ir-ref body 'schema) +contextual-parser-ir-schema+) + (fields? recognition '(dialect program) '()) + (memq dialect '(lr command lines)) + (contextual-ir-ref recognition 'program) + (rows? (contextual-ir-ref body 'context)) + (fields? construction '(syntax-kinds terminals projections) '()) + (symbol? root) (list? kinds) + (andmap (lambda (row) (and (list? row) (>= (length row) 2) + (symbol? (car row)) (symbol? (cadr row)))) kinds) + (list? (contextual-ir-ref construction 'terminals)) + (list? (contextual-ir-ref construction 'projections)) + (let (row (assq root kinds)) (and row (memq (cadr row) '(node result)))) + (if (eq? dialect 'lr) + (and (string? (contextual-ir-ref body 'base-grammar-digest)) + (string? (contextual-ir-ref body 'parser-ir-digest))) + (not (or (assq 'base-grammar-digest body) (assq 'parser-ir-digest body)))))))) +(def (make-contextual-parser-ir recognition scanner context construction root (binding '())) + (let (body (contextual-ir-copy + (append (list (cons 'schema +contextual-parser-ir-schema+) + (cons 'recognition recognition) (cons 'scanner scanner) + (cons 'context context) (cons 'construction construction) + (cons 'root-kind root)) binding))) + (unless (shape? body) (error "invalid contextual recognition/context/construction product")) + (append body (list (cons 'digest (sha256-text (canonical body))))))) +(def (contextual-parser-ir-valid? ir) + (and (list? ir) (shape? ir) + (= (length (filter (lambda (row) (eq? (car row) 'digest)) ir)) 1) + (equal? (contextual-ir-ref ir 'digest) + (sha256-text (canonical (filter (lambda (row) (not (eq? (car row) 'digest))) ir)))))) diff --git a/src/runtime/contextual-scanner.ss b/src/runtime/contextual-scanner.ss new file mode 100644 index 00000000..7fb2a181 --- /dev/null +++ b/src/runtime/contextual-scanner.ss @@ -0,0 +1,901 @@ +;;; -*- Gerbil -*- +;;; Generic executor for validated contextual scanner IR. All recognizers are +;;; closed engine opcodes; a language contributes data, never a scan callback. + +(import (only-in :std/string/utf8 string-utf8-length) + (only-in ./scan + make-literal-end-scanner identifier-start? horizontal-whitespace? newline? + scan-balanced-word scan-horizontal-whitespace scan-identifier + scan-line scan-longest-literal scan-newline + scan-quoted-strings) + (only-in ./region-scanner prepare-region-plan region-plan-end source-prefix-at?) + (only-in ./token make-token token-kind token-start token-end) + (only-in ./identity sha256-text)) +(export +empty-delimiter-queue+ + delimiter-queue-empty? delimiter-queue-list + delimiter-queue-enqueue delimiter-queue-take + decode-marker delimiter-obligation? + delimiter-obligation-marker delimiter-obligation-strip-tabs? delimiter-obligation-quoted? + +contextual-scanner-opcode-contract+ + prepare-contextual-scanner + prepare-contextual-scanner-plan + contextual-scanner-initial-state + contextual-scanner-step + contextual-scan-state? + contextual-scan-state-character-offset + contextual-scan-state-byte-offset + contextual-scan-state-mode + contextual-scan-state-with-mode + contextual-scan-state-canonical + contextual-scan-state-converged? + contextual-scanner-replay-prefix + + contextual-scan-state-source-length contextual-scan-state-common-suffix-length + prepare-contextual-scan-suffix contextual-scan-suffix-state + restore-contextual-scan-state) + +;;; Compiler products bind this semantic opcode contract. Changing opcode +;;; meaning requires a new contract, independently of lookup optimizations. +(def +contextual-scanner-opcode-contract+ + "gerbil-parser.contextual-scanner-opcodes.v1") + +(defstruct contextual-scanner (source source-identity digest initial-mode modes positions rules cells plan-owner) + transparent: #t) +;;; Static indexes are owned by the opaque plan and read only after admission. +(defstruct contextual-scanner-plan + (digest initial-mode modes positions short-rules long-rules cells)) + +(defstruct contextual-scan-state + (scanner character-offset byte-offset mode pending active expecting) + transparent: #t) +(defstruct runtime-scan-rule (name mode form matcher executor rank action result action-step byte-width) + transparent: #t) +(defstruct contextual-matcher-plan (expression executor)) +(defstruct scan-match (rule end) transparent: #t) +(defstruct delimiter-obligation (marker strip-tabs? quoted?) + transparent: #t) + +;;; Immutable FIFO: front is ordered, rear is reversed. Enqueue never copies +;;; the pending batch. A linear scan reverses each rear at most once; replaying +;;; an old checkpoint can repeat that work, so no branching amortized bound is +;;; claimed. Canonical checkpoints retain their ordered list representation. +(defstruct delimiter-queue (front rear) transparent: #t) +(def +empty-delimiter-queue+ (make-delimiter-queue '() '())) + +(def (delimiter-queue-empty? queue) + (and (null? (delimiter-queue-front queue)) + (null? (delimiter-queue-rear queue)))) + +(def (delimiter-queue-list queue) + (append (delimiter-queue-front queue) (reverse (delimiter-queue-rear queue)))) + +(def (delimiter-queue-enqueue queue obligation) + (unless (delimiter-obligation? obligation) + (error "invalid deferred delimiter obligation" obligation)) + (make-delimiter-queue (delimiter-queue-front queue) + (cons obligation (delimiter-queue-rear queue)))) + +(def (delimiter-queue-take queue) + (when (delimiter-queue-empty? queue) (error "empty deferred delimiter queue")) + (let* ((front (delimiter-queue-front queue)) + (rear (delimiter-queue-rear queue)) + (ready (if (null? front) (reverse rear) front))) + (values (car ready) + (make-delimiter-queue (cdr ready) (if (null? front) '() rear))))) + +(def (ir-ref ir key) + (let (row (assq key ir)) (and row (cdr row)))) + +(def (canonical value) + (call-with-output-string (lambda (port) (write value port)))) + +(def (scanner-ir-digest-valid? ir) + (and (list? ir) + (string? (ir-ref ir 'digest)) + (equal? + (sha256-text + (canonical (filter (lambda (row) (not (eq? (car row) 'digest))) + ir))) + (ir-ref ir 'digest)))) + +;;; Trie construction pays off for larger catalogs scanned over longer sources. +;;; Keep small catalogs and short one-shot inputs on the linear path. +(def (trie-matcher? matcher source-length) + (and (>= source-length 64) + (match matcher + (['literals values] (>= (length values) 128)) + (else #f)))) + +(def (lower-rule-matcher matcher) + (match matcher + (['region-word spec] (list 'prepared-region (prepare-region-plan spec))) + (['unless-prefix prefixes exceptions child] + (match child + (['region-word spec] + (list 'unless-prefix prefixes exceptions (list 'prepared-region (prepare-region-plan spec)))) + ((or ['literal-trie . _] ['prepared-region . _]) + (error "private contextual scanner matcher in input IR")) + (['unless-prefix . _] (error "nested contextual scanner guard in input IR")) + (else matcher))) + ((or ['literal-trie . _] ['prepared-region . _]) + (error "private contextual scanner matcher in input IR")) + (else matcher))) + +;;; Region/guard configuration owns expensive prepared indexes. Source, mode +;;; and delimiter state remain executor arguments; rule policy is never interned. +(def (prepare-rule-matcher matcher cache) + (or (table-ref cache matcher #f) + (let* ((prepared (lower-rule-matcher matcher)) + (plan (make-contextual-matcher-plan prepared (prepare-matcher prepared)))) + (table-set! cache matcher plan) + plan))) + +(def (decode-rule row source-length cache positions cells) + (match row + ([name mode form matcher rank action] + ;; Stage record construction with lexical rule metadata, without a closure + ;; or multiple-value carrier for each primitive rule. + (defrule (finish expression executor) + (make-runtime-scan-rule name mode form expression executor rank action + (prepare-rule-result mode form positions cells) + (if (eq? action 'keep) (lambda (_action _lexeme state) state) apply-scan-action) + (match expression (['literal value] (string-utf8-length value)) (else #f)))) + (match matcher + (['literals values] + ;; Literal catalogs already share their trie. Preserve that admission + ;; and leave short-input executors on their existing linear path. + (let (prepared + (if (trie-matcher? matcher source-length) + (let (scanner (or (table-ref cache values #f) + (let (scanner (make-literal-end-scanner values)) + (table-set! cache values scanner) + scanner))) + (list 'literal-trie scanner values)) + matcher)) + (finish prepared (prepare-matcher prepared)))) + ((or ['region-word _] ['unless-prefix . _]) + (let (plan (prepare-rule-matcher matcher cache)) + (finish (contextual-matcher-plan-expression plan) (contextual-matcher-plan-executor plan)))) + ((or ['literal-trie . _] ['prepared-region . _]) + (error "private contextual scanner matcher in input IR")) + (else (finish matcher (prepare-matcher matcher))))) + (else (error "invalid contextual scanner rule" row)))) + +(def (index-rules rows source-length (prefix-index? #f) (matcher-cache (make-table test: equal?)) (positions '()) (cells #f)) + (let (index (make-table test: eq?)) + (for-each + (lambda (row) + (let* ((rule (decode-rule row source-length matcher-cache positions cells)) + (mode (runtime-scan-rule-mode rule))) + (table-set! index mode + (cons rule (table-ref index mode '()))))) + (reverse rows)) + (when prefix-index? + ;; Publish only after the entire ordered mode row is prepared. + (for-each (lambda (entry) + (table-set! index (car entry) (prepare-first-character-rules (cdr entry)))) + (table->list index))) + index)) + +;;; Prefix pruning admits every rule that could match. Stateful and unknown +;;; opcode families remain universal candidates, retaining their executor and +;;; source-order tie/ambiguity semantics. No candidate list is built per token. +(defstruct first-character-rules (ascii wide alpha other)) +(def (literal-first-characters values) + (if (any (lambda (value) (zero? (string-length value))) values) + 'any + (let (characters (make-table test: eqv?)) + (for-each (lambda (value) (table-set! characters (string-ref value 0) #t)) values) + characters))) +(def (rule-first-characters rule) + (match (runtime-scan-rule-matcher rule) + (['literal value] (literal-first-characters (list value))) + (['literals values] (literal-first-characters values)) + (['literal-trie _ values] (literal-first-characters values)) + (['quoted-string delimiters] (literal-first-characters delimiters)) + (['identifier] 'identifier) + (['horizontal-whitespace+] 'horizontal) + ((or ['newline] ['newline-one]) 'newline) + (else 'any))) +(def (first-character-admits? hint character) + (case hint + ((any) #t) + ((identifier) (identifier-start? character)) + ((horizontal) (horizontal-whitespace? character)) + ((newline) (newline? character)) + (else (table-ref hint character #f)))) +(def (merge-ordered-rule-references left right) + (cond ((null? left) right) + ((null? right) left) + ((< (caar left) (caar right)) + (cons (car left) (merge-ordered-rule-references (cdr left) right))) + (else (cons (car right) (merge-ordered-rule-references left (cdr right)))))) +(def (prepare-first-character-rules rules) + (let* ((references (map cons (iota (length rules)) rules)) + (hints (map (lambda (reference) + (cons reference (rule-first-characters (cdr reference)))) references)) + (ascii (make-vector 128 '())) + (wide (make-table test: eqv?)) + (alpha (map car (filter (lambda (entry) (memq (cdr entry) '(any identifier))) hints))) + (other (map car (filter (lambda (entry) (eq? (cdr entry) 'any)) hints)))) + (let loop ((code 0)) + (when (< code 128) + (let (character (integer->char code)) + (vector-set! ascii code + (map (lambda (entry) (cdar entry)) + (filter (lambda (entry) (first-character-admits? (cdr entry) character)) hints)))) + (loop (+ code 1)))) + ;; Each finite prefix is inserted once per rule. Universal/class rows are + ;; merged in original order; Unicode prefix construction avoids U*R scans. + (for-each + (lambda (entry) + (let ((reference (car entry)) (hint (cdr entry))) + (unless (symbol? hint) + (for-each + (lambda (prefix) + (let (character (car prefix)) + (when (>= (char->integer character) 128) + (table-set! wide character + (cons reference (table-ref wide character '())))))) + (table->list hint))))) + (reverse hints)) + (for-each + (lambda (entry) + (let ((character (car entry)) (specific (cdr entry))) + (table-set! wide character + (map cdr (merge-ordered-rule-references + (if (identifier-start? character) alpha other) specific))))) + (table->list wide)) + (make-first-character-rules ascii wide (map cdr alpha) (map cdr other)))) +(def (first-character-candidates index character) + (let (code (char->integer character)) + (if (< code 128) + (vector-ref (first-character-rules-ascii index) code) + (or (table-ref (first-character-rules-wide index) character #f) + (if (identifier-start? character) + (first-character-rules-alpha index) + (first-character-rules-other index)))))) + +(def (index-cells/hash cells) + (let ((index (make-table test: equal?)) + (missing (gensym 'missing-cell))) + (for-each + (lambda (cell) + (let* ((key (take cell 3)) + (result (list-ref cell 3))) + (unless (eq? (table-ref index key missing) missing) + (error "duplicate contextual scanner dispatch cell" key)) + (table-set! index key (and result (car result))))) + cells) + index)) + +(defstruct grouped-dispatch (modes)) + +(def (multiple-dispatch-rows? cells) + (and (pair? cells) + (let ((mode (caar cells)) (position (cadar cells))) + (any (lambda (cell) (or (not (eq? mode (car cell))) + (not (eq? position (cadr cell))))) + (cdr cells))))) + +(def (short-symbolic-cells? cells) + (let loop ((rest cells) (forms '())) + (or (null? rest) + (let* ((cell (car rest)) (form (caddr cell))) + (and (symbol? (car cell)) (symbol? (cadr cell)) (symbol? form) + (let (next (if (memq form forms) forms (cons form forms))) + (and (<= (length next) 8) (loop (cdr rest) next)))))))) + +;;; Factor the composite key into mode/position indexes and short form rows. +;;; Multiple rows and at most eight forms bound row search. Single rows, wide +;;; form catalogs, and structural keys retain the original hash path. +(def (index-cells cells) + (if (and (multiple-dispatch-rows? cells) (short-symbolic-cells? cells)) + (let (modes (make-table test: eq?)) + (for-each + (lambda (cell) + (let* ((mode (car cell)) (position (cadr cell)) (form (caddr cell)) + (positions (or (table-ref modes mode #f) + (let (index (make-table test: eq?)) + (table-set! modes mode index) index))) + (row (table-ref positions position '())) + (result (list-ref cell 3))) + (when (assq form row) + (error "duplicate contextual scanner dispatch cell" (take cell 3))) + (table-set! positions position + (cons (cons form (and result (car result))) row)))) + cells) + (make-grouped-dispatch modes)) + (index-cells/hash cells))) + +(def (validate-scanner-ir! ir) + (unless (and (equal? (ir-ref ir 'schema) + "gerbil-parser.contextual-scanner-ir.v1") + (equal? (ir-ref ir 'opcode-contract) + +contextual-scanner-opcode-contract+) + (scanner-ir-digest-valid? ir)) + (error "contextual scanner requires compiled IR and source"))) + +;;; Preserve DAG sharing and symbol identities while taking ownership of every +;;; pair and string. Cycles and non-data values cannot enter a reusable plan. +(def (snapshot-scanner-ir ir) + (let ((copies (make-table test: eq?)) (active (make-table test: eq?))) + (def (copy value) + (cond + ((pair? value) + (when (table-ref active value #f) + (error "cyclic contextual scanner plan IR")) + (or (table-ref copies value #f) + (begin + (table-set! active value #t) + (let (result (cons (copy (car value)) (copy (cdr value)))) + (table-set! active value #f) + (table-set! copies value result) + result)))) + ((string? value) + (or (table-ref copies value #f) + (let (result (string-copy value)) + (table-set! copies value result) result))) + ((or (null? value) (symbol? value) (number? value) + (boolean? value) (char? value)) value) + (else (error "contextual scanner plan requires closed IR data")))) + (copy ir))) + +(def (prepare-contextual-scanner-plan ir) + (let (owned (snapshot-scanner-ir ir)) + (validate-scanner-ir! owned) + (let* ((rules (ir-ref owned 'rules)) + (matcher-cache (make-table test: equal?)) + (positions (ir-ref owned 'positions)) + (cells (index-cells (ir-ref owned 'cells))) + (short-rules (index-rules rules 0 #t matcher-cache positions cells)) + (has-trie? (any (lambda (row) + (match (list-ref row 3) + (['literals values] (>= (length values) 128)) + (else #f))) rules))) + (make-contextual-scanner-plan + (ir-ref owned 'digest) (ir-ref owned 'initial-mode) + (ir-ref owned 'modes) (ir-ref owned 'positions) + short-rules (if has-trie? (index-rules rules 64 #t matcher-cache positions cells) short-rules) + cells)))) + +(def (prepare-contextual-scanner ir source) + (unless (string? source) + (error "contextual scanner requires compiled IR and source")) + ;; A request owns one source snapshot. Its identity is shared lazily by every + ;; checkpoint; caller mutation cannot change scanning or invalidate that hash. + (let* ((owned (string-copy source)) + (identity (delay (sha256-text owned)))) + (if (contextual-scanner-plan? ir) + (make-contextual-scanner + owned identity (string-copy (contextual-scanner-plan-digest ir)) + (contextual-scanner-plan-initial-mode ir) + (contextual-scanner-plan-modes ir) (contextual-scanner-plan-positions ir) + (if (< (string-length owned) 64) + (contextual-scanner-plan-short-rules ir) (contextual-scanner-plan-long-rules ir)) + (contextual-scanner-plan-cells ir) ir) + (begin + (validate-scanner-ir! ir) + (make-contextual-scanner + owned identity (string-copy (ir-ref ir 'digest)) (ir-ref ir 'initial-mode) + (ir-ref ir 'modes) (ir-ref ir 'positions) + (index-rules (ir-ref ir 'rules) (string-length owned) #f (make-table test: equal?) + (ir-ref ir 'positions) (index-cells (ir-ref ir 'cells))) (index-cells (ir-ref ir 'cells)) (list 'request-local)))))) + +(def (contextual-scanner-initial-state scanner) + (make-contextual-scan-state scanner 0 0 + (contextual-scanner-initial-mode scanner) + +empty-delimiter-queue+ #f #f)) + +(def (contextual-scan-state-with-mode state mode) + (unless (and (contextual-scan-state? state) + (memq mode + (contextual-scanner-modes + (contextual-scan-state-scanner state)))) + (error "unknown contextual scanner mode" mode)) + (make-contextual-scan-state + (contextual-scan-state-scanner state) + (contextual-scan-state-character-offset state) + (contextual-scan-state-byte-offset state) + mode + (contextual-scan-state-pending state) + (contextual-scan-state-active state) + (contextual-scan-state-expecting state))) + +(def (contextual-scan-state-source-length state) + (string-length (contextual-scanner-source (contextual-scan-state-scanner state)))) + +;;; Alignment reads only engine-owned snapshots, never caller-mutable strings. +(def (contextual-scan-state-common-suffix-length before after) + (let ((left (contextual-scanner-source (contextual-scan-state-scanner before))) + (right (contextual-scanner-source (contextual-scan-state-scanner after)))) + (let loop ((i (- (string-length left) 1)) (j (- (string-length right) 1)) (count 0)) + (if (and (>= i 0) (>= j 0) (char=? (string-ref left i) (string-ref right j))) + (loop (- i 1) (- j 1) (+ count 1)) count)))) + +;;; Opaque proof binds two independently reached boundaries. The source worker +;;; supplies the same fixed parser position for the old and new continuations. +(defstruct contextual-scan-suffix (before after)) +(def (prepare-contextual-scan-suffix before after) + (and (contextual-scan-state-converged? before after) + (make-contextual-scan-suffix before after))) + +;;; Only states from the admitted old continuation may be rebound. Prefix +;;; checkpoints remain foreign; no matching-prefix restart is authorized here. +(def (contextual-scan-suffix-state suffix state) + (unless (and (contextual-scan-suffix? suffix) (contextual-scan-state? state)) + (error "invalid contextual suffix state")) + (let ((before (contextual-scan-suffix-before suffix)) + (after (contextual-scan-suffix-after suffix))) + (unless (and (eq? (contextual-scan-state-scanner before) (contextual-scan-state-scanner state)) + (<= (contextual-scan-state-character-offset before) (contextual-scan-state-character-offset state)) + (<= (contextual-scan-state-byte-offset before) (contextual-scan-state-byte-offset state))) + (error "checkpoint is outside the admitted scanner suffix")) + (make-contextual-scan-state + (contextual-scan-state-scanner after) + (+ (contextual-scan-state-character-offset state) + (- (contextual-scan-state-character-offset after) (contextual-scan-state-character-offset before))) + (+ (contextual-scan-state-byte-offset state) + (- (contextual-scan-state-byte-offset after) (contextual-scan-state-byte-offset before))) + (contextual-scan-state-mode state) (contextual-scan-state-pending state) + (contextual-scan-state-active state) (contextual-scan-state-expecting state)))) + +(def (same-delimiter-obligation? left right) + (or (eq? left right) + (and left right + (string=? (delimiter-obligation-marker left) (delimiter-obligation-marker right)) + (eq? (delimiter-obligation-strip-tabs? left) (delimiter-obligation-strip-tabs? right)) + (eq? (delimiter-obligation-quoted? left) (delimiter-obligation-quoted? right))))) + +(def (same-scanner-context? before after) + (and + (eq? (contextual-scanner-plan-owner (contextual-scan-state-scanner before)) + (contextual-scanner-plan-owner (contextual-scan-state-scanner after))) + (eq? (contextual-scan-state-mode before) (contextual-scan-state-mode after)) + (eq? (contextual-scan-state-expecting before) (contextual-scan-state-expecting after)) + (same-delimiter-obligation? (contextual-scan-state-active before) + (contextual-scan-state-active after)) + (let loop ((a (delimiter-queue-list (contextual-scan-state-pending before))) + (b (delimiter-queue-list (contextual-scan-state-pending after)))) + (if (null? a) (null? b) + (and (pair? b) (same-delimiter-obligation? (car a) (car b)) + (loop (cdr a) (cdr b))))))) + +;;; Re-execute the recorded parser positions on both owned sources. Equal text +;;; alone cannot certify decisions that inspected past the checkpoint boundary. +;;; Returns a newly reached checkpoint, never a relocated old state. +(def (contextual-scanner-replay-prefix scanner checkpoint positions) + (unless (and (contextual-scan-state? checkpoint) (list? positions)) + (error "scanner prefix replay requires a checkpoint and position schedule")) + (let* ((old (contextual-scan-state-scanner checkpoint)) + (left (contextual-scanner-source old)) + (right (contextual-scanner-source scanner)) + (end (contextual-scan-state-character-offset checkpoint))) + (unless (and (eq? (contextual-scanner-plan-owner old) (contextual-scanner-plan-owner scanner)) + (<= end (string-length right)) + (let loop ((i 0)) + (or (= i end) (and (char=? (string-ref left i) (string-ref right i)) + (loop (+ i 1)))))) + (error "scanner prefix source or plan mismatch")) + (let loop ((rest positions) (a (contextual-scanner-initial-state old)) + (b (contextual-scanner-initial-state scanner))) + (if (null? rest) + (begin + (unless (and (= (contextual-scan-state-character-offset a) end) + (= (contextual-scan-state-byte-offset a) (contextual-scan-state-byte-offset checkpoint)) + (same-scanner-context? a checkpoint) + (same-scanner-context? a b)) + (error "scanner checkpoint not reproduced by position schedule")) + b) + (let-values (((token-a next-a) (contextual-scanner-step old a (car rest))) + ((token-b next-b) (contextual-scanner-step scanner b (car rest)))) + (unless (and (<= (contextual-scan-state-character-offset next-a) end) + (= (contextual-scan-state-character-offset next-a) (contextual-scan-state-character-offset next-b)) + (if token-a + (and token-b (eq? (token-kind token-a) (token-kind token-b)) + (= (token-start token-a) (token-start token-b)) + (= (token-end token-a) (token-end token-b))) + (not token-b))) + (error "scanner prefix lexical decision changed")) + (loop (cdr rest) next-a next-b)))))) + +;;; Both states must have been reached independently. This proves identical +;;; future scanner decisions only under the same parser-position schedule; +;;; it neither relocates a checkpoint nor admits parser/subtree reuse. +(def (contextual-scan-state-converged? before after) + (unless (and (contextual-scan-state? before) (contextual-scan-state? after)) + (error "scanner convergence requires two checkpoints")) + (let* ((old (contextual-scan-state-scanner before)) + (new (contextual-scan-state-scanner after)) + (left (contextual-scanner-source old)) + (right (contextual-scanner-source new)) + (start-left (contextual-scan-state-character-offset before)) + (start-right (contextual-scan-state-character-offset after))) + (and + (same-scanner-context? before after) + (= (- (string-length left) start-left) (- (string-length right) start-right)) + ;; Compare the owned suffix directly: no full source hash or suffix copy. + (let loop ((i start-left) (j start-right)) + (or (= i (string-length left)) + (and (char=? (string-ref left i) (string-ref right j)) + (loop (+ i 1) (+ j 1)))))))) + +(def (obligation-row obligation) + (list (string-copy (delimiter-obligation-marker obligation)) + (delimiter-obligation-strip-tabs? obligation) + (delimiter-obligation-quoted? obligation))) + +;;; The canonical checkpoint binds source and machine digests, but contains no +;;; closure, pointer, or POO object. Receipt strings belong to the recipient; +;;; restore validates identity and owns its reconstructed delimiter obligations. +(def (contextual-scan-state-canonical state) + (let (scanner (contextual-scan-state-scanner state)) + (list (cons 'schema (string-copy "gerbil-parser.contextual-scan-state.v1")) + (cons 'scannerDigest (string-copy (contextual-scanner-digest scanner))) + (cons 'sourceDigest (string-copy (force (contextual-scanner-source-identity scanner)))) + (cons 'characterOffset + (contextual-scan-state-character-offset state)) + (cons 'byteOffset (contextual-scan-state-byte-offset state)) + (cons 'mode (contextual-scan-state-mode state)) + (cons 'pending + (map obligation-row + (delimiter-queue-list (contextual-scan-state-pending state)))) + (cons 'active + (and (contextual-scan-state-active state) + (obligation-row (contextual-scan-state-active state)))) + (cons 'expecting (contextual-scan-state-expecting state))))) + +(def (receipt-ref receipt key) + (let (row (assq key receipt)) (and row (cdr row)))) + +(def (restore-obligation row) + (match row + ([marker strip-tabs? quoted?] + (unless (and (string? marker) + (boolean? strip-tabs?) (boolean? quoted?) + (or (> (string-length marker) 0) quoted?)) + (error "invalid scanner delimiter obligation" row)) + (make-delimiter-obligation (string-copy marker) strip-tabs? quoted?)) + (else (error "invalid scanner delimiter obligation" row)))) + +(def (restore-contextual-scan-state scanner receipt) + (unless (and (contextual-scanner? scanner) + (list? receipt) + (equal? (receipt-ref receipt 'schema) + "gerbil-parser.contextual-scan-state.v1") + (equal? (receipt-ref receipt 'scannerDigest) + (contextual-scanner-digest scanner)) + (equal? (receipt-ref receipt 'sourceDigest) + (force (contextual-scanner-source-identity scanner)))) + (error "contextual scanner checkpoint identity mismatch")) + (let* ((source (contextual-scanner-source scanner)) + (character-offset (receipt-ref receipt 'characterOffset)) + (byte-offset (receipt-ref receipt 'byteOffset)) + (mode (receipt-ref receipt 'mode)) + (pending-rows (receipt-ref receipt 'pending)) + (active-row (receipt-ref receipt 'active)) + (expecting (receipt-ref receipt 'expecting))) + (unless (and (integer? character-offset) + (<= 0 character-offset (string-length source)) + (integer? byte-offset) + (= byte-offset + (string-utf8-length source 0 character-offset)) + (memq mode (contextual-scanner-modes scanner)) + (list? pending-rows) + (memq expecting '(#f plain strip-tabs))) + (error "invalid contextual scanner checkpoint" receipt)) + (make-contextual-scan-state + scanner character-offset byte-offset mode + (make-delimiter-queue (map restore-obligation pending-rows) '()) + (and active-row (restore-obligation active-row)) + expecting))) + +(def (single-newline-end source start) + (and (< start (string-length source)) + (case (string-ref source start) + ((#\newline) (+ start 1)) + ((#\return) + (if (and (< (+ start 1) (string-length source)) + (char=? (string-ref source (+ start 1)) #\newline)) + (+ start 2) + (+ start 1))) + (else #f)))) + +(def (line-content-end source start end) + (cond + ((and (> end (+ start 1)) + (char=? (string-ref source (- end 2)) #\return) + (char=? (string-ref source (- end 1)) #\newline)) + (- end 2)) + ((and (> end start) + (or (char=? (string-ref source (- end 1)) #\return) + (char=? (string-ref source (- end 1)) #\newline))) + (- end 1)) + (else end))) + +;;; Filter impossible marker lengths before materializing a comparison string. +;;; Keep Gambit string=? for equal-length candidates; SDK string= range dispatch +;;; is more expensive for the short markers in the matched component controls. +(def (marker-slice=? source start end marker) + (and (= (- end start) (string-length marker)) + (string=? (substring source start end) marker))) + +(def (marker-line-end source start state) + (let (active (contextual-scan-state-active state)) + (and active + (let* ((end (scan-line source start)) + (content-end (and end (line-content-end source start end))) + (content-start + (if (delimiter-obligation-strip-tabs? active) + (let loop ((cursor start)) + (if (and (< cursor content-end) + (char=? (string-ref source cursor) #\tab)) + (loop (+ cursor 1)) cursor)) + start))) + (and end + (marker-slice=? source content-start content-end + (delimiter-obligation-marker active)) + end))))) + +(def (profile-line-end source start separator) + (let loop ((at start)) + (cond ((= at (string-length source)) at) + ((char=? (string-ref source at) separator) (+ at 1)) + (else (loop (+ at 1)))))) +(def (profile-marker-end source start separator state) + (let (active (contextual-scan-state-active state)) + (and active + (let* ((end (profile-line-end source start separator)) + (content-end (if (and (> end start) (char=? (string-ref source (- end 1)) separator)) (- end 1) end)) + (content-start (if (delimiter-obligation-strip-tabs? active) + (let loop ((at start)) (if (and (< at content-end) (char=? (string-ref source at) #\tab)) (loop (+ at 1)) at)) start))) + (and (marker-slice=? source content-start content-end (delimiter-obligation-marker active)) end))))) +;;; Bind static matcher operands once, after closed-IR admission. The executor +;;; captures plan data only; source/checkpoint/marker state remain per request. +;;; Language data never supplies a callback and the public opcode contract is +;;; unchanged. Prefix hints still inspect the retained declarative matcher. +(def (prepare-matcher expression) + (match expression + (['unless-prefix prefixes exceptions child] + (let ((scan (prepare-matcher child)) + (excluded (make-literal-end-scanner prefixes)) + (admitted (make-literal-end-scanner exceptions))) + (lambda (source start state) + (and (or (admitted source start) + (not (excluded source start))) + (scan source start state))))) + (['line-prefix prefix separator] + (let ((scan (prepare-matcher (list 'literal prefix))) (ch (string-ref separator 0))) + (lambda (source start state) + (and (scan source start state) + (let (end (profile-line-end source start ch)) + (if (and (> end start) (char=? (string-ref source (- end 1)) ch)) + (- end 1) end)))))) + (['marker-line-at separator] + (let (ch (string-ref separator 0)) + (lambda (source start state) (profile-marker-end source start ch state)))) + (['body-line-at separator] + (let (ch (string-ref separator 0)) + (lambda (source start state) + (and (contextual-scan-state-active state) (profile-line-end source start ch))))) + (['literal value] + (let (width (string-length value)) + (if (= width 1) + (let (character (string-ref value 0)) + (lambda (source start _state) + (and (< start (string-length source)) + (char=? character (string-ref source start)) (+ start 1)))) + (lambda (source start _state) + (and (source-prefix-at? source start value) (+ start width)))))) + (['literal-trie scanner _values] + (lambda (source start _state) (scanner source start))) + (['literals values] + (lambda (source start _state) + (let (found (scan-longest-literal source start values)) + (and found (+ start (string-length found)))))) + (['horizontal-whitespace+] (lambda (source start _state) (scan-horizontal-whitespace source start))) + (['newline] (lambda (source start _state) (scan-newline source start))) + (['newline-one] (lambda (source start _state) (single-newline-end source start))) + (['identifier] (lambda (source start _state) (scan-identifier source start))) + (['marker-line] (lambda (source start state) (marker-line-end source start state))) + (['body-line] + (lambda (source start state) (and (contextual-scan-state-active state) (scan-line source start)))) + (['quoted-string delimiters] + (lambda (source start _state) (scan-quoted-strings source start delimiters))) + (['prepared-region plan] + (lambda (source start _state) (region-plan-end plan source start))) + (['balanced-word stops quotes pairs] + (lambda (source start _state) (scan-balanced-word source start stops quotes pairs))) + (else (error "unknown contextual scanner opcode" expression)))) + +;;; Candidate selection carries borrowed rule data and an endpoint. Only the +;;; winning match is materialized; dominated candidates need no result object. +(def (prefer-rule? current current-end candidate candidate-end) + (cond + ((not current) #t) + ((> candidate-end current-end) #t) + ((< candidate-end current-end) #f) + ((> (runtime-scan-rule-rank candidate) (runtime-scan-rule-rank current)) #t) + ((< (runtime-scan-rule-rank candidate) (runtime-scan-rule-rank current)) #f) + ((and (eq? (runtime-scan-rule-form candidate) (runtime-scan-rule-form current)) + (equal? (runtime-scan-rule-action candidate) (runtime-scan-rule-action current))) #f) + (else + (error "ambiguous contextual scanner match" + (runtime-scan-rule-name current) (runtime-scan-rule-name candidate))))) + +(def (best-match scanner state) + (let* ((source (contextual-scanner-source scanner)) + (start (contextual-scan-state-character-offset state)) + (indexed (table-ref (contextual-scanner-rules scanner) + (contextual-scan-state-mode state) '())) + (candidates (if (first-character-rules? indexed) + (first-character-candidates indexed (string-ref source start)) indexed))) + (let select ((rules candidates) (current #f) (end #f)) + (if (null? rules) + (and current (make-scan-match current end)) + (let* ((rule (car rules)) + (candidate-end ((runtime-scan-rule-executor rule) source start state))) + (if (and candidate-end (prefer-rule? current end rule candidate-end)) + (select (cdr rules) rule candidate-end) + (select (cdr rules) current end))))))) + +(def (find-result-kind index mode position form) + (let* ((kind (if (table? index) + (table-ref index (list mode position form) #f) + (let* ((positions (table-ref (grouped-dispatch-modes index) mode #f)) + (row (and positions (table-ref positions position '()))) + (cell (and row (assq form row)))) + (and cell (cdr cell)))))) + kind)) + +;;; Resolve mode/form once for each rule. Missing cells still fail only when +;;; that rule wins, preserving longest-match and ambiguity admission semantics. +(def (prepare-rule-result mode form positions cells) + (let (entries + (map (lambda (position) + (cons position + (find-result-kind cells mode position form))) positions)) + (if (and (pair? entries) (null? (cdr entries)) (cdar entries)) + (let (kind (cdar entries)) (lambda (_scanner _position) kind)) + (lambda (_scanner position) + (let (entry (assq position entries)) + (or (and entry (cdr entry)) + (error "missing contextual scanner dispatch" mode position form))))))) + +(def (decode-shell-delimiter word) + (let (length (string-length word)) + (let loop ((offset 0) (quote-mode #f) (quoted? #f) (characters '())) + (if (= offset length) + (begin + (when quote-mode + (error "unterminated delimiter quote" word)) + (values (list->string (reverse characters)) quoted?)) + (let (character (string-ref word offset)) + (cond + ((and (not (eq? quote-mode 'single)) + (char=? character #\\)) + (if (< (+ offset 1) length) + (let (escaped (string-ref word (+ offset 1))) + (cond + ((char=? escaped #\newline) + (loop (+ offset 2) quote-mode quoted? characters)) + ((and (eq? quote-mode 'double) + (not (memv escaped '(#\$ #\` #\" #\\)))) + (loop (+ offset 2) quote-mode quoted? + (cons escaped (cons character characters)))) + (else + (loop (+ offset 2) quote-mode #t + (cons escaped characters))))) + (loop (+ offset 1) quote-mode #t + (cons character characters)))) + ((and (not (eq? quote-mode 'double)) + (char=? character #\')) + (loop (+ offset 1) + (if quote-mode #f 'single) #t characters)) + ((and (not (eq? quote-mode 'single)) + (char=? character #\")) + (loop (+ offset 1) + (if quote-mode #f 'double) #t characters)) + (else + (loop (+ offset 1) quote-mode quoted? + (cons character characters))))))))) + +(def (decode-marker policy word strip-tabs?) + (unless (and (memq policy '(raw shell-quote-removal)) + (string? word) (boolean? strip-tabs?)) + (error "invalid deferred delimiter declaration" policy word strip-tabs?)) + (let-values (((marker quoted?) + (if (eq? policy 'raw) + (values word #f) + (decode-shell-delimiter word)))) + (when (and (string=? marker "") (not quoted?)) + (error "empty deferred delimiter" word)) + (make-delimiter-obligation (string-copy marker) strip-tabs? quoted?))) + +(def (updated-state state mode pending active expecting) + (make-contextual-scan-state + (contextual-scan-state-scanner state) + (contextual-scan-state-character-offset state) + (contextual-scan-state-byte-offset state) + mode pending active expecting)) + +(def (apply-scan-action action lexeme state) + (let ((mode (contextual-scan-state-mode state)) + (pending (contextual-scan-state-pending state)) + (active (contextual-scan-state-active state)) + (expecting (contextual-scan-state-expecting state))) + (match action + ('keep state) + (['expect-marker-in strip-tabs? target-mode] + (when expecting (error "deferred delimiter already expected")) + (updated-state state target-mode pending active (if strip-tabs? 'strip-tabs 'plain))) + (['enqueue-marker-in policy target-mode] + (unless expecting (error "no deferred delimiter expected")) + (updated-state state target-mode + (delimiter-queue-enqueue pending (decode-marker policy lexeme (eq? expecting 'strip-tabs))) active #f)) + (['expect-marker strip-tabs?] + (when expecting + (error "deferred delimiter already expected")) + (updated-state state mode pending active + (if strip-tabs? 'strip-tabs 'plain))) + (['enqueue-if-expecting policy] + (if expecting + (updated-state + state mode + (delimiter-queue-enqueue + pending (decode-marker policy lexeme (eq? expecting 'strip-tabs))) + active #f) + state)) + (['activate-next body-mode] + (when expecting + (error "missing deferred delimiter before newline")) + (if (not (delimiter-queue-empty? pending)) + (let-values (((active rest) (delimiter-queue-take pending))) + (updated-state state body-mode rest active #f)) + state)) + (['finish-marker base-mode body-mode] + (unless active + (error "deferred delimiter closed without an active marker")) + (if (not (delimiter-queue-empty? pending)) + (let-values (((next rest) (delimiter-queue-take pending))) + (updated-state state body-mode rest next #f)) + (updated-state state base-mode +empty-delimiter-queue+ #f #f))) + (else (error "unknown contextual scan action" action))))) + +;;; A checkpoint is immutable and bound to the prepared scanner. Parser +;;; reductions may change only its declared mode, without changing offsets. +(def (contextual-scanner-step scanner state position) + (unless (and (contextual-scan-state? state) + (eq? (contextual-scan-state-scanner state) scanner) + (memq position (contextual-scanner-positions scanner))) + (error "invalid contextual scanner checkpoint or position")) + (let* ((source (contextual-scanner-source scanner)) + (start (contextual-scan-state-character-offset state)) + (byte-start (contextual-scan-state-byte-offset state)) + (mode (contextual-scan-state-mode state))) + (if (= start (string-length source)) + (begin + (when (or (contextual-scan-state-active state) + (not (delimiter-queue-empty? (contextual-scan-state-pending state))) + (contextual-scan-state-expecting state)) + (error "unfinished contextual scanner obligation")) + (values #f state)) + (let (match (best-match scanner state)) + (unless match + (error "contextual scanner has no match" mode start)) + (let* ((rule (scan-match-rule match)) + (end (scan-match-end match)) + (lexeme (substring source start end)) + (byte-end (+ byte-start + (or (runtime-scan-rule-byte-width rule) (string-utf8-length lexeme)))) + (kind ((runtime-scan-rule-result rule) scanner position))) + (unless (< start end) + (error "contextual scanner did not advance" mode start)) + (values + (make-token kind lexeme byte-start byte-end) + ((runtime-scan-rule-action-step rule) (runtime-scan-rule-action rule) lexeme + (make-contextual-scan-state + scanner end byte-end mode + (contextual-scan-state-pending state) + (contextual-scan-state-active state) + (contextual-scan-state-expecting state))))))))) diff --git a/src/runtime/embedded-image.ss b/src/runtime/embedded-image.ss new file mode 100644 index 00000000..5700965e --- /dev/null +++ b/src/runtime/embedded-image.ss @@ -0,0 +1,55 @@ +;;; -*- Gerbil -*- +;;; Native constant storage for compressed language images; byte identity is unchanged. +(import (only-in :std/io open-buffered-reader BufferedReader-read-u64 BufferedReader-read-u8 + open-buffered-writer BufferedWriter-write-u64 BufferedWriter-flush)) +(export pack-language-artifact-image unpack-language-artifact-image) + +;;; The first unsigned word is byte length; remaining words use std/io's network order. +;;; Final zero padding is storage framing and never enters the compressed stream. +(def (pack-language-artifact-image bytes) + (unless (u8vector? bytes) (error "language artifact image requires bytes")) + (let* ((size (u8vector-length bytes)) + (full (quotient size 8)) + (tail (modulo size 8)) + (count (+ full (if (zero? tail) 0 1))) + (words (make-u64vector (+ count 1) 0))) + (u64vector-set! words 0 size) + ;; Read complete words from the original input. Only the final partial word + ;; needs padding; never allocate/copy a padded image of the entire payload. + (call-with-input-u8vector bytes + (lambda (port) + (let (reader (open-buffered-reader port)) + (let loop ((index 1)) + (when (<= index full) + (u64vector-set! words index (BufferedReader-read-u64 reader)) + (loop (fx+ index 1)))) + (unless (zero? tail) + (let loop ((remaining tail) (word 0)) + (if (zero? remaining) + (u64vector-set! words count (arithmetic-shift word (* 8 (- 8 tail)))) + (loop (fx- remaining 1) + (bitwise-ior (arithmetic-shift word 8) (BufferedReader-read-u8 reader))))))))) + words)) + +(def (unpack-language-artifact-image words) + (unless (and (u64vector? words) (> (u64vector-length words) 0)) + (error "invalid native language artifact image")) + (let ((size (u64vector-ref words 0)) (count (u64vector-length words))) + ;; Validate size before allocation; a hostile length cannot request a larger buffer. + (unless (= count (+ 1 (quotient (+ size 7) 8))) + (error "invalid native language artifact image length" size count)) + (let (bytes + (call-with-output-u8vector + (lambda (port) + (let (writer (open-buffered-writer port)) + (let loop ((index 1)) + (when (< index count) + (BufferedWriter-write-u64 writer (u64vector-ref words index)) + (loop (fx+ index 1)))) + (BufferedWriter-flush writer))))) + (let loop ((index size)) + (when (< index (u8vector-length bytes)) + (unless (zero? (u8vector-ref bytes index)) + (error "invalid native language artifact image padding")) + (loop (fx+ index 1)))) + (subu8vector bytes 0 size)))) diff --git a/src/runtime/event-fold-lines.ss b/src/runtime/event-fold-lines.ss new file mode 100644 index 00000000..c86d6623 --- /dev/null +++ b/src/runtime/event-fold-lines.ss @@ -0,0 +1,120 @@ +;;; -*- Gerbil -*- +;;; Native byte-view primitives shared by generated code and conformance. +(import (only-in ../compiler/event-fold-future ascii-lower-byte) + (only-in ./byte-spans byte-span-skip byte-span-all?)) +(export current-fold-line-view fold-line-bytes fold-key-byte? fold-ascii-prefix? + line-starts-with-ascii-ci? fold-line-prefix-boundary? fold-line-blank? + fold-line-has-key-after-prefix? fold-line-has-word-after-prefix? + fold-line-name-set-contains? fold-byte-prefix? fold-byte-marker-level) + +(def (byte-slice-name-compare bytes from until name) + (let ((size (string-length name))) + (let loop ((cursor from) (index 0)) + (cond + ((= cursor until) (if (= index size) 0 -1)) + ((= index size) 1) + (else + (let (difference (- (u8vector-ref bytes cursor) + (char->integer (string-ref name index)))) + (if (= difference 0) (loop (+ cursor 1) (+ index 1)) difference))))))) + +(def (fold-line-name-set-contains? bytes from until names) + ;; Names are admitted sorted unique ASCII constants. Compare source bytes + ;; directly: no slice strings, dense trie, request cache, or unrolled arms. + (unless (<= 0 from until (u8vector-length bytes)) + (error "native fold name range outside source line")) + (let search ((lower 0) (upper (vector-length names))) + (and (< lower upper) + (let* ((middle (quotient (+ lower upper) 2)) + (comparison (byte-slice-name-compare bytes from until (vector-ref names middle)))) + (cond ((zero? comparison) #t) + ((negative? comparison) (search lower middle)) + (else (search (+ middle 1) upper))))))) + +;; The immutable UTF-8 view belongs to the active statement frame, not a +;; process-wide memo. Nested helpers bind their own view and restore callers. +(def current-fold-line-view (make-parameter #f)) + +(def (fold-line-bytes line) + (let (view (current-fold-line-view)) + (if (and view (eq? line (car view))) + (cdr view) + (string->utf8 line)))) + + +(def (line-starts-with-ascii-ci? line prefix + (actual (fold-line-bytes line)) + (expected (string->utf8 prefix)) + (from 0) (until (u8vector-length actual))) + (fold-byte-prefix? actual from until expected #t)) + +(def (fold-byte-prefix? bytes from until expected ascii-ci?) + (let (size (u8vector-length expected)) + (and (>= (- until from) size) + (let loop ((index 0)) + (or (= index size) + (and (let ((actual (u8vector-ref bytes (+ from index))) + (wanted (u8vector-ref expected index))) + (= (if ascii-ci? (ascii-lower-byte actual) actual) + (if ascii-ci? (ascii-lower-byte wanted) wanted))) + (loop (+ index 1)))))))) + +(def (fold-byte-marker-level bytes from until marker boundary) + (let loop ((cursor from)) + (if (and (< cursor until) (= (u8vector-ref bytes cursor) marker)) + (loop (+ cursor 1)) + (if (and (> cursor from) (< cursor until) + (= (u8vector-ref bytes cursor) boundary)) + (- cursor from) 0)))) + +(def (fold-line-blank? line (bytes (fold-line-bytes line)) + (from 0) (until (u8vector-length bytes))) + (byte-span-all? bytes from until fold-space-byte?)) + +(def (fold-space-byte? byte) + (memv byte '(9 10 13 32))) + +(def (fold-horizontal-byte? byte) + (memv byte '(9 32))) + +(def (fold-line-prefix-boundary? line prefix marker? + (bytes (fold-line-bytes line)) + (expected (string->utf8 prefix)) + (from 0) (end (u8vector-length bytes))) + (and (line-starts-with-ascii-ci? line prefix bytes expected from end) + (let ((start (+ from (u8vector-length expected)))) + (if marker? + (byte-span-all? bytes start end fold-space-byte?) + (or (= start end) + (memv (u8vector-ref bytes start) '(9 10 13 32))))))) + + +(def (fold-ascii-prefix? value) + (and (string? value) + (every (lambda (character) (< (char->integer character) 128)) + (string->list value)))) + +(def (fold-key-byte? byte) + (or (and (<= 48 byte) (<= byte 57)) + (and (<= 65 byte) (<= byte 90)) + (and (<= 97 byte) (<= byte 122)) + (memv byte '(45 95)))) + +(def (fold-line-has-key-after-prefix? line prefix + (bytes (fold-line-bytes line)) + (expected (string->utf8 prefix)) + (from 0) (end (u8vector-length bytes))) + (and (line-starts-with-ascii-ci? line prefix bytes expected from end) + (let ((begin (+ from (u8vector-length expected)))) + (let (cursor (byte-span-skip bytes begin end fold-key-byte?)) + (and (> cursor begin) (< cursor end) + (= (u8vector-ref bytes cursor) 58)))))) + +(def (fold-line-has-word-after-prefix? line prefix + (bytes (fold-line-bytes line)) + (expected (string->utf8 prefix)) + (from 0) (end (u8vector-length bytes))) + (and (line-starts-with-ascii-ci? line prefix bytes expected from end) + (let (cursor (byte-span-skip bytes (+ from (u8vector-length expected)) end + fold-horizontal-byte?)) + (and (< cursor end) (not (fold-space-byte? (u8vector-ref bytes cursor))))))) diff --git a/src/runtime/event-program.ss b/src/runtime/event-program.ss new file mode 100644 index 00000000..e060fcfd --- /dev/null +++ b/src/runtime/event-program.ss @@ -0,0 +1,330 @@ +;;; -*- Gerbil -*- +;;; Engine-owned persistent event instructions. No mutable instruction buffer +;;; is exposed. Semantic frames share bodies; token instructions reuse tokens. +(import (only-in ./token token? token-start token-end)) +(export event-program-value? event-program-value-kind event-program-value-start + event-program-value-end event-program-value-code + event-program-token event-program-append event-program-append/avl event-program-relocate + event-program-field event-program-node-value event-program-fragment-value + event-program-walk event-program-walk/inline + event-program-sequence? event-program-sequence-arity + event-program-sequence-start event-program-sequence-end + event-program-sequence-field? event-program-sequence-for-each + event-program-sequence-height + event-program-value-body) + +(defstruct event-code-branch (left right start end height) final: #t) +(defstruct event-code-view (code delta) final: #t) +(defstruct event-code-append (prefix blocks tail start end height) final: #t) +(defstruct event-code-chunk (items start end) final: #t) +(defstruct event-field-frame (name start end body) final: #t) +(defstruct event-program-value (kind start end body) final: #t) + +;; A value is itself its node/boundary instruction: no duplicate opcode packet. +(def (event-program-value-code value) value) +(def (event-program-token token) token) +(def +event-chunk-capacity+ 16) +;;; Only bounded leaf blocks may be copied. Persistent AVL join copies the +;;; logarithmic boundary path, retaining every untouched subtree and block. +(def (event-block-width code) + (cond ((event-code-chunk? code) (vector-length (event-code-chunk-items code))) + ((or (event-code-branch? code) (event-code-append? code)) (+ +event-chunk-capacity+ 1)) + ((event-code-view? code) + (if (= (event-program-sequence-arity code) 1) 1 (+ +event-chunk-capacity+ 1))) + (else 1))) +(def (event-block-join left right) + (let* ((left-width (event-block-width left)) (right-width (event-block-width right)) + (items (make-vector (+ left-width right-width)))) + (def (copy-block code width offset) + (if (event-code-chunk? code) + (let (source (event-code-chunk-items code)) + (let loop ((i 0)) + (when (< i width) + (vector-set! items (+ offset i) (vector-ref source i)) + (loop (+ i 1))))) + (vector-set! items offset code))) + (copy-block left left-width 0) + (copy-block right right-width left-width) + (make-event-code-chunk items (event-program-sequence-start left 0) + (event-program-sequence-end right 0)))) +(def (event-rope-height code) + ;; Semantic nodes, fields and position views are opaque rope leaves. Their + ;; bodies retain their own balanced storage and depth fallback independently. + (if (event-code-branch? code) (event-code-branch-height code) 0)) +(def (event-program-sequence-height code) + (if (event-code-append? code) (event-code-append-height code) + (event-rope-height code))) +(def (event-branch left right) + (make-event-code-branch left right (event-program-sequence-start left 0) + (event-program-sequence-end right 0) + (+ 1 (max (event-rope-height left) (event-rope-height right))))) +(def (event-balance left right) + (let ((left-height (event-rope-height left)) + (right-height (event-rope-height right))) + (cond + ((> left-height (+ right-height 1)) + (let ((a (event-code-branch-left left)) (b (event-code-branch-right left))) + (if (>= (event-rope-height a) (event-rope-height b)) + (event-branch a (event-branch b right)) + (event-branch (event-branch a (event-code-branch-left b)) + (event-branch (event-code-branch-right b) right))))) + ((> right-height (+ left-height 1)) + (let ((b (event-code-branch-left right)) (c (event-code-branch-right right))) + (if (>= (event-rope-height c) (event-rope-height b)) + (event-branch (event-branch left b) c) + (event-branch (event-branch left (event-code-branch-left b)) + (event-branch (event-code-branch-right b) c))))) + (else (event-branch left right))))) +(def (event-rope-join left right) + (let ((left-height (event-rope-height left)) + (right-height (event-rope-height right))) + (cond + ((> left-height (+ right-height 1)) + (event-balance (event-code-branch-left left) + (event-rope-join (event-code-branch-right left) right))) + ((> right-height (+ left-height 1)) + (event-balance (event-rope-join left (event-code-branch-left right)) + (event-code-branch-right right))) + ((and (= left-height 0) (= right-height 0) + (<= (+ (event-block-width left) (event-block-width right)) +event-chunk-capacity+)) + (event-block-join left right)) + (else (event-branch left right))))) +(def (event-program-append/avl left right) + (cond ((or (not left) (null? left)) right) + ((or (not right) (null? right)) left) + (else (event-rope-join left right)))) +;;; Equal-rank complete leaf blocks form binary digits, newest first. +;;; Carry merges share their operands. Along a linear append chain the total +;;; carry count is linear; repeated branches from one old root may cost log N. +(def (event-append-block blocks block) + (if (and (pair? blocks) (= (event-rope-height (car blocks)) (event-rope-height block))) + (event-append-block (cdr blocks) (event-branch (car blocks) block)) + (cons block blocks))) +(def (event-snoc left right) + (cond + ((event-code-append? left) + (let* ((tail (event-code-append-tail left)) + (full? (= (event-block-width tail) +event-chunk-capacity+)) + (blocks (if full? (event-append-block (event-code-append-blocks left) tail) + (event-code-append-blocks left))) + (next-tail (if full? right (event-block-join tail right))) + (height (max (event-code-append-height left) + (if full? (+ 1 (event-rope-height (car blocks))) 1)))) + (make-event-code-append (event-code-append-prefix left) blocks next-tail + (event-code-append-start left) (event-program-sequence-end right 0) height))) + ((or (= (event-program-sequence-arity left) 1) + (and (event-code-chunk? left) (< (event-block-width left) +event-chunk-capacity+))) + ;; Small grammar actions retain one leaf block, with no empty envelope. + (event-block-join left right)) + ((event-code-chunk? left) + (make-event-code-append #f (list left) right + (event-code-chunk-start left) (event-program-sequence-end right 0) 1)) + (else + (make-event-code-append left '() right + (event-program-sequence-start left 0) (event-program-sequence-end right 0) + (+ 1 (event-program-sequence-height left)))))) +(def (event-append-tree code) + ;; Normalize only at multi-sequence concat: newest/smallest blocks first. + ;; Increasing ranks make the carry/join work telescope along the end path. + ;; Prefixes are already AVL ropes; singleton append never nests envelopes. + (if (event-code-append? code) + (let (tree (event-code-append-tail code)) + (for-each (lambda (block) (set! tree (event-rope-join block tree))) + (event-code-append-blocks code)) + (let (prefix (event-code-append-prefix code)) + (if prefix (event-rope-join prefix tree) tree))) + code)) +(def (event-program-append left right) + (cond ((or (not left) (null? left)) right) + ((or (not right) (null? right)) left) + ((= (event-program-sequence-arity right) 1) (event-snoc left right)) + (else (event-rope-join (event-append-tree left) (event-append-tree right))))) +(def (event-program-relocate code delta (moved? #t)) + (cond ((not code) #f) ((not moved?) code) + ((event-code-view? code) + (make-event-code-view (event-code-view-code code) + (+ delta (event-code-view-delta code)))) + (else (make-event-code-view code delta)))) +(def (event-program-field name start end body) + (make-event-field-frame name start end body)) +(def (event-program-node-value kind start end body) + (make-event-program-value kind start end body)) +(def (event-program-fragment-value start end body) + ;; Flush final trivia before an enclosing field close. Bare tokens do not + ;; carry this boundary, preserving the original field/trivia order. + (make-event-program-value #f start end body)) + +;;; The same DAG is the LR semantic sequence. Node/boundary values and tokens +;;; are opaque semantic leaves; field frames freeze their label. Bounds on an +;;; append branch are computed once instead of walking its children at actions. +(def (event-program-sequence? value) + (or (not value) (event-program-value? value) (token? value) + (event-field-frame? value) (event-code-branch? value) (event-code-view? value) + (event-code-chunk? value) (event-code-append? value))) +(def (event-program-sequence-arity value) + (cond ((not value) 0) + ((event-code-view? value) (event-program-sequence-arity (event-code-view-code value))) + ((or (event-code-branch? value) (event-code-chunk? value) (event-code-append? value)) 2) (else 1))) +(def (event-program-sequence-bound value default-offset end?) + (cond + ((not value) default-offset) + ((event-code-view? value) + (+ (event-code-view-delta value) + (event-program-sequence-bound (event-code-view-code value) default-offset end?))) + ((event-code-append? value) + ((if end? event-code-append-end event-code-append-start) value)) + ((event-code-chunk? value) + ((if end? event-code-chunk-end event-code-chunk-start) value)) + ((event-code-branch? value) + ((if end? event-code-branch-end event-code-branch-start) value)) + ((event-field-frame? value) + ((if end? event-field-frame-end event-field-frame-start) value)) + ((event-program-value? value) + ((if end? event-program-value-end event-program-value-start) value)) + ((token? value) ((if end? token-end token-start) value)) + (else (error "invalid event semantic sequence")))) +(def (event-program-sequence-start value default-offset) + (event-program-sequence-bound value default-offset #f)) +(def (event-program-sequence-end value default-offset) + (event-program-sequence-bound value default-offset #t)) +(def (event-program-sequence-field? value) + (if (event-code-view? value) + (event-program-sequence-field? (event-code-view-code value)) + (event-field-frame? value))) + +;;; This visitor is the explicit materialization boundary, not a reduction. +;;; Preserve relocation provenance even when its summed offset is zero. +(def (event-program-sequence-for-each visit sequence) + (let loop ((current sequence) (delta 0) (moved? #f) (pending '())) + (cond + ((event-code-view? current) + (loop (event-code-view-code current) (+ delta (event-code-view-delta current)) #t pending)) + ((event-code-append? current) + (let (frames (cons (vector (event-code-append-tail current) delta moved?) pending)) + (for-each (lambda (block) (set! frames (cons (vector block delta moved?) frames))) + (event-code-append-blocks current)) + (loop (event-code-append-prefix current) delta moved? frames))) + ((event-code-branch? current) + (loop (event-code-branch-left current) delta moved? + (cons (vector (event-code-branch-right current) delta moved?) pending))) + ((event-code-chunk? current) + (loop (vector-ref (event-code-chunk-items current) 0) delta moved? + (cons (vector current delta moved? 1) pending))) + (else + (when current + (let unwrap ((value (if (event-field-frame? current) (event-field-frame-body current) current)) + (delta delta) (moved? moved?)) + (if (event-code-view? value) + (unwrap (event-code-view-code value) (+ delta (event-code-view-delta value)) #t) + (visit (and (event-field-frame? current) (event-field-frame-name current)) value delta moved?)))) + (unless (null? pending) + (let* ((frame (car pending)) (code (vector-ref frame 0))) + (if (and (= (vector-length frame) 4) (event-code-chunk? code)) + (let* ((items (event-code-chunk-items code)) (i (vector-ref frame 3)) + (last? (= (+ i 1) (vector-length items)))) + (unless last? (vector-set! frame 3 (+ i 1))) + (loop (vector-ref items i) (vector-ref frame 1) (vector-ref frame 2) + (if last? (cdr pending) pending))) + (loop code (vector-ref frame 1) (vector-ref frame 2) (cdr pending))))))))) + +;;; Execute the instruction DAG with an explicit continuation stack. No flat +;;; intermediate tape or translated token objects are allocated. Position views +;;; retain moved provenance even when repeated translations cancel to zero. +;;; One traversal template specializes both inspection and publication. Ordinary +;;; depth uses native call frames; explicit continuations handle deep subgraphs. +;;; An opening node may return consumer state, retained only until its footer. +(defsyntax (event-program-walk/inline stx) + (syntax-case stx (lambda) + ((_ (lambda (op name offset shift moved context) effect ...) initial-code) + #'(let () + (def (iterative initial delta moved?) + (let loop ((code initial) (finish? #f) (delta delta) (moved? moved?) + (state #f) (pending '())) + (cond + ((not code) + (unless (null? pending) + (let* ((frame (car pending)) (next (vector-ref frame 0))) + (if (and (event-code-chunk? next) (integer? (vector-ref frame 1))) + (let* ((items (event-code-chunk-items next)) (i (vector-ref frame 1)) + (last? (= (+ i 1) (vector-length items)))) + (unless last? (vector-set! frame 1 (+ i 1))) + (loop (vector-ref items i) #f (vector-ref frame 2) (vector-ref frame 3) + #f (if last? (cdr pending) pending))) + (loop next (vector-ref frame 1) (vector-ref frame 2) (vector-ref frame 3) + (if (= (vector-length frame) 5) (vector-ref frame 4) #f) + (cdr pending)))))) + (finish? + (cond + ((event-program-value? code) + (if (event-program-value-kind code) + ((lambda (op name offset shift moved context) effect ...) 'close-node (event-program-value-kind code) (event-program-value-end code) delta moved? state) + ((lambda (op name offset shift moved context) effect ...) 'boundary #f (event-program-value-end code) delta moved? #f))) + ((event-field-frame? code) + ((lambda (op name offset shift moved context) effect ...) 'close-field (event-field-frame-name code) (event-field-frame-end code) delta moved? #f)) + (else (error "invalid event continuation"))) + (loop #f #f delta moved? #f pending)) + ((event-code-view? code) + (loop (event-code-view-code code) #f (+ delta (event-code-view-delta code)) #t #f pending)) + ((event-code-append? code) + (let (frames (cons (vector (event-code-append-tail code) #f delta moved?) pending)) + (for-each (lambda (block) (set! frames (cons (vector block #f delta moved?) frames))) + (event-code-append-blocks code)) + (loop (event-code-append-prefix code) #f delta moved? #f frames))) + ((event-code-branch? code) + (loop (event-code-branch-left code) #f delta moved? #f + (cons (vector (event-code-branch-right code) #f delta moved?) pending))) + ((event-code-chunk? code) + (loop (vector-ref (event-code-chunk-items code) 0) #f delta moved? #f + (cons (vector code 1 delta moved?) pending))) + ((event-program-value? code) + (let (node-state (and (event-program-value-kind code) + ((lambda (op name offset shift moved context) effect ...) 'open-node (event-program-value-kind code) + (event-program-value-start code) delta moved? #f))) + (loop (event-program-value-body code) #f delta moved? #f + (cons (vector code #t delta moved? node-state) pending)))) + ((event-field-frame? code) + ((lambda (op name offset shift moved context) effect ...) 'open-field (event-field-frame-name code) (event-field-frame-start code) delta moved? #f) + (loop (event-field-frame-body code) #f delta moved? #f + (cons (vector code #t delta moved? #f) pending))) + ((token? code) + ((lambda (op name offset shift moved context) effect ...) 'token code (token-start code) delta moved? #f) + (loop #f #f delta moved? #f pending)) + (else (error "invalid event instruction"))))) + (def (walk code delta moved? depth) + (cond + ((not code) (void)) + ((>= depth 64) (iterative code delta moved?)) + ((event-program-value? code) + (let* ((kind (event-program-value-kind code)) + (node-state (and kind ((lambda (op name offset shift moved context) effect ...) 'open-node kind (event-program-value-start code) delta moved? #f)))) + (walk (event-program-value-body code) delta moved? (+ depth 1)) + (if kind + ((lambda (op name offset shift moved context) effect ...) 'close-node kind (event-program-value-end code) delta moved? node-state) + ((lambda (op name offset shift moved context) effect ...) 'boundary #f (event-program-value-end code) delta moved? #f)))) + ((event-field-frame? code) + ((lambda (op name offset shift moved context) effect ...) 'open-field (event-field-frame-name code) (event-field-frame-start code) delta moved? #f) + (walk (event-field-frame-body code) delta moved? (+ depth 1)) + ((lambda (op name offset shift moved context) effect ...) 'close-field (event-field-frame-name code) (event-field-frame-end code) delta moved? #f)) + ((token? code) ((lambda (op name offset shift moved context) effect ...) 'token code (token-start code) delta moved? #f)) + ((event-code-view? code) + (walk (event-code-view-code code) (+ delta (event-code-view-delta code)) #t (+ depth 1))) + ((event-code-chunk? code) + (let* ((items (event-code-chunk-items code)) (n (vector-length items))) + (let loop ((i 0)) + (when (< i n) + (walk (vector-ref items i) delta moved? (+ depth 1)) + (loop (+ i 1)))))) + ((event-code-append? code) + (walk (event-code-append-prefix code) delta moved? (+ depth 1)) + (for-each (lambda (block) (walk block delta moved? (+ depth 1))) + (reverse (event-code-append-blocks code))) + (walk (event-code-append-tail code) delta moved? (+ depth 1))) + ((event-code-branch? code) + (walk (event-code-branch-left code) delta moved? (+ depth 1)) + (walk (event-code-branch-right code) delta moved? (+ depth 1))) + (else (error "invalid event instruction")))) + (walk initial-code 0 #f 0))))) +(def (event-program-walk visit code) + (event-program-walk/inline + (lambda (operation name offset delta moved? state) + (visit operation name offset delta moved?)) code)) diff --git a/src/runtime/event-reduce.ss b/src/runtime/event-reduce.ss new file mode 100644 index 00000000..867aa5f1 --- /dev/null +++ b/src/runtime/event-reduce.ss @@ -0,0 +1,85 @@ +;;; -*- Gerbil -*- +;;; Original LR field/alias actions lower directly to persistent event programs. +(import (only-in ./event-program + event-program-value? event-program-value-code + event-program-token event-program-append event-program-relocate + event-program-field event-program-node-value event-program-fragment-value + event-program-sequence? event-program-sequence-arity + event-program-sequence-start event-program-sequence-end + event-program-sequence-field?) + (only-in ./recognition + recognition-child-field recognition-child-value make-recognition-child + recognition-value-start recognition-value-end + recognition-relocation? recognition-relocation-value recognition-relocation-delta + recognition-node? recognition-node-kind recognition-node-children + recognition-node-start recognition-node-end + recognition-fragment? recognition-fragment-children recognition-fragment-start recognition-fragment-end) + (only-in ./funcs recognition-sequence-arity recognition-sequence-start + recognition-sequence-end recognition-sequence-for-each recognition-sequence->list) + (only-in ./token token?)) +(export event-children-field event-children-alias) + +;;; Conversion freezes the current semantic action's field and value. It never +;;; caches bytecode by a mutable recognition child/node identity. Already lowered +;;; program values retain their private code by identity, including reused LR +;;; fragments; token identity is revalidated against the request at publication. +(def (value-code value) + (cond + ((event-program-value? value) (event-program-value-code value)) + ((token? value) (event-program-token value)) + ((recognition-relocation? value) + (event-program-relocate (value-code (recognition-relocation-value value)) + (recognition-relocation-delta value))) + ((recognition-node? value) + (event-program-value-code + (event-program-node-value (recognition-node-kind value) + (recognition-node-start value) (recognition-node-end value) + (children-code (recognition-node-children value))))) + ((recognition-fragment? value) + (event-program-value-code + (event-program-fragment-value (recognition-fragment-start value) + (recognition-fragment-end value) + (children-code (recognition-fragment-children value))))) + (else (error "invalid semantic value for event lowering")))) +(def (children-code children) + (if (event-program-sequence? children) children + (let (code #f) + (recognition-sequence-for-each + (lambda (child delta moved?) + (let* ((value (recognition-child-value child)) + (field (recognition-child-field child)) + (body (value-code value))) + (set! code + (event-program-append code + (event-program-relocate + (if field + (event-program-field field (recognition-value-start value) + (recognition-value-end value) body) + body) delta moved?))))) children) + code))) + +(def (event-children-field name children offset (unused-fragment-constructor #f)) + (let (code (children-code children)) + (case (event-program-sequence-arity code) + ((0) #f) + ((1) + (event-program-field name + (event-program-sequence-start code offset) + (event-program-sequence-end code offset) + (if (event-program-sequence-field? code) + (event-program-fragment-value + (event-program-sequence-start code offset) + (event-program-sequence-end code offset) code) + code))) + (else + (event-program-field name + (event-program-sequence-start code offset) + (event-program-sequence-end code offset) + (event-program-fragment-value + (event-program-sequence-start code offset) + (event-program-sequence-end code offset) code)))))) +(def (event-children-alias kind children offset) + (let (code (children-code children)) + (event-program-node-value kind + (event-program-sequence-start code offset) + (event-program-sequence-end code offset) code))) diff --git a/src/runtime/fold-cost.ss b/src/runtime/fold-cost.ss new file mode 100644 index 00000000..889425ee --- /dev/null +++ b/src/runtime/fold-cost.ss @@ -0,0 +1,63 @@ +;;; Diagnostic products only. Ordinary generated products do not import this. +(import :std/os/time) +(export current-fold-cost-session make-fold-cost-session + call-with-fold-cost fold-cost-count! fold-cost-rows) + +;; Session = private aggregation table and current frame. Frames never escape +;; a synchronous call. Nested parsers and exceptions restore their caller. +(def current-fold-cost-session (make-parameter #f)) +(def (make-fold-cost-session) (vector (make-hash-table-eq) #f)) +(def (clock-ns) + (let (time (current-system-time-precise)) + (+ (* (timespec-tv_sec time) 1000000000) (timespec-tv_nsec time)))) + +;; Count-only products avoid timing thunks and frame allocation at every leaf. +;; They still incur table updates and are never admitted as timing products. +(def (fold-cost-count! name (amount 1)) + (unless (and (exact-integer? amount) (>= amount 0)) + (error "invalid diagnostic count increment" amount)) + (let (session (current-fold-cost-session)) + (when session + (let* ((table (vector-ref session 0)) + (row (or (hash-get table name) + (let (created (vector 0 0 0 0)) + (hash-put! table name created) created)))) + (vector-set! row 0 (+ amount (vector-ref row 0))))))) + +(def (call-with-fold-cost name thunk) + (let (session (current-fold-cost-session)) + (if (not session) (thunk) + (let* ((parent (vector-ref session 1)) + (frame (vector (clock-ns) 0))) + (def (finish completed?) + (let* ((elapsed (- (clock-ns) (vector-ref frame 0))) + (children (vector-ref frame 1)) + (table (vector-ref session 0)) + (row (or (hash-get table name) + (let (created (vector 0 0 0 0)) + (hash-put! table name created) created)))) + (vector-set! session 1 parent) + (when parent + (vector-set! parent 1 (+ (vector-ref parent 1) elapsed))) + (unless (<= 0 children elapsed) + (error "invalid diagnostic Fold clock interval" name elapsed children)) + (vector-set! row 0 (+ 1 (vector-ref row 0))) + (vector-set! row 1 (+ elapsed (vector-ref row 1))) + (vector-set! row 2 (+ (- elapsed children) (vector-ref row 2))) + (unless completed? (vector-set! row 3 (+ 1 (vector-ref row 3)))))) + (vector-set! session 1 frame) + (let (value (with-catch + (lambda (exception) (finish #f) (raise exception)) thunk)) + (finish #t) value))))) + +(def (fold-cost-rows session) + (unless (not (vector-ref session 1)) (error "Fold diagnostic frame still active")) + (let (rows '()) + (hash-for-each + (lambda (name row) + (set! rows (cons (list name (vector-ref row 0) + (/ (vector-ref row 1) 1000.0) + (/ (vector-ref row 2) 1000.0) + (vector-ref row 3)) rows))) + (vector-ref session 0)) + (list-sort (lambda (left right) (> (list-ref left 3) (list-ref right 3))) rows))) diff --git a/src/runtime/fold-tape.ss b/src/runtime/fold-tape.ss new file mode 100644 index 00000000..1b4df8c9 --- /dev/null +++ b/src/runtime/fold-tape.ss @@ -0,0 +1,71 @@ +;;; Request-owned direct OEV1 consumer; no intermediate event descriptors. +(import :std/ffi) +(export make-fold-tape fold-tape-start! fold-tape-token! fold-tape-finish! + fold-tape-publish!) + +;; Fixed OEV1 records are one synchronous foreign leaf operation. The owner +;; reserves the complete record first; no object/body pointer escapes the +;; leaf, which neither allocates Scheme storage nor calls back into Scheme. +;; Byte stores are portable at every alignment and on either host byte order. +(C-ffi-macrology) +(C-include "") +(C-declare "static void fold_wire_le64(uint8_t *p, uint64_t value) { + for (unsigned int i = 0; i < 8; ++i) p[i] = (uint8_t)(value >> (8 * i)); +}") +(def-C-lambda write-start-record! (scheme-object unsigned-int64 unsigned-int16) void + "uint8_t *p = U8_DATA(___arg1) + ___arg2; + p[0] = 1; p[1] = (uint8_t)___arg3; p[2] = (uint8_t)(___arg3 >> 8); + ___return;") +(def-C-lambda write-token-record! + (scheme-object unsigned-int64 unsigned-int16 unsigned-int64 unsigned-int64) void + "uint8_t *p = U8_DATA(___arg1) + ___arg2; + p[0] = 2; p[1] = (uint8_t)___arg3; p[2] = (uint8_t)(___arg3 >> 8); + fold_wire_le64(p + 3, ___arg4); fold_wire_le64(p + 11, ___arg5); + ___return;") + +(defstruct fold-tape-owner (bytes used sealed?)) +(def (make-fold-tape header) + (unless (u8vector? header) (error "invalid Fold tape header")) + (let* ((size (u8vector-length header)) (bytes (make-u8vector (max 64 size) 0))) + (u8vector-copy! bytes 0 header) + (make-fold-tape-owner bytes size #f))) + +(def (reserve! tape width) + (when (fold-tape-owner-sealed? tape) (error "published Fold tape")) + (let* ((offset (fold-tape-owner-used tape)) (end (+ offset width)) + (bytes (fold-tape-owner-bytes tape)) (size (u8vector-length bytes))) + (when (> end size) + (let (grown (make-u8vector (max end (* 2 size)) 0)) + (u8vector-copy! grown 0 bytes 0 offset) + (set! (fold-tape-owner-bytes tape) grown))) + (set! (fold-tape-owner-used tape) end) + offset)) +(def (unsigned? value width) + (and (exact-integer? value) (<= 0 value) + (< value (if (= width 2) 65536 18446744073709551616)))) +(def (fold-tape-start! tape kind) + (unless (unsigned? kind 2) (error "invalid Fold tape kind" kind)) + (let* ((offset (reserve! tape 3)) (bytes (fold-tape-owner-bytes tape))) + (write-start-record! bytes offset kind)) + tape) +(def (fold-tape-token! tape kind from until) + ;; Backward spans are part of the canonical Fold contract; retain them. + (unless (and (unsigned? kind 2) (unsigned? from 8) (unsigned? until 8)) + (error "invalid Fold tape token" kind from until)) + (let* ((offset (reserve! tape 19)) (bytes (fold-tape-owner-bytes tape))) + (write-token-record! bytes offset kind from until)) + tape) +(def (fold-tape-finish! tape) + (let ((offset (reserve! tape 1))) + (u8vector-set! (fold-tape-owner-bytes tape) offset 0)) + tape) +(def (fold-tape-publish! tape) + (when (fold-tape-owner-sealed? tape) (error "published Fold tape")) + (set! (fold-tape-owner-sealed? tape) #t) + (let ((bytes (fold-tape-owner-bytes tape)) (used (fold-tape-owner-used tape))) + ;; Same ownership transfer as std/io/bio get-memory-output-u8vector: + ;; bytes were allocated privately; no alias escaped before publication. + ;; Sealing and clearing the owner prevent future writers retaining them. + (set! (fold-tape-owner-bytes tape) #f) + (when (< used (u8vector-length bytes)) (u8vector-shrink! bytes used)) + bytes)) diff --git a/src/runtime/funcs.ss b/src/runtime/funcs.ss index 5e246cd5..7a1f8a1d 100644 --- a/src/runtime/funcs.ss +++ b/src/runtime/funcs.ss @@ -1,7 +1,14 @@ ;;; -*- Gerbil -*- ;;; Small immutable sequence algorithms for the LR semantic hot path. -(import (only-in :std/list/list-builder with-list-builder) +(import (only-in ./event-program + event-program-sequence? event-program-sequence-arity + event-program-sequence-start event-program-sequence-end + event-program-sequence-for-each event-program-relocate) + (only-in ./recognition + make-recognition-child recognition-child-field recognition-child-value + relocate-recognition-value recognition-value-start recognition-value-end) + (only-in :std/list/list-builder with-list-builder) (only-in :std/vector/vector vector-map/index)) (export association-row-vector->index association-row-index-ref @@ -12,7 +19,13 @@ vector-intern-map recognition-sequence-append recognition-sequence-concatenate - recognition-sequence->list) + recognition-sequence->list + recognition-sequence-root-child + recognition-sequence-for-action + recognition-sequence-relocate + current-recognition-sequence-fusion-enabled? + recognition-sequence-arity recognition-sequence-start recognition-sequence-end + recognition-sequence-for-each) ;;; Request-local hash-consing over immutable structural keys. Gerbil's ;;; standard equal?-table owns lookup; callers provide the canonical value @@ -94,7 +107,9 @@ ;; : (-> (Vector (Or (List Pair) HashTable)) Fixnum Value (OrFalse Pair)) (def (association-row-index-ref indexes state key) (let (index (vector-ref indexes state)) - (if (list? index) + ;; Preparation admits only proper lists or tables. Inspect the outer + ;; representation rather than traversing a whole list before assoc. + (if (or (pair? index) (null? index)) (assoc key index) (hash-get index key)))) @@ -102,13 +117,30 @@ ;; singleton shifts and already-materialized field/alias results allocate no ;; wrapper. This turns left-recursive repetition from repeated list copying ;; into constant-time concatenation. -(defstruct recognition-sequence-branch (left right) transparent: #t) +(def current-recognition-sequence-fusion-enabled? (make-parameter #f)) +(defstruct recognition-sequence-branch (left right) transparent: #t final: #t) + +(defstruct recognition-sequence-position-view (value delta) transparent: #t final: #t) +(def (recognition-sequence-relocate value delta) + (cond + ((event-program-sequence? value) (event-program-relocate value delta)) + ((null? value) '()) + ((recognition-sequence-position-view? value) + (make-recognition-sequence-position-view + (recognition-sequence-position-view-value value) + (+ delta (recognition-sequence-position-view-delta value)))) + (else (make-recognition-sequence-position-view value delta)))) ;; : (-> RecognitionSequence RecognitionSequence RecognitionSequence) (def (recognition-sequence-append left right) (cond ((null? left) right) ((null? right) left) + ;; A singleton list prepends to an ordinary list without a branch record + ;; or later materialization. Both borrowed operands remain immutable. + ;; Views and arbitrary sequence variants retain their explicit structure. + ((and (pair? left) (null? (cdr left)) (pair? right)) + (cons (car left) right)) (else (make-recognition-sequence-branch left right)))) ;; : (-> (List RecognitionSequence) RecognitionSequence) @@ -118,21 +150,93 @@ '() sequences)) -;; Iterative depth-first traversal avoids both quadratic append and stack -;; growth when a repeat production has built a deeply left-associated rope. -;; : (-> RecognitionSequence List) +;;; Cardinality is capped at two: semantic fields distinguish empty, +;;; singleton and multiple children, not the total length of a sequence. +(def (recognition-sequence-arity sequence) + (cond + ((recognition-sequence-position-view? sequence) + (recognition-sequence-arity (recognition-sequence-position-view-value sequence))) + ;; Append removes empty operands and relocate preserves empty as empty. + ;; Therefore a branch always contains at least two semantic children. + ((recognition-sequence-branch? sequence) 2) + ((and (not (pair? sequence)) (event-program-sequence? sequence)) + (event-program-sequence-arity sequence)) + ((null? sequence) 0) ((null? (cdr sequence)) 1) (else 2))) +(def (recognition-sequence-bound sequence default-offset end?) + (let loop ((current sequence) (delta 0)) + (cond + ((and (not (pair? current)) (event-program-sequence? current)) + (+ delta ((if end? event-program-sequence-end event-program-sequence-start) current default-offset))) + ((null? current) default-offset) + ((recognition-sequence-position-view? current) + (loop (recognition-sequence-position-view-value current) + (+ delta (recognition-sequence-position-view-delta current)))) + ((recognition-sequence-branch? current) + (loop (if end? (recognition-sequence-branch-right current) + (recognition-sequence-branch-left current)) delta)) + (else (+ delta ((if end? recognition-value-end recognition-value-start) + (recognition-child-value (if end? (last current) (car current))))))))) +(def (recognition-sequence-start sequence default-offset) + (recognition-sequence-bound sequence default-offset #f)) +(def (recognition-sequence-end sequence default-offset) + (recognition-sequence-bound sequence default-offset #t)) + +;;; Consume original child order and accumulated position views without +;;; building a translated child list. The visitor owns publication/materialization. +(def (recognition-sequence-for-each visit sequence) + ;; Engine sequences admit proper-list leaves and private record variants. + ;; Classify the outer representation; list? would rescan every leaf. + (if (or (pair? sequence) (null? sequence)) + (for-each (lambda (child) (visit child 0 #f)) sequence) + (if (event-program-sequence? sequence) + (event-program-sequence-for-each + (lambda (field value delta moved?) + (visit (make-recognition-child field value) delta moved?)) sequence) + (let loop ((current sequence) (delta 0) (moved? #f) (pending '())) + (cond + ((recognition-sequence-position-view? current) + (loop (recognition-sequence-position-view-value current) + (+ delta (recognition-sequence-position-view-delta current)) #t pending)) + ((recognition-sequence-branch? current) + ;; Descend left in registers; only the deferred right side needs a frame. + (loop (recognition-sequence-branch-left current) delta moved? + (cons (vector (recognition-sequence-branch-right current) delta moved?) pending))) + (else + (for-each (lambda (child) (visit child delta moved?)) current) + (unless (null? pending) + (let (frame (car pending)) + (loop (vector-ref frame 0) (vector-ref frame 1) (vector-ref frame 2) + (cdr pending)))))))))) (def (recognition-sequence->list sequence) - (if (not (recognition-sequence-branch? sequence)) - sequence + (if (or (pair? sequence) (null? sequence)) sequence (with-list-builder (collect!) - (let loop ((pending (list sequence))) - (unless (null? pending) - (let (current (car pending)) - (if (recognition-sequence-branch? current) - (loop - (cons (recognition-sequence-branch-left current) - (cons (recognition-sequence-branch-right current) - (cdr pending)))) - (begin - (for-each collect! current) - (loop (cdr pending)))))))))) + ;; Ordinary branch traversal needs neither relocation state nor heap + ;; frames. Bound recursion before delegating a deep/view subtree to the + ;; existing iterative visitor, which owns translated child publication. + (def (collect-sequence! current depth) + (cond + ((and (< depth 64) (recognition-sequence-branch? current)) + (collect-sequence! (recognition-sequence-branch-left current) (+ depth 1)) + (collect-sequence! (recognition-sequence-branch-right current) (+ depth 1))) + ((or (pair? current) (null? current)) (for-each collect! current)) + (else + (recognition-sequence-for-each + (lambda (child delta moved?) + (collect! (if (not moved?) child + (make-recognition-child (recognition-child-field child) + (relocate-recognition-value (recognition-child-value child) delta #t))))) + current)))) + (collect-sequence! sequence 0)))) + +;;; Acceptance requires one unfielded child. Return the child rather than its +;;; payload so rejection remains distinct from a payload whose value is #f. +(def (recognition-sequence-root-child sequence) + (let (children (recognition-sequence->list sequence)) + (and (pair? children) (null? (cdr children)) + (not (recognition-child-field (car children))) + (car children)))) + +;;; The interpreter and generated operand actions share this one boundary. +(def (recognition-sequence-for-action children) + (if (current-recognition-sequence-fusion-enabled?) children + (recognition-sequence->list children))) diff --git a/src/runtime/identity.ss b/src/runtime/identity.ss index 6029f974..7c69dc34 100644 --- a/src/runtime/identity.ss +++ b/src/runtime/identity.ss @@ -2,10 +2,30 @@ ;;; Stable SHA-256 text identities shared by generated machines and artifacts. (import (only-in :std/crypto/digest sha256)) -(export sha256-text) +(export sha256-bytes sha256-text) (def +hex-digits+ "0123456789abcdef") +;; : (-> U8Vector String) +;; Encode one canonical prefixed digest from bytes shared by artifact fields. +(def (sha256-bytes input) + (let ((output (make-string 71 #\0)) + (bytes (sha256 input))) + (let prefix ((index 0)) + (when (fx< index 7) + (string-set! output index (string-ref "sha256:" index)) + (prefix (fx+ index 1)))) + (let encode ((index 0)) + (when (fx< index (u8vector-length bytes)) + (let* ((byte (u8vector-ref bytes index)) + (offset (fx+ 7 (fx* index 2)))) + (string-set! output offset + (string-ref +hex-digits+ (fxquotient byte 16))) + (string-set! output (fx+ offset 1) + (string-ref +hex-digits+ (fxmodulo byte 16)))) + (encode (fx+ index 1)))) + output)) + ;; sha256-text ;; : (-> String String) ;; | doc m% @@ -19,18 +39,4 @@ ;; ``` ;; % (def (sha256-text text) - (let ((output (make-string 71 #\0)) - (bytes (sha256 (string->utf8 text)))) - (for-each (lambda (index) - (string-set! output index (string-ref "sha256:" index))) - (iota 7)) - (for-each - (lambda (index) - (let* ((byte (u8vector-ref bytes index)) - (offset (fx+ 7 (fx* index 2)))) - (string-set! output offset - (string-ref +hex-digits+ (fxquotient byte 16))) - (string-set! output (fx+ offset 1) - (string-ref +hex-digits+ (fxmodulo byte 16))))) - (iota (u8vector-length bytes))) - output)) + (sha256-bytes (string->utf8 text))) diff --git a/src/runtime/incremental.ss b/src/runtime/incremental.ss index 099948b3..6a67ba58 100644 --- a/src/runtime/incremental.ss +++ b/src/runtime/incremental.ss @@ -1,21 +1,20 @@ ;;; -*- Gerbil -*- -;;; Conservative prefix token reuse with fresh-parse-equivalent publication. - -(import (only-in ../compiler/machine - parser-machine-grammar-digest parser-machine-runtime) - (only-in ./artifact - event-end event-start parse-artifact-events - parse-artifact-ref parse-artifact-valid? token-event? - token-event-lexeme token-event-token-kind) - (only-in ./identity sha256-text) - (only-in ./lexer lex-source-from) - (only-in ./lr-parser - lr-checkpoint-advance-shifts lr-initial-checkpoint) - (only-in ./parser parse-tokenized/checkpoint) - (only-in ./significant parser-significant-tokens) - (only-in ./token - make-token token-end token-kind token-lexeme token-start)) -(export +edit-schema+ +;;; Stable incremental facade; owner modules retain the original execution forms. +(import (only-in ./probe + current-lr-lexical-plan-reuse-enabled?) + (only-in ./reuse + current-lr-probe-reuse-enabled?) + (only-in "./incremental/state.ss" + current-lr-source-index-enabled? incremental-session-source-index incremental-session-source-modes current-lr-fragment-reuse-enabled? +edit-schema+ +incremental-receipt-schema+ edit? edit-start-byte edit-delete-byte-length edit-inserted-text incremental-session? incremental-session-artifact incremental-session-recognition-root) + (only-in "./incremental/session.ss" + make-incremental-session incremental-session-project-artifact incremental-session-publication-comparison) + (only-in "./incremental/support.ss" + make-edit apply-edit) + (only-in "./incremental/driver.ss" + parse-source/incremental parse-incremental-session)) +(export current-lr-lexical-plan-reuse-enabled? current-lr-source-index-enabled? incremental-session-source-index incremental-session-source-modes + current-lr-probe-reuse-enabled? current-lr-fragment-reuse-enabled? + +edit-schema+ +incremental-receipt-schema+ make-edit edit? @@ -23,205 +22,11 @@ edit-delete-byte-length edit-inserted-text apply-edit - parse-source/incremental) - -(def +edit-schema+ "gerbil-parser.edit.v1") -(def +incremental-receipt-schema+ - "gerbil-parser.incremental-receipt.v1") - -;; : (-> String Nat Nat String EditRecord) -(defstruct edit-record (schema start-byte delete-byte-length inserted-text) - transparent: #t) - -;; : (-> Datum Boolean) -(def edit? edit-record?) -;; : (-> Edit Nat) -(def edit-start-byte edit-record-start-byte) -;; : (-> Edit Nat) -(def edit-delete-byte-length edit-record-delete-byte-length) -;; : (-> Edit String) -(def edit-inserted-text edit-record-inserted-text) - -;; : (-> Nat Nat String Edit) -(def (make-edit start-byte delete-byte-length inserted-text) - (unless (and (integer? start-byte) (>= start-byte 0) - (integer? delete-byte-length) (>= delete-byte-length 0) - (string? inserted-text)) - (error "invalid incremental edit" start-byte delete-byte-length - inserted-text)) - (make-edit-record +edit-schema+ start-byte delete-byte-length - inserted-text)) - -;; : (-> String Nat Nat) -(def (byte-index->character-index source target) - (let ((character-length (string-length source)) - (source-byte-length (u8vector-length (string->utf8 source)))) - (unless (<= 0 target source-byte-length) - (error "edit byte offset is outside source" target source-byte-length)) - (let loop ((character 0) (byte 0)) - (cond - ((= byte target) character) - ((= character character-length) - (error "edit byte offset splits a UTF-8 character" target)) - (else - (let (next - (+ byte - (u8vector-length - (string->utf8 (string (string-ref source character)))))) - (when (> next target) - (error "edit byte offset splits a UTF-8 character" target)) - (loop (+ character 1) next))))))) - -;; : (-> String Edit String) -(def (apply-edit source source-edit) - (unless (and (string? source) - (edit? source-edit) - (equal? (edit-record-schema source-edit) +edit-schema+)) - (error "apply-edit requires source and Edit v1")) - (let* ((start-byte (edit-start-byte source-edit)) - (end-byte (+ start-byte (edit-delete-byte-length source-edit))) - (start (byte-index->character-index source start-byte)) - (end (byte-index->character-index source end-byte))) - (string-append (substring source 0 start) - (edit-inserted-text source-edit) - (substring source end (string-length source))))) - -;; : (-> ParseArtifact (List Token)) -(def (artifact-tokens artifact) - (filter-map - (lambda (event) - (and (token-event? event) - (make-token (token-event-token-kind event) - (token-event-lexeme event) - (event-start event) (event-end event)))) - (parse-artifact-events artifact))) - -;; : (-> (List Token) Nat (List Token)) -(def (reusable-prefix tokens edit-start) - (filter (lambda (source-token) - (< (token-end source-token) edit-start)) - tokens)) - -;; : (-> Token Token Integer Boolean) -(def (shift-equivalent-token? old-token new-token byte-delta) - (and (eq? (token-kind old-token) (token-kind new-token)) - (equal? (token-lexeme old-token) (token-lexeme new-token)) - (= (+ (token-start old-token) byte-delta) - (token-start new-token)) - (= (+ (token-end old-token) byte-delta) - (token-end new-token)))) - -;;; Computes maximal suffix convergence in one reverse vector walk. Absolute -;;; token objects are reusable only when the edit has zero net byte delta. -;; : (-> (List Token) (List Token) Integer Nat) -(def (converged-suffix-count old-suffix new-suffix byte-delta) - (let* ((old (list->vector old-suffix)) - (new (list->vector new-suffix))) - (let loop ((old-index (- (vector-length old) 1)) - (new-index (- (vector-length new) 1)) - (count 0)) - (if (and (>= old-index 0) - (>= new-index 0) - (shift-equivalent-token? - (vector-ref old old-index) - (vector-ref new new-index) - byte-delta)) - (loop (- old-index 1) (- new-index 1) (+ count 1)) - count)))) - -;;; Reuses the maximal prefix that cannot touch the edit, re-lexes from its -;;; terminal boundary, and submits the complete token stream to the ordinary -;;; parser/ParseArtifact v1 commit path. -;; parse-source/incremental -;; : (-> ParserMachine String ParseArtifact Edit -;; (Values ParseArtifact IncrementalReceipt)) -;; | doc m% -;; Reuses a safe token prefix and emits an ordinary ParseArtifact v1. -;; -;; # Examples -;; -;; ```scheme -;; (parse-source/incremental machine old-source artifact edit) -;; ;; => fresh-equivalent artifact plus reuse receipt -;; ``` -;; % -(def (parse-source/incremental machine old-source old-artifact source-edit) - (unless (and (parse-artifact-valid? old-artifact) - (equal? (parse-artifact-ref old-artifact 'sourceDigest) - (sha256-text old-source)) - (equal? (parse-artifact-ref old-artifact 'grammarDigest) - (parser-machine-grammar-digest machine))) - (error "incremental base artifact does not match source and grammar")) - (let* ((new-source (apply-edit old-source source-edit)) - (old-tokens (artifact-tokens old-artifact)) - (prefix (reusable-prefix old-tokens (edit-start-byte source-edit))) - (old-suffix (drop old-tokens (length prefix))) - (restart-byte (if (pair? prefix) (token-end (last prefix)) 0)) - (restart-character - (byte-index->character-index new-source restart-byte)) - (suffix - (lex-source-from machine new-source restart-character restart-byte)) - (byte-delta - (- (u8vector-length (string->utf8 new-source)) - (u8vector-length (string->utf8 old-source)))) - (converged-count - (converged-suffix-count old-suffix suffix byte-delta)) - (changed-suffix-count (- (length suffix) converged-count)) - (reused-suffix-count (if (zero? byte-delta) converged-count 0)) - (relocated-suffix-count - (if (zero? byte-delta) 0 converged-count)) - (converged-tail - (if (zero? byte-delta) - (drop old-suffix (- (length old-suffix) converged-count)) - (drop suffix changed-suffix-count))) - (tokens - (append prefix - (take suffix changed-suffix-count) - converged-tail)) - (significant (parser-significant-tokens machine tokens)) - (reused-significant-count - (length (parser-significant-tokens machine prefix)))) - (let-values - (((checkpoint-status checkpoint-payload) - (if (positive? reused-significant-count) - (lr-checkpoint-advance-shifts - (lr-initial-checkpoint - (parser-machine-runtime machine) significant) - reused-significant-count) - (values - 'checkpoint - (lr-initial-checkpoint - (parser-machine-runtime machine) significant))))) - (unless (eq? checkpoint-status 'checkpoint) - (error "incremental prefix did not end at an LR checkpoint" - checkpoint-status)) - (let* ((artifact - (parse-tokenized/checkpoint - machine (parser-machine-grammar-digest machine) - new-source tokens checkpoint-payload)) - (receipt - (list - (cons 'schema +incremental-receipt-schema+) - (cons 'grammarDigest (parser-machine-grammar-digest machine)) - (cons 'baseSourceDigest (sha256-text old-source)) - (cons 'sourceDigest (sha256-text new-source)) - (cons 'edit - (list (cons 'schema +edit-schema+) - (cons 'startByte (edit-start-byte source-edit)) - (cons 'deleteByteLength - (edit-delete-byte-length source-edit)) - (cons 'insertedText - (edit-inserted-text source-edit)))) - (cons 'relexStartByte restart-byte) - (cons 'reusedTokenIds (iota (length prefix))) - (cons 'reusedTokenCount (length prefix)) - (cons 'suffixByteDelta byte-delta) - (cons 'convergedSuffixTokenCount converged-count) - (cons 'reusedSuffixTokenCount reused-suffix-count) - (cons 'relocatedSuffixTokenCount relocated-suffix-count) - (cons 'resumedSignificantTokenCount reused-significant-count) - (cons 'remainingSignificantTokenCount - (- (length significant) reused-significant-count)) - (cons 'publicationSchema - (parse-artifact-ref artifact 'schema))))) - (values artifact receipt))))) + parse-source/incremental + make-incremental-session + incremental-session? + incremental-session-artifact + incremental-session-recognition-root + incremental-session-project-artifact + incremental-session-publication-comparison + parse-incremental-session) diff --git a/src/runtime/incremental/driver.ss b/src/runtime/incremental/driver.ss new file mode 100644 index 00000000..d4e27eca --- /dev/null +++ b/src/runtime/incremental/driver.ss @@ -0,0 +1,308 @@ +;;; -*- Gerbil -*- +;;; incremental: driver owner. Private execution state and named algorithms. + +(import (only-in :std/string/utf8 + string-utf8-length) + (only-in ../source-index + source-index-count source-index-height source-index-end source-index-slice make-source-index-builder source-index-builder-token! source-index-builder-shared! source-index-builder-finish source-index-builder-fresh source-index-builder-shared source-index-builder-chunks source-index-builder-mode-chunks make-source-index-cursor source-index-cursor-token source-index-cursor-mode source-index-cursor-rank source-index-cursor-next! source-index-cursor-seek!) + (only-in ../../compiler/machine + parser-machine-runtime) + (only-in ../artifact + parse-artifact-ref parse-artifact-success?) + (only-in ../reuse + make-fragment-reuser) + (only-in ../lr-parser + lr-lexical-mode-id lr-runtime-fragment-reuse-safe? lr-prefix-snapshot-rebind) + (only-in ../parser + current-source-stream-observer parse-source/checkpoints/resume) + (only-in ../significant + parser-significant-tokens) + (only-in ../token + token-end token-kind token-lexeme token-start) + (only-in "./state.ss" + +checkpoint-spacing+ current-lr-fragment-reuse-enabled? current-lr-source-index-enabled? make-incremental-session-state incremental-session-state-machine incremental-session-state-source incremental-session-state-tokens incremental-session-state-modes incremental-session-state-checkpoints incremental-session-state-capture? incremental-session? incremental-session-artifact incremental-session-source-index incremental-session-recognition-root edit-start-byte edit-delete-byte-length) + (only-in "./reuse.ss" + source-mode-compatible? certified-token-reuse certified-token-window-reuse) + (only-in "./session.ss" + capture-session-parse session-from-directed) + (only-in "./support.ss" + byte-index->character-index apply-edit closed-lexer? relocate-token incremental-receipt require-incremental-base split-checkpoint-prefix checkpoint-before-byte edit-byte-delta directed-edit-fields)) +(export parse-source/incremental + parse-incremental-session) + +(def (parse-source/incremental machine old-source old-artifact source-edit) + (require-incremental-base machine old-source old-artifact) + (let-values (((next receipt) + (parse-incremental-session + (session-from-directed machine old-source) source-edit))) + (values (incremental-session-artifact next) receipt))) + +(def (parse-incremental-session session source-edit) + (unless (incremental-session? session) + (error "incremental edit requires a session" session)) + (let* ((machine (incremental-session-state-machine session)) + (old-source (incremental-session-state-source session)) + (old-tokens (incremental-session-state-tokens session)) + (old-modes (incremental-session-state-modes session)) + (checkpoints (incremental-session-state-checkpoints session)) + (old-index (and (current-lr-source-index-enabled?) (incremental-session-source-index session))) + (index-builder #f) (index-origin #f) (provenance-cursor #f) + (new-source (apply-edit old-source source-edit)) + (source-byte-length (string-utf8-length new-source)) + (fragment-stats (make-vector 11 0))) + (def (finish next prefix-count shifted reused-count reused-bytes restart-byte + fresh? (reused-events #f) (replaced-significant-count 0)) + (let* ((artifact (incremental-session-artifact next)) + (significant-count + (length + (parser-significant-tokens + machine (incremental-session-state-tokens next)))) + (fields + (directed-edit-fields + machine old-source new-source source-edit + prefix-count shifted reused-count reused-bytes + restart-byte significant-count + (if reused-events + (- significant-count replaced-significant-count) + (vector-ref fragment-stats 1)) + (and reused-events (- source-byte-length reused-bytes))))) + (values + next + (incremental-receipt + (append + (if (and index-builder (or (incremental-session-source-index next) (zero? source-byte-length))) + (list (cons 'sourceIndexFreshTokenCount (source-index-builder-fresh index-builder)) + (cons 'sourceIndexSharedTokenCount (source-index-builder-shared index-builder)) + (cons 'sourceIndexNewChunkCount (source-index-builder-chunks index-builder)) + (cons 'sourceIndexNewModeChunkCount (source-index-builder-mode-chunks index-builder)) + (cons 'sourceIndexHeight (source-index-height (incremental-session-source-index next)))) '()) + (if (or (positive? (vector-ref fragment-stats 0)) + (positive? (vector-ref fragment-stats 6)) + (positive? (vector-ref fragment-stats 7)) + (positive? (vector-ref fragment-stats 9))) + (list (cons 'reusedRecognitionFragmentCount (vector-ref fragment-stats 0)) + (cons 'fragmentCertificateProbeByteCount (vector-ref fragment-stats 4)) + (cons 'fragmentRejectedProbeCount (vector-ref fragment-stats 5)) + (cons 'fragmentControlProbeCount (vector-ref fragment-stats 6)) + (cons 'fragmentProbeReuseTokenCount (vector-ref fragment-stats 7)) + (cons 'fragmentProbeReuseByteCount (vector-ref fragment-stats 8)) + (cons 'fragmentCursorVisitCount (vector-ref fragment-stats 9)) + (cons 'certifiedLexicalProbeTokenCount (vector-ref fragment-stats 10))) '()) + (cond + (fresh? (cons (cons 'freshFallback? #t) fields)) + (reused-events + (cons (cons 'reusedRecognitionEventCount reused-events) fields)) + (else fields))) + artifact (and (not reused-events) shifted))))) + (def (fallback) + (set! fragment-stats (make-vector 11 0)) + (set! index-builder #f) (set! index-origin #f) + (finish (session-from-directed machine new-source + (incremental-session-state-capture? session)) + 0 0 0 0 0 #t)) + (let (token-reuse + (certified-token-reuse session source-edit new-source)) + (if token-reuse + (finish (vector-ref token-reuse 0) + (vector-ref token-reuse 1) 0 + (vector-ref token-reuse 2) + (vector-ref token-reuse 3) + (vector-ref token-reuse 4) #f + (vector-ref token-reuse 5) + (vector-ref token-reuse 6)) + (let (window-reuse + (certified-token-window-reuse session source-edit new-source)) + (if window-reuse + (finish (vector-ref window-reuse 0) + (vector-ref window-reuse 1) 0 + (vector-ref window-reuse 2) + (vector-ref window-reuse 3) + (vector-ref window-reuse 4) #f + (vector-ref window-reuse 5) + (vector-ref window-reuse 6)) + (if (or (zero? (vector-length checkpoints)) + (not old-modes) + (not (closed-lexer? machine))) + (fallback) + (let* ((index + (checkpoint-before-byte + checkpoints (edit-start-byte source-edit))) + (saved (vector-ref checkpoints index)) + (shifted (vector-ref saved 0))) + (let-values (((prefix prefix-modes old-rest mode-rest) + (split-checkpoint-prefix + old-tokens old-modes (vector-ref saved 2)))) + (let* ((restart-byte (vector-ref saved 3)) + (restart-character + (byte-index->character-index + (string->utf8 new-source) restart-byte)) + (byte-delta (edit-byte-delta source-edit)) + (edit-end + (+ (edit-start-byte source-edit) + (edit-delete-byte-length source-edit))) + (reused-count 0) + (reused-bytes 0) + (source-cursor (and old-index (make-source-index-cursor old-index (vector-ref saved 2)))) + (reuse-token + (lambda (character byte mode) + (if source-cursor + (begin + (source-index-cursor-seek! source-cursor (max edit-end (- byte byte-delta))) + (let ((old-token (source-index-cursor-token source-cursor)) + (old-mode (source-index-cursor-mode source-cursor))) + (and old-token (= (+ (token-start old-token) byte-delta) byte) + (source-mode-compatible? machine old-mode mode old-token) + (let ((rank (source-index-cursor-rank source-cursor)) + (token (if (zero? byte-delta) old-token (relocate-token old-token byte-delta)))) + (set! index-origin (vector rank 1 byte-delta + (and (not (= old-mode (lr-lexical-mode-id mode))) (lr-lexical-mode-id mode)))) + (source-index-cursor-next! source-cursor) + (set! reused-count (+ reused-count 1)) + (set! reused-bytes (+ reused-bytes (- (token-end old-token) (token-start old-token)))) + (cons token (+ character (string-length (token-lexeme old-token)))))))) + (begin + (when (null? mode-rest) + (set! old-rest '())) + (let skip () + (when (and (pair? old-rest) + (or (< (token-start (car old-rest)) edit-end) + (< (+ (token-start (car old-rest)) + byte-delta) + byte))) + (set! old-rest (cdr old-rest)) + (when (pair? mode-rest) + (set! mode-rest (cdr mode-rest))) + (skip))) + (and (pair? old-rest) + (pair? mode-rest) + (= (+ (token-start (car old-rest)) byte-delta) + byte) + (source-mode-compatible? machine (car mode-rest) mode (car old-rest)) + (let* ((old-token (car old-rest)) + (new-token + (if (zero? byte-delta) + old-token + (relocate-token old-token byte-delta)))) + (set! old-rest (cdr old-rest)) + (set! mode-rest (cdr mode-rest)) + (set! reused-count (+ reused-count 1)) + (set! reused-bytes + (+ reused-bytes + (- (token-end old-token) + (token-start old-token)))) + (cons new-token + (+ character + (string-length (token-lexeme old-token)))))))))) + (rebound + (lr-prefix-snapshot-rebind + (vector-ref saved 1) prefix '() restart-byte))) + (let-values (((reuse-fragment stats reuse-probe) + (if (and (incremental-session-state-capture? session) + (incremental-session-recognition-root session) + (current-lr-fragment-reuse-enabled?) + ;; A short checkpoint tail costs less to replay than + ;; to index/certify. Keep its existing LR path. + (> (- (if old-index (source-index-count old-index) (length old-tokens)) (vector-ref saved 2)) + (* 2 +checkpoint-spacing+)) + (lr-runtime-fragment-reuse-safe? (parser-machine-runtime machine))) + (make-fragment-reuser + machine new-source old-tokens old-modes + (incremental-session-recognition-root session) + (edit-start-byte source-edit) edit-end byte-delta restart-byte + old-index (and old-index (lambda (start count delta) + (set! index-origin (vector start count delta #f)))) + #t) + (values #f fragment-stats #f)))) + (set! fragment-stats stats) + (when (and (incremental-session-state-capture? session) (current-lr-source-index-enabled?)) + (set! provenance-cursor (and old-index (make-source-index-cursor old-index (vector-ref saved 2)))) + (set! index-builder (make-source-index-builder + (and old-index (source-index-slice old-index 0 (vector-ref saved 2))))) + ;; A newly enabled index has no old prefix tree: capture that + ;; already-required prefix once, rather than invent provenance. + (when (and (not old-index) (pair? prefix)) + (let seed ((tokens prefix) (modes prefix-modes)) + (unless (null? tokens) + (source-index-builder-token! index-builder (car tokens) (car modes)) + (seed (cdr tokens) (cdr modes)))))) + (let-values + (((artifact tokens modes records root) + (parameterize + ((current-source-stream-observer + (and index-builder + (lambda (piece modes fragment?) + ;; A scanner certificate may consume the boundary + ;; token before ordinary convergence runs. Recover + ;; source provenance only by exact token equality; + ;; a changed mode copies only bounded mode storage. + ;; Shared positions must be outside the changed bytes. + (when (and provenance-cursor (not fragment?) (not index-origin) + (or (<= (token-end piece) (edit-start-byte source-edit)) + (>= (token-start piece) (+ edit-end byte-delta)))) + (let (shift (if (<= (token-end piece) (edit-start-byte source-edit)) 0 byte-delta)) + (source-index-cursor-seek! provenance-cursor (- (token-start piece) shift)) + (let (old (source-index-cursor-token provenance-cursor)) + (when (and old + (= (token-start piece) (+ shift (token-start old))) + (= (token-end piece) (+ shift (token-end old))) + (eq? (token-kind piece) (token-kind old)) + (equal? (token-lexeme piece) (token-lexeme old))) + (set! index-origin (vector (source-index-cursor-rank provenance-cursor) 1 shift + (and (not (= modes (source-index-cursor-mode provenance-cursor))) modes))))))) + (if index-origin + (begin + (when (and fragment? (not (= (length piece) (vector-ref index-origin 1)))) + (error "source provenance disagrees with fragment extent")) + (source-index-builder-shared! index-builder old-index + (vector-ref index-origin 0) (vector-ref index-origin 1) (vector-ref index-origin 2) (vector-ref index-origin 3)) + (set! index-origin #f)) + (if fragment? + ;; The vector control has no persistent origin. + (let append ((tokens piece) (modes modes)) + (unless (null? tokens) + (source-index-builder-token! index-builder (car tokens) (car modes)) + (append (cdr tokens) (cdr modes)))) + (source-index-builder-token! index-builder piece modes))))))) + (capture-session-parse + (lambda () + (parse-source/checkpoints/resume + machine new-source +checkpoint-spacing+ rebound + prefix prefix-modes restart-character restart-byte + (if reuse-probe + (lambda (character byte mode) + (or (reuse-probe character byte mode) + (reuse-token character byte mode))) + reuse-token) + reuse-fragment)) + (incremental-session-state-capture? session))))) + (if (not (parse-artifact-success? artifact)) + (begin + (when (getenv "GERBIL_PARSER_LR_TRACE" #f) + (write (list 'CHECKPOINT-RESUME-REJECTED 'index index + 'checkpoint-count (vector-length checkpoints) + 'restart-byte restart-byte + 'diagnostics (parse-artifact-ref artifact 'diagnostics))) + (newline) (force-output)) + (fallback)) + (let* ((next-checkpoints + ;; Every checkpoint strictly before the selected one + ;; belongs to the unchanged prefix. A successful short + ;; tail may emit no new sample; retain that prefix so + ;; the next edit can resume rather than reparse in full. + (list->vector + (append + (take (vector->list checkpoints) index) + (vector->list records)))) + (next + (make-incremental-session-state + machine new-source artifact tokens modes + next-checkpoints (incremental-session-state-capture? session) root + (and index-builder root modes + (let (tree (source-index-builder-finish index-builder)) + (unless (or (and (not tree) (zero? source-byte-length)) + (= (source-index-end tree) source-byte-length)) + (error "source index does not cover successful source")) tree))))) + (finish next (vector-ref saved 2) shifted + (+ reused-count (vector-ref fragment-stats 2)) + (+ reused-bytes (vector-ref fragment-stats 3)) restart-byte #f))))))))))))))) + diff --git a/src/runtime/incremental/reuse.ss b/src/runtime/incremental/reuse.ss new file mode 100644 index 00000000..fd93e2e9 --- /dev/null +++ b/src/runtime/incremental/reuse.ss @@ -0,0 +1,451 @@ +;;; -*- Gerbil -*- +;;; incremental: reuse owner. Private execution state and named algorithms. + +(import (only-in ../probe + current-lr-lexical-plan-reuse-enabled?) + (only-in ../../compiler/machine + parser-machine-runtime parser-machine-trivia parser-machine-lexical-modes-compatible?) + (only-in ../artifact + make-same-width-token-artifact make-shifted-token-artifact make-certified-window-artifact parse-artifact-events parse-artifact-success?) + (only-in ../lr-parser + lr-lexical-mode-id lr-runtime-lexical-mode-catalog) + (only-in ../lexer + scan-source-token) + (only-in ../token + token-end token-kind token-lexeme token-start) + (only-in "./state.ss" + make-incremental-session-state incremental-session-state-machine incremental-session-state-artifact incremental-session-state-tokens incremental-session-state-modes incremental-session-state-checkpoints incremental-session-state-capture? edit-start-byte edit-delete-byte-length) + (only-in "./support.ss" + byte-index->character-index closed-lexer? relocate-token edit-byte-delta generic-lr-lexemes? same-scanned-token?) + (only-in "./session.ss" + splice-session-index)) +(export source-mode-compatible? + certified-token-reuse + scan-certified-window + scan-certified-trivia-window + scan-aligned-window + certified-token-window-reuse) + +;;; Compiler candidate-set certificates affect lexical reuse only. +(def (source-mode-compatible? machine old-mode mode old-token) + (let (new-mode (lr-lexical-mode-id mode)) + (or (= old-mode new-mode) + (and (current-lr-lexical-plan-reuse-enabled?) + (parser-machine-lexical-modes-compatible? machine old-mode new-mode + (string-ref (token-lexeme old-token) 0)))))) + +;;; A closed scanner and one-token edit certify unchanged suffix tokenization +;;; modulo its byte delta. Generic significant lexemes preserve LR actions. +;;; Snapshots after a shifted or significant token hold stale semantic values. +(def (certified-token-reuse session source-edit new-source) + (let* ((machine (incremental-session-state-machine session)) + (old-artifact (incremental-session-state-artifact session)) + (old-tokens (incremental-session-state-tokens session)) + (old-modes (incremental-session-state-modes session)) + (edit-start (edit-start-byte source-edit)) + (edit-end (+ edit-start (edit-delete-byte-length source-edit))) + (delta (edit-byte-delta source-edit))) + (and (parse-artifact-success? old-artifact) + old-modes + (closed-lexer? machine) + (let loop ((tokens old-tokens) (modes old-modes) (index 0) + (previous #f) (previous-mode #f)) + (and (pair? tokens) (pair? modes) + (let ((old-token (car tokens)) + (start (token-start (car tokens))) + (end (token-end (car tokens)))) + (cond + ((<= end edit-start) + (loop (cdr tokens) (cdr modes) (+ index 1) + old-token (car modes))) + ((and (<= start edit-start) (<= edit-end end)) + (let (new-token + (with-catch + (lambda (_condition) #f) + (lambda () + (let* ((mode + (vector-ref + (lr-runtime-lexical-mode-catalog + (parser-machine-runtime machine)) + (car modes))) + (character + (byte-index->character-index + (string->utf8 new-source) start))) + (let-values (((token _next-character) + (scan-source-token + machine new-source character + start mode))) + token))))) + (and new-token + (= (token-start new-token) start) + (= (token-end new-token) (+ end delta)) + (eq? (token-kind new-token) + (token-kind old-token)) + (eq? (not ((parser-machine-trivia machine) + old-token)) + (not ((parser-machine-trivia machine) + new-token))) + (or ((parser-machine-trivia machine) new-token) + (generic-lr-lexemes? + (parser-machine-runtime machine) + (token-lexeme old-token) + (token-lexeme new-token))) + (or (zero? delta) + (and + (or (not previous) + (same-scanned-token? + machine new-source previous + previous-mode 0)) + (or (null? (cdr tokens)) + (same-scanned-token? + machine new-source (cadr tokens) + (cadr modes) delta)))) + (let* ((artifact+shared + (if (zero? delta) + (cons + (make-same-width-token-artifact + old-artifact new-source index new-token) + (- (length (parse-artifact-events + old-artifact)) 1)) + (let-values (((shifted shared) + (make-shifted-token-artifact + old-artifact new-source + index start end new-token + delta))) + (cons shifted shared)))) + (artifact (car artifact+shared)) + (next-tokens + (append (take old-tokens index) + (cons new-token + (if (zero? delta) + (cdr tokens) + (map + (lambda (token) + (relocate-token token + delta)) + (cdr tokens)))))) + (trivia? + ((parser-machine-trivia machine) new-token)) + (next-checkpoints + (if (and trivia? (zero? delta)) + (incremental-session-state-checkpoints + session) + (list->vector + (filter + (lambda (record) + (<= (vector-ref record 3) start)) + (vector->list + (incremental-session-state-checkpoints + session)))))) + (next + (make-incremental-session-state + machine new-source artifact next-tokens + old-modes next-checkpoints + (incremental-session-state-capture? session) #f + (splice-session-index session index 1 (list new-token) (list (car modes)) delta)))) + (vector next index (length (cdr tokens)) + (- (u8vector-length + (string->utf8 new-source)) + (token-end new-token)) + start + (cdr artifact+shared) + (if trivia? 0 1)))))) + (else #f)))))))) + +;;; Re-lex exactly the old token window in its recorded LR modes. The same +;;; number of tokens, kinds, trivia membership, and LR actions certify that +;;; recognition events have the same topology after a multi-token edit. +(def (scan-certified-window machine source old-window mode-ids start target-end) + (with-catch + (lambda (_condition) #f) + (lambda () + (let loop ((old old-window) (modes mode-ids) + (byte start) + (character + (byte-index->character-index (string->utf8 source) start)) + (found '()) (changed-significant 0)) + (if (null? old) + (and (= byte target-end) + (cons (reverse found) changed-significant)) + (and (pair? modes) + (let (mode + (vector-ref + (lr-runtime-lexical-mode-catalog + (parser-machine-runtime machine)) + (car modes))) + (let-values (((new-token next-character) + (scan-source-token + machine source character byte mode))) + (and (eq? (token-kind (car old)) + (token-kind new-token)) + (eq? (not ((parser-machine-trivia machine) + (car old))) + (not ((parser-machine-trivia machine) + new-token))) + (or ((parser-machine-trivia machine) new-token) + (equal? (token-lexeme (car old)) + (token-lexeme new-token)) + (generic-lr-lexemes? + (parser-machine-runtime machine) + (token-lexeme (car old)) + (token-lexeme new-token))) + (<= (token-end new-token) target-end) + (loop (cdr old) (cdr modes) + (token-end new-token) next-character + (cons new-token found) + (+ changed-significant + (if ((parser-machine-trivia machine) + new-token) + 0 1)))))))))))) + +;;; Trivia leaves the LR checkpoint unchanged. A replacement can therefore +;;; contain a different number of trivia tokens when all old modes agree. +(def (scan-certified-trivia-window machine source mode-id start target-end) + (with-catch + (lambda (_condition) #f) + (lambda () + (let (mode + (vector-ref + (lr-runtime-lexical-mode-catalog + (parser-machine-runtime machine)) mode-id)) + (let loop ((byte start) + (character + (byte-index->character-index (string->utf8 source) start)) + (found '())) + (cond + ((= byte target-end) (reverse found)) + ((> byte target-end) #f) + (else + (let-values (((token next-character) + (scan-source-token machine source character byte mode))) + (and ((parser-machine-trivia machine) token) + (> (token-end token) byte) + (<= (token-end token) target-end) + (loop (token-end token) next-character + (cons token found))))))))))) + +;;; Align significant tokens while allowing each existing trivia run to grow +;;; or shrink. Every run keeps its LR lexical mode; only significant tokens +;;; advance LR state. Boundaries inside a changed trivia run remain unmapped +;;; and cause event reuse to fail closed. +(def (scan-aligned-window machine source old-window mode-ids start target-end) + (with-catch + (lambda (_condition) #f) + (lambda () + (let ((catalog + (lr-runtime-lexical-mode-catalog + (parser-machine-runtime machine))) + (trivia? (parser-machine-trivia machine))) + (def (scan byte character mode-id) + (scan-source-token + machine source character byte (vector-ref catalog mode-id))) + (let loop ((old old-window) (modes mode-ids) + (byte start) + (character + (byte-index->character-index (string->utf8 source) start)) + (tokens-rev '()) (modes-rev '()) (groups-rev '()) + (boundaries '()) (significant 0)) + (if (null? old) + (and (= byte target-end) + (positive? significant) + (vector (reverse tokens-rev) (reverse modes-rev) + (reverse groups-rev) boundaries significant)) + (and (pair? modes) + (if (trivia? (car old)) + (let gather ((run old) (run-modes modes) + (count 0) (last #f)) + (if (and (pair? run) (trivia? (car run))) + (and (pair? run-modes) + (= (car run-modes) (car modes)) + (gather (cdr run) (cdr run-modes) + (+ count 1) (car run))) + (and (or (null? run) + (and (pair? run-modes) + (= (car run-modes) (car modes)))) + (let () + (def (resume at at-character found) + (let (replacement (reverse found)) + (loop + run run-modes at at-character + (append found tokens-rev) + (append (make-list (length found) + (car modes)) + modes-rev) + (append (make-list (- count 1) '()) + (cons replacement groups-rev)) + (cons (cons (token-end last) at) + (cons (cons (token-start (car old)) byte) + boundaries)) + significant))) + (let collect ((at byte) (at-character character) + (found '())) + (if (= at target-end) + (resume at at-character found) + (and (< at target-end) + (let-values (((token next-character) + (scan at at-character + (car modes)))) + (if (trivia? token) + (and (> (token-end token) at) + (<= (token-end token) target-end) + (collect (token-end token) + next-character + (cons token found))) + (resume at at-character found)))))))))) + (let-values (((new-token next-character) + (scan byte character (car modes)))) + (and (not (trivia? new-token)) + (eq? (token-kind (car old)) + (token-kind new-token)) + (or (equal? (token-lexeme (car old)) + (token-lexeme new-token)) + (generic-lr-lexemes? + (parser-machine-runtime machine) + (token-lexeme (car old)) + (token-lexeme new-token))) + (<= (token-end new-token) target-end) + (loop + (cdr old) (cdr modes) + (token-end new-token) next-character + (cons new-token tokens-rev) + (cons (car modes) modes-rev) + (cons (list new-token) groups-rev) + (cons (cons (token-end (car old)) + (token-end new-token)) + (cons (cons (token-start (car old)) + (token-start new-token)) + boundaries)) + (+ significant 1)))))))))))) + +(def (certified-token-window-reuse session source-edit new-source) + (let* ((machine (incremental-session-state-machine session)) + (old-artifact (incremental-session-state-artifact session)) + (old-tokens (incremental-session-state-tokens session)) + (old-modes (incremental-session-state-modes session)) + (edit-start (edit-start-byte source-edit)) + (edit-end (+ edit-start (edit-delete-byte-length source-edit))) + (delta (edit-byte-delta source-edit))) + (and (parse-artifact-success? old-artifact) + old-modes + (closed-lexer? machine) + (let locate ((tokens old-tokens) (modes old-modes) + (prefix '()) (prefix-modes '()) (index 0)) + (and (pair? tokens) (pair? modes) + (if (<= (token-end (car tokens)) edit-start) + (locate (cdr tokens) (cdr modes) + (cons (car tokens) prefix) + (cons (car modes) prefix-modes) (+ index 1)) + (let collect ((rest tokens) (rest-modes modes) + (window '()) (window-modes '()) + (count 0)) + (if (and (pair? rest) + (or (zero? count) + (< (token-start (car rest)) edit-end))) + (and (pair? rest-modes) + (collect (cdr rest) (cdr rest-modes) + (cons (car rest) window) + (cons (car rest-modes) window-modes) + (+ count 1))) + (and (positive? count) + (let* ((old-window (reverse window)) + (start (token-start (car old-window))) + (target-end + (+ (token-end (car window)) delta)) + (trivia-window? + (and (every + (parser-machine-trivia machine) + old-window) + (every + (lambda (mode-id) + (= mode-id (car window-modes))) + window-modes))) + (scanned + (and (>= count 2) + (scan-certified-window + machine new-source old-window + (reverse window-modes) + start target-end))) + (trivia-scanned + (and (not scanned) trivia-window? + (scan-certified-trivia-window + machine new-source + (car window-modes) + start target-end))) + (aligned + (and (not scanned) (not trivia-scanned) + (scan-aligned-window + machine new-source old-window + (reverse window-modes) + start target-end))) + (new-window + (cond (scanned (car scanned)) + (trivia-scanned trivia-scanned) + (aligned (vector-ref aligned 0)) + (else #f)))) + (and new-window + (or (null? prefix) + (same-scanned-token? + machine new-source (car prefix) + (car prefix-modes) 0)) + (or (null? rest) + (and (pair? rest-modes) + (same-scanned-token? + machine new-source (car rest) + (car rest-modes) delta))) + (let* ((next-tokens + (append + (reverse prefix) new-window + (if (zero? delta) rest + (map + (lambda (token) + (relocate-token token delta)) + rest)))) + (next-modes + (if scanned old-modes + (append + (reverse prefix-modes) + (if aligned + (vector-ref aligned 1) + (make-list + (length new-window) + (car window-modes))) + rest-modes))) + (next-checkpoints + (list->vector + (filter + (lambda (record) + (<= (vector-ref record 3) start)) + (vector->list + (incremental-session-state-checkpoints + session))))) + (suffix-bytes + (- (u8vector-length + (string->utf8 new-source)) + target-end))) + (with-catch + (lambda (_condition) #f) + (lambda () + (let-values (((artifact shared) + (make-certified-window-artifact + old-artifact new-source + index old-window new-window + delta + (and aligned + (vector-ref aligned 2)) + (and aligned + (vector-ref aligned 3))))) + (vector + (make-incremental-session-state + machine new-source artifact next-tokens + next-modes next-checkpoints + (incremental-session-state-capture? session) #f + (splice-session-index session index (length old-window) new-window + (cond (scanned (reverse window-modes)) + (aligned (vector-ref aligned 1)) + (else (make-list (length new-window) (car window-modes)))) delta)) + index (length rest) suffix-bytes + start shared + (cond (scanned (cdr scanned)) + (aligned (vector-ref aligned 4)) + (else 0)))))))))))))))))) + diff --git a/src/runtime/incremental/session.ss b/src/runtime/incremental/session.ss new file mode 100644 index 00000000..8bd385f1 --- /dev/null +++ b/src/runtime/incremental/session.ss @@ -0,0 +1,127 @@ +;;; -*- Gerbil -*- +;;; incremental: session owner. Private execution state and named algorithms. + +(import (only-in ../source-index + source-index-count source-index-slice make-source-index-builder source-index-builder-token! source-index-builder-shared! source-index-builder-finish) + (only-in ../../compiler/machine + parser-machine-grammar-digest parser-machine-trivia parser-machine-for-current-semantic-backend) + (only-in ../artifact + make-success-parse-artifact parse-artifact-success?) + (only-in ../recognition + recognition-child-value relocate-recognition-value) + (only-in ../event-program + event-program-value? event-program-relocate) + (only-in ../funcs + recognition-sequence->list) + (only-in ../lr-parser + current-lr-recognition-observer lr-recognition-project lr-recognition-view? lr-recognition-view-base lr-recognition-view-delta lr-recognition-fragment-value) + (only-in ../parser + current-source-stream-observer parse-source/checkpoints) + (only-in "./state.ss" + +checkpoint-spacing+ current-lr-source-index-enabled? make-incremental-session-state incremental-session-state-machine incremental-session-state-source incremental-session-state-tokens incremental-session? incremental-session-source-index incremental-session-recognition-root) + (only-in "./support.ss" + artifact-tokens)) +(export incremental-session-project-artifact + incremental-session-publication-comparison + capture-session-parse + session-from-directed + splice-session-index + make-incremental-session) + +(def (incremental-session-project-artifact session) + (let* ((root (incremental-session-recognition-root session)) + (machine (incremental-session-state-machine session))) + (unless root (error "session has no certified grammar snapshot")) + (let (children (recognition-sequence->list (lr-recognition-project root + (incremental-session-state-tokens session)))) + (unless (and (pair? children) (null? (cdr children))) + (error "grammar snapshot projection does not have one root")) + ;; Publication consumes the exact captured source token instances, whose + ;; identity is checked by the artifact owner against recognition leaves. + (make-success-parse-artifact + (parser-machine-grammar-digest machine) + (incremental-session-state-source session) + (incremental-session-state-tokens session) + (recognition-child-value (car children)) (parser-machine-trivia machine))))) + +;;; Internal diagnostic: projection is outside timing; publication thunks retain +;;; the session's exact machine, source and token identities. No token vector or +;;; mutable session execution field is exposed. Generic publication proofs stay. +(def (incremental-session-publication-comparison session) + (unless (incremental-session? session) (error "publication requires a session")) + (let* ((root (incremental-session-recognition-root session)) + (machine (incremental-session-state-machine session)) + (source (incremental-session-state-source session)) + (tokens (incremental-session-state-tokens session))) + (unless root (error "publication requires a captured grammar root")) + (def (single-value sequence) + (let (children (recognition-sequence->list sequence)) + (unless (and (pair? children) (null? (cdr children))) + (error "publication requires one semantic root")) + (recognition-child-value (car children)))) + (let unwrap ((piece root) (delta 0) (moved? #f)) + (if (lr-recognition-view? piece) + (unwrap (lr-recognition-view-base piece) + (+ delta (lr-recognition-view-delta piece)) #t) + (let* ((value (single-value (lr-recognition-fragment-value piece))) + (canonical (single-value (lr-recognition-project root tokens))) + (publish (lambda (value) + (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens value (parser-machine-trivia machine))))) + (unless (event-program-value? value) + (error "publication comparison requires the event backend")) + (let (program (if moved? (relocate-recognition-value value delta #t) value)) + (values (lambda () (publish canonical)) (lambda () (publish program)) + (if moved? (event-program-relocate value delta #t) value)))))))) + +(def (capture-session-parse thunk capture?) + (if (not capture?) + (let-values (((artifact tokens modes checkpoints) (thunk))) + (values artifact tokens modes checkpoints #f)) + (let ((root #f) (observer (current-lr-recognition-observer))) + (parameterize + ((current-lr-recognition-observer + (lambda (value) + (set! root value) + (when observer (observer value))))) + (let-values (((artifact tokens modes checkpoints) (thunk))) + (values artifact tokens modes checkpoints + (and (parse-artifact-success? artifact) root))))))) + +(def (session-from-directed machine source (capture? #f)) + (let (builder (and capture? (current-lr-source-index-enabled?) (make-source-index-builder))) + (let-values (((artifact source-tokens source-modes checkpoints root) + (parameterize + ((current-source-stream-observer + (and builder (lambda (token mode fragment?) + (when fragment? (error "fresh source driver transferred a fragment")) + (source-index-builder-token! builder token mode))))) + (capture-session-parse + (lambda () (parse-source/checkpoints machine source +checkpoint-spacing+)) capture?)))) + (make-incremental-session-state + machine source artifact (or source-tokens (artifact-tokens artifact)) + source-modes checkpoints capture? root + (and builder root source-modes (parse-artifact-success? artifact) + (source-index-builder-finish builder)))))) + +;;; Certified event-window edits have an exact source-token splice. Preserve +;;; outside chunks even when the fast artifact path drops its grammar forest. +(def (splice-session-index session start deleted tokens modes delta) + (let (old (and (current-lr-source-index-enabled?) (incremental-session-source-index session))) + (and old + (let (builder (make-source-index-builder (source-index-slice old 0 start))) + (let loop ((tokens tokens) (modes modes)) + (unless (null? tokens) + (unless (pair? modes) (error "source splice has missing lexical modes")) + (source-index-builder-token! builder (car tokens) (car modes)) + (loop (cdr tokens) (cdr modes)))) + (source-index-builder-shared! builder old (+ start deleted) + (- (source-index-count old) start deleted) delta) + (source-index-builder-finish builder))))) + +(def (make-incremental-session machine source (capture? #f)) + (unless (boolean? capture?) (error "invalid recognition capture option" capture?)) + (session-from-directed + (if capture? (parser-machine-for-current-semantic-backend machine) machine) + source capture?)) + diff --git a/src/runtime/incremental/state.ss b/src/runtime/incremental/state.ss new file mode 100644 index 00000000..60751794 --- /dev/null +++ b/src/runtime/incremental/state.ss @@ -0,0 +1,100 @@ +;;; -*- Gerbil -*- +;;; incremental: state owner. Private execution state and named algorithms. + +(export +edit-schema+ + +incremental-receipt-schema+ + +checkpoint-spacing+ + current-lr-fragment-reuse-enabled? + current-lr-source-index-enabled? + incremental-session-state + make-incremental-session-state + incremental-session-state? + incremental-session-state-machine + incremental-session-state-machine-set! + incremental-session-state-source + incremental-session-state-source-set! + incremental-session-state-artifact + incremental-session-state-artifact-set! + incremental-session-state-tokens + incremental-session-state-tokens-set! + incremental-session-state-modes + incremental-session-state-modes-set! + incremental-session-state-checkpoints + incremental-session-state-checkpoints-set! + incremental-session-state-capture? + incremental-session-state-capture?-set! + incremental-session-state-recognition-root + incremental-session-state-recognition-root-set! + incremental-session-state-source-index + incremental-session-state-source-index-set! + incremental-session? + incremental-session-artifact + incremental-session-source-index + incremental-session-source-modes + incremental-session-recognition-root + edit-record + make-edit-record + edit-record? + edit-record-schema + edit-record-schema-set! + edit-record-start-byte + edit-record-start-byte-set! + edit-record-delete-byte-length + edit-record-delete-byte-length-set! + edit-record-inserted-text + edit-record-inserted-text-set! + edit? + edit-start-byte + edit-delete-byte-length + edit-inserted-text) + +;;; -*- Gerbil -*- +;;; LR checkpoint resume with lexical-mode-certified token reuse. +(def +edit-schema+ "gerbil-parser.edit.v1") + +(def +incremental-receipt-schema+ + "gerbil-parser.incremental-receipt.v1") + +(def +checkpoint-spacing+ 32) + +;;; Experimental capture sessions can compare the same executor with transfers +;;; enabled/disabled. Ordinary sessions still do not capture a grammar forest. +(def current-lr-fragment-reuse-enabled? (make-parameter #t)) + +;;; Same-head native measurements currently favor vector convergence for full +;;; ParseArtifact edits. Persistent provenance remains an explicit experiment +;;; for the later local-recognition/publication boundary (RFC 0011). +(def current-lr-source-index-enabled? (make-parameter #f)) + +(defstruct incremental-session-state + (machine source artifact tokens modes checkpoints capture? recognition-root source-index) + transparent: #t) + +(def incremental-session? incremental-session-state?) + +(def incremental-session-artifact incremental-session-state-artifact) + +(def incremental-session-source-index incremental-session-state-source-index) + +(def incremental-session-source-modes incremental-session-state-modes) + +;;; Private session execution metadata for conformance and future reuse. +;;; ParseArtifact remains the only published parse result. +(def incremental-session-recognition-root incremental-session-state-recognition-root) + +;; : (-> String Nat Nat String EditRecord) +(defstruct edit-record (schema start-byte delete-byte-length inserted-text) + transparent: #t) + +;; : (-> Datum Boolean) +(def edit? edit-record?) + +;; : (-> Edit Nat) +(def edit-start-byte edit-record-start-byte) + +;; : (-> Edit Nat) +(def edit-delete-byte-length edit-record-delete-byte-length) + +;; : (-> Edit String) +(def edit-inserted-text edit-record-inserted-text) + diff --git a/src/runtime/incremental/support.ss b/src/runtime/incremental/support.ss new file mode 100644 index 00000000..ba06a167 --- /dev/null +++ b/src/runtime/incremental/support.ss @@ -0,0 +1,224 @@ +;;; -*- Gerbil -*- +;;; incremental: support owner. Private execution state and named algorithms. + +(import (only-in :std/string/utf8 + string-utf8-length) + (only-in ../../compiler/machine + parser-machine-grammar-digest parser-machine-ir parser-machine-runtime) + (only-in ../parser-ir-data + parser-ir-ref) + (only-in ../artifact + event-end event-start parse-artifact-events parse-artifact-ref parse-artifact-valid? token-event? token-event-lexeme token-event-token-kind) + (only-in ../identity + sha256-text) + (only-in ../lr-parser + lr-lexical-mode-terminals lr-runtime-lexical-mode-catalog) + (only-in ../lexer + scan-source-token) + (only-in ../token + make-token token-end token-kind token-lexeme token-start) + (only-in "./state.ss" + +edit-schema+ +incremental-receipt-schema+ make-edit-record edit-record-schema edit? edit-start-byte edit-delete-byte-length edit-inserted-text)) +(export make-edit + byte-index->character-index + apply-edit + artifact-tokens + closed-lexical-expression? + closed-lexer? + relocate-token + incremental-receipt + require-incremental-base + split-checkpoint-prefix + checkpoint-before-byte + edit-byte-delta + directed-edit-fields + generic-lr-lexemes? + same-scanned-token?) + +;; : (-> Nat Nat String Edit) +(def (make-edit start-byte delete-byte-length inserted-text) + (unless (and (integer? start-byte) (>= start-byte 0) + (integer? delete-byte-length) (>= delete-byte-length 0) + (string? inserted-text)) + (error "invalid incremental edit" start-byte delete-byte-length + inserted-text)) + (make-edit-record +edit-schema+ start-byte delete-byte-length + inserted-text)) + +;; : (-> String Nat Nat) +(def (byte-index->character-index source-bytes target) + (let (source-byte-length (u8vector-length source-bytes)) + (unless (<= 0 target source-byte-length) + (error "edit byte offset is outside source" target source-byte-length)) + (when (and (< target source-byte-length) + (<= 128 (u8vector-ref source-bytes target) 191)) + (error "edit byte offset splits a UTF-8 character" target)) + (let loop ((byte 0) (character 0)) + (if (= byte target) + character + (loop (+ byte 1) + (if (<= 128 (u8vector-ref source-bytes byte) 191) + character + (+ character 1))))))) + +;; : (-> String Edit String) +(def (apply-edit source source-edit) + (unless (and (string? source) + (edit? source-edit) + (equal? (edit-record-schema source-edit) +edit-schema+)) + (error "apply-edit requires source and Edit v1")) + (let* ((start-byte (edit-start-byte source-edit)) + (end-byte (+ start-byte (edit-delete-byte-length source-edit))) + (source-bytes (string->utf8 source)) + (start (byte-index->character-index source-bytes start-byte)) + (end (byte-index->character-index source-bytes end-byte))) + (string-append (substring source 0 start) + (edit-inserted-text source-edit) + (substring source end (string-length source))))) + +;; : (-> ParseArtifact (List Token)) +(def (artifact-tokens artifact) + (filter-map + (lambda (event) + (and (token-event? event) + (make-token (token-event-token-kind event) + (token-event-lexeme event) + (event-start event) (event-end event)))) + (parse-artifact-events artifact))) + +;; Only the closed lexical algebra has position-local scanners. An external +;; scanner may consult the source prefix, so a matching byte boundary alone +;; cannot prove that its later tokens are unchanged. +(def (closed-lexical-expression? expression) + (case (car expression) + ((external) #f) + ((choice) (every closed-lexical-expression? (cdr expression))) + ((precedence) (closed-lexical-expression? (caddr expression))) + (else #t))) + +(def (closed-lexer? machine) + (every (lambda (row) (closed-lexical-expression? (cadr row))) + (parser-ir-ref (parser-machine-ir machine) 'lexical-rules))) + +(def (relocate-token source-token byte-delta) + (make-token (token-kind source-token) (token-lexeme source-token) + (+ (token-start source-token) byte-delta) + (+ (token-end source-token) byte-delta))) + +(def (incremental-receipt fields artifact (checkpoint-shifts #f)) + (append + fields + (if checkpoint-shifts + (list (cons 'checkpointReusedShiftCount checkpoint-shifts)) + '()) + (list (cons 'publicationSchema + (parse-artifact-ref artifact 'schema))))) + +(def (require-incremental-base machine old-source old-artifact) + (unless (and (parse-artifact-valid? old-artifact) + (equal? (parse-artifact-ref old-artifact 'sourceDigest) + (sha256-text old-source)) + (equal? (parse-artifact-ref old-artifact 'grammarDigest) + (parser-machine-grammar-digest machine))) + (error "incremental base artifact does not match source and grammar"))) + +;; The checkpoint stores its exact source-token cursor, including trivia. +;; Return both sides so the edit driver never scans the accepted prefix again. +(def (split-checkpoint-prefix tokens modes count) + (let loop ((remaining tokens) (remaining-modes modes) + (left count) (prefix '()) (prefix-modes '())) + (if (zero? left) + (values (reverse prefix) (reverse prefix-modes) + remaining remaining-modes) + (if (or (null? remaining) (null? remaining-modes)) + (error "checkpoint exceeds source token stream" count) + (loop (cdr remaining) (cdr remaining-modes) (- left 1) + (cons (car remaining) prefix) + (cons (car remaining-modes) prefix-modes)))))) + +;; Snapshot byte ends are monotone. The strict boundary matches the old +;; reusable-prefix contract: a token ending at the edit start is rescanned. +(def (checkpoint-before-byte checkpoints edit-start) + (let loop ((index (- (vector-length checkpoints) 1))) + (if (or (zero? index) + (< (vector-ref (vector-ref checkpoints index) 3) + edit-start)) + index + (loop (- index 1))))) + +(def (edit-byte-delta source-edit) + (- (string-utf8-length (edit-inserted-text source-edit)) + (edit-delete-byte-length source-edit))) + +(def (directed-edit-fields machine old-source new-source source-edit + prefix-count shifted reused-tail-count + reused-tail-bytes restart-byte significant-count + (reused-significant-count 0) + (relex-stop-byte #f)) + (let* ((source-byte-length (string-utf8-length new-source)) + (byte-delta (edit-byte-delta source-edit))) + (list + (cons 'schema +incremental-receipt-schema+) + (cons 'grammarDigest (parser-machine-grammar-digest machine)) + (cons 'baseSourceDigest (sha256-text old-source)) + (cons 'sourceDigest (sha256-text new-source)) + (cons 'edit + (list (cons 'schema +edit-schema+) + (cons 'startByte (edit-start-byte source-edit)) + (cons 'deleteByteLength + (edit-delete-byte-length source-edit)) + (cons 'insertedText (edit-inserted-text source-edit)))) + (cons 'relexStartByte restart-byte) + (cons 'relexStopByte (or relex-stop-byte source-byte-length)) + (cons 'relexedByteCount + (- source-byte-length restart-byte reused-tail-bytes)) + (cons 'reusedTokenIds (iota prefix-count)) + (cons 'reusedTokenCount prefix-count) + (cons 'suffixByteDelta byte-delta) + (cons 'convergedSuffixTokenCount reused-tail-count) + (cons 'reusedSuffixTokenCount + (if (zero? byte-delta) reused-tail-count 0)) + (cons 'relocatedSuffixTokenCount + (if (zero? byte-delta) 0 reused-tail-count)) + (cons 'resumedSignificantTokenCount shifted) + (cons 'reusedSignificantTokenCount reused-significant-count) + (cons 'remainingSignificantTokenCount + (- significant-count shifted reused-significant-count))))) + +;;; Literal actions refine token kinds. A changed significant token can retain +;;; the LR action sequence only when neither lexeme matches any literal in the +;;; runtime's interned lexical modes (including case-insensitive lookup). +(def (generic-lr-lexemes? runtime old-lexeme new-lexeme) + (let ((catalog (lr-runtime-lexical-mode-catalog runtime)) + (old-upper (string-upcase old-lexeme)) + (new-upper (string-upcase new-lexeme))) + (let mode-loop ((index 0)) + (or (= index (vector-length catalog)) + (and (every + (lambda (terminal) + (or (not (eq? (cadr terminal) 'literal)) + (and (not (equal? (caddr terminal) old-lexeme)) + (not (equal? (caddr terminal) old-upper)) + (not (equal? (caddr terminal) new-lexeme)) + (not (equal? (caddr terminal) new-upper))))) + (lr-lexical-mode-terminals + (vector-ref catalog index))) + (mode-loop (+ index 1))))))) + +(def (same-scanned-token? machine source old-token mode-id delta) + (with-catch + (lambda (_condition) #f) + (lambda () + (let* ((start (+ (token-start old-token) delta)) + (mode + (vector-ref + (lr-runtime-lexical-mode-catalog + (parser-machine-runtime machine)) mode-id)) + (character + (byte-index->character-index (string->utf8 source) start))) + (let-values (((token _next-character) + (scan-source-token machine source character start mode))) + (and (eq? (token-kind token) (token-kind old-token)) + (equal? (token-lexeme token) (token-lexeme old-token)) + (= (token-end token) (+ (token-end old-token) delta)))))))) + diff --git a/src/runtime/language-artifact.ss b/src/runtime/language-artifact.ss index 0ec7aad7..6c455185 100644 --- a/src/runtime/language-artifact.ss +++ b/src/runtime/language-artifact.ss @@ -1,9 +1,9 @@ ;;; -*- Gerbil -*- ;;; Runtime admission boundary for materialized language IR sidecars. -(import (only-in ./identity sha256-text) +(import (only-in ./identity sha256-bytes) (only-in :std/misc/ports read-all-as-u8vector) - (only-in :std/encoding/base64 base64-decode) + (only-in ./embedded-image unpack-language-artifact-image) (only-in :std/string/utf8 utf8->string) (only-in :std/encoding/zlib uncompress)) (export sha256-identity-filename @@ -15,6 +15,11 @@ (def +compiled-language-artifact-prefix+ "gerbil-parser/compiled-language-artifacts/") +(def (trace-artifact phase schema count) + (when (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1") + (displayln "[gerbil-parser-artifact] phase=" phase " schema=" schema " count=" count) + (force-output))) + ;; : (-> Datum Boolean) (def (sha256-identity? value) (and (string? value) @@ -59,13 +64,18 @@ (def (decode-compiled-language-artifact expected-schema locator compressed origin) (validate-compiled-language-artifact-locator expected-schema locator) + (trace-artifact 'decode expected-schema (u8vector-length compressed)) (let* ((relative-path (car locator)) (expected-digest (cadr locator)) - (serialized (utf8->string (uncompress compressed))) - (actual-digest (sha256-text serialized))) + (bytes (uncompress compressed)) + (_uncompressed (trace-artifact 'uncompressed expected-schema (u8vector-length bytes))) + (serialized (utf8->string bytes)) + (_decoded (trace-artifact 'utf8-decoded expected-schema (string-length serialized))) + (actual-digest (sha256-bytes bytes))) (unless (equal? actual-digest expected-digest) (error "compiled language artifact digest mismatch" expected-schema expected-digest actual-digest origin)) + (trace-artifact 'verified expected-schema (string-length serialized)) (call-with-input-string serialized (lambda (port) @@ -79,6 +89,7 @@ (equal? (cdr row) expected-schema))) (error "compiled language artifact schema mismatch" expected-schema value)) + (trace-artifact 'admitted expected-schema (length value)) value))))) ;;; Admission is fail-closed: storage location, content identity, complete @@ -105,13 +116,13 @@ ;;; AOT language packs carry the same compressed, content-addressed bytes in ;;; their module image. Runtime selection therefore consumes the linked ;;; language without consulting compiler caches or filesystem sidecars. -;; : (forall (a) (-> String [String] String [(Pair Symbol a)])) -;; : (-> String List String Alist) +;; : (forall (a) (-> String [String] U64Vector [(Pair Symbol a)])) +;; : (-> String List U64Vector Alist) (def (load-compiled-language-artifact/embedded expected-schema locator encoded) - (unless (string? encoded) + (unless (u64vector? encoded) (error "invalid embedded compiled language artifact" expected-schema)) (decode-compiled-language-artifact - expected-schema locator (base64-decode encoded) 'aot-image)) + expected-schema locator (unpack-language-artifact-image encoded) 'aot-image)) ;; : (forall (a) (-> String [String] [(Pair Symbol a)])) ;; : (-> String List Alist) diff --git a/src/runtime/layout.ss b/src/runtime/layout.ss new file mode 100644 index 00000000..32e933ea --- /dev/null +++ b/src/runtime/layout.ss @@ -0,0 +1,213 @@ +;;; -*- Gerbil -*- +;;; Request-local source columns and branch-local indentation references. + +(import (only-in ./lr-action-index + lookup-action-entry lookup-literal-action-entry + lookup-casefolded-literal-action-entry + lookup-layout-start-action-entry + lookup-layout-next-action-entry lr-action-row-tokens lr-action-row-eof) + (only-in ./token token-kind token-lexeme token-start token-end)) +(export make-layout-columns + current-layout-columns current-layout-frames + layout-token-column layout-shift-allowed? + layout-marker-eligible? layout-after-shift layout-after-end + layout-current-action-row prepare-layout-action-selector) + +(def current-layout-columns (make-parameter #f)) +(def current-layout-frames (make-parameter '())) + +;;; Stage storage operations once, rather than dispatch for every source byte. +;;; Each UTF-8 byte retains its leading character's column, including EOF. +(defrule (build-layout-columns input storage put!) + (let* ((bytes input) (columns storage) (size (u8vector-length bytes))) + (let loop ((offset 0) (column 0)) + (if (= offset size) + (begin (put! columns offset column) columns) + (let* ((byte (u8vector-ref bytes offset)) + (width (cond ((< byte 128) 1) ((< byte 224) 2) + ((< byte 240) 3) (else 4))) + (end (+ offset width)) + (next-column + (case byte + ((10 13) 0) + ((9) (* (+ (quotient column 8) 1) 8)) + (else (+ column 1))))) + (let fill ((cursor offset)) + (when (< cursor end) + (put! columns cursor column) + (fill (+ cursor 1)))) + (loop end next-column)))))) + +;;; Tabs advance at most eight columns per character. Choose packed storage +;;; only when that bound fits u32; larger sources retain exact Scheme integers. +;;; The native UTF-8 encoder retains its ownership of character conversion. +;;; Its valid widths keep private write indices within the allocated buffer; +;;; only that bounded builder uses the native setter. Token reads stay checked. +(def (make-layout-columns source) + (let* ((bytes (string->utf8 source)) (size (+ (u8vector-length bytes) 1))) + (if (<= (string-length source) (quotient #xffffffff 8)) + (build-layout-columns bytes (make-u32vector size 0) ##u32vector-set!) + (build-layout-columns bytes (make-vector size 0) vector-set!)))) + +(def (layout-column-ref columns offset) + (if (u32vector? columns) + (u32vector-ref columns offset) + (vector-ref columns offset))) + +(def (layout-token-column token) + (let (columns (current-layout-columns)) + (and columns (layout-column-ref columns (token-start token))))) + +(def (layout-token-end-column token) + ;; JavaCC uses the final characters column, not the exclusive token end. + ;; UTF-8 continuation bytes already carry their leading characters column. + (layout-column-ref (current-layout-columns) (- (token-end token) 1))) + +(def (layout-shift-allowed? token) + (let (frames (current-layout-frames)) + (or (null? frames) + (> (layout-token-column token) (caar frames))))) + +(def (layout-marker-eligible? role token) + (let ((frames (current-layout-frames)) + (column (layout-token-column token))) + (case role + ((layout-start) + (or (null? frames) (> column (caar frames)))) + ((layout-next) + (and (pair? frames) + (= (layout-token-end-column token) (caar frames)) + (equal? (token-lexeme token) (cdar frames)))) + (else #f)))) + +(def (layout-after-shift role token) + (if (eq? role 'layout-start) + (cons (cons (layout-token-end-column token) + (token-lexeme token)) + (current-layout-frames)) + (current-layout-frames))) + +(def (layout-after-end next-token (boundaries '())) + (let (frames (current-layout-frames)) + (and (pair? frames) + (or (not next-token) + (and (not (layout-marker-eligible? 'layout-next next-token)) + (or (<= (layout-token-column next-token) (caar frames)) + (member (token-lexeme next-token) boundaries)))) + (cdr frames)))) + +(def (layout-ordinary-entry entry shift-allowed?) + (if (or (not entry) shift-allowed?) + (if (and entry (eq? (cadr entry) 'layout-guard)) + (cons (car entry) (caddr entry)) entry) + (let (action (cdr entry)) + (case (car action) + ((layout-guard) (cons (car entry) (caddr action))) + ((shift) #f) + ((fork) + (let (admitted + (filter (lambda (branch) (not (eq? (car branch) 'shift))) + (cdr action))) + (cond ((null? admitted) #f) + ((null? (cdr admitted)) + (cons (car entry) (car admitted))) + (else (cons (car entry) (cons 'fork admitted)))))) + (else entry))))) + +(def (layout-mark-action action role) + (case (car action) + ((layout-guard) (layout-mark-action (cadr action) role)) + ((shift) (list 'layout-shift role (cadr action))) + ((fork) + (cons 'fork + (map (lambda (branch) (layout-mark-action branch role)) + (cdr action)))) + (else action))) + +(def (layout-mark-entry entry role) + (and entry + (let (action (layout-mark-action (cdr entry) role)) + (if (eq? action (cdr entry)) entry (cons (car entry) action))))) + +(def (layout-current-action-row row token case-insensitive?) + (let* ((lexeme (token-lexeme token)) + (next-entry (lookup-layout-next-action-entry row lexeme)) + (next (and next-entry (layout-marker-eligible? 'layout-next token) + (layout-mark-entry next-entry 'layout-next))) + (start-entry (and (not next) (lookup-layout-start-action-entry row lexeme))) + (start (and start-entry (layout-marker-eligible? 'layout-start token) + (layout-mark-entry start-entry 'layout-start))) + (ordinary + (layout-ordinary-entry + (or (lookup-literal-action-entry row lexeme) + (and case-insensitive? + (lookup-casefolded-literal-action-entry row lexeme)) + (lookup-action-entry (lr-action-row-tokens row) (token-kind token))) + (layout-shift-allowed? token)))) + (or next start ordinary + (layout-ordinary-entry next-entry #f) + (layout-ordinary-entry start-entry #f)))) + + +;;; Layout action projections are grammar constants. Prepare both eligibility +;;; outcomes once; requests retain only columns and branch-local frames. +(def (layout-project-entry entry allowed?) + (case (cadr (car entry)) + ((eof) entry) + ((layout-start layout-next) + (if allowed? (layout-mark-entry entry (cadr (car entry))) + (layout-ordinary-entry entry #f))) + (else (layout-ordinary-entry entry allowed?)))) + +(defrule (layout-projected-entry pool-expression entry-expression) + (let ((pool pool-expression) (entry entry-expression)) + (and entry + ;; Reductions cannot contain a shift. Keep their original entry and + ;; avoid even the prepared hash lookup on the dominant LR action. + (if (eq? (cadr entry) 'reduce) entry (table-ref pool entry entry))))) + +(def (prepared-layout-current-action-row row allowed blocked token case-insensitive? columns frames) + (let* ((lexeme (token-lexeme token)) + (next (lookup-layout-next-action-entry row lexeme))) + (or (and next (pair? frames) + (= (layout-column-ref columns (- (token-end token) 1)) (caar frames)) + (equal? lexeme (cdar frames)) + (layout-projected-entry allowed next)) + (let* ((start (lookup-layout-start-action-entry row lexeme)) + (entry (or (lookup-literal-action-entry row lexeme) + (and case-insensitive? (lookup-casefolded-literal-action-entry row lexeme)) + (lookup-action-entry (lr-action-row-tokens row) (token-kind token))))) + ;; Once an eligible next marker has lost and no start marker exists, + ;; a pure reduction is independent of the source column relation. + (if (and (not start) entry (eq? (cadr entry) 'reduce)) entry + (let (shift-allowed? (or (null? frames) + (> (layout-column-ref columns (token-start token)) (caar frames)))) + ;; Resolve refinement before projection; denied literals retain + ;; their priority over generic tokens and later duplicates. + (or (and start shift-allowed? (layout-projected-entry allowed start)) + (layout-projected-entry (if shift-allowed? allowed blocked) entry) + (layout-projected-entry blocked next) + (layout-projected-entry blocked start)))))))) + +(def (prepare-layout-action-selector rows original case-insensitive?) + (let ((allowed (make-table test: eq?)) (blocked (make-table test: eq?))) + (let loop ((state 0)) + (when (< state (vector-length rows)) + (for-each (lambda (entry) + (let ((positive (layout-project-entry entry #t)) + (negative (layout-project-entry entry #f))) + ;; Preserve unchanged entry identity and retain #f rejections. + (unless (eq? positive entry) (table-set! allowed entry positive)) + (unless (eq? negative entry) (table-set! blocked entry negative)))) + (vector-ref rows state)) + (loop (+ state 1)))) + (lambda (state token) + (let (row (vector-ref original state)) + (if (not token) (lr-action-row-eof row) + (let ((columns (current-layout-columns))) + (if columns + (prepared-layout-current-action-row row allowed blocked token case-insensitive? + columns (current-layout-frames)) + (or (lookup-literal-action-entry row (token-lexeme token)) + (and case-insensitive? (lookup-casefolded-literal-action-entry row (token-lexeme token))) + (lookup-action-entry (lr-action-row-tokens row) (token-kind token)))))))))) diff --git a/src/runtime/lexer.ss b/src/runtime/lexer.ss index 279cc608..86d1b669 100644 --- a/src/runtime/lexer.ss +++ b/src/runtime/lexer.ss @@ -1,17 +1,19 @@ ;;; Thin execution boundary over an AOT-generated lexer machine. -(import (only-in ../compiler/machine parser-machine-lex)) +(import (only-in ../compiler/machine parser-machine-lex call-with-parser-machine-source)) (export lex-source lex-source-from scan-source-token) ;; : (-> ParserMachine String (List Token)) (def (lex-source machine source) - ((parser-machine-lex machine) source)) + (call-with-parser-machine-source machine source + (lambda () ((parser-machine-lex machine) source)))) ;; : (-> ParserMachine String Nat Nat (List Token)) (def (lex-source-from machine source character-offset byte-offset) - ((parser-machine-lex machine) source character-offset byte-offset)) + (call-with-parser-machine-source machine source + (lambda () ((parser-machine-lex machine) source character-offset byte-offset)))) ;; : (-> ParserMachine String Nat Nat LexicalMode (Values Token Nat)) (def (scan-source-token machine source character-offset byte-offset mode) diff --git a/src/runtime/lexical-source.ss b/src/runtime/lexical-source.ss new file mode 100644 index 00000000..4890b311 --- /dev/null +++ b/src/runtime/lexical-source.ss @@ -0,0 +1,82 @@ +;;; Immutable source preparation for closed lexical capture opcodes. +(import (only-in ./module-source prepare-module-source scan-prepared-module-source)) +(export scan-module-text prepare-lexical-source-plan call-with-lexical-source scan-header-delimiter scan-header-data) +(defstruct lexical-source-context (owner source headers)) +(def current-lexical-source-context (make-parameter #f)) +(def (prepare-lexical-source-plan rules) + (let visit ((expressions (map cadr rules)) (headers '())) + (if (null? expressions) (reverse headers) + (let (expression (car expressions)) + (case (car expression) + ((header-delimiter header-data) + (let (capture (list (string-copy (cadr expression)) (caddr expression))) + (visit (cdr expressions) (if (member capture headers) headers (cons capture headers))))) + ((module-text) + (let (capture (cons 'module-text (map string-copy (cdr expression)))) + (visit (cdr expressions) (if (member capture headers) headers (cons capture headers))))) + ((choice) (visit (append (cdr expression) (cdr expressions)) headers)) + ((precedence) (visit (cons (caddr expression) (cdr expressions)) headers)) + (else (visit (cdr expressions) headers))))))) +(def (call-with-lexical-source plan source thunk) + (if (null? plan) (thunk) + (let (current (current-lexical-source-context)) + (if (and current (eq? plan (lexical-source-context-owner current)) + (eq? source (lexical-source-context-source current))) + (thunk) + (let (prepared (make-lexical-source-context plan source + (map (lambda (capture) + (cons capture (if (eq? (car capture) 'module-text) (prepare-module-source source capture) + (validate-header-base source (car capture) (cadr capture))))) plan))) + (parameterize ((current-lexical-source-context prepared)) (thunk))))))) +(def (header-delimiter-base source prefix count) + (let (context (current-lexical-source-context)) + (if (and context (eq? source (lexical-source-context-source context))) + (let loop ((rows (lexical-source-context-headers context))) + (if (null? rows) (validate-header-base source prefix count) + (let* ((row (car rows)) (capture (car row))) + (if (and (equal? prefix (car capture)) (= count (cadr capture))) + (cdr row) (loop (cdr rows)))))) + (validate-header-base source prefix count)))) + +;;; Header-derived delimiter rules are closed data, independent of language. +;;; Compare directly in the source: no delimiter list, substring or hash table. +;;; A bounded header contains distinct non-alphabetic, non-numeric characters. +(def (validate-header-base source prefix count) + (let (base (string-length prefix)) + (and (<= (+ base count) (string-length source)) (string-prefix? prefix source) + (let loop ((index 0)) + (if (= index count) base + (let (character (string-ref source (+ base index))) + (and (not (char-alphabetic? character)) (not (char-numeric? character)) + (let unique ((previous 0)) + (or (= previous index) + (and (not (char=? character (string-ref source (+ base previous)))) + (unique (+ previous 1))))) + (loop (+ index 1))))))))) +(def (scan-header-delimiter source start prefix count index) + (and (exact-integer? start) (<= 0 start) (< start (string-length source)) + (let (base (header-delimiter-base source prefix count)) + (and base (char=? (string-ref source start) (string-ref source (+ base index))) + (+ start 1))))) +(def (scan-header-data source start prefix count stops) + (and (exact-integer? start) (<= 0 start) (< start (string-length source)) + (let (base (header-delimiter-base source prefix count)) + (and base + (let loop ((end start)) + (if (and (< end (string-length source)) + (let (character (string-ref source end)) + (and (let stop ((index 0)) + (or (= index (string-length stops)) + (and (not (char=? character (string-ref stops index))) + (stop (+ index 1))))) + (let delimiter ((index 0)) + (or (= index count) + (and (not (char=? character (string-ref source (+ base index)))) + (delimiter (+ index 1)))))))) + (loop (+ end 1)) (and (> end start) end))))))) + +(def (scan-module-text source start expression) + (let* ((context (current-lexical-source-context)) + (row (and context (eq? source (lexical-source-context-source context)) + (assoc expression (lexical-source-context-headers context))))) + (scan-prepared-module-source (if row (cdr row) (prepare-module-source source expression)) start))) diff --git a/src/runtime/lr-action-index.ss b/src/runtime/lr-action-index.ss new file mode 100644 index 00000000..24f6d184 --- /dev/null +++ b/src/runtime/lr-action-index.ss @@ -0,0 +1,140 @@ +;;; -*- Gerbil -*- +;;; Indexed LR action rows are shared by prepared and checkpoint execution. + +(import (only-in ./scan make-literal-trie literal-trie-terminal literal-trie-child)) + +(export index-action-rows index-action-row lookup-casefolded-literal-action-entry + lookup-action-entry + lookup-literal-action-entry + lookup-layout-start-action-entry + lookup-layout-next-action-entry + lr-action-row-eof + lr-action-row-tokens lr-action-row-has-literals?) + +;;; ASCII punctuation has a direct action slot; longer/Unicode literals and +;;; token kinds retain their existing indexes. Entries remain the original +;;; (terminal . action) pairs for GLR receipt and action identity semantics. +(defstruct lr-action-row + (ascii-literals literals tokens eof layout-start layout-next casefold-literals) + transparent: #t final: #t) + +(def (index-action-entries entries) + (if (<= (length entries) 8) + entries + (let (index (make-table test: equal?)) + (for-each + (lambda (entry) + (unless (table-ref index (car entry) #f) + (table-set! index (car entry) (cdr entry)))) + entries) + index))) + +(def (index-action-row/with-casefold row casefold-index) + (let ((ascii-literals #f) (literals '()) (tokens '()) (eof #f) + (layout-start '()) (layout-next '())) + (for-each + (lambda (entry) + (let (terminal (car entry)) + (case (cadr terminal) + ((literal) + (let (literal (caddr terminal)) + (if (and (string? literal) + (= (string-length literal) 1) + (< (char->integer (string-ref literal 0)) 128)) + (begin + (unless ascii-literals + (set! ascii-literals (make-vector 128 #f))) + (let (code (char->integer (string-ref literal 0))) + (unless (vector-ref ascii-literals code) + (vector-set! ascii-literals code entry)))) + (set! literals (cons (cons literal entry) literals))))) + ((token) + (set! tokens (cons (cons (caddr terminal) entry) tokens))) + ((layout-start) + (set! layout-start + (cons (cons (caddr terminal) entry) layout-start))) + ((layout-next) + (set! layout-next + (cons (cons (caddr terminal) entry) layout-next))) + ((eof) + (unless eof (set! eof entry))) + (else (error "unsupported LR action terminal" terminal))))) + row) + (make-lr-action-row + ascii-literals + (index-action-entries (reverse literals)) + (index-action-entries (reverse tokens)) + eof + (index-action-entries (reverse layout-start)) + (index-action-entries (reverse layout-next)) casefold-index))) + +;;; One grammar vocabulary is shared by every state; row lookup still admits +;;; only that state's original entry and therefore retains first duplicates. +(def (prepare-casefold-literals rows) + (let (entries '()) + (for-each + (lambda (row) + (for-each + (lambda (entry) + (let (terminal (car entry)) + (when (and (eq? (cadr terminal) 'literal) (string? (caddr terminal))) + (set! entries (cons (cons (caddr terminal) (caddr terminal)) entries))))) row)) rows) + (and (pair? entries) (make-literal-trie entries)))) +(def (index-action-row row) + (index-action-row/with-casefold row (prepare-casefold-literals (list row)))) +(def (index-action-rows rows case-insensitive?) + (let* ((trie (and case-insensitive? (prepare-casefold-literals (vector->list rows)))) + (result (make-vector (vector-length rows)))) + (let loop ((index 0)) + (when (< index (vector-length rows)) + (vector-set! result index (index-action-row/with-casefold (vector-ref rows index) trie)) + (loop (+ index 1)))) + result)) + +;;; A prepared row without literals cannot match either the source lexeme or +;;; its case-normalized form. Bind the row once; retain the indexed protocol. +(defrule (lr-action-row-has-literals? row-expr) + (let (row row-expr) + (or (not (not (lr-action-row-ascii-literals row))) + (not (null? (lr-action-row-literals row)))))) + +(def (lookup-action-entry index key) + ;; Preparation admits only proper lists or tables. Inspect the outer + ;; representation rather than traversing a whole list before assoc. + (if (or (pair? index) (null? index)) + (let (found (assoc key index)) + (and found (cdr found))) + (table-ref index key #f))) + +(def (lookup-literal-action-entry row literal) + (if (and (string? literal) + (= (string-length literal) 1) + (< (char->integer (string-ref literal 0)) 128)) + (let (ascii (lr-action-row-ascii-literals row)) + (and ascii + (vector-ref ascii (char->integer (string-ref literal 0))))) + (lookup-action-entry (lr-action-row-literals row) literal))) + +;;; ASCII case conversion is length preserving. Failed prefixes cannot become +;;; a declared literal. Non-ASCII input delegates to the original whole-string +;;; Unicode conversion, including mappings that do not preserve character count. +(def (lookup-casefolded-literal/at row node source index width) + (if (= index width) + (let (literal (literal-trie-terminal node)) + (and literal (lookup-literal-action-entry row literal))) + (let (code (char->integer (string-ref source index))) + (if (>= code 128) + (lookup-literal-action-entry row (string-upcase source)) + (let (child (literal-trie-child node + (integer->char (if (<= 97 code 122) (- code 32) code)))) + (and child (lookup-casefolded-literal/at row child source (+ index 1) width))))))) +(def (lookup-casefolded-literal-action-entry row source) + (let (trie (lr-action-row-casefold-literals row)) + (and trie (lr-action-row-has-literals? row) (string? source) + (lookup-casefolded-literal/at row trie source 0 (string-length source))))) + +(def (lookup-layout-start-action-entry row literal) + (lookup-action-entry (lr-action-row-layout-start row) literal)) + +(def (lookup-layout-next-action-entry row literal) + (lookup-action-entry (lr-action-row-layout-next row) literal)) diff --git a/src/runtime/lr-completion.ss b/src/runtime/lr-completion.ss new file mode 100644 index 00000000..01a5f2ca --- /dev/null +++ b/src/runtime/lr-completion.ss @@ -0,0 +1,59 @@ +;;; -*- Gerbil -*- +;;; Private completion values and pure selective-GLR ranking policy. + +(export make-candidate candidate-root candidate-rest candidate-score + candidate-ambiguities candidate-winner-reason candidate-completion-count + select-candidate) + +;;; The retained token suffix is immutable. Cache its length at completion, +;;; rather than traverse it again at every comparison in an enclosing fork. +(defstruct lr-completion + (root rest score ambiguities winner-reason completion-count remaining-count) + final: #t) + +(def (make-candidate root rest score (ambiguities 0) + (winner-reason 'unique-completion) (completion-count 1)) + (make-lr-completion root rest score ambiguities winner-reason completion-count + (length rest))) + +(def candidate-root lr-completion-root) +(def candidate-rest lr-completion-rest) +(def candidate-score lr-completion-score) +(def candidate-ambiguities lr-completion-ambiguities) +(def candidate-winner-reason lr-completion-winner-reason) +(def candidate-completion-count lr-completion-completion-count) + +(def (ranked-candidate winner reason count) + (using (winner : lr-completion) + (make-lr-completion winner.root winner.rest winner.score winner.ambiguities + reason count winner.remaining-count))) + +;;; Return the chosen representative and whether equal completed products +;;; merged. Request-local counters belong to the executor, not this policy. +(def (select-candidate current candidate) + (if (not current) + (values candidate #f) + (using (current : lr-completion) + (using (candidate : lr-completion) + (let (count (+ current.completion-count candidate.completion-count)) + (cond + ((> candidate.score current.score) + (values (ranked-candidate candidate 'dynamic-precedence count) #f)) + ((< candidate.score current.score) + (values (ranked-candidate current 'dynamic-precedence count) #f)) + ((< candidate.remaining-count current.remaining-count) + (values (ranked-candidate candidate 'maximal-consumption count) #f)) + ((> candidate.remaining-count current.remaining-count) + (values (ranked-candidate current 'maximal-consumption count) #f)) + (else + (let* ((equivalent? (and (equal? candidate.root current.root) + (equal? candidate.rest current.rest))) + (ambiguities (+ current.ambiguities candidate.ambiguities + (if equivalent? 0 1)))) + ;; Keep an ambiguous representative: an enclosing dynamic fork + ;; may outrank the whole set with another completed branch. + (values + (make-lr-completion current.root current.rest current.score ambiguities + (if (zero? ambiguities) 'equivalent-merge 'ambiguous) + count current.remaining-count) + equivalent?))))))))) diff --git a/src/runtime/lr-parser.ss b/src/runtime/lr-parser.ss index 4ad3f611..596db1b2 100644 --- a/src/runtime/lr-parser.ss +++ b/src/runtime/lr-parser.ss @@ -1,47 +1,56 @@ ;;; -*- Gerbil -*- -;;; Immutable LR table execution and lossless recognition reduction. - -(import (only-in ../compiler/lr - lr-spec-ref operand-actions production-action - production-lhs production-precedence production-rhs - production-table) - (only-in ./recognition - make-recognition-child make-recognition-fragment - recognition-child-field - recognition-child-value) - (only-in ./reduce - recognition-children-alias recognition-children-field) - (only-in ./funcs - association-row-index-ref - association-row-vector->index - make-value-interner - recognition-sequence-concatenate - recognition-sequence->list - value-interner-created-count - value-interner-hit-count - value-interner-intern - vector-intern-map) - (only-in ./observability - call-with-parser-observed-phase) - (only-in ./token - token-end token-kind token-lexeme token-start)) -(export lr-parse +;;; Stable lr-parser facade; owner modules retain the original execution forms. +(import (only-in "./lr-parser/state.ss" + lr-recognition-view? lr-recognition-view-base lr-recognition-view-delta lr-recognition-fragment-offset current-lr-recognition-observer current-lr-transfer-yield-observer lr-recognition-fragment? lr-recognition-fragment-production-id lr-recognition-fragment-runtime lr-recognition-fragment-executor lr-recognition-fragment-entry-state lr-recognition-fragment-exit-state lr-recognition-fragment-children lr-recognition-fragment-lookahead lr-recognition-fragment-value lr-recognition-fragment-start lr-recognition-fragment-end lr-recognition-fragment-token-count current-lr-branch-budget lr-checkpoint? lr-lexical-mode? lr-lexical-mode-id lr-lexical-mode-terminals lr-runtime-lexical-mode-catalog lr-runtime-layout? lr-runtime-direct-step lr-runtime-event-step current-lr-event-program-enabled? lr-checkpoint-deterministic-actions lr-checkpoint-deterministic-shifts lr-failure-frontier? lr-failure-frontier-state lr-failure-frontier-expected-terminals) + (only-in "./lr-parser/prepare.ss" + lr-runtime-with-action-selector lr-prepare install-lr-runtime-direct-step! install-lr-runtime-event-step! lr-runtime-event-program? lr-runtime-for-current-semantic-backend) + (only-in "./lr-parser/recognition.ss" + lr-recognition-relocate lr-recognition-project) + (only-in "./lr-parser/checkpoint-base.ss" + lr-checkpoint-state lr-initial-checkpoint require-lr-checkpoint-runtime! lr-checkpoint-lexical-mode lr-checkpoint-interned-lexical-mode-count lr-checkpoint-remaining-token-count lr-failure-frontier-remaining-tokens lr-rejection-condition?) + (only-in "./lr-parser/batch.ss" + lr-parse/receipt lr-parse/prepared/receipt) + (only-in "./lr-parser/checkpoint.ss" + lr-checkpoint-advance lr-checkpoint-advance-shifts lr-checkpoint-feed) + (only-in "./lr-parser/transfer.ss" + lr-checkpoint-before-shift lr-checkpoint-fragment-compatible? lr-checkpoint-inject-fragment lr-runtime-fragment-reuse-safe? lr-recognition-fragment-exit-mode lr-checkpoint-resume lr-checkpoint-frontier lr-checkpoint-drive lr-checkpoint-drive/contextual lr-checkpoint-prefix-snapshot lr-prefix-snapshot-rebind lr-checkpoint-rebind-suffix lr-checkpoint-resume-suffix lr-failure-frontier-resume) + (only-in "./lr-parser/entry.ss" + lr-parse lr-parse/prepared)) +(export lr-runtime-with-action-selector + lr-checkpoint-state lr-recognition-view? lr-recognition-view-base lr-recognition-view-delta + lr-recognition-relocate lr-checkpoint-before-shift + lr-checkpoint-fragment-compatible? lr-checkpoint-inject-fragment + lr-runtime-fragment-reuse-safe? + lr-recognition-fragment-offset lr-recognition-fragment-exit-mode + current-lr-recognition-observer current-lr-transfer-yield-observer + lr-recognition-fragment? lr-recognition-fragment-production-id + lr-recognition-fragment-runtime lr-recognition-fragment-executor + lr-recognition-fragment-entry-state + lr-recognition-fragment-exit-state lr-recognition-fragment-children + lr-recognition-fragment-lookahead lr-recognition-fragment-value + lr-recognition-fragment-start lr-recognition-fragment-end + lr-recognition-fragment-token-count lr-recognition-project + current-lr-branch-budget lr-parse lr-parse/receipt + lr-parse/prepared/receipt lr-prepare lr-parse/prepared - lr-checkpoint? - lr-initial-checkpoint - lr-checkpoint-advance - lr-checkpoint-advance-shifts - lr-checkpoint-resume - lr-checkpoint-frontier - lr-checkpoint-feed + lr-checkpoint? lr-initial-checkpoint require-lr-checkpoint-runtime! lr-checkpoint-advance + lr-checkpoint-advance-shifts lr-checkpoint-resume lr-checkpoint-frontier + lr-checkpoint-feed lr-checkpoint-drive + lr-checkpoint-drive/contextual lr-checkpoint-lexical-mode + lr-checkpoint-prefix-snapshot + lr-prefix-snapshot-rebind + lr-checkpoint-rebind-suffix lr-checkpoint-resume-suffix lr-lexical-mode? lr-lexical-mode-id lr-lexical-mode-terminals - lr-runtime-lexical-mode-catalog + lr-runtime-lexical-mode-catalog lr-runtime-layout? lr-runtime-direct-step lr-runtime-event-step + install-lr-runtime-direct-step! install-lr-runtime-event-step! + current-lr-event-program-enabled? lr-runtime-event-program? + lr-runtime-for-current-semantic-backend lr-checkpoint-interned-lexical-mode-count lr-checkpoint-deterministic-actions lr-checkpoint-deterministic-shifts @@ -52,744 +61,3 @@ lr-failure-frontier-remaining-tokens lr-failure-frontier-resume lr-rejection-condition?) - -(def +lr-eof+ '(terminal eof)) - -;; Immutable execution data derived once for generated parser machines. -(defstruct lr-runtime - (productions table actions action-index gotos goto-index - case-insensitive? dynamic? lexical-modes lexical-mode-catalog) - transparent: #t) - -;;; Interned parser-directed lexical expectation shared by LR states with the -;;; same terminal row. -(defstruct lr-lexical-mode (id terminals) transparent: #t) - -;;; Immutable continuation of the deterministic LR machine. The constructor -;;; remains private: every public checkpoint is tied to the exact prepared -;;; runtime and original token sequence that produced it. -(defstruct lr-checkpoint - (runtime tokens states semantic-values rest - deterministic-actions deterministic-shifts) - transparent: #t) - -;;; A deterministic failure frontier retains the exact immutable continuation -;;; and the terminals admitted by its LR state. Recovery can therefore test a -;;; local edit without replaying the already accepted prefix. -(defstruct lr-failure-frontier (checkpoint state expected-terminals) - transparent: #t) - -(def (lr-initial-checkpoint runtime tokens) - (make-lr-checkpoint runtime tokens '(0) '() tokens 0 0)) - -(def (lr-checkpoint-remaining-token-count checkpoint) - (length (lr-checkpoint-rest checkpoint))) - -(def (lr-checkpoint-lexical-mode checkpoint) - (vector-ref - (lr-runtime-lexical-modes (lr-checkpoint-runtime checkpoint)) - (car (lr-checkpoint-states checkpoint)))) - -(def (lr-checkpoint-interned-lexical-mode-count checkpoint) - (vector-length - (lr-runtime-lexical-mode-catalog (lr-checkpoint-runtime checkpoint)))) - -(def (lr-failure-frontier-remaining-tokens frontier) - (lr-checkpoint-rest (lr-failure-frontier-checkpoint frontier))) - -;;; Distinguishes an expected typed parser rejection from an implementation or -;;; contract exception. Recovery probes may discard the former only. -(def (lr-rejection-condition? condition) - (any (lambda (irritant) - (and (list? irritant) (assq 'failureKind irritant))) - (error-irritants condition))) - -(def (lr-prepare spec) - (let* ((productions (lr-spec-ref spec 'productions)) - (actions (lr-spec-ref spec 'actions)) - (gotos (lr-spec-ref spec 'gotos)) - (dynamic? - (any (lambda (production) - (let (precedence (production-precedence production)) - (and precedence (eq? (car precedence) 'dynamic)))) - productions))) - (let-values (((modes mode-catalog) - (vector-intern-map - actions - (lambda (row) (map car row)) - (lambda (terminals id) - (make-lr-lexical-mode id terminals))))) - (make-lr-runtime - productions - (production-table productions) - actions - (association-row-vector->index actions) - gotos - (association-row-vector->index gotos) - (lr-spec-ref spec 'case-insensitive?) - dynamic? - modes - mode-catalog)))) - -;; lookup-action-row -;; : (-> (Vector (Or (List Pair) HashTable)) Fixnum Datum (OrFalse Pair)) -(def lookup-action-row association-row-index-ref) - -;; current-action-row -;; : (-> Vector Fixnum List Boolean (OrFalse Pair)) -(def (current-action-row actions state tokens case-insensitive?) - (if (null? tokens) - (lookup-action-row actions state +lr-eof+) - ;; A literal is a contextual keyword/punctuation refinement of its lexical - ;; token kind. It has deterministic priority over the generic kind action. - (or (lookup-action-row actions state - (list 'terminal 'literal - (token-lexeme (car tokens)))) - (and case-insensitive? - (string? (token-lexeme (car tokens))) - (lookup-action-row actions state - (list 'terminal 'literal - (string-upcase - (token-lexeme (car tokens)))))) - (lookup-action-row actions state - (list 'terminal 'token - (token-kind (car tokens))))))) - -;; apply-operand-action -;; : (-> List List Fixnum List) -(def (apply-operand-action action children default-offset - fragment-constructor) - (let (materialized (recognition-sequence->list children)) - (case (car action) - ((field) - (recognition-children-field - (cadr action) materialized default-offset fragment-constructor)) - ((alias) - (recognition-children-alias (cadr action) materialized default-offset)) - (else (error "unknown LR operand action" action))))) - -;; apply-operand-actions -;; : (-> List List Fixnum List) -(def (apply-operand-actions value actions default-offset fragment-constructor) - (foldl (lambda (action children) - (apply-operand-action - action children default-offset fragment-constructor)) - value - actions)) - -;; reduce-value -;; : (-> List List Fixnum List) -(def (reduce-value production reversed-values default-offset - fragment-constructor) - (let* ((rhs (production-rhs production)) - (source-values (reverse reversed-values)) - (reduced-values - (map (lambda (operand value) - (apply-operand-actions - value (operand-actions operand) default-offset - fragment-constructor)) - rhs source-values)) - (children (recognition-sequence-concatenate reduced-values)) - (action (production-action production))) - (cond - ((eq? action 'concat) children) - ((eq? action 'pass) - (if (= (length reduced-values) 1) - (car reduced-values) - children)) - (else (error "unknown LR semantic action" action))))) - -;; goto-target -;; : (-> (Vector (Or (List Pair) HashTable)) Fixnum Symbol (OrFalse Fixnum)) -(def (goto-target gotos state lhs) - (let (row (association-row-index-ref gotos state lhs)) - (and row (cdr row)))) - -;; : (-> Datum List Integer [Nat] [Symbol] [Nat] List) -(def (make-candidate root rest score (ambiguities 0) - (winner-reason 'unique-completion) - (completion-count 1)) - (list root rest score ambiguities winner-reason completion-count)) -(def candidate-root car) -(def candidate-rest cadr) -(def candidate-score caddr) -(def candidate-ambiguities cadddr) -(def (candidate-winner-reason candidate) (car (cddddr candidate))) -(def (candidate-completion-count candidate) (cadr (cddddr candidate))) - -;;; Canonical request-local GLR configuration. Parser stacks, semantic values, -;;; and token suffixes are immutable, so structural interning cannot leak -;;; mutation between branches or requests. -(defstruct glr-configuration (states semantic-values rest score) - transparent: #t) - -;;; Executes immutable tables and evaluates every admitted fork within a -;;; deterministic branch budget. Dynamic precedence scores complete branches; -;;; structurally identical ties merge and distinct equal-score ties fail closed. -;; lr-parse/receipt -;; : (-> List List Integer (Values Datum List Alist)) -;; | doc m% -;; Executes immutable LR tables and publishes selective-GLR evidence. -;; -;; # Examples -;; -;; ```scheme -;; (let-values (((root rest receipt) -;; (lr-parse/receipt spec tokens))) -;; (assq 'schema receipt)) -;; ;; => (schema . "gerbil-parser.selective-glr-receipt.v1") -;; ``` -;; % -(def (lr-parse/prepared/receipt runtime tokens (branch-budget 256) - (initial-states '(0)) - (initial-semantic-values '()) - (initial-rest tokens) - (deterministic-prefix-actions 0) - (deterministic-prefix-shifts 0)) - (unless (and (integer? branch-budget) (positive? branch-budget)) - (error "selective GLR branch budget must be positive" branch-budget)) - (let* ((productions (lr-runtime-productions runtime)) - (table (lr-runtime-table runtime)) - (actions (lr-runtime-actions runtime)) - (action-index (lr-runtime-action-index runtime)) - (goto-index (lr-runtime-goto-index runtime)) - (case-insensitive? (lr-runtime-case-insensitive? runtime)) - (input-end-offset - (fold (lambda (input-token offset) - (max offset (token-end input-token))) - 0 tokens)) - (branches-explored 0) - ;; The preferred action is the deterministic continuation of a fork. - ;; Only fallback actions consume the speculative branch budget; a - ;; long preferred path must not fail merely because it visits many - ;; conflict cells. - (speculative-branches-explored 0) - (speculative-depth 0) - (max-speculative-depth 0) - (branch-identities '()) - (branch-sites '()) - (merge-count 0) - (successful-completions 0) - (completion-identities '()) - (configuration-interner (make-value-interner)) - (completion-interner (make-value-interner)) - (fragment-interner (make-value-interner)) - (configuration-results (make-table test: eq?)) - (configuration-result-missing (cons #f #f)) - (configuration-result-visiting (cons #f #t)) - (configuration-result-failed (cons #t #f)) - (configuration-memo-hits 0) - (budget-exhausted? #f) - (best-failure #f)) - (def (intern-configuration states semantic-values rest score) - (value-interner-intern - configuration-interner - (list states semantic-values rest score) - (lambda () - (make-glr-configuration states semantic-values rest score)))) - (def (intern-fragment start end children) - (value-interner-intern - fragment-interner - (list start end children) - (lambda () (make-recognition-fragment start end children)))) - ;; Branch sites identify the actual LR conflict cell; schema/version data - ;; stays in the receipt while state and terminal remain pure identities. - (def (record-branch-site! state terminal) - (let (found - (find (lambda (row) - (and (= (car row) state) - (equal? (cadr row) terminal))) - branch-sites)) - (if found - (set-car! (cddr found) (+ 1 (caddr found))) - (set! branch-sites - (cons (list state terminal 1) branch-sites))))) - (def (failure-offset rest) - (if (pair? rest) - (token-start (car rest)) - input-end-offset)) - (def (record-failure! state rest) - (let (offset (failure-offset rest)) - (when (or (not best-failure) - (> offset (cdr (assq 'byteOffset best-failure)))) - (set! best-failure - (list - (cons 'failureKind 'lr-no-action) - (cons 'state state) - (cons 'byteOffset offset) - (cons 'tokenKind - (if (pair? rest) (token-kind (car rest)) 'eof)) - (cons 'tokenLexeme - (and (pair? rest) (token-lexeme (car rest)))) - (cons 'tokenStart - (and (pair? rest) (token-start (car rest)))) - (cons 'tokenEnd - (and (pair? rest) (token-end (car rest)))) - (cons 'expectedTerminals - (map car (vector-ref actions state)))))))) - (def (record-completion! root rest) - (set! successful-completions (+ successful-completions 1)) - (let* ((identity (list root rest)) - (canonical - (value-interner-intern - completion-interner identity (lambda () identity)))) - (unless (memq canonical completion-identities) - (set! completion-identities - (cons canonical completion-identities))))) - (def (candidate-with-reason candidate reason) - (make-candidate - (candidate-root candidate) (candidate-rest candidate) - (candidate-score candidate) (candidate-ambiguities candidate) reason - (candidate-completion-count candidate))) - (def (candidate-with-completion-count candidate count) - (make-candidate - (candidate-root candidate) (candidate-rest candidate) - (candidate-score candidate) (candidate-ambiguities candidate) - (candidate-winner-reason candidate) count)) - (def (better-candidate current candidate) - (if (not current) - candidate - (let* ((completion-count - (+ (candidate-completion-count current) - (candidate-completion-count candidate))) - (winner - (cond - ((> (candidate-score candidate) (candidate-score current)) - (candidate-with-reason candidate 'dynamic-precedence)) - ((< (candidate-score candidate) (candidate-score current)) - (candidate-with-reason current 'dynamic-precedence)) - ((< (length (candidate-rest candidate)) - (length (candidate-rest current))) - (candidate-with-reason candidate 'maximal-consumption)) - ((> (length (candidate-rest candidate)) - (length (candidate-rest current))) - (candidate-with-reason current 'maximal-consumption)) - ((and (equal? (candidate-root candidate) - (candidate-root current)) - (equal? (candidate-rest candidate) - (candidate-rest current))) - (set! merge-count (+ merge-count 1)) - (make-candidate - (candidate-root current) (candidate-rest current) - (candidate-score current) - (+ (candidate-ambiguities current) - (candidate-ambiguities candidate)) - (if (or (> (candidate-ambiguities current) 0) - (> (candidate-ambiguities candidate) 0)) - 'ambiguous - 'equivalent-merge))) - (else - ;; Retain a representative so an outer dynamic-precedence - ;; fork can still outrank this complete ambiguous set. - (make-candidate - (candidate-root current) (candidate-rest current) - (candidate-score current) - (+ 1 (candidate-ambiguities current) - (candidate-ambiguities candidate)) - 'ambiguous))))) - (candidate-with-completion-count winner completion-count)))) - (def (try-action action terminal states semantic-values rest score) - (case (car action) - ((shift) - (and (pair? rest) - (try-parse - (cons (cadr action) states) - (cons (list (make-recognition-child #f (car rest))) - semantic-values) - (cdr rest) score))) - ((reduce) - (let* ((production (vector-ref table (cadr action))) - (count (length (production-rhs production)))) - (let-values (((popped-values remaining-values) - (split-at semantic-values count)) - ((_popped-states remaining-states) - (split-at states count))) - (let* ( - (offset (if (pair? rest) (token-start (car rest)) - input-end-offset)) - (value - (reduce-value - production popped-values offset intern-fragment)) - (precedence (production-precedence production)) - (next-score - (if (and precedence (eq? (car precedence) 'dynamic)) - (+ score (cadr precedence)) - score)) - (target - (and (pair? remaining-states) - (goto-target goto-index (car remaining-states) - (production-lhs production))))) - (and target - (try-parse (cons target remaining-states) - (cons value remaining-values) rest next-score)))))) - ((accept) - (and (pair? semantic-values) - (let (children - (recognition-sequence->list (car semantic-values))) - (and (pair? children) - (null? (cdr children)) - (not (recognition-child-field (car children))) - (let (root (recognition-child-value (car children))) - (record-completion! root rest) - (make-candidate root rest score)))))) - ((fork) - (let loop ((branches (cdr action)) (best #f) (preferred? #t)) - (if (null? branches) - (begin - (when budget-exhausted? - (error "selective GLR branch budget exhausted" - (list - (cons 'failureKind 'selective-glr-budget-exhausted) - (cons 'branchBudget branch-budget) - (cons 'branchesExplored branches-explored) - (cons 'speculativeBranchesExplored - speculative-branches-explored) - (cons 'maxSpeculativeDepth max-speculative-depth) - (cons 'branchSites (reverse branch-sites))))) - best) - (begin - (set! branches-explored (+ branches-explored 1)) - (unless preferred? - (set! speculative-branches-explored - (+ speculative-branches-explored 1)) - (set! speculative-depth (+ speculative-depth 1)) - (when (> speculative-depth max-speculative-depth) - (set! max-speculative-depth speculative-depth))) - (record-branch-site! (car states) terminal) - (set! branch-identities - (cons branches-explored branch-identities)) - (when (> speculative-depth branch-budget) - (set! budget-exhausted? #t)) - (let (candidate - (and (not budget-exhausted?) - (with-catch - (lambda (_) #f) - (lambda () - (try-action (car branches) terminal - states semantic-values rest score))))) - (unless preferred? - (set! speculative-depth (- speculative-depth 1))) - (loop (cdr branches) - (if candidate - (better-candidate best candidate) - best) - #f)))))) - ((reject-nonassoc) - (error "non-associative operator cannot be chained" - (list - (cons 'failureKind 'non-associative-chain) - (cons 'precedence (cadr action)) - (cons 'associativity (caddr action))))) - (else (error "unknown LR action" action)))) - (def (try-parse states semantic-values rest score) - (let* ((configuration - (intern-configuration states semantic-values rest score)) - (cached - (table-ref configuration-results configuration - configuration-result-missing))) - (cond - ((eq? cached configuration-result-failed) #f) - ((eq? cached configuration-result-visiting) #f) - ((not (eq? cached configuration-result-missing)) - (set! configuration-memo-hits (fx+ configuration-memo-hits 1)) - (set! successful-completions - (+ successful-completions - (candidate-completion-count cached))) - cached) - (else - (table-set! configuration-results configuration - configuration-result-visiting) - (let* ((states (glr-configuration-states configuration)) - (semantic-values - (glr-configuration-semantic-values configuration)) - (rest (glr-configuration-rest configuration)) - (score (glr-configuration-score configuration)) - (state (car states)) - (action-row - (current-action-row - action-index state rest case-insensitive?)) - (result - (if action-row - (try-action (cdr action-row) (car action-row) - states semantic-values rest score) - (begin - (record-failure! state rest) - #f)))) - (table-set! configuration-results configuration - (or result configuration-result-failed)) - result))))) - ;; The prepared fast path hands its immutable checkpoint to selective GLR. - ;; Starting from that checkpoint avoids replaying the deterministic prefix - ;; from token zero whenever the first admitted fork is encountered. - (let (result (try-parse initial-states initial-semantic-values - initial-rest 0)) - (unless result - (error "input does not match LR parser" - (or best-failure - '((failureKind . lr-no-complete-parse) - (byteOffset . 0))))) - (when (> (candidate-ambiguities result) 0) - (error - "selective GLR ambiguity is unresolved" - (list - (cons 'failureKind 'selective-glr-ambiguity) - (cons 'successfulCompletions successful-completions) - (cons 'distinctCompletions (length completion-identities)) - (cons 'ambiguousBranches (candidate-ambiguities result)) - (cons 'dynamicScore (candidate-score result)) - (cons 'branchSites (reverse branch-sites))))) - (values - (candidate-root result) - (candidate-rest result) - (list - (cons 'schema "gerbil-parser.selective-glr-receipt.v1") - (cons 'branchBudget branch-budget) - (cons 'deterministicPrefixActions deterministic-prefix-actions) - (cons 'deterministicPrefixShifts deterministic-prefix-shifts) - (cons 'branchesExplored branches-explored) - (cons 'speculativeBranchesExplored - speculative-branches-explored) - (cons 'maxSpeculativeDepth max-speculative-depth) - (cons 'branchIdentities (reverse branch-identities)) - (cons 'branchSites (reverse branch-sites)) - (cons 'mergedBranches merge-count) - (cons 'ambiguousBranches (candidate-ambiguities result)) - (cons 'successfulCompletions successful-completions) - (cons 'equivalentCompletions - (- successful-completions (length completion-identities))) - (cons 'distinctCompletions (length completion-identities)) - (cons 'internedConfigurations - (value-interner-created-count configuration-interner)) - (cons 'configurationInternHits - (value-interner-hit-count configuration-interner)) - (cons 'configurationMemoHits configuration-memo-hits) - (cons 'internedCompletionIdentities - (value-interner-created-count completion-interner)) - (cons 'completionIdentityInternHits - (value-interner-hit-count completion-interner)) - (cons 'internedRecognitionFragments - (value-interner-created-count fragment-interner)) - (cons 'recognitionFragmentInternHits - (value-interner-hit-count fragment-interner)) - (cons 'winnerReason (candidate-winner-reason result)) - (cons 'dynamicScore (candidate-score result))))))) - -(def (lr-parse/receipt spec tokens (branch-budget 256)) - (lr-parse/prepared/receipt (lr-prepare spec) tokens branch-budget)) - -;;; Sole deterministic executor. State remains in local variables on the hot -;;; path; an immutable checkpoint object is allocated only when an explicit -;;; action budget pauses execution. Fork/failure fallback passes the raw local -;;; frontier directly to selective GLR without replay. -;; : (-> LRCheckpoint (OrFalse Nat) PooFlowDebugCallPolicy -;; (Values Symbol Datum)) -(def (lr-run-checkpoint checkpoint action-budget observability - (stop-at-failure? #f) (shift-target #f) - (stop-at-fork? #f)) - (unless (lr-checkpoint? checkpoint) - (error "LR execution requires an immutable checkpoint" checkpoint)) - (let* ((runtime (lr-checkpoint-runtime checkpoint)) - (table (lr-runtime-table runtime)) - (actions-table (lr-runtime-actions runtime)) - (action-index (lr-runtime-action-index runtime)) - (goto-index (lr-runtime-goto-index runtime)) - (case-insensitive? (lr-runtime-case-insensitive? runtime)) - (tokens (lr-checkpoint-tokens checkpoint)) - (input-end-offset - (fold (lambda (input-token offset) - (max offset (token-end input-token))) - 0 tokens))) - (def (fallback states semantic-values rest actions shifts) - (let-values - (((root remaining _receipt) - (call-with-parser-observed-phase - observability 'selective-glr-execution - (lambda () - (lr-parse/prepared/receipt - runtime tokens 256 states semantic-values rest - actions shifts))))) - (values 'accepted (list root remaining)))) - (let loop ((states (lr-checkpoint-states checkpoint)) - (semantic-values - (lr-checkpoint-semantic-values checkpoint)) - (rest (lr-checkpoint-rest checkpoint)) - (actions (lr-checkpoint-deterministic-actions checkpoint)) - (shifts (lr-checkpoint-deterministic-shifts checkpoint)) - (remaining-budget action-budget)) - (if (or (and remaining-budget (zero? remaining-budget)) - (and shift-target (>= shifts shift-target))) - (values - 'checkpoint - (make-lr-checkpoint - runtime tokens states semantic-values rest actions shifts)) - (let* ((state (car states)) - (action-row - (current-action-row - action-index state rest case-insensitive?))) - (if (not action-row) - (if stop-at-failure? - (values - 'failure - (make-lr-failure-frontier - (make-lr-checkpoint - runtime tokens states semantic-values rest actions shifts) - state - (map car (vector-ref actions-table state)))) - (fallback states semantic-values rest actions shifts)) - (let (action (cdr action-row)) - (case (car action) - ((shift) - (if (pair? rest) - (loop - (cons (cadr action) states) - (cons (list (make-recognition-child #f (car rest))) - semantic-values) - (cdr rest) (fx+ actions 1) (fx+ shifts 1) - (and remaining-budget (fx- remaining-budget 1))) - (fallback states semantic-values rest actions shifts))) - ((reduce) - (let* ((production (vector-ref table (cadr action))) - (count (length (production-rhs production)))) - (let-values (((popped-values remaining-values) - (split-at semantic-values count)) - ((_popped-states remaining-states) - (split-at states count))) - (let* ((offset - (if (pair? rest) (token-start (car rest)) - input-end-offset)) - (value - (reduce-value - production popped-values offset - make-recognition-fragment)) - (target - (and (pair? remaining-states) - (goto-target - goto-index (car remaining-states) - (production-lhs production))))) - (if target - (loop - (cons target remaining-states) - (cons value remaining-values) - rest (fx+ actions 1) shifts - (and remaining-budget - (fx- remaining-budget 1))) - (fallback - states semantic-values rest actions shifts)))))) - ((fork) - (if stop-at-fork? - (values - 'fork - (make-lr-checkpoint - runtime tokens states semantic-values rest actions shifts)) - (fallback states semantic-values rest actions shifts))) - ((accept) - (let (children - (and (pair? semantic-values) - (recognition-sequence->list - (car semantic-values)))) - (if (and (pair? children) - (null? (cdr children)) - (not (recognition-child-field (car children)))) - (values - 'accepted - (list (recognition-child-value (car children)) rest)) - (fallback - states semantic-values rest actions shifts)))) - (else - (fallback states semantic-values rest actions shifts)))))))))) - -;;; Advances through the same executor and materializes one checkpoint only at -;;; the requested deterministic action boundary. -(def (lr-checkpoint-advance checkpoint (action-budget 1) - (observability #f)) - (unless (and (integer? action-budget) (positive? action-budget)) - (error "LR checkpoint action budget must be positive" action-budget)) - (lr-run-checkpoint checkpoint action-budget observability)) - -;;; Advances by consumed significant tokens rather than raw actions. This is -;;; the stable boundary used by incremental parsing because reductions do not -;;; consume source input. -(def (lr-checkpoint-advance-shifts checkpoint (shift-budget 1) - (observability #f)) - (unless (and (integer? shift-budget) (positive? shift-budget)) - (error "LR checkpoint shift budget must be positive" shift-budget)) - (lr-run-checkpoint - checkpoint #f observability #f - (+ (lr-checkpoint-deterministic-shifts checkpoint) shift-budget))) - -;;; Feeds exactly one significant token through reductions and one shift. If -;;; reductions expose a selective-GLR cell first, returns its exact immutable -;;; frontier instead of starting GLR with a truncated one-token suffix. -(def (lr-checkpoint-feed checkpoint input-token (observability #f)) - (let* ((shift-target - (fx+ (lr-checkpoint-deterministic-shifts checkpoint) 1)) - (fed - (make-lr-checkpoint - (lr-checkpoint-runtime checkpoint) - (cons input-token (lr-checkpoint-tokens checkpoint)) - (lr-checkpoint-states checkpoint) - (lr-checkpoint-semantic-values checkpoint) - (list input-token) - (lr-checkpoint-deterministic-actions checkpoint) - (lr-checkpoint-deterministic-shifts checkpoint)))) - (lr-run-checkpoint fed #f observability #f shift-target #t))) - -;;; Rebinds an unconsumed suffix to the same immutable deterministic frontier. -;;; This is used only when a streaming path reaches a selective-GLR cell. -(def (lr-checkpoint-resume-suffix checkpoint tokens rest (observability #f)) - (lr-checkpoint-resume - (make-lr-checkpoint - (lr-checkpoint-runtime checkpoint) - tokens - (lr-checkpoint-states checkpoint) - (lr-checkpoint-semantic-values checkpoint) - rest - (lr-checkpoint-deterministic-actions checkpoint) - (lr-checkpoint-deterministic-shifts checkpoint)) - observability)) - -;;; Completes parsing from an initial or advanced immutable checkpoint. -(def (lr-checkpoint-resume checkpoint (observability #f)) - (let-values (((status payload) - (lr-run-checkpoint checkpoint #f observability))) - (unless (eq? status 'accepted) - (error "LR resume did not reach a terminal result" status)) - (values (car payload) (cadr payload)))) - -;;; Runs to acceptance or to the first deterministic LR failure frontier. An -;;; admitted GLR fork still delegates to the selective-GLR owner. -(def (lr-checkpoint-frontier checkpoint (observability #f)) - (lr-run-checkpoint checkpoint #f observability #t)) - -;;; Tests one edited suffix from the retained failure state. The returned CST -;;; remains private to recovery; only acceptance is exposed. -(def (lr-failure-frontier-resume frontier edited-rest (observability #f)) - (unless (lr-failure-frontier? frontier) - (error "LR recovery requires a failure frontier" frontier)) - (let* ((checkpoint (lr-failure-frontier-checkpoint frontier)) - (edited - (make-lr-checkpoint - (lr-checkpoint-runtime checkpoint) - (lr-checkpoint-tokens checkpoint) - (lr-checkpoint-states checkpoint) - (lr-checkpoint-semantic-values checkpoint) - edited-rest - (lr-checkpoint-deterministic-actions checkpoint) - (lr-checkpoint-deterministic-shifts checkpoint)))) - (with-catch - (lambda (condition) - (if (lr-rejection-condition? condition) - #f - (raise condition))) - (lambda () - (let-values (((_root rest) (lr-checkpoint-resume edited observability))) - (null? rest)))))) - -(def (lr-parse/prepared runtime tokens (observability #f)) - (lr-checkpoint-resume - (lr-initial-checkpoint runtime tokens) observability)) - -;; lr-parse -;; : (-> List List (Values Datum List)) -(def (lr-parse spec tokens) - (lr-parse/prepared (lr-prepare spec) tokens)) diff --git a/src/runtime/lr-parser/batch.ss b/src/runtime/lr-parser/batch.ss new file mode 100644 index 00000000..d8b7b237 --- /dev/null +++ b/src/runtime/lr-parser/batch.ss @@ -0,0 +1,400 @@ +;;; -*- Gerbil -*- +;;; lr-parser: batch owner. Private execution state and named algorithms. + +(import (only-in ../reduction-plan + reduce-lr-source-plan) + (only-in ../lr-completion + make-candidate candidate-root candidate-rest candidate-score candidate-ambiguities candidate-winner-reason candidate-completion-count select-candidate) + (only-in ../../compiler/lr + production-action layout-end-action? production-lhs production-precedence) + (only-in ../recognition + make-recognition-child make-recognition-fragment recognition-child-value) + (only-in ../funcs + make-value-interner current-recognition-sequence-fusion-enabled? recognition-sequence-root-child value-interner-created-count value-interner-hit-count value-interner-intern) + (only-in ../layout + current-layout-frames layout-after-shift layout-after-end) + (only-in ../token + token-end token-kind token-lexeme token-start) + (only-in "./state.ss" + lr-runtime-productions lr-runtime-table lr-runtime-reduction-widths lr-runtime-actions lr-runtime-action-index lr-runtime-goto-index lr-runtime-goto-columns lr-runtime-case-insensitive? lr-runtime-layout? lr-runtime-reduction-plans lr-runtime-action-selector current-lr-branch-budget) + (only-in "./prepare.ss" + lr-prepare goto-target) + (only-in "./reduction.ss" + with-pop-reduction)) +(export lr-parse/prepared/receipt + lr-parse/prepared/receipt/materialized + lr-parse/receipt) + +;;; GLR equivalence compares canonical trees, never rope association shapes. +(def (lr-parse/prepared/receipt runtime tokens (branch-budget (current-lr-branch-budget)) + (initial-states '(0)) + (initial-semantic-values '()) + (initial-rest tokens) + (deterministic-prefix-actions 0) + (deterministic-prefix-shifts 0)) + (parameterize ((current-recognition-sequence-fusion-enabled? #f)) + (lr-parse/prepared/receipt/materialized runtime tokens branch-budget + initial-states initial-semantic-values initial-rest + deterministic-prefix-actions deterministic-prefix-shifts))) + +(def (lr-parse/prepared/receipt/materialized runtime tokens (branch-budget (current-lr-branch-budget)) + (initial-states '(0)) + (initial-semantic-values '()) + (initial-rest tokens) + (deterministic-prefix-actions 0) + (deterministic-prefix-shifts 0)) + (unless (and (integer? branch-budget) (positive? branch-budget)) + (error "selective GLR branch budget must be positive" branch-budget)) + (let* ((productions (lr-runtime-productions runtime)) + (table (lr-runtime-table runtime)) + (widths (lr-runtime-reduction-widths runtime)) + (actions (lr-runtime-actions runtime)) + (action-index (lr-runtime-action-index runtime)) + (select-action (lr-runtime-action-selector runtime)) + (goto-index (lr-runtime-goto-index runtime)) + (goto-columns (lr-runtime-goto-columns runtime)) + (case-insensitive? (lr-runtime-case-insensitive? runtime)) + (layout? (lr-runtime-layout? runtime)) + (input-end-offset + (fold (lambda (input-token offset) + (max offset (token-end input-token))) + 0 tokens)) + (branches-explored 0) + ;; Only fallback actions consume the speculative depth budget. + (speculative-branches-explored 0) + (speculative-depth 0) + (max-speculative-depth 0) + (branch-identities '()) + (branch-sites '()) + (merge-count 0) + (successful-completions 0) + (completion-identities '()) + ;; Every remaining stream is a shared tail of this request's input. + ;; Partition by suffix identity and top LR state. Small state buckets + ;; use assoc; wider buckets retain structural hashing. + (configuration-tables (make-table test: eq?)) + (deterministic-memo-fuel 1024) + (interned-configurations 0) + (configuration-intern-hits 0) + (completion-interner (make-value-interner)) + (fragment-interner (make-value-interner)) + (configuration-result-visiting (cons #f #t)) + (configuration-result-failed (cons #t #f)) + (configuration-memo-hits 0) + (budget-exhausted? #f) + (best-failure-state #f) + (best-failure-rest #f) + (best-failure-offset -1)) + (def (configuration-table rest) + (let (found (table-ref configuration-tables rest #f)) + (or found + (let (created (make-table test: eq?)) + (table-set! configuration-tables rest created) + created)))) + (def (configuration-bucket-ref bucket key) + (if (table? bucket) + (table-ref bucket key #f) + (let (entry (assoc key bucket)) + (and entry (cdr entry))))) + (def (configuration-bucket-set! results state bucket key memo) + (cond + ((table? bucket) + (table-set! bucket key memo)) + ((>= (length bucket) 8) + (let (wide (make-table test: equal?)) + (for-each + (lambda (entry) + (table-set! wide (car entry) (cdr entry))) + bucket) + (table-set! wide key memo) + (table-set! results state wide))) + (else + (table-set! results state (cons (cons key memo) bucket))))) + (def (intern-fragment start end children) + (value-interner-intern + fragment-interner + (list start end children) + (lambda () (make-recognition-fragment start end children)))) + ;; Branch sites identify the actual LR conflict cell; schema/version data + ;; stays in the receipt while state and terminal remain pure identities. + (def (record-branch-site! state terminal) + (let (found + (find (lambda (row) + (and (= (car row) state) + (equal? (cadr row) terminal))) + branch-sites)) + (if found + (set-car! (cddr found) (+ 1 (caddr found))) + (set! branch-sites + (cons (list state terminal 1) branch-sites))))) + (def (failure-offset rest) + (if (pair? rest) + (token-start (car rest)) + input-end-offset)) + (def (record-failure! state rest) + (let (offset (failure-offset rest)) + (when (> offset best-failure-offset) + (set! best-failure-state state) + (set! best-failure-rest rest) + (set! best-failure-offset offset)))) + ;; Failed branches are common even in successful GLR parses. Materialize + ;; the diagnostic only if every admitted branch has failed. + (def (best-failure-evidence) + (if best-failure-state + (list + (cons 'failureKind 'lr-no-action) + (cons 'state best-failure-state) + (cons 'byteOffset best-failure-offset) + (cons 'tokenKind + (if (pair? best-failure-rest) + (token-kind (car best-failure-rest)) 'eof)) + (cons 'tokenLexeme + (and (pair? best-failure-rest) + (token-lexeme (car best-failure-rest)))) + (cons 'tokenStart + (and (pair? best-failure-rest) + (token-start (car best-failure-rest)))) + (cons 'tokenEnd + (and (pair? best-failure-rest) + (token-end (car best-failure-rest)))) + (cons 'expectedTerminals + (map car (vector-ref actions best-failure-state)))) + '((failureKind . lr-no-complete-parse) + (byteOffset . 0)))) + (def (record-completion! root rest) + (set! successful-completions (+ successful-completions 1)) + (let* ((identity (list root rest)) + (canonical + (value-interner-intern + completion-interner identity (lambda () identity)))) + (unless (memq canonical completion-identities) + (set! completion-identities + (cons canonical completion-identities))))) + (def (better-candidate current candidate) + (let-values (((winner merged?) (select-candidate current candidate))) + (when merged? (set! merge-count (+ merge-count 1))) + winner)) + (def (try-action action terminal states semantic-values rest score fuel) + (case (car action) + ((shift) + (and (pair? rest) + (try-parse + (cons (cadr action) states) + (cons (list (make-recognition-child #f (car rest))) + semantic-values) + (cdr rest) score fuel))) + ((layout-shift) + (and (pair? rest) + (parameterize + ((current-layout-frames + (layout-after-shift (cadr action) (car rest)))) + (try-parse + (cons (caddr action) states) + (cons (list (make-recognition-child #f (car rest))) + semantic-values) + (cdr rest) score fuel)))) + ((reduce) + (let* ((production-id (cadr action)) + (production (vector-ref table production-id)) + (count (vector-ref widths production-id))) + (with-pop-reduction states semantic-values count + (source-values remaining-values remaining-states) + (let* ((offset (if (pair? rest) (token-start (car rest)) + input-end-offset)) + (value + (reduce-lr-source-plan + (vector-ref (lr-runtime-reduction-plans runtime) production-id) + source-values offset intern-fragment)) + (precedence (production-precedence production)) + (next-score + (if (and precedence (eq? (car precedence) 'dynamic)) + (+ score (cadr precedence)) + score)) + (target + (and (pair? remaining-states) + (if goto-columns + (vector-ref (vector-ref goto-columns production-id) (car remaining-states)) + (goto-target goto-index (car remaining-states) (production-lhs production)))))) + (and target + (if (layout-end-action? (production-action production)) + (alet (frames (layout-after-end (and (pair? rest) (car rest)) + (if (pair? (production-action production)) + (cdr (production-action production)) '()))) + (parameterize ((current-layout-frames frames)) + (try-parse (cons target remaining-states) + (cons value remaining-values) + rest next-score fuel))) + (try-parse (cons target remaining-states) + (cons value remaining-values) + rest next-score fuel))))))) + ((accept) + (and (or (not (lr-runtime-layout? runtime)) + (null? (current-layout-frames))) + (pair? semantic-values) + (let (child (recognition-sequence-root-child (car semantic-values))) + (and child + (let (root (recognition-child-value child)) + (record-completion! root rest) + (make-candidate root rest score)))))) + ((fork) + (let loop ((branches (cdr action)) (best #f) (preferred? #t)) + (if (null? branches) + (begin + (when budget-exhausted? + (error "selective GLR branch budget exhausted" + (list + (cons 'failureKind 'selective-glr-budget-exhausted) + (cons 'branchBudget branch-budget) + (cons 'branchesExplored branches-explored) + (cons 'speculativeBranchesExplored + speculative-branches-explored) + (cons 'maxSpeculativeDepth max-speculative-depth) + (cons 'branchSites (reverse branch-sites))))) + best) + (begin + (set! branches-explored (+ branches-explored 1)) + (when (and (zero? (modulo branches-explored 1000)) + (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1")) + (displayln "[gerbil-parser-lr] branches=" branches-explored + " speculative-depth=" speculative-depth) (force-output)) + (unless preferred? + (set! speculative-branches-explored + (+ speculative-branches-explored 1)) + (set! speculative-depth (+ speculative-depth 1)) + (when (> speculative-depth max-speculative-depth) + (set! max-speculative-depth speculative-depth))) + (record-branch-site! (car states) terminal) + (set! branch-identities + (cons branches-explored branch-identities)) + (when (> speculative-depth branch-budget) + (set! budget-exhausted? #t)) + (let (candidate + (and (not budget-exhausted?) + (with-catch + (lambda (_) #f) + (lambda () + (try-action (car branches) terminal + states semantic-values rest score + deterministic-memo-fuel))))) + (unless preferred? + (set! speculative-depth (- speculative-depth 1))) + (loop (cdr branches) + (if candidate + (better-candidate best candidate) + best) + #f)))))) + ((reject-nonassoc) + (error "non-associative operator cannot be chained" + (list + (cons 'failureKind 'non-associative-chain) + (cons 'precedence (cadr action)) + (cons 'associativity (caddr action))))) + (else (error "unknown LR action" action)))) + ;; Most configurations follow one action. Delay chart allocation until a + ;; fork or a long linear path needs cycle detection and result reuse. + ;; The bound sends a deterministic grammar cycle to the memoized engine. + (def (try-parse states semantic-values rest score + (fuel deterministic-memo-fuel)) + (if (positive? fuel) + (let (action-row + (select-action (car states) (and (pair? rest) (car rest)))) + (cond + ((not action-row) + (record-failure! (car states) rest) + #f) + ((memq (cadr action-row) '(fork accept)) + (try-parse/memo states semantic-values rest score)) + (else + (try-action (cdr action-row) (car action-row) + states semantic-values rest score (fx- fuel 1))))) + (try-parse/memo states semantic-values rest score))) + (def (try-parse/memo states semantic-values rest score) + (let* ((results (configuration-table rest)) + (state (car states)) + (bucket (table-ref results state '())) + (key (if (lr-runtime-layout? runtime) + (list states semantic-values score + (current-layout-frames)) + (list states semantic-values score))) + (memo (configuration-bucket-ref bucket key)) + (cached (and memo (car memo)))) + (if memo + (set! configuration-intern-hits + (fx+ configuration-intern-hits 1)) + (set! interned-configurations (fx+ interned-configurations 1))) + (cond + ((eq? cached configuration-result-failed) #f) + ((eq? cached configuration-result-visiting) #f) + (memo + (set! configuration-memo-hits (fx+ configuration-memo-hits 1)) + (set! successful-completions + (+ successful-completions + (candidate-completion-count cached))) + cached) + (else + (let (memo (cons configuration-result-visiting #f)) + (configuration-bucket-set! results state bucket key memo) + (let* ((action-row + (select-action state (and (pair? rest) (car rest)))) + (result + (if action-row + (try-action (cdr action-row) (car action-row) + states semantic-values rest score 0) + (begin + (record-failure! state rest) + #f)))) + (set-car! memo (or result configuration-result-failed)) + result)))))) + ;; Resume selective GLR from the immutable deterministic checkpoint. + (let (result (try-parse initial-states initial-semantic-values + initial-rest 0)) + (unless result + (error "input does not match LR parser" + (best-failure-evidence))) + (when (> (candidate-ambiguities result) 0) + (error + "selective GLR ambiguity is unresolved" + (list + (cons 'failureKind 'selective-glr-ambiguity) + (cons 'successfulCompletions successful-completions) + (cons 'distinctCompletions (length completion-identities)) + (cons 'ambiguousBranches (candidate-ambiguities result)) + (cons 'dynamicScore (candidate-score result)) + (cons 'branchSites (reverse branch-sites))))) + (values + (candidate-root result) + (candidate-rest result) + (list + (cons 'schema "gerbil-parser.selective-glr-receipt.v1") + (cons 'branchBudget branch-budget) + (cons 'deterministicPrefixActions deterministic-prefix-actions) + (cons 'deterministicPrefixShifts deterministic-prefix-shifts) + (cons 'branchesExplored branches-explored) + (cons 'speculativeBranchesExplored + speculative-branches-explored) + (cons 'maxSpeculativeDepth max-speculative-depth) + (cons 'branchIdentities (reverse branch-identities)) + (cons 'branchSites (reverse branch-sites)) + (cons 'mergedBranches merge-count) + (cons 'ambiguousBranches (candidate-ambiguities result)) + (cons 'successfulCompletions successful-completions) + (cons 'equivalentCompletions + (- successful-completions (length completion-identities))) + (cons 'distinctCompletions (length completion-identities)) + (cons 'internedConfigurations interned-configurations) + (cons 'configurationInternHits configuration-intern-hits) + (cons 'configurationMemoHits configuration-memo-hits) + (cons 'internedCompletionIdentities + (value-interner-created-count completion-interner)) + (cons 'completionIdentityInternHits + (value-interner-hit-count completion-interner)) + (cons 'internedRecognitionFragments + (value-interner-created-count fragment-interner)) + (cons 'recognitionFragmentInternHits + (value-interner-hit-count fragment-interner)) + (cons 'winnerReason (candidate-winner-reason result)) + (cons 'dynamicScore (candidate-score result))))))) + +(def (lr-parse/receipt spec tokens (branch-budget (current-lr-branch-budget))) + (lr-parse/prepared/receipt (lr-prepare spec) tokens branch-budget)) + diff --git a/src/runtime/lr-parser/checkpoint-base.ss b/src/runtime/lr-parser/checkpoint-base.ss new file mode 100644 index 00000000..ea560427 --- /dev/null +++ b/src/runtime/lr-parser/checkpoint-base.ss @@ -0,0 +1,63 @@ +;;; -*- Gerbil -*- +;;; lr-parser: checkpoint-base owner. Private execution state and named algorithms. + +(import (only-in ../token + token-end) + (only-in "./state.ss" + lr-runtime-layout? lr-runtime-lexical-modes lr-runtime-lexical-mode-catalog make-lr-checkpoint lr-checkpoint-runtime lr-checkpoint-states lr-checkpoint-rest current-lr-recognition-observer lr-failure-frontier-checkpoint) + (only-in "./prepare.ss" + lr-runtime-for-current-semantic-backend)) +(export require-lr-checkpoint-runtime! + lr-initial-checkpoint + lr-initial-checkpoint/selected + lr-checkpoint-remaining-token-count + lr-checkpoint-lexical-mode + lr-checkpoint-interned-lexical-mode-count + lr-failure-frontier-remaining-tokens + lr-rejection-condition? + lr-checkpoint-state) + +;;; Checkpoints own all continuation state. Layout execution also needs source +;;; columns and branch frames, which this record does not capture. +(def (require-lr-checkpoint-runtime! runtime) + (when (lr-runtime-layout? runtime) + (error "LR checkpoints require a runtime without layout"))) + +(def (lr-initial-checkpoint runtime tokens) + (require-lr-checkpoint-runtime! runtime) + (lr-initial-checkpoint/selected (lr-runtime-for-current-semantic-backend runtime) tokens)) + +(def (lr-initial-checkpoint/selected runtime tokens) + (make-lr-checkpoint + runtime tokens + (fold (lambda (input-token offset) + (max offset (token-end input-token))) + 0 tokens) + '(0) '() tokens 0 0 + (and (current-lr-recognition-observer) + (not (lr-runtime-layout? runtime)) '()))) + +(def (lr-checkpoint-remaining-token-count checkpoint) + (length (lr-checkpoint-rest checkpoint))) + +(def (lr-checkpoint-lexical-mode checkpoint) + (vector-ref + (lr-runtime-lexical-modes (lr-checkpoint-runtime checkpoint)) + (car (lr-checkpoint-states checkpoint)))) + +(def (lr-checkpoint-interned-lexical-mode-count checkpoint) + (vector-length + (lr-runtime-lexical-mode-catalog (lr-checkpoint-runtime checkpoint)))) + +(def (lr-failure-frontier-remaining-tokens frontier) + (lr-checkpoint-rest (lr-failure-frontier-checkpoint frontier))) + +;;; Distinguishes an expected typed parser rejection from an implementation or +;;; contract exception. Recovery probes may discard the former only. +(def (lr-rejection-condition? condition) + (any (lambda (irritant) + (and (list? irritant) (assq 'failureKind irritant))) + (error-irritants condition))) + +(def (lr-checkpoint-state checkpoint) (car (lr-checkpoint-states checkpoint))) + diff --git a/src/runtime/lr-parser/checkpoint.ss b/src/runtime/lr-parser/checkpoint.ss new file mode 100644 index 00000000..64415095 --- /dev/null +++ b/src/runtime/lr-parser/checkpoint.ss @@ -0,0 +1,358 @@ +;;; -*- Gerbil -*- +;;; lr-parser: checkpoint owner. Private execution state and named algorithms. + +(import (only-in ../reduction-plan + reduce-lr-vector-plan) + (only-in ../../compiler/lr + production-lhs) + (only-in ../recognition + make-recognition-child recognition-child-value) + (only-in ../funcs + recognition-sequence-root-child) + (only-in ../observability + call-with-parser-observed-phase) + (only-in ../token + token-end token-start) + (only-in ../lr-stack-storage + with-lr-stack-storage lr-stack-policy-value lr-stack-current lr-stack-present? lr-semantic-stack-present? lr-stack-value lr-stack-freeze-states lr-stack-freeze-values lr-stack-push-state lr-stack-shift-value lr-stack-push-value with-pop-driver-reduction reduce-driver-plan lr-token-observer? lr-stack-shift-observer lr-stack-notify-shift) + (only-in "./state.ss" + lr-runtime-table lr-runtime-reduction-widths lr-runtime-actions lr-runtime-goto-index lr-runtime-goto-columns lr-runtime-lexical-modes lr-runtime-direct-step lr-runtime-semantic-reducer lr-runtime-reduction-plans lr-runtime-action-selector lr-runtime-owned-stack? lr-checkpoint make-lr-checkpoint lr-checkpoint-runtime lr-checkpoint-tokens lr-checkpoint-input-end-offset lr-checkpoint-states lr-checkpoint-semantic-values lr-checkpoint-rest lr-checkpoint-deterministic-actions lr-checkpoint-deterministic-shifts lr-checkpoint-recognition-stack current-lr-recognition-observer make-lr-failure-frontier current-lr-branch-budget) + (only-in "./recognition.ss" + prepare-capture-plans capture-reduction) + (only-in "./reduction.ss" + reduce-value/events reduce-prepared-plan with-pop-runtime-reduction) + (only-in "./prepare.ss" + goto-target) + (only-in "./batch.ss" + lr-parse/prepared/receipt)) +(export define-lr-driver + lr-run-persistent-checkpoint + lr-run-owned-checkpoint + lr-run-checkpoint + lr-checkpoint-advance + lr-checkpoint-advance-shifts + lr-checkpoint-feed) + +;;; Sole deterministic executor. State remains in local variables on the hot +;;; path; an immutable checkpoint object is allocated only when an explicit +;;; action budget pauses execution. Fork/failure fallback passes the raw local +;;; frontier directly to selective GLR without replay. +;; : (-> LRCheckpoint (OrFalse Nat) PooFlowDebugCallPolicy +;; (Values Symbol Datum)) +;;; The input representation is selected at expansion time. Streaming keeps +;;; token/#f in the loop; borrowed token tails retain their physical identity. +(defrules lr-input-start () + ((_ #f rest) rest) + ((_ #t rest) (if (pair? rest) (car rest) #f))) +(defrules lr-input-empty? () + ((_ #f rest) (null? rest)) + ((_ #t input) (not input))) +(defrules lr-input-token () + ((_ #f rest) (car rest)) + ((_ #t input) input)) +(defrules lr-input-tail () + ((_ #f rest) (cdr rest)) + ((_ #t _input) #f)) +(defrules lr-input-admit () + ((_ #f input) (if input (list input) '())) + ((_ #t input) input)) +(defrules lr-input-list () + ((_ #f rest) rest) + ((_ #t input) (if input (list input) '()))) + +(defrule (define-lr-driver policy scalar? name) + (def (name (checkpoint : lr-checkpoint) action-budget observability + (stop-at-failure? #f) (shift-target #f) + (stop-at-fork? #f) (feed-token #f) + (next-input #f) (after-shift #f) + (direct-step-override 'installed) + (next-input-state? #f) (stop-before-shift? #f) + (snapshot-spacing #f) (capture-snapshot #f)) + (let* ((runtime (lr-checkpoint-runtime checkpoint)) + (selected-observer (lr-stack-shift-observer policy after-shift)) + (direct-step + (if (eq? direct-step-override 'installed) + (lr-runtime-direct-step runtime) + direct-step-override)) + (capture-plans + (and (lr-checkpoint-recognition-stack checkpoint) + (prepare-capture-plans runtime))) + (event-semantics? (lr-stack-policy-value policy + (eq? (lr-runtime-semantic-reducer runtime) reduce-value/events) #f)) + (table (lr-runtime-table runtime)) + (widths (lr-runtime-reduction-widths runtime)) + (modes (lr-runtime-lexical-modes runtime)) + (reduction-plans (lr-runtime-reduction-plans runtime)) + (actions-table (lr-runtime-actions runtime)) + (select-action (lr-runtime-action-selector runtime)) + (goto-index (lr-runtime-goto-index runtime)) + (goto-columns (lr-runtime-goto-columns runtime)) + (tokens + (if feed-token + (cons feed-token (lr-checkpoint-tokens checkpoint)) + (lr-checkpoint-tokens checkpoint))) + (input-end-offset + (if feed-token + (max (lr-checkpoint-input-end-offset checkpoint) + (token-end feed-token)) + (lr-checkpoint-input-end-offset checkpoint)))) + (with-lr-stack-storage policy + (lr-checkpoint-states checkpoint) (lr-checkpoint-semantic-values checkpoint) + (state-storage value-storage initial-states initial-values) + (def (fallback states semantic-values rest actions shifts) + (when (current-lr-recognition-observer) + ((current-lr-recognition-observer) #f)) + (let-values + (((root remaining _receipt) + (call-with-parser-observed-phase + observability 'selective-glr-execution + (lambda () + (lr-parse/prepared/receipt + runtime tokens (current-lr-branch-budget) + (lr-stack-freeze-states policy states state-storage) + (lr-stack-freeze-values policy semantic-values states value-storage) (lr-input-list scalar? rest) + actions shifts))))) + (values 'accepted (list root remaining)))) + (let loop ((states initial-states) + (semantic-values initial-values) + (rest (lr-input-start scalar? (if feed-token + (list feed-token) + (lr-checkpoint-rest checkpoint)))) + (actions (lr-checkpoint-deterministic-actions checkpoint)) + (shifts (lr-checkpoint-deterministic-shifts checkpoint)) + (remaining-budget (lr-stack-policy-value policy action-budget #f)) + (recognition-stack (lr-checkpoint-recognition-stack checkpoint))) + (if (lr-stack-policy-value policy + (or (and remaining-budget (zero? remaining-budget)) + (and shift-target (>= shifts shift-target))) #f) + (values + 'checkpoint + (make-lr-checkpoint + runtime tokens input-end-offset + (lr-stack-freeze-states policy states state-storage) + (lr-stack-freeze-values policy semantic-values states value-storage) + (lr-input-list scalar? rest) actions shifts recognition-stack)) + (let* ((state (lr-stack-current policy states state-storage)) + (rest + (if (and next-input (lr-input-empty? scalar? rest)) + (let (input-token + (if next-input-state? + (next-input + (vector-ref modes state) + state) + (next-input + (vector-ref modes state)))) + (if input-token + (begin + (set! tokens (cons input-token tokens)) + (set! input-end-offset + (max input-end-offset + (token-end input-token))) + (lr-input-admit scalar? input-token)) + (lr-input-admit scalar? #f))) + rest)) + (action-row + (select-action state (and (not (lr-input-empty? scalar? rest)) (lr-input-token scalar? rest))))) + (if (not action-row) + (if stop-at-failure? + (values + 'failure + (make-lr-failure-frontier + (make-lr-checkpoint + runtime tokens input-end-offset + (lr-stack-freeze-states policy states state-storage) + (lr-stack-freeze-values policy semantic-values states value-storage) + (lr-input-list scalar? rest) actions shifts recognition-stack) + state + (map car (vector-ref actions-table state)))) + (fallback states semantic-values rest actions shifts)) + (let (action (cdr action-row)) + (case (car action) + ((shift) + (if (lr-stack-policy-value policy stop-before-shift? #f) + (values 'checkpoint + (make-lr-checkpoint runtime tokens input-end-offset + (lr-stack-freeze-states policy states state-storage) + (lr-stack-freeze-values policy semantic-values states value-storage) + (lr-input-list scalar? rest) actions shifts recognition-stack)) + (if (not (lr-input-empty? scalar? rest)) + (let ((next-states (lr-stack-push-state policy (cadr action) states state-storage value-storage)) + (next-values + (lr-stack-shift-value policy + (if event-semantics? (lr-input-token scalar? rest) (list (make-recognition-child #f (lr-input-token scalar? rest)))) + semantic-values states value-storage)) + (next-actions (fx+ actions 1)) + (next-shifts (fx+ shifts 1))) + (when selected-observer + (lr-stack-notify-shift policy selected-observer (lr-input-token scalar? rest) next-states next-values + next-actions next-shifts)) + (let (next-recognition + (and recognition-stack (cons (lr-input-token scalar? rest) recognition-stack))) + ;; Sparse captures freeze exactly the same post-shift + ;; frontier; other shifts allocate no checkpoint. + (when (and capture-snapshot (zero? (modulo next-shifts snapshot-spacing))) + (capture-snapshot + (make-lr-checkpoint runtime tokens input-end-offset + (lr-stack-freeze-states policy next-states state-storage) + (lr-stack-freeze-values policy next-values next-states value-storage) + (lr-input-list scalar? (lr-input-tail scalar? rest)) next-actions next-shifts next-recognition))) + (loop next-states next-values (lr-input-tail scalar? rest) + next-actions next-shifts + (lr-stack-policy-value policy (and remaining-budget (fx- remaining-budget 1)) #f) + next-recognition))) + (fallback states semantic-values rest actions shifts)))) + ((reduce) + (let (production-id (cadr action)) + (if direct-step + (let-values (((target next-states next-values) + (direct-step + production-id states semantic-values (lr-input-list scalar? rest) + input-end-offset goto-index))) + (if target + (loop next-states next-values rest + (fx+ actions 1) shifts + (lr-stack-policy-value policy (and remaining-budget (fx- remaining-budget 1)) #f) + (and recognition-stack + (capture-reduction + (vector-ref capture-plans production-id) direct-step + (lr-stack-policy-value policy + (list-ref states (vector-ref widths production-id)) + (vector-ref state-storage (fx- states (vector-ref widths production-id)))) + recognition-stack (and (not (lr-input-empty? scalar? rest)) (lr-input-token scalar? rest)) + (if (not (lr-input-empty? scalar? rest)) (token-start (lr-input-token scalar? rest)) input-end-offset) + (lr-stack-value policy next-values next-states value-storage) target))) + (fallback states semantic-values rest + actions shifts))) + (let* ((production (vector-ref table production-id)) + (count (vector-ref widths production-id))) + (with-pop-driver-reduction policy with-pop-runtime-reduction + states semantic-values count event-semantics? + (source-values remaining-values remaining-states) + (let* ((offset + (if (not (lr-input-empty? scalar? rest)) (token-start (lr-input-token scalar? rest)) + input-end-offset)) + (value + (reduce-driver-plan policy reduce-prepared-plan reduce-lr-vector-plan + (vector-ref reduction-plans production-id) + source-values offset event-semantics? value-storage)) + (target + (and (lr-stack-present? policy remaining-states) + (if goto-columns + (vector-ref (vector-ref goto-columns production-id) + (lr-stack-current policy remaining-states state-storage)) + (goto-target goto-index + (lr-stack-current policy remaining-states state-storage) + (production-lhs production)))))) + (if target + (loop + (lr-stack-push-state policy target remaining-states state-storage value-storage) + (lr-stack-push-value policy value remaining-values remaining-states states value-storage) + rest (fx+ actions 1) shifts + (lr-stack-policy-value policy (and remaining-budget (fx- remaining-budget 1)) #f) + (and recognition-stack + (capture-reduction (vector-ref capture-plans production-id) #f + (lr-stack-current policy remaining-states state-storage) recognition-stack (and (not (lr-input-empty? scalar? rest)) (lr-input-token scalar? rest)) offset value target))) + (fallback + states semantic-values rest + actions shifts)))))))) + ((fork) + (if stop-at-fork? + (begin + (when (current-lr-recognition-observer) + ((current-lr-recognition-observer) #f)) + (values + 'fork + (make-lr-checkpoint + runtime tokens input-end-offset + (lr-stack-freeze-states policy states state-storage) + (lr-stack-freeze-values policy semantic-values states value-storage) + (lr-input-list scalar? rest) actions shifts #f))) + (fallback states semantic-values rest actions shifts))) + ((accept) + (let (child + (and (lr-semantic-stack-present? policy semantic-values states) + (recognition-sequence-root-child + (lr-stack-value policy semantic-values states value-storage)))) + (if child + (begin + (when (current-lr-recognition-observer) + ((current-lr-recognition-observer) + (and recognition-stack (pair? recognition-stack) + (null? (cdr recognition-stack)) + (car recognition-stack)))) + (values + 'accepted + (list (recognition-child-value child) (lr-input-list scalar? rest)))) + (fallback + states semantic-values rest actions shifts)))) + (else + (fallback states semantic-values rest actions shifts)))))))))))) + +;;; One transition body has two compile-time storage policies. Immutable +;;; checkpoints and arbitrary stack observers retain persistent frontiers; +;;; unbounded private recognition may own reusable vectors within one call. +(define-lr-driver #f #f lr-run-persistent-checkpoint) +(define-lr-driver #f #t lr-run-stream-checkpoint) + +(define-lr-driver #t #f lr-run-owned-checkpoint) +(define-lr-driver #t #t lr-run-owned-stream-checkpoint) + +(def (lr-run-checkpoint (checkpoint : lr-checkpoint) action-budget observability + (stop-at-failure? #f) (shift-target #f) + (stop-at-fork? #f) (feed-token #f) + (next-input #f) (after-shift #f) + (direct-step-override 'installed) + (next-input-state? #f) (stop-before-shift? #f) + (snapshot-spacing #f) (capture-snapshot #f)) + (let* ((runtime (lr-checkpoint-runtime checkpoint)) + (stream? (and next-input (not feed-token) (null? (lr-checkpoint-rest checkpoint)) + ;; Generated reduction steps consume a borrowed list. + ;; Do not rebuild it on every reduction query. + (if (eq? direct-step-override 'installed) + (not (lr-runtime-direct-step runtime)) + (not direct-step-override))))) + ;; Keep each generated executor call syntactically direct. The SDK's + ;; procedure-join specialization must not choose an executor for this owner. + (defrule (invoke-driver driver) + (driver checkpoint action-budget observability stop-at-failure? shift-target + stop-at-fork? feed-token next-input after-shift direct-step-override next-input-state? stop-before-shift? + snapshot-spacing capture-snapshot)) + (if (and (lr-runtime-owned-stack? runtime) (not action-budget) (not shift-target) + (not observability) (not stop-before-shift?) + (or (eq? direct-step-override 'installed) (not direct-step-override)) + (or (not after-shift) (lr-token-observer? after-shift))) + (if stream? + (invoke-driver lr-run-owned-stream-checkpoint) + (invoke-driver lr-run-owned-checkpoint)) + (if stream? + (invoke-driver lr-run-stream-checkpoint) + (invoke-driver lr-run-persistent-checkpoint))))) + +;;; Advances through the same executor and materializes one checkpoint only at +;;; the requested deterministic action boundary. +(def (lr-checkpoint-advance checkpoint (action-budget 1) + (observability #f)) + (unless (and (integer? action-budget) (positive? action-budget)) + (error "LR checkpoint action budget must be positive" action-budget)) + (lr-run-checkpoint checkpoint action-budget observability)) + +;;; Advances by consumed significant tokens rather than raw actions. This is +;;; the stable boundary used by incremental parsing because reductions do not +;;; consume source input. +(def (lr-checkpoint-advance-shifts checkpoint (shift-budget 1) + (observability #f)) + (unless (and (integer? shift-budget) (positive? shift-budget)) + (error "LR checkpoint shift budget must be positive" shift-budget)) + (lr-run-checkpoint + checkpoint #f observability #f + (+ (lr-checkpoint-deterministic-shifts checkpoint) shift-budget))) + +;;; Feeds exactly one significant token through reductions and one shift. If +;;; reductions expose a selective-GLR cell first, returns its exact immutable +;;; frontier instead of starting GLR with a truncated one-token suffix. +(def (lr-checkpoint-feed checkpoint input-token (observability #f)) + (lr-run-checkpoint + checkpoint #f observability #f + (fx+ (lr-checkpoint-deterministic-shifts checkpoint) 1) + #t input-token)) + diff --git a/src/runtime/lr-parser/entry.ss b/src/runtime/lr-parser/entry.ss new file mode 100644 index 00000000..19c7a14e --- /dev/null +++ b/src/runtime/lr-parser/entry.ss @@ -0,0 +1,27 @@ +;;; -*- Gerbil -*- +;;; lr-parser: entry owner. Private execution state and named algorithms. + +(import (only-in "./state.ss" + lr-runtime-layout?) + (only-in "./checkpoint-base.ss" + lr-initial-checkpoint) + (only-in "./prepare.ss" + lr-prepare) + (only-in "./batch.ss" + lr-parse/prepared/receipt) + (only-in "./transfer.ss" + lr-checkpoint-resume)) +(export lr-parse/prepared + lr-parse) + +(def (lr-parse/prepared runtime tokens (observability #f)) + (if (lr-runtime-layout? runtime) + (let-values (((root rest _receipt) + (lr-parse/prepared/receipt runtime tokens))) + (values root rest)) + (lr-checkpoint-resume + (lr-initial-checkpoint runtime tokens) observability))) + +(def (lr-parse spec tokens) + (lr-parse/prepared (lr-prepare spec) tokens)) + diff --git a/src/runtime/lr-parser/prepare.ss b/src/runtime/lr-parser/prepare.ss new file mode 100644 index 00000000..0a7419f6 --- /dev/null +++ b/src/runtime/lr-parser/prepare.ss @@ -0,0 +1,246 @@ +;;; -*- Gerbil -*- +;;; lr-parser: prepare owner. Private execution state and named algorithms. + +(import (only-in ../reduction-plan + prepare-lr-reduction-plans) + (only-in ../../compiler/lr-table + validate-lr-tables layout-productions?) + (only-in :std/vector/vector + vector-map/index) + (only-in :core/poo-clos/interface + .defgeneric .defmethod) + (only-in ../../compiler/lr + lr-spec-ref production-action validate-production-semantics production-precedence production-rhs production-table production-lhs) + (only-in ../funcs + association-row-vector->index vector-intern-map) + (only-in ../lr-action-index + index-action-rows lr-action-row-has-literals? lookup-literal-action-entry lr-action-row-eof lr-action-row-tokens lookup-casefolded-literal-action-entry) + (only-in ../layout + prepare-layout-action-selector) + (only-in ../token + token-kind token-lexeme) + (only-in "./state.ss" + make-lr-runtime lr-runtime? lr-runtime-productions lr-runtime-table lr-runtime-reduction-widths lr-runtime-actions lr-runtime-action-index lr-runtime-gotos lr-runtime-goto-index lr-runtime-goto-columns lr-runtime-case-insensitive? lr-runtime-dynamic? lr-runtime-layout? lr-runtime-lexical-modes lr-runtime-lexical-mode-catalog lr-runtime-direct-step lr-runtime-direct-step-set! lr-runtime-semantic-reducer lr-runtime-event-reducer lr-runtime-event-step lr-runtime-event-step-set! lr-runtime-event-runtime lr-runtime-event-runtime-set! lr-runtime-reduction-plans lr-runtime-action-selector lr-runtime-owned-stack?-set! current-lr-event-program-enabled? make-lr-lexical-mode) + (only-in "./reduction.ss" + reduce-value reduce-value/events)) +(export install-lr-runtime-direct-step! + select-lr-semantic-reducer + lr-semantic-capability + lr-runtime-event-program? + install-lr-runtime-event-step! + lr-runtime-for-current-semantic-backend + lr-prepare + prepare-token-action-lookup + make-lr-action-selector + lr-runtime-with-action-selector + prepare-lr-goto-columns + goto-target) + +;;; Install a generated reduction step once, before the runtime is shared. +(def (install-lr-runtime-direct-step! runtime step) + (unless (and (lr-runtime? runtime) + (procedure? step) + (not (lr-runtime-direct-step runtime))) + (error "invalid generated LR reduction step")) + (lr-runtime-direct-step-set! runtime step) + (lr-runtime-owned-stack?-set! runtime #f) + (lr-runtime-event-runtime-set! runtime #f)) + +;;; Closed preparation protocol: ordinary values specialize two independent +;;; axes. Capture the selected procedures in the runtime, never redispatch in +;;; a request or reduction. These methods are private and cannot evolve through +;;; a public extension API; retained runtimes own their admitted executors. +(.defgeneric select-lr-semantic-reducer (backend capability)) + +(.defmethod (select-lr-semantic-reducer (backend (eql 'recognition)) capability) + reduce-value) + +(.defmethod (select-lr-semantic-reducer (backend (eql 'event)) capability) + #f) + +(.defmethod (select-lr-semantic-reducer (backend (eql 'event)) + (capability (eql 'plain))) + reduce-value/events) + +;;; Grammar eligibility belongs to preparation, including negative admission. +;;; It is independent of the request's optional backend preference. +(def (lr-semantic-capability table actions dynamic? layout?) + (cond + (layout? 'layout) + (dynamic? 'dynamic) + ((vector-any (lambda (row) + (any (lambda (entry) (eq? (cadr entry) 'fork)) row)) actions) + 'branching) + ((vector-any (lambda (production) + (not (memq (production-action production) '(pass concat)))) table) + 'general) + (else 'plain))) + +(def (lr-runtime-event-program? runtime) + (eq? (lr-runtime-semantic-reducer runtime) reduce-value/events)) + +(def (install-lr-runtime-event-step! runtime step) + (unless (and (lr-runtime? runtime) (procedure? step) + (not (lr-runtime-event-step runtime)) + (not (lr-runtime-event-program? runtime))) + (error "invalid generated LR event step")) + (lr-runtime-event-step-set! runtime step) + (lr-runtime-event-runtime-set! runtime #f)) + +;;; The backend is selected at the prepared-runtime boundary. Checkpoints and +;;; retained fragments keep that runtime identity across edits; no semantic +;;; action reads a dynamic switch. Only deterministic pass/concat LR is admitted. +(def (lr-runtime-for-current-semantic-backend runtime) + (cond + ((or (not (current-lr-event-program-enabled?)) + (lr-runtime-event-program? runtime)) runtime) + ;; Check the selected runtime before scanning grammar eligibility again. + ((lr-runtime-event-runtime runtime) => identity) + ((not (lr-runtime-event-reducer runtime)) runtime) + (else + (let (selected + (make-lr-runtime + (lr-runtime-productions runtime) (lr-runtime-table runtime) + (lr-runtime-reduction-widths runtime) (lr-runtime-actions runtime) + (lr-runtime-action-index runtime) (lr-runtime-gotos runtime) + (lr-runtime-goto-index runtime) (lr-runtime-case-insensitive? runtime) + #f #f (lr-runtime-lexical-modes runtime) + (lr-runtime-lexical-mode-catalog runtime) + (lr-runtime-event-step runtime) (lr-runtime-event-reducer runtime) #f #f #f + (prepare-lr-reduction-plans 'event (lr-runtime-table runtime)) + (lr-runtime-action-selector runtime) #f (lr-runtime-goto-columns runtime) (void))) + (lr-runtime-event-runtime-set! runtime selected) + selected)))) + +(def (lr-prepare spec) + (let* ((productions (validate-production-semantics (lr-spec-ref spec 'productions))) + (actions (lr-spec-ref spec 'actions)) + (gotos (lr-spec-ref spec 'gotos)) + (dynamic? + (any (lambda (production) + (let (precedence (production-precedence production)) + (and precedence (eq? (car precedence) 'dynamic)))) + productions)) + (layout? (layout-productions? productions))) + (validate-lr-tables productions actions gotos) + (let-values (((modes mode-catalog) + (vector-intern-map + actions + (lambda (row) (map car row)) + (lambda (terminals id) + (make-lr-lexical-mode id terminals))))) + (let* ((table (production-table productions)) + (plans (prepare-lr-reduction-plans 'recognition table)) + (capability (lr-semantic-capability table actions dynamic? layout?)) + (case-insensitive? (lr-spec-ref spec 'case-insensitive?)) + (action-index (index-action-rows actions case-insensitive?)) + (goto-index (association-row-vector->index gotos))) + (make-lr-runtime + productions + table + (vector-map/index + (lambda (_index production) + (length (production-rhs production))) table) + actions + action-index + gotos + goto-index + (lr-spec-ref spec 'case-insensitive?) + dynamic? + layout? + modes + mode-catalog + #f (select-lr-semantic-reducer 'recognition capability) + (select-lr-semantic-reducer 'event capability) #f #f plans + (make-lr-action-selector actions action-index case-insensitive? layout?) + (not layout?) (prepare-lr-goto-columns table goto-index) (void)))))) + +;;; Token terminals are admitted symbols. Specialize the owned row shape once; +;;; first duplicate entries and exact action-pair identity remain unchanged. +(def (prepare-token-action-lookup entries) + (cond + ((null? entries) (lambda (_kind) #f)) + ((and (pair? entries) (null? (cdr entries))) + (let ((kind (caar entries)) (entry (cdar entries))) + (lambda (candidate) (and (eq? kind candidate) entry)))) + ((pair? entries) + (lambda (kind) (let (entry (assq kind entries)) (and entry (cdr entry))))) + (else (lambda (kind) (table-ref entries kind #f))))) + +(def (make-lr-action-selector rows index case-insensitive? layout?) + (if layout? + (prepare-layout-action-selector rows index case-insensitive?) + (let (selectors + (vector-map/index + (lambda (_state row) + (let ((eof (lr-action-row-eof row)) (lookup-token (prepare-token-action-lookup (lr-action-row-tokens row)))) + (if (lr-action-row-has-literals? row) + (lambda (input) + (if (not input) eof + (let (token input) + (or (lookup-literal-action-entry row (token-lexeme token)) + (and case-insensitive? + (lookup-casefolded-literal-action-entry row (token-lexeme token))) + (lookup-token (token-kind token)))))) + (lambda (input) + (if (not input) eof + (lookup-token (token-kind input))))))) index)) + (lambda (state input) ((vector-ref selectors state) input))))) + +;;; Qualification/extension preparation replaces only the action lookup +;;; primitive. Immutable grammar tables, semantic plans and modes stay owned. +;;; Generated steps cannot bypass the supplied selection procedure. +(def (lr-runtime-with-action-selector runtime factory) + (unless (procedure? factory) (error "LR action selector factory must be a procedure")) + (let (selector (factory (lr-runtime-actions runtime) (lr-runtime-action-index runtime) + (lr-runtime-case-insensitive? runtime) (lr-runtime-layout? runtime))) + (unless (procedure? selector) (error "LR action selector factory must produce a procedure")) + (make-lr-runtime + (lr-runtime-productions runtime) (lr-runtime-table runtime) + (lr-runtime-reduction-widths runtime) (lr-runtime-actions runtime) + (lr-runtime-action-index runtime) (lr-runtime-gotos runtime) (lr-runtime-goto-index runtime) + (lr-runtime-case-insensitive? runtime) (lr-runtime-dynamic? runtime) (lr-runtime-layout? runtime) + (lr-runtime-lexical-modes runtime) (lr-runtime-lexical-mode-catalog runtime) + #f (lr-runtime-semantic-reducer runtime) (lr-runtime-event-reducer runtime) #f #f + (lr-runtime-reduction-plans runtime) selector + (and (not (lr-runtime-layout? runtime)) + (not (lr-runtime-event-program? runtime))) + (lr-runtime-goto-columns runtime) (void)))) + +;; goto-target +;; : (-> (Vector (Or (List Pair) HashTable)) Fixnum Symbol (OrFalse Fixnum)) +;;; Canonical production admission requires symbolic nonterminals. Stage the +;;; private lookup into each transition: symbol equality is identity equality, +;;; and the original first entry remains authoritative. General association +;;; indexes retain their equal?-keyed contract. +(defrule (goto-target gotos-expr state-expr lhs-expr) + (let* ((gotos gotos-expr) (state state-expr) (lhs lhs-expr) + (index (vector-ref gotos state)) + (entry (if (or (pair? index) (null? index)) + (assq lhs index) (hash-get index lhs)))) + (and entry (cdr entry)))) + + +;;; Preparation owns these columns. Share each LHS column across productions; +;;; cap retained cells before allocation and keep sparse rows above the budget. +(def (prepare-lr-goto-columns table gotos (cell-budget 65536)) + (unless (and (exact-integer? cell-budget) (>= cell-budget 0)) + (error "goto column cell budget must be a nonnegative integer")) + (let ((by-lhs (make-table test: eq?)) (keys '())) + (vector-for-each + (lambda (production) + (let (lhs (production-lhs production)) + (unless (table-ref by-lhs lhs #f) + (table-set! by-lhs lhs #t) + (set! keys (cons lhs keys))))) table) + (and (<= (* (length keys) (vector-length gotos)) cell-budget) + (begin + (for-each + (lambda (lhs) + (let (column (make-vector (vector-length gotos) #f)) + (let loop ((state 0)) + (when (< state (vector-length gotos)) + (vector-set! column state (goto-target gotos state lhs)) + (loop (+ state 1)))) + (table-set! by-lhs lhs column))) keys) + (vector-map/index (lambda (_id production) (table-ref by-lhs (production-lhs production))) table))))) diff --git a/src/runtime/lr-parser/recognition.ss b/src/runtime/lr-parser/recognition.ss new file mode 100644 index 00000000..3d2d87d5 --- /dev/null +++ b/src/runtime/lr-parser/recognition.ss @@ -0,0 +1,259 @@ +;;; -*- Gerbil -*- +;;; lr-parser: recognition owner. Private execution state and named algorithms. + +(import (only-in ../recognition + make-recognition-child make-recognition-fragment) + (only-in ../funcs + current-recognition-sequence-fusion-enabled?) + (only-in ../token + make-token token? token-end token-kind token-lexeme token-start) + (only-in "./state.ss" + make-lr-capture-children lr-capture-plan make-lr-capture-plan lr-runtime-capture-plans lr-runtime-capture-plans-set! lr-runtime-actions make-lr-capture-context lr-capture-context lr-capture-context-executor lr-capture-context-exit-state lr-runtime-table lr-runtime-reduction-widths make-lr-recognition-fragment lr-recognition-fragment lr-recognition-fragment? lr-recognition-fragment-runtime lr-recognition-fragment-production-id lr-recognition-fragment-children lr-recognition-fragment-start lr-recognition-fragment-end lr-recognition-fragment-token-count lr-recognition-fragment-offset lr-recognition-fragment-yield-cache lr-recognition-fragment-yield-cache-set! make-lr-recognition-view lr-recognition-view? lr-recognition-view-base lr-recognition-view-delta make-lr-terminal-yield-branch make-lr-terminal-yield-view lr-terminal-yield-view? lr-terminal-yield-view-base lr-terminal-yield-view-delta) + (only-in "./reduction.ss" + reduce-value)) +(export prepare-capture-plans capture-context-from-plan + lr-recognition-fragment-terminal-yield + lr-recognition-relocate + terminal-yield-append + terminal-yield-relocate + recognition-piece-terminal-yield + recognition-piece-start + recognition-piece-end + recognition-piece-token-count + capture-reduction + lr-recognition-project + lr-recognition-project/materialized) + +(def (lr-recognition-relocate piece delta) + (if (lr-recognition-view? piece) + (make-lr-recognition-view (lr-recognition-view-base piece) + (+ delta (lr-recognition-view-delta piece))) + (make-lr-recognition-view piece delta))) + +(def (terminal-yield-append left right) + (cond ((not left) right) ((not right) left) + (else (make-lr-terminal-yield-branch left right)))) + +(def (terminal-yield-relocate yield delta) + (cond ((or (not yield) (zero? delta)) yield) + ((lr-terminal-yield-view? yield) + (make-lr-terminal-yield-view (lr-terminal-yield-view-base yield) + (+ delta (lr-terminal-yield-view-delta yield)))) + (else (make-lr-terminal-yield-view yield delta)))) + +(def (recognition-piece-terminal-yield piece) + (cond ((lr-recognition-view? piece) + (terminal-yield-relocate + (recognition-piece-terminal-yield (lr-recognition-view-base piece)) + (lr-recognition-view-delta piece))) + ((token? piece) piece) + (else (lr-recognition-fragment-terminal-yield piece)))) + +(def (recognition-piece-start piece) + (cond ((lr-recognition-view? piece) + (+ (lr-recognition-view-delta piece) + (recognition-piece-start (lr-recognition-view-base piece)))) + ((token? piece) (token-start piece)) + (else (lr-recognition-fragment-start piece)))) + +(def (recognition-piece-end piece) + (cond ((lr-recognition-view? piece) + (+ (lr-recognition-view-delta piece) + (recognition-piece-end (lr-recognition-view-base piece)))) + ((token? piece) (token-end piece)) + (else (lr-recognition-fragment-end piece)))) + +(def (recognition-piece-token-count piece) + (cond ((lr-recognition-view? piece) + (recognition-piece-token-count (lr-recognition-view-base piece))) + ((token? piece) 1) + (else (lr-recognition-fragment-token-count piece)))) + +;;; Terminal transfer queries own this derived index. Fresh recognition already +;;; retains immutable grammar topology and exact lookahead; building a second +;;; terminal tree before anyone requests it adds allocation and GC pressure. +;;; The cache is published only when complete. An explicit postorder stack +;;; handles deep left-recursive grammar trees without a Scheme call chain. +(def (lr-recognition-fragment-terminal-yield fragment) + (when (void? (lr-recognition-fragment-yield-cache fragment)) + (let walk ((pending (list (cons fragment #f)))) + (unless (null? pending) + (let* ((frame (car pending)) (piece (car frame)) (ready? (cdr frame)) + (rest (cdr pending))) + (cond + ((lr-recognition-view? piece) + (walk (cons (cons (lr-recognition-view-base piece) #f) rest))) + ((token? piece) (walk rest)) + ((not (void? (lr-recognition-fragment-yield-cache piece))) (walk rest)) + (ready? + (lr-recognition-fragment-yield-cache-set! piece + (foldl (lambda (child yield) + (terminal-yield-append yield (recognition-piece-terminal-yield child))) + #f (lr-recognition-fragment-children piece))) + (walk rest)) + (else + (walk (append (map (lambda (child) (cons child #f)) + (lr-recognition-fragment-children piece)) + (cons (cons piece #t) rest))))))))) + (lr-recognition-fragment-yield-cache fragment)) + +(defrule (with-capture-bounds expression (start end tokens) body ...) + (let (piece expression) + (cond + ((lr-recognition-view? piece) + (let ((start (recognition-piece-start piece)) (end (recognition-piece-end piece)) + (tokens (recognition-piece-token-count piece))) body ...)) + ((token? piece) + (let ((start (token-start piece)) (end (token-end piece)) (tokens 1)) body ...)) + (else + ;; Official checked class binding admits the final record once; its + ;; slot reads then share that fact, as std/struct/pqueue does. + (using (piece : lr-recognition-fragment) + (let ((start piece.start) (end piece.end) (tokens piece.token-count)) body ...)))))) + +;;; A prepared immutable production plan carries static capture input once. +;;; Only the bounded row cells mutate. Uncaptured execution prepares nothing. +(def (prepare-capture-plans runtime) + (let (cached (lr-runtime-capture-plans runtime)) + (if (void? cached) + (let* ((width (vector-length (lr-runtime-table runtime))) + (states (vector-length (lr-runtime-actions runtime))) + (widths (lr-runtime-reduction-widths runtime)) + (indexed? (<= (* 2 width states) 65536)) + (plans (list->vector + (map (lambda (production) + (make-lr-capture-plan runtime production (vector-ref widths production) + (and indexed? (make-vector (* 2 states) #f)))) + (iota width))))) + (lr-runtime-capture-plans-set! runtime plans) plans) + cached))) + +(definline (capture-context-from-plan (plan : lr-capture-plan) executor entry target kind) => :t + ;; Disjoint halves preserve custom reuse without weakening canonical admission. + ;; Only canonical goto transitions enter the first half of a bounded row. + (if executor + (let* ((row plan.contexts) + (index (and row + (let (states (quotient (vector-length row) 2)) + (unless (and (exact-integer? entry) (<= 0 entry) (< entry states)) + (error "custom capture entry is outside its prepared row" entry)) + (+ entry states)))) + (context (and row (vector-ref row index)))) + (if (and context + (using (context : lr-capture-context) + (and (eq? executor context.executor) (eqv? target context.exit-state) + (eq? kind context.lookahead-kind)))) + context + (let (next (make-lr-capture-context plan.runtime executor plan.production-id entry target kind)) + (when row (vector-set! row index next)) next))) + (let* ((row plan.contexts) (context (and row (vector-ref row entry)))) + (if (and context (using (context : lr-capture-context) (eq? kind context.lookahead-kind))) + context + (let (next (make-lr-capture-context plan.runtime #f plan.production-id entry target kind)) + (when row (vector-set! row entry next)) next))))) + +(definline (capture-reduction plan executor entry-state stack input offset value target) => :t + (if (not stack) #f + (using (plan : lr-capture-plan) + (let* ((count plan.width) + (kind (and input (token-kind input))) + (lookahead (and input (token-lexeme input)))) + ;; The pending boundary representation reserves false for EOF and pairs + ;; for a published signature. Admit its live token values before storage. + (when (and input (not (and (symbol? kind) (string? lookahead)))) + (error "capture lookahead requires symbolic kind and string lexeme" input)) + (let (context (capture-context-from-plan plan executor entry-state target kind)) + (case count + ((0) + (cons (make-lr-recognition-fragment + context '() lookahead value + offset offset 0 offset #f) stack)) + ((1) + (let (piece (car stack)) + (with-capture-bounds piece (start end tokens) + (cons (make-lr-recognition-fragment + context piece lookahead value + start end tokens offset (void)) (cdr stack))))) + ((2 3) + (let split ((left count) (remaining stack) (children '()) + (tokens 0) (start offset) (end offset)) + (if (zero? left) + (cons (make-lr-recognition-fragment context children lookahead value + start end tokens offset (void)) remaining) + (let (piece (car remaining)) + (with-capture-bounds piece (piece-start piece-end piece-count) + (split (- left 1) (cdr remaining) (cons piece children) + (+ tokens piece-count) piece-start + (if (= left count) piece-end end))))))) + (else + ;; Copy only child references into one source-order array. Unlike a + ;; borrowed stack prefix, this cannot pin an unrelated stack suffix. + (let (children (make-vector count)) + (let split ((left count) (remaining stack) + (tokens 0) (start offset) (end offset)) + (if (zero? left) + (cons (make-lr-recognition-fragment + context (make-lr-capture-children children) lookahead value + start end tokens offset (void)) remaining) + (let (piece (car remaining)) + (with-capture-bounds piece (piece-start piece-end piece-count) + (vector-set! children (- left 1) piece) + (split (- left 1) (cdr remaining) + (+ tokens piece-count) piece-start + (if (= left count) piece-end end)))))))))))))) + +;;; Iterative postorder projection reevaluates semantic actions from retained +;;; productions and token leaves. It deliberately does not trust cached values. +;;; Left-recursive trees therefore do not consume the Scheme call stack. +;;; Keep the independent replay on the materialized semantic path. +(def (lr-recognition-project root (source-tokens #f)) + (parameterize ((current-recognition-sequence-fusion-enabled? #f)) + (lr-recognition-project/materialized root source-tokens))) + +(def (lr-recognition-project/materialized root source-tokens) + (unless (or (lr-recognition-fragment? root) (lr-recognition-view? root)) + (error "recognition projection requires a grammar fragment" root)) + (let (tokens-by-start (and source-tokens (make-table test: eqv?))) + (when tokens-by-start + (for-each (lambda (token) (table-set! tokens-by-start (token-start token) token)) + source-tokens)) + (let loop ((pending (list (vector #f root 0))) (values-stack '())) + (if (null? pending) + (if (and (pair? values-stack) (null? (cdr values-stack))) + (car values-stack) (error "invalid recognition projection stack")) + (let* ((frame (car pending)) (piece (vector-ref frame 1)) + (delta (vector-ref frame 2))) + (cond + ((lr-recognition-view? piece) + (loop (cons (vector #f (lr-recognition-view-base piece) + (+ delta (lr-recognition-view-delta piece))) (cdr pending)) + values-stack)) + ((token? piece) + (let (token + (if tokens-by-start + (table-ref tokens-by-start (+ delta (token-start piece)) #f) + (if (zero? delta) piece + (make-token (token-kind piece) (token-lexeme piece) + (+ delta (token-start piece)) (+ delta (token-end piece)))))) + (unless (and token (eq? (token-kind token) (token-kind piece)) + (equal? (token-lexeme token) (token-lexeme piece)) + (= (token-end token) (+ delta (token-end piece)))) + (error "grammar position view does not bind the current token")) + (loop (cdr pending) + (cons (list (make-recognition-child #f token)) values-stack)))) + ((vector-ref frame 0) + (let* ((count (length (lr-recognition-fragment-children piece))) + (children (reverse (take values-stack count))) + (runtime (lr-recognition-fragment-runtime piece)) + (production (vector-ref (lr-runtime-table runtime) + (lr-recognition-fragment-production-id piece))) + (value (reduce-value production children + (+ delta (lr-recognition-fragment-offset piece)) + make-recognition-fragment))) + (loop (cdr pending) (cons value (drop values-stack count))))) + (else + (loop (append (map (lambda (child) (vector #f child delta)) + (lr-recognition-fragment-children piece)) + (cons (vector #t piece delta) (cdr pending))) + values-stack)))))))) + diff --git a/src/runtime/lr-parser/reduction.ss b/src/runtime/lr-parser/reduction.ss new file mode 100644 index 00000000..f1c99251 --- /dev/null +++ b/src/runtime/lr-parser/reduction.ss @@ -0,0 +1,172 @@ +;;; -*- Gerbil -*- +;;; lr-parser: reduction owner. Private execution state and named algorithms. + +(import (only-in ../reduction-plan + reduce-lr-stack-plan reduce-lr-event-plan source-operand-offsets) + (only-in ../event-program + event-program-append) + (only-in ../event-reduce + event-children-field event-children-alias) + (only-in ../../compiler/lr + operand-actions production-action layout-end-action? production-rhs) + (only-in ../reduce + recognition-children-alias recognition-children-field) + (only-in ../funcs + recognition-sequence-append recognition-sequence-start recognition-sequence-for-action)) +(export apply-operand-action + define-operand-action-fold + apply-operand-actions + reduce-operands + reduce-value + apply-operand-action/events + apply-operand-actions/events + reduce-operands/events + reduce-value/events + reduce-prepared-plan + with-pop-reduction + reduction-stack-tail + reduction-source-values + with-pop-runtime-reduction) + +;; apply-operand-action +;; : (-> List List Fixnum List) +(def (apply-operand-action action children default-offset + fragment-constructor) + (let (materialized (recognition-sequence-for-action children)) + (case (car action) + ((field) + (recognition-children-field + (cadr action) materialized default-offset fragment-constructor)) + ((alias) + (recognition-children-alias (cadr action) materialized default-offset)) + (else (error "unknown LR operand action" action))))) + +;;; An admitted chain is an ordered fold, independent of its semantic backend. +;;; Stage the backend operation into a closed tail loop, as std/iter stages +;;; iterator dispatch. No request-time callback captures offset or constructor. +(defrule (define-operand-action-fold fold-name action-step) + (def (fold-name value actions offset constructor) + (if (pair? actions) + (fold-name (action-step (car actions) value offset constructor) + (cdr actions) offset constructor) + value))) + +(define-operand-action-fold apply-operand-actions apply-operand-action) + +;;; Keep the two-list reduction loop closed: offset and constructor travel +;;; as explicit parameters, so no callback captures them for each reduction. +;;; Like foldl2, consume the lists together in source order. +(def (reduce-operands operands values offset constructor children offsets index) + (if (and (pair? operands) (pair? values)) + (reduce-operands + (cdr operands) (cdr values) offset constructor + (recognition-sequence-append + children (apply-operand-actions + (car values) (operand-actions (car operands)) + (if offsets (vector-ref offsets index) offset) constructor)) + offsets (fx+ index 1)) + children)) + +;; reduce-value +;; : (-> List List Fixnum List) +(def (reduce-value production source-values default-offset + fragment-constructor) + (let* ((rhs (production-rhs production)) + (action (production-action production))) + (cond + ((and (eq? action 'pass) (pair? rhs) (null? (cdr rhs))) + (apply-operand-actions + (car source-values) (operand-actions (car rhs)) + default-offset fragment-constructor)) + ((or (eq? action 'concat) (eq? action 'pass) + (layout-end-action? action)) + (reduce-operands rhs source-values default-offset fragment-constructor '() + (and (any null? source-values) + (source-operand-offsets (length rhs) source-values default-offset recognition-sequence-start)) 0)) + (else (error "unknown LR semantic action" action))))) + +(defrule (apply-operand-action/events action-expr value offset ignored-constructor) + (let (action action-expr) + (case (car action) + ((field) (event-children-field (cadr action) value offset)) + ((alias) (event-children-alias (cadr action) value offset)) + (else (error "unknown event semantic action" action))))) + +(define-operand-action-fold apply-operand-actions/events apply-operand-action/events) + +(def (reduce-operands/events operands values offset children offsets index) + (if (and (pair? operands) (pair? values)) + (reduce-operands/events + (cdr operands) (cdr values) offset + (event-program-append + children (apply-operand-actions/events + (car values) (operand-actions (car operands)) + (if offsets (vector-ref offsets index) offset) #f)) + offsets (fx+ index 1)) + children)) + +(def (reduce-value/events production source-values offset ignored-constructor) + (let ((rhs (production-rhs production)) (action (production-action production))) + (cond + ((and (eq? action 'pass) (pair? rhs) (null? (cdr rhs))) + (apply-operand-actions/events (car source-values) + (operand-actions (car rhs)) offset #f)) + ((memq action '(pass concat)) + (reduce-operands/events rhs source-values offset #f + (and (any (lambda (value) (or (not value) (null? value))) source-values) + (source-operand-offsets (length rhs) source-values offset recognition-sequence-start)) 0)) + (else (error "unsupported event LR production" action))))) + +;;; Keep prepared-plan access outside the LR continuation's generated body. +(def (reduce-prepared-plan plan values offset events?) + (if events? + (reduce-lr-event-plan plan values offset #f) + (reduce-lr-stack-plan plan values offset))) + +;;; Bind the popped immutable stack prefixes directly into the reduction body. +;;; Both private reducers stop at the RHS width, so unary reductions borrow the +;;; first semantic cell. Wider reductions build a source-order value list. +;;; The named loop passes the suffixes as locals instead of returning an +;;; intermediate multiple-value container to an immediately unpacking caller. +(defrule (with-pop-reduction states semantic-values count + (source-values remaining-values remaining-states) + body ...) + (let* ((initial-states states) (initial-values semantic-values) + (width count) (unary? (eqv? width 1))) + (let pop ((remaining width) (remaining-states initial-states) + (remaining-values initial-values) + (source-values (if unary? initial-values '()))) + (if (zero? remaining) + (begin body ...) + (if (and (pair? remaining-states) (pair? remaining-values)) + (pop (fx- remaining 1) (cdr remaining-states) (cdr remaining-values) + (if unary? source-values + (cons (car remaining-values) source-values))) + (error "LR reduction exceeds parser stack" width)))))) + +;;; Recognition borrows the stack. Event programs retain source-order operands +;;; for their amortized sequential append builder; right-associated joins are +;;; semantically equivalent but need not have the same construction cost. +(def (reduction-stack-tail stack width) + (let loop ((stack stack) (remaining width)) + (cond ((zero? remaining) stack) + ((pair? stack) (loop (cdr stack) (fx- remaining 1))) + (else (error "LR reduction exceeds parser stack" width))))) + +(def (reduction-source-values stack width) + ;; The caller already validated the prefix. Unary event reductions borrow + ;; one value; wider event reductions retain their sequential append order. + (if (eqv? width 1) stack + (let loop ((stack stack) (remaining width) (values '())) + (if (zero? remaining) values + (loop (cdr stack) (fx- remaining 1) (cons (car stack) values)))))) + +(defrule (with-pop-runtime-reduction states semantic-values count events? + (source-values remaining-values remaining-states) + body ...) + (let* ((width count) (stack semantic-values) + (remaining-states (reduction-stack-tail states width)) + (remaining-values (reduction-stack-tail stack width)) + (source-values (if events? (reduction-source-values stack width) stack))) + body ...)) + diff --git a/src/runtime/lr-parser/state.ss b/src/runtime/lr-parser/state.ss new file mode 100644 index 00000000..97e81fbb --- /dev/null +++ b/src/runtime/lr-parser/state.ss @@ -0,0 +1,257 @@ +;;; -*- Gerbil -*- +;;; lr-parser: state owner. Private execution state and named algorithms. + +(export lr-capture-children make-lr-capture-children lr-capture-children? lr-capture-children-items + lr-capture-plan make-lr-capture-plan + lr-capture-plan-runtime lr-capture-plan-production-id lr-capture-plan-width lr-capture-plan-contexts + lr-capture-context make-lr-capture-context + lr-capture-context-runtime lr-capture-context-executor + lr-capture-context-production-id lr-capture-context-entry-state lr-capture-context-exit-state lr-capture-context-lookahead-kind + lr-runtime + make-lr-runtime + lr-runtime? + lr-runtime-productions + lr-runtime-productions-set! + lr-runtime-table + lr-runtime-table-set! + lr-runtime-reduction-widths + lr-runtime-reduction-widths-set! + lr-runtime-actions + lr-runtime-actions-set! + lr-runtime-action-index + lr-runtime-action-index-set! + lr-runtime-gotos + lr-runtime-gotos-set! + lr-runtime-goto-index + lr-runtime-goto-columns + lr-runtime-capture-plans + lr-runtime-capture-plans-set! + lr-runtime-goto-index-set! + lr-runtime-case-insensitive? + lr-runtime-case-insensitive?-set! + lr-runtime-dynamic? + lr-runtime-dynamic?-set! + lr-runtime-layout? + lr-runtime-layout?-set! + lr-runtime-lexical-modes + lr-runtime-lexical-modes-set! + lr-runtime-lexical-mode-catalog + lr-runtime-lexical-mode-catalog-set! + lr-runtime-direct-step + lr-runtime-direct-step-set! + lr-runtime-semantic-reducer + lr-runtime-semantic-reducer-set! + lr-runtime-event-reducer + lr-runtime-event-reducer-set! + lr-runtime-event-step + lr-runtime-event-step-set! + lr-runtime-event-runtime + lr-runtime-event-runtime-set! + lr-runtime-reduction-plans + lr-runtime-reduction-plans-set! + lr-runtime-action-selector + lr-runtime-action-selector-set! + lr-runtime-owned-stack? + lr-runtime-owned-stack?-set! + current-lr-event-program-enabled? + lr-lexical-mode + make-lr-lexical-mode + lr-lexical-mode? + lr-lexical-mode-id + lr-lexical-mode-id-set! + lr-lexical-mode-terminals + lr-lexical-mode-terminals-set! + lr-checkpoint + make-lr-checkpoint + lr-checkpoint? + lr-checkpoint-runtime + lr-checkpoint-runtime-set! + lr-checkpoint-tokens + lr-checkpoint-tokens-set! + lr-checkpoint-input-end-offset + lr-checkpoint-input-end-offset-set! + lr-checkpoint-states + lr-checkpoint-states-set! + lr-checkpoint-semantic-values + lr-checkpoint-semantic-values-set! + lr-checkpoint-rest + lr-checkpoint-rest-set! + lr-checkpoint-deterministic-actions + lr-checkpoint-deterministic-actions-set! + lr-checkpoint-deterministic-shifts + lr-checkpoint-deterministic-shifts-set! + lr-checkpoint-recognition-stack + lr-checkpoint-recognition-stack-set! + lr-prefix-snapshot + make-lr-prefix-snapshot + lr-prefix-snapshot? + lr-prefix-snapshot-runtime + lr-prefix-snapshot-runtime-set! + lr-prefix-snapshot-states + lr-prefix-snapshot-states-set! + lr-prefix-snapshot-semantic-values + lr-prefix-snapshot-semantic-values-set! + lr-prefix-snapshot-deterministic-actions + lr-prefix-snapshot-deterministic-actions-set! + lr-prefix-snapshot-deterministic-shifts + lr-prefix-snapshot-deterministic-shifts-set! + lr-prefix-snapshot-recognition-stack + lr-prefix-snapshot-recognition-stack-set! + lr-recognition-fragment + make-lr-recognition-fragment + lr-recognition-fragment? + lr-recognition-fragment-runtime + lr-recognition-fragment-executor + lr-recognition-fragment-production-id + lr-recognition-fragment-entry-state + lr-recognition-fragment-exit-state + lr-recognition-fragment-children + lr-recognition-fragment-children-cache + lr-recognition-fragment-lookahead + lr-recognition-fragment-lookahead-cache + lr-recognition-fragment-value + lr-recognition-fragment-value-set! + lr-recognition-fragment-start + lr-recognition-fragment-start-set! + lr-recognition-fragment-end + lr-recognition-fragment-end-set! + lr-recognition-fragment-token-count + lr-recognition-fragment-token-count-set! + lr-recognition-fragment-offset + lr-recognition-fragment-offset-set! + lr-recognition-fragment-yield-cache + lr-recognition-fragment-yield-cache-set! + lr-recognition-view + make-lr-recognition-view + lr-recognition-view? + lr-recognition-view-base + lr-recognition-view-base-set! + lr-recognition-view-delta + lr-recognition-view-delta-set! + lr-terminal-yield-branch + make-lr-terminal-yield-branch + lr-terminal-yield-branch? + lr-terminal-yield-branch-left + lr-terminal-yield-branch-left-set! + lr-terminal-yield-branch-right + lr-terminal-yield-branch-right-set! + lr-terminal-yield-view + make-lr-terminal-yield-view + lr-terminal-yield-view? + lr-terminal-yield-view-base + lr-terminal-yield-view-base-set! + lr-terminal-yield-view-delta + lr-terminal-yield-view-delta-set! + current-lr-transfer-yield-observer + current-lr-recognition-observer + lr-failure-frontier + make-lr-failure-frontier + lr-failure-frontier? + lr-failure-frontier-checkpoint + lr-failure-frontier-checkpoint-set! + lr-failure-frontier-state + lr-failure-frontier-state-set! + lr-failure-frontier-expected-terminals + lr-failure-frontier-expected-terminals-set! + current-lr-branch-budget) + +;;; -*- Gerbil -*- +;;; Immutable LR table execution and lossless recognition reduction. +;; Prepared execution data. A generated reduction step may be installed once +;; during language-module initialization before the runtime is shared. +(defstruct lr-runtime + (productions table reduction-widths actions action-index gotos goto-index + case-insensitive? dynamic? layout? + lexical-modes lexical-mode-catalog direct-step semantic-reducer event-reducer event-step event-runtime + reduction-plans action-selector owned-stack? goto-columns capture-plans) + transparent: #t final: #t) + +(def current-lr-event-program-enabled? (make-parameter #f)) + +;;; Interned parser-directed lexical expectation shared by LR states with the +;;; same terminal row. +(defstruct lr-lexical-mode (id terminals) transparent: #t final: #t) + +;;; Immutable continuation of the deterministic LR machine. The constructor +;;; remains private: every public checkpoint is tied to the exact prepared +;;; runtime and original token sequence that produced it. +(defstruct lr-checkpoint + (runtime tokens input-end-offset states semantic-values rest + deterministic-actions deterministic-shifts recognition-stack) + transparent: #t final: #t) + +;;; A reusable prefix keeps parser state and recognition values but does not +;;; retain a complete historical token stream across incremental edits. +(defstruct lr-prefix-snapshot + (runtime states semantic-values deterministic-actions deterministic-shifts recognition-stack) + transparent: #t final: #t) + +;;; Private grammar structure, captured before pass/concat/alias projection. +;;; This is execution metadata, never a second published parse authority. +;;; Immutable prepared transition identity is shared; request-local payloads +;;; remain separate in each fragment. No context mutator is exported. +(defstruct lr-capture-plan (runtime production-id width contexts) + transparent: #t final: #t) +(defstruct lr-capture-context (runtime executor production-id entry-state exit-state lookahead-kind) + transparent: #t final: #t) +;;; Owned packed children contain only the current production's operands. +;;; No recognition-stack link or unrelated suffix escapes into this payload. +(defstruct lr-capture-children (items) transparent: #t final: #t) +(defstruct lr-recognition-fragment + (context children-cache lookahead-cache value start end token-count offset yield-cache) + transparent: #t final: #t) +(definline (lr-recognition-fragment-lookahead (piece : lr-recognition-fragment)) => :t + (let (stored piece.lookahead-cache) + (if (or (not stored) (pair? stored)) stored + (let (signature (cons (lr-capture-context-lookahead-kind piece.context) stored)) + (lr-recognition-fragment-lookahead-cache-set! piece signature) + signature)))) +(definline (lr-recognition-fragment-children (piece : lr-recognition-fragment)) => :t + (let (stored piece.children-cache) + (if (or (pair? stored) (null? stored)) stored + (let (children + (if (lr-capture-children? stored) + (vector->list (lr-capture-children-items stored)) + (list stored))) + (lr-recognition-fragment-children-cache-set! piece children) + children)))) +(definline (lr-recognition-fragment-runtime (piece : lr-recognition-fragment)) => :t + (lr-capture-context-runtime piece.context)) +(definline (lr-recognition-fragment-executor (piece : lr-recognition-fragment)) => :t + (lr-capture-context-executor piece.context)) +(definline (lr-recognition-fragment-production-id (piece : lr-recognition-fragment)) => :t + (lr-capture-context-production-id piece.context)) +(definline (lr-recognition-fragment-entry-state (piece : lr-recognition-fragment)) => :t + (lr-capture-context-entry-state piece.context)) +(definline (lr-recognition-fragment-exit-state (piece : lr-recognition-fragment)) => :t + (lr-capture-context-exit-state piece.context)) + +(defstruct lr-recognition-view (base delta) transparent: #t final: #t) + +;;; A reduction shares its terminal yield instead of materializing a flat +;;; token list at every ancestor. Empty/unary grammar wrappers allocate no +;;; yield nodes; only concatenation and coordinate views need records. +(defstruct lr-terminal-yield-branch (left right) final: #t) + +(defstruct lr-terminal-yield-view (base delta) final: #t) + +;;; Optional transfer-work observation, independent of admission and publication. +;;; Receives the visited yield-node count once per attempted yield validation. +(def current-lr-transfer-yield-observer (make-parameter #f)) + +;;; Observer receives the accepted deterministic root, or #f when execution +;;; enters a path whose structure cannot be certified (GLR/layout/recovery). +(def current-lr-recognition-observer (make-parameter #f)) + +;;; A deterministic failure frontier retains the exact immutable continuation +;;; and the terminals admitted by its LR state. Recovery can therefore test a +;;; local edit without replaying the already accepted prefix. +(defstruct lr-failure-frontier (checkpoint state expected-terminals) + transparent: #t final: #t) + +;;; Executes immutable tables and evaluates every admitted fork within a +;;; deterministic branch budget. Dynamic precedence scores complete branches; +;;; structurally identical ties merge and distinct equal-score ties fail closed. +;; : (-> List List Integer (Values Datum List Alist)) +(def current-lr-branch-budget (make-parameter 256)) + diff --git a/src/runtime/lr-parser/transfer.ss b/src/runtime/lr-parser/transfer.ss new file mode 100644 index 00000000..4fde6095 --- /dev/null +++ b/src/runtime/lr-parser/transfer.ss @@ -0,0 +1,237 @@ +;;; -*- Gerbil -*- +;;; lr-parser: transfer owner. Private execution state and named algorithms. + +(import (only-in ../../compiler/lr + production-action production-lhs production-rhs base-symbol) + (only-in ../funcs + recognition-sequence-relocate) + (only-in ../token + token? token-end token-kind token-lexeme token-start) + (only-in "./state.ss" + lr-runtime-table lr-runtime-actions lr-runtime-goto-index lr-runtime-dynamic? lr-runtime-layout? lr-runtime-lexical-modes lr-runtime-direct-step make-lr-checkpoint lr-checkpoint? lr-checkpoint-runtime lr-checkpoint-tokens lr-checkpoint-input-end-offset lr-checkpoint-states lr-checkpoint-semantic-values lr-checkpoint-rest lr-checkpoint-deterministic-actions lr-checkpoint-deterministic-shifts lr-checkpoint-recognition-stack make-lr-prefix-snapshot lr-prefix-snapshot? lr-prefix-snapshot-runtime lr-prefix-snapshot-states lr-prefix-snapshot-semantic-values lr-prefix-snapshot-deterministic-actions lr-prefix-snapshot-deterministic-shifts lr-prefix-snapshot-recognition-stack lr-recognition-fragment? lr-recognition-fragment-runtime lr-recognition-fragment-executor lr-recognition-fragment-production-id lr-recognition-fragment-entry-state lr-recognition-fragment-exit-state lr-recognition-fragment-value lr-recognition-fragment-token-count lr-terminal-yield-branch? lr-terminal-yield-branch-left lr-terminal-yield-branch-right lr-terminal-yield-view? lr-terminal-yield-view-base lr-terminal-yield-view-delta current-lr-transfer-yield-observer lr-failure-frontier? lr-failure-frontier-checkpoint) + (only-in "./recognition.ss" lr-recognition-fragment-terminal-yield + lr-recognition-relocate recognition-piece-end) + (only-in "./checkpoint-base.ss" + lr-rejection-condition?) + (only-in "./prepare.ss" + goto-target) + (only-in "./checkpoint.ss" + lr-run-checkpoint)) +(export lr-checkpoint-before-shift + lr-runtime-fragment-reuse-safe? + lr-checkpoint-fragment-compatible? + lr-recognition-fragment-exit-mode + same-transfer-token? + fragment-yield-matches? + lr-checkpoint-inject-fragment + lr-checkpoint-drive + lr-checkpoint-drive/contextual + lr-checkpoint-prefix-snapshot + lr-prefix-snapshot-rebind + lr-checkpoint-rebind-suffix + lr-checkpoint-resume-suffix + lr-checkpoint-resume + lr-checkpoint-frontier + lr-failure-frontier-resume) + +(def (lr-checkpoint-before-shift checkpoint token) + (let-values (((status next) + (lr-run-checkpoint checkpoint #f #f #t #f #t token + #f #f 'installed #f #t))) + (and (eq? status 'checkpoint) next))) + +(def (lr-runtime-fragment-reuse-safe? runtime) + (and (not (lr-runtime-layout? runtime)) (not (lr-runtime-dynamic? runtime)) + ;; Admit the repeated-structure family first. Operator ancestry has no + ;; such concat recurrence and does not justify building a reuse catalog. + (vector-any + (lambda (production) + (let (rhs (production-rhs production)) + (and (eq? (production-action production) 'concat) + (= (length rhs) 2) + (equal? (base-symbol (car rhs)) + (list 'nonterminal (production-lhs production)))))) + (lr-runtime-table runtime)) + (not (vector-any + (lambda (row) (any (lambda (entry) (eq? (cadr entry) 'fork)) row)) + (lr-runtime-actions runtime))))) + +(def (lr-checkpoint-fragment-compatible? checkpoint fragment) + (let ((runtime (lr-checkpoint-runtime checkpoint)) + (state (car (lr-checkpoint-states checkpoint)))) + (and (lr-recognition-fragment? fragment) + (eq? runtime (lr-recognition-fragment-runtime fragment)) + (eq? (lr-runtime-direct-step runtime) (lr-recognition-fragment-executor fragment)) + (not (lr-runtime-layout? runtime)) (not (lr-runtime-dynamic? runtime)) + (= state (lr-recognition-fragment-entry-state fragment)) + (let (target + (goto-target (lr-runtime-goto-index runtime) state + (production-lhs (vector-ref (lr-runtime-table runtime) + (lr-recognition-fragment-production-id fragment))))) + (and target (= target (lr-recognition-fragment-exit-state fragment))))))) + +(def (lr-recognition-fragment-exit-mode fragment) + (vector-ref (lr-runtime-lexical-modes (lr-recognition-fragment-runtime fragment)) + (lr-recognition-fragment-exit-state fragment))) + +;;; Compare retained terminal leaves without projecting the published AST. +;;; Views accumulate byte relocation; pending siblings keep their own delta. +(def (same-transfer-token? actual retained delta) + (and (token? actual) (token? retained) + (eq? (token-kind actual) (token-kind retained)) + (equal? (token-lexeme actual) (token-lexeme retained)) + (= (token-start actual) (+ delta (token-start retained))) + (= (token-end actual) (+ delta (token-end retained))))) + +(def (fragment-yield-matches? fragment delta tokens) + (let ((yield (lr-recognition-fragment-terminal-yield fragment)) + (observer (current-lr-transfer-yield-observer))) + (def (finish matched? visited) + (when observer (observer visited)) + matched?) + (let loop ((pending (if yield (list (cons yield delta)) '())) (rest tokens) (visited 0)) + (if (null? pending) (finish (null? rest) visited) + (let* ((frame (car pending)) (piece (car frame)) (shift (cdr frame)) + (visited (if observer (+ visited 1) visited))) + (cond + ((lr-terminal-yield-view? piece) + (loop (cons (cons (lr-terminal-yield-view-base piece) + (+ shift (lr-terminal-yield-view-delta piece))) + (cdr pending)) rest visited)) + ((lr-terminal-yield-branch? piece) + (loop (cons (cons (lr-terminal-yield-branch-left piece) shift) + (cons (cons (lr-terminal-yield-branch-right piece) shift) (cdr pending))) + rest visited)) + ((and (pair? rest) (same-transfer-token? (car rest) piece shift)) + (loop (cdr pending) (cdr rest) visited)) + (else (finish #f visited)))))))) + +;;; The source owner certifies unchanged text and both lexical boundaries. +;;; The LR owner independently verifies the transferred terminal yield. +;;; Transfer one actual nonterminal, not an exported AST kind or an LR suffix. +(def (lr-checkpoint-inject-fragment checkpoint fragment delta significant-tokens) + (unless (and (lr-checkpoint-fragment-compatible? checkpoint fragment) + (pair? (lr-checkpoint-rest checkpoint)) + (pair? significant-tokens) + (same-transfer-token? (car significant-tokens) + (car (lr-checkpoint-rest checkpoint)) 0) + (positive? (lr-recognition-fragment-token-count fragment)) + (= (length significant-tokens) (lr-recognition-fragment-token-count fragment)) + (fragment-yield-matches? fragment delta significant-tokens)) + (error "uncertified LR fragment transfer")) + (let* ((piece (lr-recognition-relocate fragment delta)) + (end (recognition-piece-end piece)) + (count (lr-recognition-fragment-token-count fragment))) + (make-lr-checkpoint + (lr-checkpoint-runtime checkpoint) + (append (reverse significant-tokens) (cdr (lr-checkpoint-tokens checkpoint))) + (max (lr-checkpoint-input-end-offset checkpoint) end) + (cons (lr-recognition-fragment-exit-state fragment) (lr-checkpoint-states checkpoint)) + (cons (recognition-sequence-relocate (lr-recognition-fragment-value fragment) delta) + (lr-checkpoint-semantic-values checkpoint)) + '() (+ 1 (lr-checkpoint-deterministic-actions checkpoint)) + (+ count (lr-checkpoint-deterministic-shifts checkpoint)) + (and (lr-checkpoint-recognition-stack checkpoint) + (cons piece (lr-checkpoint-recognition-stack checkpoint)))))) + +;;; Drives one deterministic LR loop while the source owner supplies tokens +;;; under the current lexical mode. The callback records shifted source tokens. +;;; A fork returns its exact checkpoint for the existing selective-GLR handoff. +;;; The optional override gives conformance tests and matched benchmarks an +;;; indexed baseline without mutating the installed language machine. +(def (lr-checkpoint-drive checkpoint next-input after-shift + (observability #f) + (direct-step-override 'installed) + (snapshot-spacing #f) (capture-snapshot #f)) + (when capture-snapshot + (unless (and (procedure? capture-snapshot) (integer? snapshot-spacing) (> snapshot-spacing 0)) + (error "streamed checkpoint capture requires positive spacing"))) + (lr-run-checkpoint + checkpoint #f observability #f #f #t #f + next-input after-shift direct-step-override #f #f snapshot-spacing capture-snapshot)) + +;;; Contextual scanning needs the exact LR state as its syntactic position +;;; axis. Existing lexer drivers keep the one-argument callback contract. +(def (lr-checkpoint-drive/contextual checkpoint next-input after-shift + (observability #f)) + (lr-run-checkpoint + checkpoint #f observability #f #f #t #f + next-input after-shift 'installed #t)) + +;;; Rebinds an unconsumed suffix to the same immutable deterministic frontier. +;;; Streaming GLR handoff and explicit incremental sessions share this path. +(def (lr-checkpoint-prefix-snapshot checkpoint) + (unless (lr-checkpoint? checkpoint) + (error "LR prefix snapshot requires a checkpoint" checkpoint)) + (make-lr-prefix-snapshot + (lr-checkpoint-runtime checkpoint) + (lr-checkpoint-states checkpoint) + (lr-checkpoint-semantic-values checkpoint) + (lr-checkpoint-deterministic-actions checkpoint) + (lr-checkpoint-deterministic-shifts checkpoint) + (lr-checkpoint-recognition-stack checkpoint))) + +(def (lr-prefix-snapshot-rebind snapshot tokens rest + (input-end-offset #f)) + (unless (lr-prefix-snapshot? snapshot) + (error "LR prefix rebind requires a snapshot" snapshot)) + (make-lr-checkpoint + (lr-prefix-snapshot-runtime snapshot) + tokens + (or input-end-offset + (fold (lambda (input-token offset) + (max offset (token-end input-token))) + 0 tokens)) + (lr-prefix-snapshot-states snapshot) + (lr-prefix-snapshot-semantic-values snapshot) + rest + (lr-prefix-snapshot-deterministic-actions snapshot) + (lr-prefix-snapshot-deterministic-shifts snapshot) + (lr-prefix-snapshot-recognition-stack snapshot))) + +(def (lr-checkpoint-rebind-suffix checkpoint tokens rest) + (lr-prefix-snapshot-rebind + (lr-checkpoint-prefix-snapshot checkpoint) tokens rest)) + +(def (lr-checkpoint-resume-suffix checkpoint tokens rest (observability #f)) + (lr-checkpoint-resume + (lr-checkpoint-rebind-suffix checkpoint tokens rest) observability)) + +;;; Completes parsing from an initial or advanced immutable checkpoint. +(def (lr-checkpoint-resume checkpoint (observability #f)) + (let-values (((status payload) + (lr-run-checkpoint checkpoint #f observability))) + (unless (eq? status 'accepted) + (error "LR resume did not reach a terminal result" status)) + (values (car payload) (cadr payload)))) + +;;; Runs to acceptance or to the first deterministic LR failure frontier. An +;;; admitted GLR fork still delegates to the selective-GLR owner. +(def (lr-checkpoint-frontier checkpoint (observability #f)) + (lr-run-checkpoint checkpoint #f observability #t)) + +;;; Tests one edited suffix from the retained failure state. The returned CST +;;; remains private to recovery; only acceptance is exposed. +(def (lr-failure-frontier-resume frontier edited-rest (observability #f)) + (unless (lr-failure-frontier? frontier) + (error "LR recovery requires a failure frontier" frontier)) + (let* ((checkpoint (lr-failure-frontier-checkpoint frontier)) + (edited + (make-lr-checkpoint + (lr-checkpoint-runtime checkpoint) + (lr-checkpoint-tokens checkpoint) + (lr-checkpoint-input-end-offset checkpoint) + (lr-checkpoint-states checkpoint) + (lr-checkpoint-semantic-values checkpoint) + edited-rest + (lr-checkpoint-deterministic-actions checkpoint) + (lr-checkpoint-deterministic-shifts checkpoint) #f))) + (with-catch + (lambda (condition) + (if (lr-rejection-condition? condition) + #f + (raise condition))) + (lambda () + (let-values (((_root rest) (lr-checkpoint-resume edited observability))) + (null? rest)))))) + diff --git a/src/runtime/lr-stack-storage.ss b/src/runtime/lr-stack-storage.ss new file mode 100644 index 00000000..97b4d17f --- /dev/null +++ b/src/runtime/lr-stack-storage.ss @@ -0,0 +1,121 @@ +;;; -*- Gerbil -*- +;;; Statically selected LR stack storage; owned buffers never escape a request. +(export make-lr-token-observer lr-token-observer? + with-lr-stack-storage lr-stack-policy-value lr-stack-current + lr-stack-present? lr-semantic-stack-present? lr-stack-value + lr-stack-freeze-states lr-stack-freeze-values + lr-stack-push-state lr-stack-shift-value lr-stack-push-value + with-pop-driver-reduction reduce-driver-plan + lr-stack-shift-observer lr-stack-notify-shift) + +(defstruct lr-token-observer (procedure) final: #t constructor: :init!) +(defmethod {:init! lr-token-observer} + (lambda (self proc) + (unless (procedure? proc) (error "LR token observer requires a procedure" proc)) + (set! (@ self procedure) proc))) + +(def (copy-lr-stack! storage stack) + (let loop ((rest stack) (index (fx- (length stack) 1))) + (unless (null? rest) + (vector-set! storage index (car rest)) + (loop (cdr rest) (fx- index 1))))) +(def (freeze-lr-stack storage end) + (let loop ((index 0) (result '())) + (if (fx= index end) result + (loop (fx+ index 1) (cons (vector-ref storage index) result))))) +(def (grow-lr-stacks states semantic-storage) + (let* ((width (vector-length states)) (capacity (fx* width 2)) + (next-states (make-vector capacity #f)) (next-values (make-vector capacity #f))) + (subvector-move! states 0 width next-states 0) + (subvector-move! semantic-storage 0 width next-values 0) + (values next-states next-values))) + +;;; Only request-owned buffers enter these forms. Initialization establishes +;;; depth < capacity; reduction checks width <= depth, and push grows before +;;; writing. Checkpoint export and borrowed storage retain checked operations. +;;; This is the local array invariant used by std/struct/pqueue, not a module- +;;; wide unchecked declaration or a replacement for runtime table admission. +(defrule (owned-vector-ref storage index) + (let () (declare (not safe)) (vector-ref storage index))) +(defrule (owned-vector-set! storage index value) + (let () (declare (not safe)) (vector-set! storage index value))) + +(defrules with-lr-stack-storage () + ((_ #f states values (state-storage value-storage initial-states initial-values) body ...) + (let ((state-storage #f) (value-storage #f) (initial-states states) (initial-values values)) body ...)) + ((_ #t states values (state-storage value-storage initial-states initial-values) body ...) + (let* ((source-states states) (source-values values) (depth (length source-values)) + (capacity (max 16 (fx+ depth 2))) + (state-storage (make-vector capacity #f)) (value-storage (make-vector capacity #f)) + (initial-states depth) (initial-values #f)) + (unless (= (length source-states) (fx+ depth 1)) + (error "LR state and semantic stack depths differ")) + (copy-lr-stack! state-storage source-states) + (copy-lr-stack! value-storage source-values) + body ...))) +(defrules lr-stack-policy-value () + ((_ #f original _owned) original) + ((_ #t _original owned) owned)) +(defrules lr-stack-current () + ((_ #f states _storage) (car states)) + ((_ #t top storage) (owned-vector-ref storage top))) +(defrules lr-stack-present? () + ((_ #f states) (pair? states)) + ((_ #t top) (fx>= top 0))) +(defrules lr-semantic-stack-present? () + ((_ #f values _top) (pair? values)) + ((_ #t _values top) (fx> top 0))) +(defrules lr-stack-value () + ((_ #f values _top _storage) (car values)) + ((_ #t _values top storage) (owned-vector-ref storage (fx- top 1)))) +(defrules lr-stack-freeze-states () + ((_ #f states _storage) states) + ((_ #t top storage) (freeze-lr-stack storage (fx+ top 1)))) +(defrules lr-stack-freeze-values () + ((_ #f values _top _storage) values) + ((_ #t _values top storage) (freeze-lr-stack storage top))) +(defrules lr-stack-push-state () + ((_ #f target states _state-storage _value-storage) (cons target states)) + ((_ #t target top state-storage value-storage) + (let ((next (fx+ top 1)) (next-state target)) + (when (fx>= next (vector-length state-storage)) + (let-values (((next-states next-values) (grow-lr-stacks state-storage value-storage))) + (set! state-storage next-states) (set! value-storage next-values))) + (owned-vector-set! state-storage next next-state) next))) +(defrules lr-stack-shift-value () + ((_ #f value values _top _storage) (cons value values)) + ((_ #t value _values top storage) (begin (owned-vector-set! storage top value) #f))) +(defrules lr-stack-push-value () + ((_ #f value remaining _base _top _storage) (cons value remaining)) + ((_ #t value _remaining base top storage) + ;; The result overwrites its slot. Clear only the discarded tail; unary + ;; and epsilon reductions need no bulk call or redundant store. + (begin + (when (fx< (fx+ base 1) top) + (subvector-fill! storage (fx+ base 1) top 0)) + (owned-vector-set! storage base value) #f))) + +;;; Prefix validation remains checked. Recognition consumes the original +;;; right-to-left operand order; owned frames need no list-prefix traversal. +(defrules with-pop-driver-reduction () + ((_ #f pop states values count events? (source remaining-values remaining-states) body ...) + (pop states values count events? (source remaining-values remaining-states) body ...)) + ((_ #t _pop states values count _events? (source remaining-values remaining-states) body ...) + (let ((width count) (source states) (remaining-values values)) + (unless (fx<= width states) (error "LR reduction exceeds parser stack" width)) + (let (remaining-states (fx- states width)) body ...)))) +(defrules reduce-driver-plan () + ((_ #f reduce _owned-reduce plan values offset events? _storage) (reduce plan values offset events?)) + ((_ #t _reduce owned-reduce plan top offset _events? storage) (owned-reduce plan storage top offset))) + +;;; The token-only capability is explicit. Arbitrary five-argument observers +;;; continue receiving immutable list frontiers in the persistent executor. +(defrules lr-stack-shift-observer () + ((_ #f observer) + (if (lr-token-observer? observer) + (let (proc (lr-token-observer-procedure observer)) + (lambda (token _states _values _actions _shifts) (proc token))) observer)) + ((_ #t observer) (and observer (lr-token-observer-procedure observer)))) +(defrules lr-stack-notify-shift () + ((_ #f observer token states values actions shifts) (observer token states values actions shifts)) + ((_ #t observer token _states _values _actions _shifts) (observer token))) diff --git a/src/runtime/module-source.ss b/src/runtime/module-source.ss new file mode 100644 index 00000000..d0ab9322 --- /dev/null +++ b/src/runtime/module-source.ss @@ -0,0 +1,94 @@ +;;; Prepared module framing. Repeated borders are indexed as intervals, not suffix scans. +(export prepare-module-source scan-prepared-module-source) +(defstruct module-source-index (source expression headers events)) +(def (prefix-at? source at text) + (and (<= (+ at (string-length text)) (string-length source)) + (let loop ((i 0)) + (or (= i (string-length text)) + (and (char=? (string-ref source (+ at i)) (string-ref text i)) (loop (+ i 1))))))) +(def (run-end source at accept?) + (let loop ((at at)) + (if (and (< at (string-length source)) (accept? (string-ref source at))) (loop (+ at 1)) at))) +(def (name-end source at extra) + (let loop ((end at) (seen? #f)) + (if (and (< end (string-length source)) + (let (ch (string-ref source end)) + (or (char-alphabetic? ch) (char-numeric? ch) (char=? ch extra)))) + (loop (+ end 1) (or seen? (char-alphabetic? (string-ref source end)) + (char=? (string-ref source end) extra))) + (and seen? end)))) +(def (header-tail-end source at border word extra) + (let (word-at (run-end source at char-whitespace?)) + (and (prefix-at? source word-at word) + (let* ((name-at (run-end source (+ word-at (string-length word)) char-whitespace?)) + (name-stop (name-end source name-at extra))) + (and name-stop + (let (close-at (run-end source name-stop char-whitespace?)) + (and (prefix-at? source close-at border) + (run-end source close-at (cut char=? <> (string-ref border 0)))))))))) +(def (module-header-intervals source border word extra) + (let ((ch (string-ref border 0)) (width (string-length border)) (size (string-length source))) + (let loop ((at 0) (rows '())) + (if (= at size) (list->vector (reverse rows)) + (if (char=? (string-ref source at) ch) + (let* ((end (run-end source at (cut char=? <> ch))) + (stop (and (>= (- end at) width) (header-tail-end source end border word extra)))) + (loop end (if stop (cons (vector at (- end width) stop) rows) rows))) + (loop (+ at 1) rows)))))) +(def (skip-block source at open close) + (let loop ((at (+ at (string-length open))) (depth 1)) + (cond ((>= at (string-length source)) at) + ((prefix-at? source at open) (loop (+ at (string-length open)) (+ depth 1))) + ((prefix-at? source at close) + (if (= depth 1) (+ at (string-length close)) (loop (+ at (string-length close)) (- depth 1)))) + (else (loop (+ at 1) depth))))) +(def (skip-string source at) + (let loop ((at (+ at 1))) + (cond ((>= at (string-length source)) at) + ((char=? (string-ref source at) #\") (+ at 1)) + ((char=? (string-ref source at) #\\) (loop (+ at 2))) + (else (loop (+ at 1)))))) +(def (module-depth-events source headers end-border open close comment) + (let loop ((at 0) (cursor 0) (depth 0) (rows '())) + (cond + ((>= at (string-length source)) (list->vector (reverse rows))) + ((and (< cursor (vector-length headers)) (> at (vector-ref (vector-ref headers cursor) 1))) + (loop at (+ cursor 1) depth rows)) + ((prefix-at? source at open) (loop (skip-block source at open close) cursor depth rows)) + ((char=? (string-ref source at) #\") (loop (skip-string source at) cursor depth rows)) + ((prefix-at? source at comment) + (loop (run-end source at (lambda (ch) (not (char=? ch #\newline)))) cursor depth rows)) + ((and (< cursor (vector-length headers)) (>= at (vector-ref (vector-ref headers cursor) 0))) + (loop (vector-ref (vector-ref headers cursor) 2) cursor (+ depth 1) (cons (vector at (+ depth 1)) rows))) + ((prefix-at? source at end-border) + (loop (run-end source at (cut char=? <> (string-ref end-border 0))) cursor (- depth 1) + (cons (vector at (- depth 1)) rows))) + (else (loop (+ at 1) cursor depth rows))))) +(def (prepare-module-source source expression) + (match expression + (['module-text border word end-border open close comment extra] + (let (headers (module-header-intervals source border word (string-ref extra 0))) + (make-module-source-index source expression headers + (module-depth-events source headers end-border open close comment)))))) +(def (lower-bound rows before?) + (let loop ((lo 0) (hi (vector-length rows))) + (if (= lo hi) lo + (let (mid (quotient (+ lo hi) 2)) + (if (before? (vector-ref rows mid)) (loop (+ mid 1) hi) (loop lo mid)))))) +(def (scan-prepared-module-source index start) + (let* ((source (module-source-index-source index)) (size (string-length source)) + (end-border (list-ref (module-source-index-expression index) 3))) + (and (exact-integer? start) (<= 0 start) (< start size) + (or (= start 0) + (let* ((prior (let loop ((at (- start 1))) + (if (and (>= at 0) (char-whitespace? (string-ref source at))) (loop (- at 1)) at))) + (border-at (- (+ prior 1) (string-length end-border))) + (events (module-source-index-events index)) + (event (lower-bound events (lambda (row) (< (vector-ref row 0) start))))) + (and (>= border-at 0) (prefix-at? source border-at end-border) + (= (if (zero? event) 0 (vector-ref (vector-ref events (- event 1)) 1)) 0)))) + (let* ((headers (module-source-index-headers index)) + (next (lower-bound headers (lambda (row) (< (vector-ref row 1) start)))) + (end (if (= next (vector-length headers)) size + (max start (vector-ref (vector-ref headers next) 0))))) + (and (> end start) end))))) diff --git a/src/runtime/observability.ss b/src/runtime/observability.ss index 94c59ab0..ff9b1a16 100644 --- a/src/runtime/observability.ss +++ b/src/runtime/observability.ss @@ -1,15 +1,23 @@ ;;; -*- Gerbil -*- ;;; Grammar-owned adapter to the POO Flow observability framework. -;;; -;;; LanguageGrammar carries either #f or a PooFlowDebugCallPolicy. This module -;;; defines no observation type, timer, sink, receipt, or enablement semantic. - -(import (only-in :core/observability/debug - call-with-poo-flow-debug-trace)) +;;; Delayed phase bodies execute directly when the policy is absent. +(import (only-in :core/observability/debug call-with-poo-flow-debug-trace)) (export call-with-parser-observed-phase) -(def (call-with-parser-observed-phase policy phase thunk) +(def (call-with-parser-observed-phase/procedure policy phase thunk) (if policy - (call-with-poo-flow-debug-trace - policy phase thunk '() emit?: #t) + (call-with-poo-flow-debug-trace policy phase thunk '() emit?: #t) (thunk))) + +;;; Follow the SDK's hygienic delayed-body and first-class identifier pattern. +;;; Evaluate policy and phase once; only observed execution needs a thunk. +(defrules call-with-parser-observed-phase (lambda) + ((_ policy phase (lambda () body ...)) + (let* ((selected-policy policy) (selected-phase phase)) + (if selected-policy + (call-with-parser-observed-phase/procedure + selected-policy selected-phase (lambda () body ...)) + (begin body ...)))) + ((_ policy phase thunk) + (call-with-parser-observed-phase/procedure policy phase thunk)) + (id (identifier? #'id) call-with-parser-observed-phase/procedure)) diff --git a/src/runtime/parse-cost.ss b/src/runtime/parse-cost.ss new file mode 100644 index 00000000..88a00214 --- /dev/null +++ b/src/runtime/parse-cost.ss @@ -0,0 +1,33 @@ +;;; Optional request-local diagnostics; disabled calls do not allocate a thunk. +(export current-parser-cost-observer with-parser-cost-stage admit-parser-allocation) +;; The SDK allocation counter includes occupied still objects and may fall at GC. +;; Retain the raw delta separately; admit an allocation comparison only without GC. +(def (admit-parser-allocation delta collections) + (and (zero? collections) (>= delta 0) delta)) +(def current-parser-cost-observer (make-parameter #f)) +(def (call-with-parser-cost-stage observer name thunk) + (let ((before (##process-statistics)) + (started (##current-time-point)) + (completed? #f)) + (dynamic-wind void + (lambda () + (let (result (call-with-values thunk list)) + (set! completed? #t) + (apply values result))) + (lambda () + (let* ((elapsed (* 1000 (- (##current-time-point) started))) + (after (##process-statistics)) + (delta (lambda (index) (- (f64vector-ref after index) (f64vector-ref before index))))) + (observer name + (list (cons 'wall-ms elapsed) + (cons 'cpu-ms (* 1000 (+ (delta 0) (delta 1)))) + (cons 'allocated-bytes (admit-parser-allocation (delta 7) (delta 6))) + (cons 'allocation-counter-delta (delta 7)) + (cons 'gc-count (delta 6)) + (cons 'completed completed?)))))))) +(defrules with-parser-cost-stage () + ((_ name expression) + (let (observer (current-parser-cost-observer)) + (if observer + (call-with-parser-cost-stage observer name (lambda () expression)) + expression)))) diff --git a/src/runtime/parser-ir-data.ss b/src/runtime/parser-ir-data.ss new file mode 100644 index 00000000..06386095 --- /dev/null +++ b/src/runtime/parser-ir-data.ss @@ -0,0 +1,5 @@ +;;; -*- Gerbil -*- +;;; Runtime IR access has no grammar construction or LR compiler dependency. +(export parser-ir-ref) +(def (parser-ir-ref ir key) + (alet (entry (assq key ir)) (cdr entry))) diff --git a/src/runtime/parser.ss b/src/runtime/parser.ss index 52e4397a..e3ed1224 100644 --- a/src/runtime/parser.ss +++ b/src/runtime/parser.ss @@ -1,28 +1,206 @@ ;;; -*- Gerbil -*- ;;; Thin request boundary over generated machines and ParseArtifact admission. -(import (only-in ../compiler/parser-ir parser-ir-ref) +(import (only-in :std/string/utf8 string-utf8-length) + (only-in ./parse-cost with-parser-cost-stage) + (only-in ./parser-ir-data parser-ir-ref) (only-in ../compiler/machine - parser-machine-grammar-digest parser-machine-ir + call-with-parser-machine-source parser-machine-grammar-digest parser-machine-ir parser-machine-parse parser-machine-runtime - parser-machine-trivia) + parser-machine-trivia parser-machine-direct-drive + parser-machine-direct-source) (only-in ./artifact +diagnostic-schema+ make-failure-parse-artifact - make-success-parse-artifact) + make-success-parse-artifact parse-artifact-success?) (only-in ./lexer lex-source lex-source-from scan-source-token) + (only-in ./contextual-scanner + prepare-contextual-scanner prepare-contextual-scanner-plan contextual-scanner-initial-state + contextual-scanner-step + contextual-scan-state-character-offset + contextual-scan-state-byte-offset) + (only-in ./contextual-ir contextual-ir-ref contextual-parser-ir-valid?) + (only-in ./identity sha256-text) (only-in ./lr-parser - lr-checkpoint-feed lr-checkpoint-lexical-mode + lr-runtime-layout? require-lr-checkpoint-runtime! + lr-checkpoint-deterministic-shifts lr-checkpoint-drive + lr-checkpoint-drive/contextual + lr-checkpoint-feed + lr-checkpoint-lexical-mode lr-checkpoint-prefix-snapshot + lr-lexical-mode-id lr-checkpoint-resume lr-checkpoint-resume-suffix lr-initial-checkpoint) + (only-in ./layout + make-layout-columns current-layout-columns + current-layout-frames) (only-in ./observability call-with-parser-observed-phase) - (only-in ./significant parser-significant-tokens) - (only-in ./token token-end token-lexeme)) -(export parse-source + (only-in ./significant + parser-significant-tokens parser-significant-joined) + (only-in ./token make-token token-end token-lexeme)) +(import (only-in ./lr-stack-storage make-lr-token-observer)) +(export current-source-stream-observer parse-source + parse-source/contextual + prepare-contextual-parser contextual-parser-grammar-digest + parse-source/contextual/prepared + parse-source/contextual/prepared/deferred + parse-source/checkpoints + parse-source/checkpoints/resume parse-tokenized parse-tokenized/checkpoint) +(def (contextual-canonical value) + (call-with-output-string (lambda (port) (write value port)))) + +(def (valid-contextual-product? machine product) + (and (contextual-parser-ir-valid? product) + (eq? (contextual-ir-ref (contextual-ir-ref product 'recognition) 'dialect) 'lr) + (equal? (contextual-ir-ref (contextual-ir-ref product 'recognition) 'program) + (parser-machine-ir machine)) + (equal? (contextual-ir-ref product 'root-kind) + (parser-ir-ref (parser-machine-ir machine) 'root-kind)) + (let (construction (contextual-ir-ref product 'construction)) + (and (equal? (contextual-ir-ref construction 'syntax-kinds) + (parser-ir-ref (parser-machine-ir machine) 'syntax-kinds)) + (equal? (contextual-ir-ref construction 'terminals) + (parser-ir-ref (parser-machine-ir machine) 'terminals)) + (equal? (contextual-ir-ref construction 'projections) + (parser-ir-ref (parser-machine-ir machine) 'rules)))) + (equal? (contextual-ir-ref product 'base-grammar-digest) + (parser-machine-grammar-digest machine)) + (equal? (contextual-ir-ref product 'parser-ir-digest) + (sha256-text + (contextual-canonical (parser-machine-ir machine)))))) + +;;; The position table is compiler output: one entry for every LR state. The +;;; scanner receives a finite position id, never the checkpoint or a callback. +(def (validate-contextual-position-table machine scanner-ir table) + (let* ((positions (contextual-ir-ref scanner-ir 'positions)) + (state-count + (vector-length + (cdr + (assq 'actions + (parser-ir-ref (parser-machine-ir machine) 'lr-spec))))) + (index (make-vector state-count #f))) + (unless (and (list? table) + (= (length table) state-count) + (andmap + (lambda (row) + (and (pair? row) (integer? (car row)) + (<= 0 (car row)) + (< (car row) state-count) + (symbol? (cdr row)) + (memq (cdr row) positions))) + table)) + (error "invalid contextual LR position table" table)) + (for-each + (lambda (row) + (when (vector-ref index (car row)) + (error "invalid contextual LR position table" table)) + (vector-set! index (car row) (cdr row))) + table) + index)) + +;;; Bind immutable generated machine data once; product-owned mutable values +;;; become private indexes, an owned scanner plan and a copied digest. +(defstruct contextual-parser-plan (machine scanner positions digest)) + +;;; Publication identity belongs to the admitted contextual product, not its base. +(def (contextual-parser-grammar-digest plan) + (string-copy (contextual-parser-plan-digest plan))) + +(def (prepare-contextual-parser machine product) + (unless (valid-contextual-product? machine product) + (error "contextual parser product does not match parser machine")) + (require-lr-checkpoint-runtime! (parser-machine-runtime machine)) + (let* ((scanner-ir (contextual-ir-ref product 'scanner)) + (positions (validate-contextual-position-table + machine scanner-ir (contextual-ir-ref (contextual-ir-ref product 'context) 'state-positions))) + (scanner (prepare-contextual-scanner-plan scanner-ir))) + (make-contextual-parser-plan machine scanner positions + (string-copy (contextual-ir-ref product 'digest))))) + +;;; Internal deterministic LR slice. Full contextual admission still requires +;;; declaration-macro binding and GLR branch state. Forks yield rejected artifacts. +(def (parse-source/contextual machine product source) + (unless (string? source) + (error "contextual parser product does not match parser machine")) + (parse-source/contextual/prepared (prepare-contextual-parser machine product) source)) + +(def (parse-source/contextual/prepared plan source) + (unless (and (contextual-parser-plan? plan) (string? source)) + (error "contextual parser requires prepared plan and source")) + (parse-contextual + (contextual-parser-plan-machine plan) + ;; Artifacts expose their digest string; keep publication input-local. + (string-copy (contextual-parser-plan-digest plan)) + (contextual-parser-plan-positions plan) + (prepare-contextual-scanner (contextual-parser-plan-scanner plan) source) + source)) + +;;; The admitted byte length belongs to this source. Builders validate/count +;;; inside the parse catch and return publication thunks. Serialization errors +;;; therefore remain publication errors, rather than becoming rejected parses. +(def (parse-source/contextual/prepared/deferred plan source source-byte-length success failure) + (unless (and (contextual-parser-plan? plan) (string? source) + (integer? source-byte-length) (>= source-byte-length 0) + (procedure? success) (procedure? failure)) + (error "contextual deferred parser requires admitted source and builders")) + ((parse-contextual + (contextual-parser-plan-machine plan) + (string-copy (contextual-parser-plan-digest plan)) + (contextual-parser-plan-positions plan) + (prepare-contextual-scanner (contextual-parser-plan-scanner plan) source) + source success failure source-byte-length))) + +(def (parse-contextual machine digest position-table scanner source + (success make-success-parse-artifact) + (failure failure-artifact) (source-byte-length #f)) + (let* ((state (contextual-scanner-initial-state scanner)) + (tokens-reversed '()) + (trivia? (parser-machine-trivia machine)) + (initial (lr-initial-checkpoint + (parser-machine-runtime machine) '()))) + (def (next-input _mode lr-state) + (let (position (vector-ref position-table lr-state)) + (let-values (((input-token next-state) + (contextual-scanner-step scanner state position))) + (set! state next-state) + (when input-token + (set! tokens-reversed (cons input-token tokens-reversed))) + (if (and input-token (trivia? input-token)) + (next-input _mode lr-state) + input-token)))) + (with-catch + (lambda (condition) + (let* ((offset (contextual-scan-state-character-offset state)) + (tokens (reverse tokens-reversed)) + (remaining (substring source offset (string-length source)))) + (failure + machine digest source + (if (zero? (string-length remaining)) + tokens + (append tokens + (list (make-token + 'unknown remaining + (contextual-scan-state-byte-offset state) + (or source-byte-length (string-utf8-length source)))))) + condition))) + (lambda () + (let-values (((status payload) + (with-parser-cost-stage 'contextual-drive + (lr-checkpoint-drive/contextual initial next-input #f)))) + (unless (eq? status 'accepted) + (error "contextual LR drive requires deterministic acceptance" + status)) + (unless (null? (cadr payload)) + (error "contextual LR accepted with trailing tokens")) + (success + digest + source (reverse tokens-reversed) (car payload) trivia?)))))) + ;; : (-> ParserMachine Exception Diagnostic) +(def current-source-stream-observer (make-parameter #f)) + (def (diagnostic machine condition) (let* ((recoveries (parser-ir-ref (parser-machine-ir machine) 'recoveries)) (row (and (pair? recoveries) (car recoveries))) @@ -42,8 +220,15 @@ ;; : (-> ParserMachine Digest String (List Token) Exception ParseArtifact) (def (failure-artifact machine grammar-digest source tokens condition) - (make-failure-parse-artifact - grammar-digest source tokens (diagnostic machine condition))) + (with-parser-cost-stage 'artifact-materialization + (make-failure-parse-artifact + grammar-digest source + ;; Atomic lexing may fail before publishing any tokens. A rejection still + ;; owns every source byte; expose it as one opaque token, never a partial CST. + (if (and (null? tokens) (positive? (string-length source))) + (list (make-token 'unknown source 0 (string-utf8-length source))) + tokens) + (diagnostic machine condition)))) ;; : (-> ParserMachine Digest String (List Token) ParseArtifact) (def (parse-tokenized/with machine grammar-digest source tokens @@ -60,13 +245,17 @@ (call-with-parser-observed-phase observability 'significant-token-filter - (lambda () (parser-significant-tokens machine tokens))))) + (lambda () + (with-parser-cost-stage 'significant-token-filter + (parser-significant-tokens machine tokens)))))) (let-values (((root rest) (call-with-parser-observed-phase observability 'lr-execution - (lambda () (parse-significant significant observability))))) + (lambda () + (with-parser-cost-stage 'lr-execution + (parse-significant significant observability)))))) (unless (null? rest) (error "unexpected trailing token" (token-lexeme (car rest)))) (call-with-parser-observed-phase @@ -79,11 +268,17 @@ (def (parse-tokenized machine grammar-digest source tokens (observability #f)) - (parse-tokenized/with - machine grammar-digest source tokens - (lambda (significant observability) - ((parser-machine-parse machine) significant observability)) - observability)) + ;; Every source request owns its columns and initial frames. Parameter + ;; restoration also protects an outer request after success or rejection. + (parameterize ((current-layout-columns + (and (lr-runtime-layout? (parser-machine-runtime machine)) + (make-layout-columns source))) + (current-layout-frames '())) + (parse-tokenized/with + machine grammar-digest source tokens + (lambda (significant observability) + ((parser-machine-parse machine) significant observability)) + observability))) ;;; Publishes through the same ParseArtifact boundary while the LR phase resumes ;;; an immutable checkpoint bound to this request's significant token stream. @@ -95,47 +290,185 @@ (lr-checkpoint-resume checkpoint observability)) observability)) +;;; A fresh parse keeps one LR execution loop open while the scanner supplies +;;; tokens under the current LR mode and freezes selected fresh captures. The +;;; ordinary checkpoint path still owns resumed, reused and observed requests. +(def (parse-source/directed/stream machine grammar-digest source initial + (use-generated? #t) (capture #f) (spacing #f)) + (let ((direct-drive (and use-generated? (parser-machine-direct-drive machine))) + (source-length (string-length source)) + (trivia? (parser-machine-trivia machine)) + (character-offset 0) + (byte-offset 0) + (pending-character #f) + (pending-byte #f) + (tokens-reversed '()) (modes-reversed '()) (token-count 0) (pending-mode #f) + (source-observer (and capture (current-source-stream-observer)))) + (def (next-input mode) + (if (= character-offset source-length) + #f + (let-values + (((input-token next-character) + (scan-source-token + machine source character-offset byte-offset mode))) + (let ((start-character character-offset) + (start-byte byte-offset)) + (set! character-offset next-character) + (set! byte-offset (token-end input-token)) + (when source-observer (source-observer input-token (lr-lexical-mode-id mode) #f)) + (if (trivia? input-token) + (begin + (set! tokens-reversed + (cons input-token tokens-reversed)) + (when capture + (set! modes-reversed (cons (lr-lexical-mode-id mode) modes-reversed)) + (set! token-count (+ token-count 1))) + (next-input mode)) + (begin + (when capture (set! pending-mode (lr-lexical-mode-id mode))) + (set! pending-character start-character) + (set! pending-byte start-byte) + input-token)))))) + (defrule (record-shift! input-token) + (begin + (set! tokens-reversed (cons input-token tokens-reversed)) + (when capture + (set! modes-reversed (cons pending-mode modes-reversed)) + (set! token-count (+ token-count 1))) + (set! pending-character #f) + (set! pending-byte #f))) + (def after-shift + (if direct-drive + (lambda (input-token _states _values _actions _shifts) (record-shift! input-token)) + (make-lr-token-observer (lambda (input-token) (record-shift! input-token))))) + (def (publish tokens root rest (modes (and capture (reverse modes-reversed)))) + (unless (null? rest) + (error "unexpected trailing token" (token-lexeme (car rest)))) + (when capture (capture #f tokens modes 0 0)) + (with-parser-cost-stage 'artifact-materialization + (make-success-parse-artifact + grammar-digest source tokens root trivia?))) + (when capture (capture initial #f #f 0 0)) + ;; Scanning and LR are interleaved; retain the selected driver and measure + ;; its whole recognition interval rather than instrumenting every token. + (let-values (((status payload) + (with-parser-cost-stage 'source-recognition + (if direct-drive + (with-parser-cost-stage 'generated-drive-execution + (direct-drive + (parser-machine-runtime machine) + next-input after-shift)) + (with-parser-cost-stage 'prepared-checkpoint-execution + (lr-checkpoint-drive + initial next-input after-shift #f 'installed spacing + (and capture + (lambda (checkpoint) + (capture checkpoint #f #f token-count byte-offset))))))))) + (case status + ((accepted) + (unless (and (= character-offset source-length) + (not pending-character)) + (error "streamed LR accepted before source end" + character-offset source-length)) + (publish (reverse tokens-reversed) + (car payload) (cadr payload))) + ((fallback) + ;; A generated driver only handles its admitted deterministic path. + ;; Restart with the original checkpoint and fresh scanner state so + ;; rejected inputs and GLR forks keep the existing diagnostic owner. + (parse-source/directed/stream + machine grammar-digest source initial #f)) + ((fork) + (when capture (capture #f #f (reverse modes-reversed) token-count (or pending-byte byte-offset))) + (let* ((suffix + (with-parser-cost-stage 'fallback-lexical-analysis + (lex-source-from + machine source + (or pending-character character-offset) + (or pending-byte byte-offset)))) + (tokens (append-reverse tokens-reversed suffix))) + (let-values (((significant remaining) + (parser-significant-joined + machine tokens-reversed suffix))) + (let-values (((root rest) + (with-parser-cost-stage 'fallback-lr-execution + (lr-checkpoint-resume-suffix + payload significant remaining)))) + (publish tokens root rest #f))))) + (else (error "streamed LR parse did not terminate" status)))))) + ;;; Deterministic source driver for the sole generated scanner and LR executor. ;;; Each state supplies its interned terminal mode. Trivia advances only the ;;; scanner; significant tokens advance exactly one LR shift. An admitted fork ;;; scans the remaining suffix once and transfers the exact checkpoint to GLR. -(def (parse-source/directed machine grammar-digest source observability) +(def (parse-source/directed machine grammar-digest source observability capture + initial prefix-tokens prefix-modes + start-character start-byte reuse-token reuse-fragment) (let ((source-length (string-length source)) - (trivia? (parser-machine-trivia machine))) - (def (publish tokens root rest) + (trivia? (parser-machine-trivia machine)) + (source-observer (current-source-stream-observer))) + (def (publish tokens modes root rest) (unless (null? rest) (error "unexpected trailing token" (token-lexeme (car rest)))) + (when capture (capture #f tokens modes 0 0)) (call-with-parser-observed-phase observability 'artifact-materialization (lambda () (make-success-parse-artifact grammar-digest source tokens root trivia?)))) - (let loop ((character-offset 0) - (byte-offset 0) - (checkpoint - (lr-initial-checkpoint - (parser-machine-runtime machine) '())) - (tokens-reversed '())) + ;; A resumed checkpoint owns the retained source-token prefix, including + ;; trivia. Its byte cursor and token cursor must describe the same prefix. + (when capture (capture initial #f #f (length prefix-tokens) start-byte)) + (let loop ((character-offset start-character) + (byte-offset start-byte) + (checkpoint initial) + (tokens-reversed (reverse prefix-tokens)) + (modes-reversed (reverse prefix-modes)) + (token-count (length prefix-tokens))) (if (= character-offset source-length) (let-values (((root rest) (call-with-parser-observed-phase observability 'lr-execution (lambda () (lr-checkpoint-resume checkpoint observability))))) - (publish (reverse tokens-reversed) root rest)) + (publish (reverse tokens-reversed) + (and capture (reverse modes-reversed)) + root rest)) + (let (fragment (and reuse-fragment (reuse-fragment character-offset byte-offset checkpoint))) + (if fragment + (let* ((next (vector-ref fragment 0)) (segment (vector-ref fragment 1)) + (segment-modes (vector-ref fragment 2)) + (next-character (vector-ref fragment 3)) (next-byte (vector-ref fragment 4)) + (next-count (+ token-count (length segment)))) + (when source-observer (source-observer segment segment-modes #t)) + (when capture (capture next #f #f next-count next-byte #t)) + (loop next-character next-byte next + (append-reverse segment tokens-reversed) + (if capture (append-reverse segment-modes modes-reversed) modes-reversed) + next-count)) (let (mode (lr-checkpoint-lexical-mode checkpoint)) (let-values (((input-token next-character-offset) (call-with-parser-observed-phase observability 'lexical-analysis (lambda () - (scan-source-token - machine source character-offset byte-offset mode))))) + (let (reused + (and reuse-token + (reuse-token character-offset byte-offset mode))) + (if reused + (values (car reused) (cdr reused)) + (scan-source-token + machine source character-offset byte-offset mode))))))) + (when source-observer (source-observer input-token (lr-lexical-mode-id mode) #f)) (if (call-with-parser-observed-phase observability 'significant-token-filter (lambda () (trivia? input-token))) (loop next-character-offset (token-end input-token) - checkpoint (cons input-token tokens-reversed)) + checkpoint (cons input-token tokens-reversed) + (if capture + (cons (lr-lexical-mode-id mode) modes-reversed) + modes-reversed) + (+ token-count 1)) (let-values (((status next-checkpoint) (call-with-parser-observed-phase @@ -145,10 +478,20 @@ checkpoint input-token observability))))) (case status ((checkpoint) + (when capture + (capture next-checkpoint #f #f + (+ token-count 1) (token-end input-token))) (loop next-character-offset (token-end input-token) next-checkpoint - (cons input-token tokens-reversed))) + (cons input-token tokens-reversed) + (if capture + (cons (lr-lexical-mode-id mode) modes-reversed) + modes-reversed) + (+ token-count 1))) ((fork) + (when capture + (capture #f #f (reverse modes-reversed) + token-count byte-offset)) (let* ((suffix (call-with-parser-observed-phase observability 'lexical-analysis @@ -157,15 +500,14 @@ machine source character-offset byte-offset)))) (tokens - (append (reverse tokens-reversed) suffix))) + (append-reverse tokens-reversed suffix))) (let-values (((significant remaining) (call-with-parser-observed-phase observability 'significant-token-filter (lambda () - (values - (parser-significant-tokens machine tokens) - (parser-significant-tokens machine suffix)))))) + (parser-significant-joined + machine tokens-reversed suffix))))) (let-values (((root rest) (call-with-parser-observed-phase @@ -174,14 +516,26 @@ (lr-checkpoint-resume-suffix next-checkpoint significant remaining observability))))) - (publish tokens root rest))))) + (publish tokens #f root rest))))) (else (error "parser-directed feed did not yield a checkpoint" - status))))))))))) + status))))))))))))) ;; : (-> ParserMachine String ParseArtifact) -(def (parse-source machine source - (observability #f)) +(def (parse-source/with-capture machine source observability capture + (checkpoint #f) (prefix-tokens '()) + (prefix-modes '()) (start-character 0) + (start-byte 0) (reuse-token #f) (reuse-fragment #f) + (capture-spacing #f)) + (call-with-parser-machine-source machine source + (lambda () (parse-source/with-prepared-capture machine source observability capture + checkpoint prefix-tokens prefix-modes start-character start-byte reuse-token reuse-fragment capture-spacing)))) + +(def (parse-source/with-prepared-capture machine source observability capture + (checkpoint #f) (prefix-tokens '()) + (prefix-modes '()) (start-character 0) + (start-byte 0) (reuse-token #f) (reuse-fragment #f) + (capture-spacing #f)) (unless (string? source) (error "parse source must be a string" source)) (let (grammar-digest (parser-machine-grammar-digest machine)) @@ -211,5 +565,87 @@ (failure-artifact machine grammar-digest source tokens condition))))))) (lambda () - (parse-source/directed - machine grammar-digest source observability))))) + (let* ((fresh-base? + (and (not observability) (not checkpoint) (null? prefix-tokens) + (zero? start-character) (zero? start-byte) + (not reuse-token) (not reuse-fragment))) + (fresh? (and fresh-base? (not capture))) + (captured-fresh? (and fresh-base? capture capture-spacing)) + (direct-source + (and fresh? (parser-machine-direct-source machine))) + (candidate + (and direct-source + (with-catch + (lambda (_condition) #f) + (lambda () + (with-parser-cost-stage 'generated-source-product + (direct-source machine source))))))) + (if candidate + candidate + (let (initial + (or checkpoint + (lr-initial-checkpoint + (parser-machine-runtime machine) '()))) + (if (or fresh? captured-fresh?) + (parse-source/directed/stream + machine grammar-digest source initial (not captured-fresh?) + (and captured-fresh? capture) capture-spacing) + (parse-source/directed + machine grammar-digest source observability capture initial + prefix-tokens prefix-modes start-character start-byte + reuse-token reuse-fragment))))))))) + +(def (parse-source machine source (observability #f)) + (if (lr-runtime-layout? (parser-machine-runtime machine)) + (let (digest (parser-machine-grammar-digest machine)) + (with-catch + (lambda (condition) + (failure-artifact machine digest source '() condition)) + (lambda () + (parse-tokenized machine digest source + (with-parser-cost-stage 'lexical-analysis + (lex-source machine source)) observability)))) + (parse-source/with-capture machine source observability #f))) + +;; Capture sparse deterministic prefixes during the initial directed parse. +;; A fork keeps the certified prefix; selective GLR owns the uncaptured suffix. +(def (parse-source/checkpoints/resume machine source spacing checkpoint + prefix-tokens prefix-modes + start-character start-byte reuse-token (reuse-fragment #f)) + (unless (and (integer? spacing) (positive? spacing)) + (error "checkpoint spacing must be positive" spacing)) + (if (lr-runtime-layout? (parser-machine-runtime machine)) + ;; Layout branches carry a column-reference stack. The incremental API + ;; reparses this grammar in full until checkpoints can retain that stack. + (values (parse-source machine source) #f #f #()) + (let ((snapshots '()) (source-tokens #f) (source-modes #f)) + (let* ((artifact + (parse-source/with-capture + machine source #f + (lambda (checkpoint tokens modes token-count byte-end (force? #f)) + (cond + (checkpoint + (let (shifts + (lr-checkpoint-deterministic-shifts checkpoint)) + (when (or force? (zero? (modulo shifts spacing))) + (set! snapshots + (cons + (vector shifts + (lr-checkpoint-prefix-snapshot checkpoint) + token-count byte-end) + snapshots))))) + (tokens + (set! source-tokens tokens) + (when modes (set! source-modes modes))) + (modes (set! source-modes modes)))) + checkpoint prefix-tokens prefix-modes + start-character start-byte reuse-token reuse-fragment spacing)) + (records + (if (parse-artifact-success? artifact) + (list->vector (reverse snapshots)) + #()))) + (values artifact source-tokens source-modes records))))) + +(def (parse-source/checkpoints machine source spacing) + (parse-source/checkpoints/resume + machine source spacing #f '() '() 0 0 #f)) diff --git a/src/runtime/probe.ss b/src/runtime/probe.ss new file mode 100644 index 00000000..e45b2241 --- /dev/null +++ b/src/runtime/probe.ss @@ -0,0 +1,44 @@ +;;; -*- Gerbil -*- +;;; One request-owned scanner result, reusable only in its exact lexical context. +(import (only-in ./lexer scan-source-token)) +(export current-lr-lexical-plan-reuse-enabled? make-source-probe-cache source-probe-scan source-probe-take!) + +;;; Complete-edit controls favor some flat prefixes, but nested and history +;;; gains are not stable. Keep stronger lexical reuse an explicit experiment. +(def current-lr-lexical-plan-reuse-enabled? (make-parameter #f)) + +(defstruct source-probe-cache-instance (machine source slot scanner)) +(def (make-source-probe-cache machine source (scanner #f)) + (make-source-probe-cache-instance machine source #f scanner)) + +;;; An internal source owner may provide a scanner that returns independently +;;; certified old tokens; the slot still binds the actual requested mode. +;;; A failed speculative read leaves no cached error or prior stale result. +;;; The canonical lexer remains responsible for diagnostics. +(def (source-probe-scan cache character byte mode) + (source-probe-cache-instance-slot-set! cache #f) + (let-values (((token next-character) + (if (source-probe-cache-instance-scanner cache) + ((source-probe-cache-instance-scanner cache) character byte mode) + (scan-source-token (source-probe-cache-instance-machine cache) + (source-probe-cache-instance-source cache) + character byte mode)))) + (source-probe-cache-instance-slot-set! cache + (vector character byte mode token next-character)) + (values token next-character))) + +;;; Modes are interned in one prepared runtime. Object identity is stronger +;;; than comparing the mode's numeric ID, which another runtime can also use. +;;; A future-position result survives intervening trivia; consumed/passed results +;;; are dropped. The cache never grows with source size or edit history. +(def (source-probe-take! cache character byte mode) + (let (slot (source-probe-cache-instance-slot cache)) + (and slot + (cond + ((and (= character (vector-ref slot 0)) (= byte (vector-ref slot 1)) + (eq? mode (vector-ref slot 2))) + (source-probe-cache-instance-slot-set! cache #f) + (cons (vector-ref slot 3) (vector-ref slot 4))) + ((> byte (vector-ref slot 1)) + (source-probe-cache-instance-slot-set! cache #f) #f) + (else #f))))) diff --git a/src/runtime/recognition.ss b/src/runtime/recognition.ss index 3aeb7ba4..43222fe5 100644 --- a/src/runtime/recognition.ss +++ b/src/runtime/recognition.ss @@ -1,8 +1,13 @@ ;;; -*- Gerbil -*- ;;; Private recognition values produced by generated parser machines. -(import (only-in ./token token? token-end token-start)) -(export make-recognition-node +(import (only-in ./token make-token token? token-lexeme token-end token-start) + (only-in ./event-program event-program-value? event-program-value-start event-program-value-end)) +(export prepare-recognition-source recognition-source? recognition-source-text + recognition-source-offset recognition-source-token recognition-source-node + relocate-recognition-value + recognition-relocation? recognition-relocation-value recognition-relocation-delta + make-recognition-node recognition-node? recognition-node-kind recognition-node-start @@ -19,24 +24,133 @@ recognition-child-field-set! recognition-child-value recognition-value-start - recognition-value-end) + recognition-value-end recognition-value-bounds with-recognition-value-bounds) -(defstruct recognition-node (kind start end children) transparent: #t) -(defstruct recognition-fragment (start end children) transparent: #t) -(defstruct recognition-child (field value) transparent: #t) +;;; These engine-owned values have no subclass protocol. Final classes let the +;;; compiler emit checked direct predicates and slot access in hot reducers. +(defstruct recognition-node (kind start end children) transparent: #t final: #t) +(defstruct recognition-fragment (start end children) transparent: #t final: #t) +(defstruct recognition-child (field value) transparent: #t final: #t) + + +;;; Position views share the semantic subtree. Collapse repeated moves of the +;;; same view so successive edits do not retain a chain of source snapshots. +(defstruct recognition-relocation (value delta) transparent: #t final: #t) +(def (relocate-recognition-value value delta (force? #f)) + (cond + ((recognition-relocation? value) + (make-recognition-relocation (recognition-relocation-value value) + (+ delta (recognition-relocation-delta value)))) + ((and (zero? delta) (not force?)) value) + (else (make-recognition-relocation value delta)))) ;; : (-> RecognitionValue Nat) (def (recognition-value-start value) (cond + ((recognition-relocation? value) + (+ (recognition-relocation-delta value) + (recognition-value-start (recognition-relocation-value value)))) ((token? value) (token-start value)) ((recognition-node? value) (recognition-node-start value)) ((recognition-fragment? value) (recognition-fragment-start value)) + ((event-program-value? value) (event-program-value-start value)) (else (error "invalid recognition value" value)))) +;;; Constructors need both endpoints. Classify the closed recognition value +;;; once rather than repeating token admission and variant dispatch for each +;;; endpoint. Position views retain their own displacement and source identity. +;;; Constructors need both endpoints. Classify the closed recognition value +;;; once rather than repeating token admission and variant dispatch for each +;;; endpoint. Position views retain their own displacement and source identity. +(defrule (with-recognition-value-bounds value-expr (start end) body ...) + (let (current value-expr) + (cond + ((recognition-relocation? current) + (let-values (((inner-start inner-end) + (recognition-value-bounds (recognition-relocation-value current)))) + (let ((start (+ (recognition-relocation-delta current) inner-start)) + (end (+ (recognition-relocation-delta current) inner-end))) body ...))) + ((token? current) + (let ((start (token-start current)) (end (token-end current))) body ...)) + ((recognition-node? current) + (let ((start (recognition-node-start current)) (end (recognition-node-end current))) body ...)) + ((recognition-fragment? current) + (let ((start (recognition-fragment-start current)) (end (recognition-fragment-end current))) body ...)) + ((event-program-value? current) + (let ((start (event-program-value-start current)) (end (event-program-value-end current))) body ...)) + (else (error "invalid recognition value" current))))) + +;;; External callers retain the multiple-value protocol. Constructor bodies +;;; expand at this classification boundary and need no transient values object. +(def (recognition-value-bounds value) + (with-recognition-value-bounds value (start end) (values start end))) + ;; : (-> RecognitionValue Nat) (def (recognition-value-end value) (cond + ((recognition-relocation? value) + (+ (recognition-relocation-delta value) + (recognition-value-end (recognition-relocation-value value)))) ((token? value) (token-end value)) ((recognition-node? value) (recognition-node-end value)) ((recognition-fragment? value) (recognition-fragment-end value)) + ((event-program-value? value) (event-program-value-end value)) (else (error "invalid recognition value" value)))) + +;;; Source-backed results share one UTF-8 boundary index. Preparing a source is +;;; linear in characters; every node/token endpoint is then a constant-time +;;; lookup, including nonmonotonic recursive publication. Own the text so a +;;; caller cannot invalidate admitted boundaries by mutating the original token. +(defstruct recognition-source (owned-text origin offsets)) + +(def (prepare-recognition-source token) + (unless (and (token? token) (exact-integer? (token-start token)) + (exact-integer? (token-end token))) + (error "invalid recognition source token" token)) + (let* ((text (string-copy (token-lexeme token))) + (length (string-length text)) (origin (token-start token))) + ;; ASCII uses origin + character and needs no vector. Allocate a boundary + ;; index only on the first multibyte scalar, then backfill the ASCII prefix. + (let loop ((at 0) (byte origin) (offsets #f)) + (when offsets (vector-set! offsets at byte)) + (if (= at length) + (begin + (unless (= byte (token-end token)) + (error "recognition source byte span disagrees with text" token)) + (make-recognition-source text origin offsets)) + (let* ((code (char->integer (string-ref text at))) + (index + (if (or offsets (< code #x80)) offsets + (let (index (make-vector (+ length 1))) + (let prefix ((before 0)) + (when (<= before at) + (vector-set! index before (+ origin before)) + (prefix (+ before 1)))) + index)))) + (unless (and (<= 0 code #x10ffff) (not (<= #xd800 code #xdfff))) + (error "invalid Unicode scalar in recognition source" at)) + (loop (+ at 1) + (+ byte (cond ((< code #x80) 1) ((< code #x800) 2) + ((< code #x10000) 3) (else 4))) index)))))) + +(def (recognition-source-text source) + (string-copy (recognition-source-owned-text source))) + +(def (recognition-source-offset source character) + (unless (and (exact-integer? character) (<= 0 character) + (<= character (string-length (recognition-source-owned-text source)))) + (error "recognition boundary outside admitted source" character)) + (let (index (recognition-source-offsets source)) + (if index (vector-ref index character) + (+ (recognition-source-origin source) character)))) + +(def (recognition-source-token source kind start end) + (unless (<= start end) (error "reversed recognition token bounds" start end)) + (let ((begin (recognition-source-offset source start)) + (finish (recognition-source-offset source end))) + (make-token kind (substring (recognition-source-owned-text source) start end) begin finish))) + +(def (recognition-source-node source kind start end children) + (unless (<= start end) (error "reversed recognition node bounds" start end)) + (make-recognition-node kind (recognition-source-offset source start) + (recognition-source-offset source end) children)) diff --git a/src/runtime/record-projection.ss b/src/runtime/record-projection.ss new file mode 100644 index 00000000..b17a2b6d --- /dev/null +++ b/src/runtime/record-projection.ss @@ -0,0 +1,108 @@ +;;; Closed record selectors and POO recipes; no language-specific projection code. +(import (only-in :clan/poo/object .o .cc .ref .slot? object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in ../language/descriptor language-grammar? language-grammar-machine) + (only-in ../compiler/machine parser-machine-grammar-digest) + (only-in ./lexical-source scan-header-delimiter) + (only-in ./artifact parse-artifact-valid? parse-artifact-success? parse-artifact-roundtrip parse-artifact-ref)) +(export RecordProjection. RecordProjectionContract bind-record-projection deflanguage-projection) +(def (selector? value count (depth 0)) + (and (< depth 64) (list? value) (pair? value) + (case (car value) + ((text) (and (= (length value) 2) (string? (cadr value)))) + ((record) (and (= (length value) 3) (string? (cadr value)) (positive? (string-length (cadr value))) + (exact-integer? (caddr value)) (<= 0 (caddr value)))) + ((split) (and (= (length value) 4) (selector? (cadr value) count (+ depth 1)) + (exact-integer? (caddr value)) (<= 0 (caddr value)) (< (caddr value) count) + (exact-integer? (cadddr value)) (<= 0 (cadddr value)))) + ((concat list) (and (pair? (cdr value)) + (andmap (lambda (child) (selector? child count (+ depth 1))) (cdr value)))) + (else #f)))) +(def (unique-columns? rows) + (let loop ((remaining rows) (names '(sourceDigest grammarDigest))) + (or (null? remaining) + (and (pair? (car remaining)) (symbol? (caar remaining)) + (not (memq (caar remaining) names)) + (loop (cdr remaining) (cons (caar remaining) names)))))) +(def (projection-shape? value) + (with-catch (lambda (_) #f) + (lambda () + (and (object? value) + (andmap (lambda (slot) (.slot? value slot)) '(descriptor prefix delimiter-count expectations constants columns metadata)) + (language-grammar? (.ref value 'descriptor)) (object? (.ref value 'metadata)) + (string? (.ref value 'prefix)) (positive? (string-length (.ref value 'prefix))) + (exact-integer? (.ref value 'delimiter-count)) (<= 1 (.ref value 'delimiter-count) 32) + (list? (.ref value 'constants)) (list? (.ref value 'columns)) (list? (.ref value 'expectations)) + (unique-columns? (append (.ref value 'constants) (.ref value 'columns))) + (andmap (lambda (row) (and (list? row) (= (length row) 2) + (selector? (car row) (.ref value 'delimiter-count)) + (or (string? (cadr row)) (list? (cadr row))))) (.ref value 'expectations)) + (andmap (lambda (row) (and (pair? row) (symbol? (car row)) + (selector? (cdr row) (.ref value 'delimiter-count)))) (.ref value 'columns)) + (andmap (lambda (row) (or (string? (cdr row)) (symbol? (cdr row)) (number? (cdr row)) (boolean? (cdr row)))) + (.ref value 'constants)))))) +(define-type (RecordProjectionContract @ PooFlowContract.) + identity: 'gerbil-parser/record-projection + .classify: (lambda (candidate context) + (let (accepted? (projection-shape? candidate)) + (poo-flow-classification-evidence 'gerbil-parser/record-projection candidate accepted? + (if accepted? '() '((expected gerbil-parser/record-projection))) context)))) +(def RecordProjection. + (.o descriptor: #f prefix: #f delimiter-count: #f expectations: '() constants: '() columns: '() metadata: (.o))) +(def (split-text text delimiter) + (let (limit (string-length text)) + (let loop ((start 0) (at 0) (parts '())) + (cond ((= at limit) (reverse (cons (substring text start at) parts))) + ((char=? (string-ref text at) delimiter) + (loop (+ at 1) (+ at 1) (cons (substring text start at) parts))) + (else (loop start (+ at 1) parts)))))) +(def (record-lines source) + (let (limit (string-length source)) + (let loop ((start 0) (at 0) (rows '())) + (if (or (= at limit) (memv (string-ref source at) '(#\return #\newline))) + (let (next-rows (if (= start at) rows (cons (substring source start at) rows))) + (if (= at limit) (reverse next-rows) (loop (+ at 1) (+ at 1) next-rows))) + (loop start (+ at 1) rows))))) +(def (required-item items index) + (let loop ((remaining items) (at index)) + (cond ((null? remaining) (error "record selector has no required item" index)) + ((zero? at) (car remaining)) + (else (loop (cdr remaining) (- at 1)))))) +(def (copy-data value) + (cond ((pair? value) (cons (copy-data (car value)) (copy-data (cdr value)))) + ((string? value) (string-copy value)) (else value))) +(def (bind-record-projection descriptor prototype) + (let (candidate (.cc prototype 'descriptor descriptor)) + (validate RecordProjectionContract candidate) + (let ((digest (parser-machine-grammar-digest (language-grammar-machine descriptor))) + (prefix (string-copy (.ref candidate 'prefix))) (count (.ref candidate 'delimiter-count)) + (expectations (copy-data (.ref candidate 'expectations))) + (constants (copy-data (.ref candidate 'constants))) (columns (copy-data (.ref candidate 'columns)))) + (lambda (artifact) + (unless (and (parse-artifact-valid? artifact) (parse-artifact-success? artifact) + (equal? (parse-artifact-ref artifact 'grammarDigest) digest)) + (error "record projection requires its own accepted artifact")) + (let* ((source (parse-artifact-roundtrip artifact)) (base (string-length prefix))) + (unless (scan-header-delimiter source base prefix count 0) + (error "record projection header violates its recipe")) + (let ((delimiters (substring source base (+ base count))) + (records (map (lambda (line) (split-text line (string-ref source base))) (record-lines source)))) + (def (evaluate expression) + (case (car expression) + ((text) (cadr expression)) + ((record) (let (row (find (lambda (row) (equal? (car row) (cadr expression))) records)) + (unless row (error "record projection has no required record" (cadr expression))) + (required-item row (caddr expression)))) + ((split) (required-item (split-text (evaluate (cadr expression)) (string-ref delimiters (caddr expression))) + (cadddr expression))) + ((concat) (apply string-append (map evaluate (cdr expression)))) + ((list) (map evaluate (cdr expression))))) + (for-each (lambda (row) (unless (equal? (evaluate (car row)) (cadr row)) + (error "record projection does not admit this message" row))) expectations) + (append (list (cons 'sourceDigest (parse-artifact-ref artifact 'sourceDigest)) + (cons 'grammarDigest digest)) constants + (map (lambda (row) (cons (car row) (evaluate (cdr row)))) columns)))))))) +(defrules deflanguage-projection (grammar strategy) + ((_ binding (grammar descriptor) (strategy prototype)) + (def binding (bind-record-projection descriptor prototype)))) diff --git a/src/runtime/recovery.ss b/src/runtime/recovery.ss index 7ae86575..3a31573f 100644 --- a/src/runtime/recovery.ss +++ b/src/runtime/recovery.ss @@ -4,16 +4,19 @@ (import (only-in ../compiler/machine parser-machine-grammar-digest parser-machine-ir parser-machine-runtime) - (only-in ../compiler/parser-ir parser-ir-ref) + (only-in ./parser-ir-data parser-ir-ref) (only-in ./artifact - parse-artifact-ref parse-artifact-success?) + parse-artifact-ref parse-artifact-success? + parse-artifact-events token-event? token-event-token-kind + token-event-lexeme event-start event-end) (only-in ./lexer lex-source) (only-in ./lr-parser lr-checkpoint-frontier lr-failure-frontier? lr-failure-frontier-expected-terminals lr-failure-frontier-remaining-tokens lr-failure-frontier-resume lr-failure-frontier-state - lr-initial-checkpoint lr-rejection-condition?) + lr-initial-checkpoint lr-rejection-condition? + lr-runtime-layout?) (only-in ./parser parse-source) (only-in ./significant parser-significant-tokens) (only-in ./token @@ -24,6 +27,25 @@ (def +recovery-receipt-schema+ "gerbil-parser.recovery-receipt.v1") +;;; The ordinary rejected artifact already owns the complete fallback lexical +;;; stream. Reuse it for private recovery instead of scanning the source again. +;;; A lexical failure may publish no tokens; retain the existing fallback then. +(def (rejected-artifact-tokens artifact) + (let ((events (parse-artifact-events artifact)) + (length (parse-artifact-ref artifact 'sourceByteLength))) + (let loop ((rest events) (offset 0) (tokens '())) + (cond + ((null? rest) (and (= offset length) (reverse tokens))) + ((and (token-event? (car rest)) + (= (event-start (car rest)) offset)) + (let (event (car rest)) + (loop (cdr rest) (event-end event) + (cons (make-token (token-event-token-kind event) + (token-event-lexeme event) + offset (event-end event)) + tokens)))) + (else #f))))) + ;; : (-> Terminal (OrFalse String)) (def (terminal-literal terminal) (and (pair? terminal) @@ -186,6 +208,10 @@ (values artifact (recovery-receipt machine artifact row 'not-needed #f 0 budget #f #f 0))) + ((lr-runtime-layout? (parser-machine-runtime machine)) + (values artifact + (recovery-receipt machine artifact row 'disabled #f 0 + budget #f #f 0))) ((not row) (values artifact (recovery-receipt machine artifact #f 'disabled #f 0 @@ -195,7 +221,8 @@ (with-catch (lambda (_) '()) (lambda () (parser-significant-tokens machine - (lex-source machine source))))) + (or (rejected-artifact-tokens artifact) + (lex-source machine source)))))) (let-values (((operation attempts exhausted? frontier) (find-recovery machine tokens budget))) (values artifact diff --git a/src/runtime/reduce.ss b/src/runtime/reduce.ss index f1b0d805..d0716cc1 100644 --- a/src/runtime/reduce.ss +++ b/src/runtime/reduce.ss @@ -4,59 +4,73 @@ (import (only-in ./recognition make-recognition-child make-recognition-fragment make-recognition-node recognition-child-field - recognition-child-value - recognition-value-end recognition-value-start)) + recognition-child-value with-recognition-value-bounds)) +(import (only-in ./funcs recognition-sequence-arity recognition-sequence-start + recognition-sequence-end recognition-sequence->list + recognition-sequence-for-action)) (export recognition-children-field - recognition-children-alias) - -;; children-start -;; : (-> List Fixnum Fixnum) -(def (children-start children default-offset) - (or (alet (child (and (pair? children) (car children))) - (recognition-value-start (recognition-child-value child))) - default-offset)) - -;; children-end -;; : (-> List Fixnum Fixnum) -(def (children-end children default-offset) - (if (null? children) - default-offset - (recognition-value-end (recognition-child-value (last children))))) - -;; recognition-children-field -;; : (-> Symbol List Fixnum [(-> Fixnum Fixnum List RecognitionFragment)] List) -(def (recognition-children-field name children default-offset - (fragment-constructor make-recognition-fragment)) - (match children - ([] '()) - ([child] - (if (recognition-child-field child) - (list - (make-recognition-child - name - (fragment-constructor - (children-start children default-offset) - (children-end children default-offset) - children))) - (list (make-recognition-child - name (recognition-child-value child))))) - (_ - (list - (make-recognition-child - name - (fragment-constructor - (children-start children default-offset) - (children-end children default-offset) - children)))))) - -;; recognition-children-alias -;; : (-> Symbol List Fixnum List) -(def (recognition-children-alias kind children default-offset) - (list - (make-recognition-child - #f - (make-recognition-node - kind - (children-start children default-offset) - (children-end children default-offset) - children)))) + recognition-children-alias recognition-action-field recognition-action-alias + make-recognition-field-kernel make-recognition-alias-kernel) + +;;; Stage one input-shape decision and one singleton-value classification into +;;; both constructor families. Empty aliases retain their invocation boundary. +(defrule (with-recognition-children-bounds children-expr offset-expr (start end) body ...) + (let ((children children-expr) (offset offset-expr)) + (cond + ((null? children) (let ((start offset) (end offset)) body ...)) + ((and (pair? children) (null? (cdr children))) + (with-recognition-value-bounds (recognition-child-value (car children)) + (start end) body ...)) + (else + (let ((start (recognition-sequence-start children offset)) + (end (recognition-sequence-end children offset))) body ...))))) + +(defrule (construct-recognition-bounded (constructor prefix ...) children offset prepare) + (with-recognition-children-bounds children offset (start end) + (constructor prefix ... start end (prepare children)))) + +(defrule (construct-recognition-field name children default-offset fragment-constructor prepare) + (case (recognition-sequence-arity children) + ((0) '()) + ((1) + (let (child (car (recognition-sequence->list children))) + (if (recognition-child-field child) + (list (make-recognition-child name + (construct-recognition-bounded (fragment-constructor) + children default-offset prepare))) + (list (make-recognition-child name (recognition-child-value child)))))) + (else + (list (make-recognition-child name + (construct-recognition-bounded (fragment-constructor) + children default-offset prepare)))))) + +(defrule (construct-recognition-alias kind children default-offset prepare) + (list (make-recognition-child #f + (construct-recognition-bounded (make-recognition-node kind) + children default-offset prepare)))) + +(defrule (define-recognition-field reducer prepare) + (def (reducer name children default-offset + (fragment-constructor make-recognition-fragment)) + (construct-recognition-field name children default-offset fragment-constructor prepare))) +(defrule (define-recognition-alias reducer prepare) + (def (reducer kind children default-offset) + (construct-recognition-alias kind children default-offset prepare))) + +;;; Bind grammar-static names outside the request. The canonical constructor +;;; body expands into the kernel, removing its extra semantic wrapper call. +(def (make-recognition-field-kernel name) + (lambda (children offset constructor) + (construct-recognition-field name children offset constructor recognition-sequence-for-action))) +(def (make-recognition-alias-kernel kind) + (lambda (children offset _constructor) + (construct-recognition-alias kind children offset recognition-sequence-for-action))) + +;;; Stage normalization into action constructors. Reading the endpoints from +;;; the shared sequence before flattening avoids a second linear list-tail +;;; scan. Ordinary reducers retain their borrowed child representation. +(defrule (borrow-recognition-children children) children) +(define-recognition-field recognition-children-field borrow-recognition-children) +(define-recognition-alias recognition-children-alias borrow-recognition-children) +(define-recognition-field recognition-action-field recognition-sequence-for-action) +(define-recognition-alias recognition-action-alias recognition-sequence-for-action) diff --git a/src/runtime/reduction-plan.ss b/src/runtime/reduction-plan.ss new file mode 100644 index 00000000..8b50d28a --- /dev/null +++ b/src/runtime/reduction-plan.ss @@ -0,0 +1,143 @@ +;;; -*- Gerbil -*- +;;; Prepared semantic kernels; the LR driver owns stacks and branch admission. +(import (only-in :core/poo-clos/interface .defgeneric .defmethod) + (only-in :std/vector/vector vector-map/index) + (only-in ../compiler/lr production-rhs operand-actions) + (only-in ./recognition make-recognition-fragment) + (only-in ./reduce make-recognition-field-kernel make-recognition-alias-kernel) + (only-in ./funcs recognition-sequence-append + recognition-sequence-start) + (only-in ./event-reduce event-children-field event-children-alias) + (only-in ./event-program event-program-append)) +(export prepare-lr-reduction-plans reduce-lr-stack-plan reduce-lr-vector-plan + reduce-lr-source-plan reduce-lr-event-plan source-operand-offsets) + +;;; An empty marked operand is anchored at the next semantic input boundary, +;;; or the production's lookahead boundary when no later value has a start. +;;; Forward executors build this suffix index only upon encountering an empty +;;; marked operand; the reverse stack already has the following children. +(def (source-operand-offsets width values offset start) + (let (offsets (make-vector width)) + (let fill ((index 0) (values values)) + (when (fx< index width) + (vector-set! offsets index (start (car values) #f)) + (fill (fx+ index 1) (cdr values)))) + (let resolve ((index (fx- width 1)) (following offset)) + (when (fx>= index 0) + (let (boundary (or (vector-ref offsets index) following)) + (vector-set! offsets index boundary) + (resolve (fx- index 1) boundary)))) + offsets)) + +;;; Canonical operands remain diagnostic data. Preparation stages the finite +;;; backend/action dispatch once; request calls receive their own offset and +;;; fragment constructor. An absent step is the identity operand. +(def (event-field-kernel name) + (lambda (value offset _constructor) (event-children-field name value offset))) +(def (event-alias-kernel name) + (lambda (value offset _constructor) (event-children-alias name value offset))) + +;;; The backend catalog is closed and private. Resolve its generic choices +;;; once at module initialization; grammar preparation binds only rule names. +(.defgeneric select-lr-action-factory (backend kind)) +(.defmethod (select-lr-action-factory (backend (eql 'recognition)) (kind (eql 'field))) + make-recognition-field-kernel) +(.defmethod (select-lr-action-factory (backend (eql 'recognition)) (kind (eql 'alias))) + make-recognition-alias-kernel) +(.defmethod (select-lr-action-factory (backend (eql 'event)) (kind (eql 'field))) + event-field-kernel) +(.defmethod (select-lr-action-factory (backend (eql 'event)) (kind (eql 'alias))) + event-alias-kernel) + +(def +recognition-action-factories+ + (cons (select-lr-action-factory 'recognition 'field) + (select-lr-action-factory 'recognition 'alias))) +(def +event-action-factories+ + (cons (select-lr-action-factory 'event 'field) + (select-lr-action-factory 'event 'alias))) + +(def (prepare-operand-actions field alias actions) + (and (pair? actions) + (let* ((action (car actions)) + (first ((case (car action) ((field) field) ((alias) alias)) (cadr action))) + (remaining (prepare-operand-actions field alias (cdr actions)))) + (if remaining + (lambda (value offset constructor) + (remaining (first value offset constructor) offset constructor)) + first)))) + +(def (prepare-reduction-plan field alias production) + (let* ((operands (production-rhs production)) + (plan (make-vector (length operands)))) + (let loop ((operands operands) (index 0)) + (when (pair? operands) + (vector-set! plan index (prepare-operand-actions field alias (operand-actions (car operands)))) + (loop (cdr operands) (fx+ index 1)))) + plan)) + +(def (prepare-lr-reduction-plans backend table) + (let (factories (case backend + ((recognition) +recognition-action-factories+) + ((event) +event-action-factories+) + (else (error "unknown LR semantic backend" backend)))) + (vector-map/index + (lambda (_index production) + (prepare-reduction-plan (car factories) (cdr factories) production)) table))) + +;;; The deterministic executor consumes the immutable semantic stack directly. +;;; Each prepared field/alias chain retains declaration order. Prepending each +;;; completed operand restores source order without an intermediate value list. +;;; The canonical projection reducer remains an independent consumer. +(defrule (reduce-lr-cursor-plan actions offset (cursor initial read-value advance)) + (let ((width (vector-length actions)) (cursor initial)) + (case width + ((0) '()) + ((1) + (let ((step (vector-ref actions 0)) (input read-value)) + (if step (step input offset make-recognition-fragment) input))) + (else + (let loop ((index (fx- width 1)) (cursor cursor) (children '())) + (if (fx>= index 0) + (let* ((step (vector-ref actions index)) (input read-value) + (value (if step + (step input (if (null? input) + (recognition-sequence-start children offset) offset) + make-recognition-fragment) + input))) + (loop (fx- index 1) advance (recognition-sequence-append value children))) + children)))))) +(def (reduce-lr-stack-plan actions stack offset) + (reduce-lr-cursor-plan actions offset (cursor stack (car cursor) (cdr cursor)))) +(def (reduce-lr-vector-plan actions storage top offset) + (reduce-lr-cursor-plan actions offset + (cursor top (vector-ref storage (fx- cursor 1)) (fx- cursor 1)))) + +;;; GLR retains its source-order association and request-local interner. +;;; Event construction retains its own sequential append discipline. Stage +;;; the join and empty value into the same ordered private traversal. +(defrule (define-source-plan-reducer name empty join empty? start) + (def (name actions values offset constructor) + (let (width (vector-length actions)) + (case width + ((0) empty) + ((1) + (let (step (vector-ref actions 0)) + (if step (step (car values) offset constructor) (car values)))) + (else + (let loop ((index 0) (remaining values) (children empty) (offsets #f)) + (if (fx< index width) + (let* ((step (vector-ref actions index)) (input (car remaining)) + (anchored? (and step (empty? input))) + (offsets (if anchored? (or offsets (source-operand-offsets width values offset start)) + offsets))) + (loop (fx+ index 1) (cdr remaining) + (join children (if step + (step input (if anchored? (vector-ref offsets index) offset) constructor) + input)) offsets)) + children))))))) + +(define-source-plan-reducer reduce-lr-source-plan '() recognition-sequence-append + null? recognition-sequence-start) +(defrule (empty-event-input? value) (or (not value) (null? value))) +(define-source-plan-reducer reduce-lr-event-plan #f event-program-append + empty-event-input? recognition-sequence-start) diff --git a/src/runtime/region-scanner.ss b/src/runtime/region-scanner.ss new file mode 100644 index 00000000..e958af86 --- /dev/null +++ b/src/runtime/region-scanner.ss @@ -0,0 +1,223 @@ +;;; -*- Gerbil -*- +;;; Prepared closed region plans. A single forward cursor and explicit frames +;;; own nested matching; callers declare delimiters, escaping and quote scope. +(import (only-in ./scan make-literal-end-scanner)) +(export defregion-plan valid-region-specification? source-prefix-at? region-plan? prepare-region-plan region-plan-end + region-plan-quote-end region-plan-pair-end region-plan-operator + region-plan-specification prepare-region-source prepare-scoped-region-source region-source? + region-source-pair-end region-source-quote-end) + +(defstruct region-plan (data stops quotes pairs operators pair-index)) + +(defrules defregion-plan (stops quotes pairs consume-initial-stop) + ((_ name (stops stop ...) (quotes quote-row ...) (pairs pair ...) (consume-initial-stop flag)) + (def name (prepare-region-plan '((stop ...) (quote-row ...) (pair ...) flag))))) + +(def (copy-data value) + (cond ((pair? value) (cons (copy-data (car value)) (copy-data (cdr value)))) + ((string? value) (string-copy value)) + (else value))) + +(def (region-plan-specification plan) (copy-data (region-plan-data plan))) + +(def (unique? rows key) + (let loop ((rest rows) (seen '())) + (or (null? rest) + (let (value (key (car rest))) + (and (not (member value seen)) + (loop (cdr rest) (cons value seen))))))) + +(def (valid-region-specification? spec) + (and (list? spec) (= (length spec) 4) + (list? (car spec)) (pair? (car spec)) + (andmap (lambda (text) (and (string? text) (positive? (string-length text)))) (car spec)) + (unique? (car spec) (lambda (value) value)) + (list? (cadr spec)) (list? (caddr spec)) (boolean? (cadddr spec)) + (andmap + (lambda (row) + (and (list? row) (= (length row) 4) (string? (car row)) + (positive? (string-length (car row))) (char? (cadr row)) + (char? (caddr row)) (exact-integer? (cadddr row)) + (<= 1 (cadddr row) (string-length (car row))) + (char=? (string-ref (car row) (- (string-length (car row)) 1)) (cadr row)))) + (caddr spec)) + (unique? (caddr spec) car) + (andmap + (lambda (row) + (and (list? row) (= (length row) 3) (char? (car row)) + (boolean? (cadr row)) (list? (caddr row)) + (andmap (lambda (prefix) (and (string? prefix) (assoc prefix (caddr spec)))) (caddr row)) + (unique? (caddr row) (lambda (value) value)))) + (cadr spec)) + (unique? (cadr spec) car))) + +(def (prepare-pair-index pairs) + ;; Buckets retain the owned declaration rows and their stable ordering. + ;; Most input characters have no opener candidate; they need no prefix walk. + (let ((ascii (make-vector 128 '())) (wide (make-hash-table-eqv))) + (for-each + (lambda (row) + (let (code (char->integer (string-ref (car row) 0))) + (if (< code 128) + (vector-set! ascii code (cons row (vector-ref ascii code))) + (hash-put! wide code (cons row (or (hash-get wide code) '())))))) + (reverse pairs)) + (cons ascii wide))) + +(def (prepare-region-plan spec) + (unless (valid-region-specification? spec) (error "invalid closed region plan" spec)) + (let (owned (copy-data spec)) + (make-region-plan owned (car owned) (cadr owned) (caddr owned) + (make-literal-end-scanner (car owned)) + (prepare-pair-index (caddr owned))))) + +(def (source-prefix-at? source at text) + (let ((width (string-length text)) (limit (string-length source))) + (and (exact-integer? at) (<= 0 at) (<= (+ at width) limit) + (let loop ((index 0)) + (or (= index width) + (and (char=? (string-ref text index) (string-ref source (+ at index))) + (loop (+ index 1)))))))) + +(def (region-plan-operator plan source start) + (let (end ((region-plan-operators plan) source start)) + (and end (substring source start end)))) + +(def (pair-at plan source at (allowed #f)) + (and (not (null? allowed)) (exact-integer? at) (<= 0 at) (< at (string-length source)) + (let* ((index (region-plan-pair-index plan)) + (code (char->integer (string-ref source at))) + (rows (if (< code 128) (vector-ref (car index) code) + (or (hash-get (cdr index) code) '())))) + (let loop ((rest rows) (selected #f)) + (if (null? rest) selected + (let (row (car rest)) + (loop (cdr rest) + (if (and (or (not allowed) (member (car row) allowed)) + (source-prefix-at? source at (car row)) + (or (not selected) + (> (string-length (car row)) (string-length (car selected))))) + row selected)))))))) + +;;; Declaration rows remain immutable. Each match owns its frame vectors and +;;; stack cells; only the active pair frame's depth changes in place. No frame +;;; escapes into the source cache, which stores completed numeric endpoints. +;;; There is no recursive call on source nesting or source-prefix substring. +(def (cached-pair-end ends start row) + (and ends (let (entries (hash-get (vector-ref ends 1) row)) (and entries (hash-get entries start))))) +(def (cache-pair-end! ends start row end) + (let (entries (or (hash-get (vector-ref ends 1) row) + (let (table (make-hash-table-eqv)) (hash-put! (vector-ref ends 1) row table) table))) + (hash-put! entries start end))) +(def (pair-frame row depth start scopes) + (vector 'pair row depth start (let (entry (assoc (car row) scopes)) (and entry (cdr entry))))) +(def (region-match plan source start frames word? (ends #f) (scopes '())) + (def (close! frame end) + (when ends + (if (eq? (vector-ref frame 0) 'quote) + (hash-put! (vector-ref ends 0) (vector-ref frame 2) end) + (cache-pair-end! ends (vector-ref frame 3) (vector-ref frame 1) end)))) + (let (limit (string-length source)) + (let loop ((at start) (stack frames)) + (cond + ((= at limit) + (if (null? stack) at (error "unterminated declared region" start))) + ((pair? stack) + (let* ((frame (car stack)) (row (vector-ref frame 1)) + (ch (string-ref source at)) (quote? (eq? (vector-ref frame 0) 'quote)) + (nested (pair-at plan source at (if quote? (caddr row) (vector-ref frame 4))))) + (cond + ((and (char=? ch #\\) (or (not quote?) (cadr row))) + (loop (min limit (+ at 2)) stack)) + ((and quote? (char=? ch (car row))) + (close! frame (+ at 1)) + (if (and (null? (cdr stack)) (not word?)) (+ at 1) + (loop (+ at 1) (cdr stack)))) + ((and (not quote?) (assv ch (region-plan-quotes plan))) => + (lambda (quote-row) (loop (+ at 1) (cons (vector 'quote quote-row at) stack)))) + ((and nested (cached-pair-end ends at nested)) => + (lambda (after) (loop after stack))) + (nested + (loop (+ at (string-length (car nested))) + (cons (pair-frame nested (cadddr nested) at scopes) stack))) + ((and (not quote?) (char=? ch (cadr row))) + (vector-set! frame 2 (+ (vector-ref frame 2) 1)) + (loop (+ at 1) stack)) + ((and (not quote?) (char=? ch (caddr row))) + (if (= (vector-ref frame 2) 1) + (begin + (close! frame (+ at 1)) + (if (and (null? (cdr stack)) (not word?)) (+ at 1) + (loop (+ at 1) (cdr stack)))) + (begin + (vector-set! frame 2 (- (vector-ref frame 2) 1)) + (loop (+ at 1) stack)))) + (else (loop (+ at 1) stack))))) + ((not word?) at) + (else + (let* ((ch (string-ref source at)) (nested (pair-at plan source at)) + (quote-row (assv ch (region-plan-quotes plan)))) + (cond + ;; Declared region openers shield overlapping operator stops. + (nested (loop (+ at (string-length (car nested))) + (list (pair-frame nested (cadddr nested) at scopes)))) + ((and (or (> at start) (not (cadddr (region-plan-data plan)))) + (or (char-whitespace? ch) ((region-plan-operators plan) source at))) + (and (> at start) at)) + ((char=? ch #\\) (loop (min limit (+ at 2)) '())) + (quote-row (loop (+ at 1) (list (vector 'quote quote-row at)))) + (else (loop (+ at 1) '()))))))))) + +(def (region-plan-end plan source start) + (and (exact-integer? start) (<= 0 start) (< start (string-length source)) + (region-match plan source start '() #t))) + +(def (region-plan-quote-end plan source start delimiter) + (let (row (assv delimiter (region-plan-quotes plan))) + (unless (and row (exact-integer? start) (<= 0 start) (< start (string-length source)) + (char=? delimiter (string-ref source start))) + (error "invalid declared quote entry" start delimiter)) + (region-match plan source (+ start 1) (list (vector 'quote row start)) #f))) + +(def (region-plan-pair-end plan source start) + (let (row (pair-at plan source start)) + (unless row (error "missing declared region opener" start)) + (region-match plan source (+ start (string-length (car row))) + (list (pair-frame row (cadddr row) start '())) #f))) + +;;; A source-local boundary cache records every nested frame closed by a match. +;;; Decomposing an outer region subsequently visits its children by lookup, +;;; rather than walking each suffix again. Literal quote characters outside a +;;; requested region are never eagerly interpreted (e.g. here-document text). +(defstruct region-source (plan text ends scopes)) +(def (prepare-region-source plan text) + (prepare-scoped-region-source plan text '())) +(def (prepare-scoped-region-source plan text scopes) + (unless (and (region-plan? plan) (string? text)) + (error "invalid region source" plan text)) + (unless (and (list? scopes) + (andmap (lambda (entry) + (and (list? entry) (pair? entry) (assoc (car entry) (region-plan-pairs plan)) + (andmap (lambda (prefix) (assoc prefix (region-plan-pairs plan))) (cdr entry)))) scopes) + (unique? scopes car)) (error "invalid region pair scopes")) + (make-region-source plan (string-copy text) + (vector (make-hash-table-eqv) (make-hash-table-eq)) (copy-data scopes))) + +(def (region-source-quote-end source start delimiter) + (let* ((plan (region-source-plan source)) (text (region-source-text source)) + (row (assv delimiter (region-plan-quotes plan)))) + (unless (and row (exact-integer? start) (<= 0 start) (< start (string-length text)) + (char=? delimiter (string-ref text start))) + (error "invalid declared quote entry" start delimiter)) + (or (hash-get (vector-ref (region-source-ends source) 0) start) + (region-match plan text (+ start 1) (list (vector 'quote row start)) #f + (region-source-ends source) (region-source-scopes source))))) + +(def (region-source-pair-end source start) + (let* ((plan (region-source-plan source)) (text (region-source-text source)) + (row (and (exact-integer? start) (pair-at plan text start)))) + (unless row (error "missing declared region opener" start)) + (or (cached-pair-end (region-source-ends source) start row) + (region-match plan text (+ start (string-length (car row))) + (list (pair-frame row (cadddr row) start (region-source-scopes source))) #f + (region-source-ends source) (region-source-scopes source))))) diff --git a/src/runtime/reuse.ss b/src/runtime/reuse.ss new file mode 100644 index 00000000..6b530728 --- /dev/null +++ b/src/runtime/reuse.ss @@ -0,0 +1,262 @@ +;;; -*- Gerbil -*- +;;; Source-owner certificates for deterministic nonterminal transfers. +(import (only-in ./source-index source-index-count make-source-index-cursor + source-index-cursor-copy source-index-cursor-token source-index-cursor-mode + source-index-cursor-rank source-index-cursor-next! source-index-cursor-seek!) + (only-in ../compiler/machine parser-machine-runtime parser-machine-trivia parser-machine-lexical-modes-compatible?) + (only-in ./lexer scan-source-token) + (only-in ./probe make-source-probe-cache source-probe-scan source-probe-take! + current-lr-lexical-plan-reuse-enabled?) + (only-in ./token make-token token? token-start token-end token-kind token-lexeme) + (only-in ./lr-parser + lr-checkpoint-lexical-mode lr-checkpoint-state lr-lexical-mode-id + lr-checkpoint-before-shift lr-checkpoint-fragment-compatible? + lr-checkpoint-inject-fragment lr-recognition-fragment-exit-mode + lr-recognition-fragment? lr-recognition-fragment-entry-state lr-recognition-fragment-runtime + lr-recognition-fragment-start lr-recognition-fragment-end + lr-recognition-fragment-offset lr-recognition-fragment-token-count + lr-recognition-fragment-lookahead lr-recognition-fragment-children + lr-recognition-view? lr-recognition-view-base lr-recognition-view-delta)) +(export make-fragment-reuser current-lr-probe-reuse-enabled?) +(def current-lr-probe-reuse-enabled? (make-parameter #t)) + +;;; Stats: fragments, significant tokens, source tokens, source bytes, +;;; scanner-probe bytes, rejected candidates, normalized control probes, +;;; consumed probe tokens/bytes, inspected grammar-cursor frames, +;;; old-source-certified lexical probe requests. +;;; With a captured source index, cursors share immutable chunks and no full +;;; token/mode vectors are built. Trial cursors do not consume rejected spans. +;;; Request cursors and the single probe slot expire at the end of the edit. +(def (make-fragment-reuser machine source old-tokens old-modes root edit-start edit-end delta restart-byte + (source-index #f) (on-shared #f) (closed-source-owner? #f)) + (let* ((tokens (and (not source-index) (list->vector old-tokens))) + (modes (and tokens (list->vector old-modes))) + (input-cursor (and source-index (make-source-index-cursor source-index))) + (count (if source-index (source-index-count source-index) (vector-length tokens))) (trivia? (parser-machine-trivia machine)) + (stats (make-vector 11 0)) (pending (list (cons root 0))) + (first-index 0) (source-length (string-length source)) + (certified-scanning? (and closed-source-owner? (current-lr-lexical-plan-reuse-enabled?))) + (probe-cache #f) (probe-index 0) + (probe-cursor (and source-index (make-source-index-cursor source-index)))) + (unless (or source-index (= count (vector-length modes))) (error "invalid captured lexical modes")) + (def (add-stat! index value) + (vector-set! stats index (+ (vector-ref stats index) value))) + (def (advance!) (set! pending (cdr pending))) + (def (descend! piece position-delta) + (set! pending + (append (map (lambda (child) (cons child position-delta)) + (lr-recognition-fragment-children piece)) + (cdr pending)))) + ;; A conservative end mapping permits skipping complete passed subtrees. + ;; Positions inside deleted text collapse to the changed range, not before it. + (def (mapped-end end) + (cond ((<= end edit-start) end) + ((< end edit-end) edit-start) + (else (+ end delta)))) + (def (same-token? fresh old shift) + (and (= (token-start fresh) (+ shift (token-start old))) + (= (token-end fresh) (+ shift (token-end old))) + (eq? (token-kind fresh) (token-kind old)) + (equal? (token-lexeme fresh) (token-lexeme old)))) + ;; The speculative lookup finger is separate from the grammar cursor. + ;; Forward exponential bracketing depends on query distance, not total + ;; source size; backward queries reset with a full binary lower bound. + (def (vector-probe-rank byte) + (def (lower-bound low high) + (if (= low high) low + (let (middle (quotient (+ low high) 2)) + (if (< (token-start (vector-ref tokens middle)) byte) + (lower-bound (+ middle 1) high) (lower-bound low middle))))) + (let (rank + (cond ((or (= probe-index count) + (< byte (token-start (vector-ref tokens probe-index)))) + (lower-bound 0 count)) + ((= byte (token-start (vector-ref tokens probe-index))) probe-index) + (else + (let bracket ((low probe-index) (step 1)) + (let (high (min count (+ low step))) + (if (and (< high count) (< (token-start (vector-ref tokens high)) byte)) + (bracket high (* step 2)) + (lower-bound (+ low 1) (min count (+ high 1))))))))) + (set! probe-index rank) rank)) + (def (certified-source-probe character byte mode) + (let* ((suffix? (>= byte (+ edit-end delta))) + (shift delta) + (old-byte (- byte shift))) + (and suffix? + (let* ((cursor + (and source-index + (begin + (let (current (source-index-cursor-token probe-cursor)) + (when (or (not current) (> (token-start current) old-byte)) + (set! probe-cursor (make-source-index-cursor source-index)))) + probe-cursor))) + (rank (if cursor + (begin (source-index-cursor-seek! cursor old-byte) + (source-index-cursor-rank cursor)) + (vector-probe-rank old-byte))) + (old (and (< rank count) (if cursor (source-index-cursor-token cursor) (vector-ref tokens rank)))) + (old-mode (and old (if cursor (source-index-cursor-mode cursor) (vector-ref modes rank))))) + (and old (= (token-start old) old-byte) + (>= old-byte edit-end) + (parser-machine-lexical-modes-compatible? machine old-mode (lr-lexical-mode-id mode) + (string-ref (token-lexeme old) 0)) + (let (token (if (zero? shift) old + (make-token (token-kind old) (token-lexeme old) + (+ shift (token-start old)) (+ shift (token-end old))))) + (add-stat! 10 1) + (cons token (+ character (string-length (token-lexeme old)))))))))) + (def (scan-certified-probe character byte mode) + (let (old (certified-source-probe character byte mode)) + (if old (values (car old) (cdr old)) + (let-values (((token next) (scan-source-token machine source character byte mode))) + (add-stat! 4 (- (token-end token) byte)) + (values token next))))) + (def (probe character byte mode) + (with-catch (lambda (_condition) #f) + (lambda () + (let-values (((token next-character) + (cond (probe-cache (source-probe-scan probe-cache character byte mode)) + (certified-scanning? (scan-certified-probe character byte mode)) + (else (scan-source-token machine source character byte mode))))) + (unless certified-scanning? (add-stat! 4 (- (token-end token) byte))) + (cons token next-character))))) + (def (boundary-matches? piece position-delta character byte eof-offset) + (let ((expected (lr-recognition-fragment-lookahead piece)) + (mode (lr-recognition-fragment-exit-mode piece))) + (let loop ((character character) (byte byte)) + (if (= character source-length) + (and (not expected) + (= eof-offset (+ position-delta (lr-recognition-fragment-offset piece)))) + (let (scanned (probe character byte mode)) + (and scanned + (let (token (car scanned)) + (if (trivia? token) + (loop (cdr scanned) (token-end token)) + (and expected + (= (token-start token) + (+ position-delta (lr-recognition-fragment-offset piece))) + (eq? (token-kind token) (car expected)) + (equal? (token-lexeme token) (cdr expected))))))))))) + ;; Source-token lookup advances monotonically. Rejected ancestor attempts + ;; leave its first index in place so a smaller same-start node can retry. + (def (old-token index cursor) + (if cursor (source-index-cursor-token cursor) (vector-ref tokens index))) + (def (old-mode index cursor) + (if cursor (source-index-cursor-mode cursor) (vector-ref modes index))) + (def (seek-token! start) + (if input-cursor + (begin (source-index-cursor-seek! input-cursor start) + (set! first-index (source-index-cursor-rank input-cursor))) + (let loop () + (when (and (< first-index count) (< (token-start (vector-ref tokens first-index)) start)) + (set! first-index (+ first-index 1)) (loop)))) + (and (< first-index count) + (= (token-start (old-token first-index input-cursor)) start) first-index)) + (def (try-fragment piece position-delta shift old-start old-end + character byte mode first-probe normalized) + (let (first (seek-token! old-start)) + (and first + (not (trivia? (old-token first input-cursor))) + (= (lr-lexical-mode-id mode) (old-mode first input-cursor)) + (same-token? (car first-probe) (old-token first input-cursor) shift) + (let collect ((index first) (segment '()) (segment-modes '()) + (significant '()) (significant-count 0) + (characters 0) (eof-offset byte) + (cursor (and input-cursor (source-index-cursor-copy input-cursor)))) + (if (and (< index count) (<= (token-end (old-token index cursor)) old-end)) + (let* ((old (old-token index cursor)) + (token (if (zero? shift) old + (make-token (token-kind old) (token-lexeme old) + (+ shift (token-start old)) (+ shift (token-end old))))) + (trivia-token? (trivia? token)) + (next-index (+ index 1)) + (next-segment (cons token segment)) + (next-modes (cons (old-mode index cursor) segment-modes)) + (next-significant (if trivia-token? significant (cons token significant))) + (next-count (+ significant-count (if trivia-token? 0 1))) + (next-characters (+ characters (string-length (token-lexeme token)))) + (next-eof-offset (if trivia-token? eof-offset (token-end token)))) + (when cursor (source-index-cursor-next! cursor)) + (if (= (token-end old) old-end) + (let ((next-character (+ character next-characters)) + (next-byte (+ old-end shift))) + (and (= next-count (lr-recognition-fragment-token-count piece)) + (boundary-matches? piece position-delta next-character next-byte next-eof-offset) + (begin + (add-stat! 0 1) (add-stat! 1 next-count) + (add-stat! 2 (- next-index first)) (add-stat! 3 (- next-byte byte)) + (set! first-index next-index) + (when cursor (set! input-cursor cursor)) + (when on-shared (on-shared first (- next-index first) shift)) + (vector + (lr-checkpoint-inject-fragment normalized piece position-delta + (reverse next-significant)) + (reverse next-segment) (reverse next-modes) next-character next-byte)))) + (collect next-index next-segment next-modes next-significant next-count + next-characters next-eof-offset cursor))) + #f))))) + (when (current-lr-probe-reuse-enabled?) + (set! probe-cache (make-source-probe-cache machine source + (and certified-scanning? scan-certified-probe)))) + (values + (lambda (character byte checkpoint) + (let ((mode (lr-checkpoint-lexical-mode checkpoint)) + (first-probe #f) (normalized #f) (normalization-tried? #f)) + (let seek () + (and (pair? pending) + (let* ((frame (car pending)) (piece (car frame)) (position-delta (cdr frame))) + (add-stat! 9 1) + (cond + ((lr-recognition-view? piece) + (set! pending + (cons (cons (lr-recognition-view-base piece) + (+ position-delta (lr-recognition-view-delta piece))) + (cdr pending))) + (seek)) + ((or (not (lr-recognition-fragment? piece)) + (< (lr-recognition-fragment-token-count piece) 2)) + (advance!) (seek)) + (else + (let* ((old-start (+ position-delta (lr-recognition-fragment-start piece))) + (old-end (+ position-delta (lr-recognition-fragment-end piece))) + (offset (+ position-delta (lr-recognition-fragment-offset piece))) + (suffix? (>= old-start edit-end)) + (prefix? (and (<= old-end edit-start) (<= offset edit-start))) + (shift (if suffix? delta 0)) + (start (+ old-start shift))) + (cond + ((<= (mapped-end old-end) (max byte restart-byte)) + (advance!) (seek)) + ;; A future unedited subtree must remain available when + ;; ordinary scanning reaches it. An overlapping ancestor + ;; can only descend; it cannot transfer across the edit. + ((and (or prefix? suffix?) (> start byte)) #f) + ((or (not (or prefix? suffix?)) (< start byte) (<= old-end old-start)) + (descend! piece position-delta) (seek)) + (else + (unless normalization-tried? + (set! normalization-tried? #t) + (set! first-probe (probe character byte mode)) + (when first-probe + (add-stat! 6 1) + (set! normalized + (lr-checkpoint-before-shift checkpoint (car first-probe))))) + (and normalized + (let (result + (and (eq? (parser-machine-runtime machine) + (lr-recognition-fragment-runtime piece)) + (lr-checkpoint-fragment-compatible? normalized piece) + (try-fragment piece (+ position-delta shift) shift old-start old-end + character byte mode first-probe normalized))) + (if result + (begin (advance!) result) + (begin (add-stat! 5 1) + (descend! piece position-delta) (seek))))))))))))))) + stats + (and probe-cache + (lambda (character byte mode) + (let (result (source-probe-take! probe-cache character byte mode)) + (when result + (add-stat! 7 1) (add-stat! 8 (- (token-end (car result)) byte))) + result)))))) diff --git a/src/runtime/scan.ss b/src/runtime/scan.ss index ba754eaf..39b21029 100644 --- a/src/runtime/scan.ss +++ b/src/runtime/scan.ss @@ -1,13 +1,22 @@ ;;; -*- Gerbil -*- -;;; Language-neutral scanner primitives used by generated lexers. - -(import (only-in :std/func any-of) - (only-in ./token make-token)) - -(export scan-whitespace +;;; Stable lexical facade; pure scanning owners preserve the original forms. +(import (only-in "./scan/profile.ss" make-bounded-text-profile-scanner make-text-profile-scanner) + (only-in "./scan/basic.ss" identifier-start? horizontal-whitespace? newline? scan-whitespace scan-horizontal-whitespace scan-newline scan-line scan-character-run scan-until-delimiters scan-identifier scan-emit) + (only-in "./scan/number.ss" scan-decimal-digits scan-number-literal scan-number-literal/profile) + (only-in "./scan/regular.ss" make-ranked-regular-scanner) + (only-in "./scan/quoted.ss" scan-quoted-string scan-quoted-strings scan-escaped-quoted-strings scan-quoted-string/profile scan-balanced-word) + (only-in "./scan/comment.ss" scan-heredoc scan-line-comment scan-block-comment scan-nested-block-comment) + (only-in "./scan/literal.ss" make-literal-trie literal-trie-terminal literal-trie-child make-bounded-literal-end-scanner make-literal-end-scanner make-ranked-literal-scanner scan-longest-literal) + (only-in "./scan/lexical.ss" make-ranked-lexical-scanner-factory)) +(export make-literal-trie literal-trie-terminal literal-trie-child + make-bounded-text-profile-scanner make-bounded-literal-end-scanner + make-text-profile-scanner + identifier-start? horizontal-whitespace? newline? + scan-whitespace scan-horizontal-whitespace scan-newline scan-line + scan-character-run scan-until-delimiters scan-decimal-digits scan-number-literal @@ -16,496 +25,14 @@ scan-quoted-string scan-quoted-strings scan-escaped-quoted-strings + scan-quoted-string/profile scan-heredoc + scan-balanced-word scan-line-comment scan-block-comment scan-nested-block-comment make-literal-end-scanner + make-ranked-literal-scanner make-ranked-lexical-scanner-factory + make-ranked-regular-scanner scan-longest-literal scan-emit) - -;; Scanner functions return the exclusive source-character end offset or #f. -;; : (-> String Nat (-> Char Boolean) (Maybe Nat)) -(def (scan-while source start predicate) - (let (length (string-length source)) - (let loop ((offset start)) - (if (and (< offset length) (predicate (string-ref source offset))) - (loop (+ offset 1)) - offset)))) - -;; identifier-start? -;; : (-> Char Boolean) -(def (identifier-start? ch) - (any-of (list char-alphabetic? (cut char=? <> #\_)) ch)) - -;; identifier-rest? -;; : (-> Char Boolean) -(def (identifier-rest? ch) - (any-of (list identifier-start? char-numeric? (cut char=? <> #\-)) ch)) - -;; scan-nonempty-while -;; : (-> Procedure Procedure String Fixnum Fixnum) -(def (scan-nonempty-while first-predicate rest-predicate source start) - (and (first-predicate (string-ref source start)) - (scan-while source start rest-predicate))) - -;; scan-whitespace -;; : (-> String Fixnum Fixnum) -(def scan-whitespace - (cut scan-nonempty-while char-whitespace? char-whitespace? <> <>)) - -;; horizontal-whitespace? -;; : (-> Char Boolean) -(def (horizontal-whitespace? ch) - (any-of (list (cut char=? <> #\space) (cut char=? <> #\tab)) ch)) - -;; scan-horizontal-whitespace -;; : (-> String Fixnum Fixnum) -(def scan-horizontal-whitespace - (cut scan-nonempty-while - horizontal-whitespace? horizontal-whitespace? <> <>)) - -;; newline? -;; : (-> Char Boolean) -(def (newline? ch) - (any-of (list (cut char=? <> #\newline) (cut char=? <> #\return)) ch)) - -;; scan-newline -;; : (-> String Fixnum Fixnum) -(def scan-newline (cut scan-nonempty-while newline? newline? <> <>)) - -;; One complete line, including its LF, CRLF, or bare CR terminator. -;; This primitive is useful for line-oriented grammars without allocating a -;; substring or emitting one token per source character. -(def (scan-line source start) - (let (length (string-length source)) - (and (< start length) - (let loop ((offset start)) - (cond - ((= offset length) length) - ((char=? (string-ref source offset) #\newline) - (+ offset 1)) - ((char=? (string-ref source offset) #\return) - (if (and (< (+ offset 1) length) - (char=? (string-ref source (+ offset 1)) #\newline)) - (+ offset 2) - (+ offset 1))) - (else (loop (+ offset 1)))))))) - -;; Maximal nonempty atom run, stopping before Unicode whitespace or a delimiter. -;; Delimiters are language-owned; this primitive knows no S-expression policy. -(def (scan-until-delimiters source start delimiters) - (and (< start (string-length source)) - (let (end (scan-while - source start - (lambda (character) - (and (not (char-whitespace? character)) - (not (let loop ((index 0)) - (and (< index (string-length delimiters)) - (or (char=? character - (string-ref delimiters index)) - (loop (+ index 1)))))))))) - (and (> end start) end)))) - -;; scan-decimal-digits -;; : (-> String Fixnum Fixnum) -(def scan-decimal-digits - (cut scan-nonempty-while char-numeric? char-numeric? <> <>)) - -;; scan-number-fraction-end -;; : (-> String Fixnum Fixnum Fixnum) -(def (scan-number-fraction-end source whole-end length) - (if (and (< whole-end length) - (char=? (string-ref source whole-end) #\.) - (< (+ whole-end 1) length) - (char-numeric? (string-ref source (+ whole-end 1)))) - (scan-while source (+ whole-end 1) char-numeric?) - whole-end)) - -;; scan-number-exponent-digits-position -;; : (-> String Fixnum Fixnum Fixnum) -(def (scan-number-exponent-digits-position source exponent-end length) - (let (position (+ exponent-end 1)) - (if (and (< position length) - (memv (string-ref source position) '(#\+ #\-))) - (+ position 1) - position))) - -;; scan-number-exponent-end -;; : (-> String Fixnum Fixnum Fixnum) -(def (scan-number-exponent-end source fraction-end length) - (if (and (< fraction-end length) - (memv (string-ref source fraction-end) '(#\e #\E))) - (let (digits-position - (scan-number-exponent-digits-position source fraction-end length)) - (if (and (< digits-position length) - (char-numeric? (string-ref source digits-position))) - (scan-while source digits-position char-numeric?) - fraction-end)) - fraction-end)) - -(def (scan-number-literal source start) - (and (char-numeric? (string-ref source start)) - (let* ((length (string-length source)) - (whole-end (scan-while source start char-numeric?)) - (fraction-end - (scan-number-fraction-end source whole-end length))) - (scan-number-exponent-end source fraction-end length)))) - -;; : (-> Char (Maybe Fixnum)) -(def (ascii-digit-value ch) - (cond - ((and (char>=? ch #\0) (char<=? ch #\9)) - (- (char->integer ch) (char->integer #\0))) - ((and (char>=? ch #\a) (char<=? ch #\f)) - (+ 10 (- (char->integer ch) (char->integer #\a)))) - ((and (char>=? ch #\A) (char<=? ch #\F)) - (+ 10 (- (char->integer ch) (char->integer #\A)))) - (else #f))) - -;; : (-> Char Fixnum Boolean) -(def (ascii-digit-for-base? ch base) - (alet (value (ascii-digit-value ch)) - (< value base))) - -;; : (-> String Fixnum Fixnum Char (Maybe Fixnum)) -(def (scan-separated-digits source start base separator) - (let (length (string-length source)) - (and (< start length) - (ascii-digit-for-base? (string-ref source start) base) - (let loop ((offset (+ start 1))) - (cond - ((>= offset length) offset) - ((ascii-digit-for-base? (string-ref source offset) base) - (loop (+ offset 1))) - ((and (char=? (string-ref source offset) separator) - (< (+ offset 1) length) - (ascii-digit-for-base? - (string-ref source (+ offset 1)) base)) - (loop (+ offset 2))) - (else offset)))))) - -;; : (-> String Fixnum) -(def (numeric-prefix-base prefix) - (let (last (char-downcase - (string-ref prefix (- (string-length prefix) 1)))) - (case last - ((#\b) 2) - ((#\o) 8) - ((#\x) 16) - (else (error "numeric radix prefix must end in b, o, or x" prefix))))) - -;; : (-> String Fixnum [String] Char (Maybe Fixnum)) -(def (scan-radix-number source start prefixes separator) - (alet (prefix (scan-longest-literal source start prefixes)) - (scan-separated-digits - source (+ start (string-length prefix)) - (numeric-prefix-base prefix) separator))) - -;; : (-> String Fixnum Char Boolean Boolean (Maybe Fixnum)) -(def (scan-profile-decimal-mantissa - source start separator leading-period? trailing-period?) - (let (length (string-length source)) - (cond - ((and leading-period? - (< (+ start 1) length) - (char=? (string-ref source start) #\.) - (ascii-digit-for-base? (string-ref source (+ start 1)) 10)) - (scan-separated-digits source (+ start 1) 10 separator)) - ((and (< start length) - (ascii-digit-for-base? (string-ref source start) 10)) - (let* ((whole-end - (scan-separated-digits source start 10 separator)) - (period? (and (< whole-end length) - (char=? (string-ref source whole-end) #\.)))) - (if period? - (or (scan-separated-digits source (+ whole-end 1) 10 separator) - (and trailing-period? (+ whole-end 1)) - whole-end) - whole-end))) - (else #f)))) - -;; : (-> String Fixnum Char Fixnum) -(def (scan-profile-exponent source mantissa-end separator) - (let (length (string-length source)) - (if (and (< mantissa-end length) - (memv (string-ref source mantissa-end) '(#\e #\E))) - (let* ((after-indicator (+ mantissa-end 1)) - (digits-start - (if (and (< after-indicator length) - (memv (string-ref source after-indicator) '(#\+ #\-))) - (+ after-indicator 1) - after-indicator))) - (or (scan-separated-digits source digits-start 10 separator) - mantissa-end)) - mantissa-end))) - -;; scan-number-literal/profile -;; : (-> String Fixnum [String] String [String] Boolean Boolean (Maybe Fixnum)) -;; | doc m% -;; Scans one grammar-declared numeric profile without assigning a -;; language identity in runtime code. -;; % -(def (scan-number-literal/profile - source start prefixes separator suffixes leading-period? trailing-period?) - (let (separator-character (string-ref separator 0)) - (or (scan-radix-number source start prefixes separator-character) - (alet* ((mantissa-end - (scan-profile-decimal-mantissa - source start separator-character - leading-period? trailing-period?)) - (number-end - (scan-profile-exponent source mantissa-end separator-character))) - (or (alet (suffix (scan-longest-literal source number-end suffixes)) - (+ number-end (string-length suffix))) - number-end))))) - -;; scan-identifier -;; : (-> String Fixnum Fixnum) -(def scan-identifier - (cut scan-nonempty-while identifier-start? identifier-rest? <> <>)) - -;; scan-quoted-string -;; : (-> String Fixnum String Fixnum) -;; | doc m% -;; `scan-quoted-string` scans one escaped, delimiter-bound string. -;; -;; # Examples -;; -;; ```scheme -;; (scan-quoted-string "'value'" 0 "'") -;; ;; => 7 -;; ``` -;; % -(def (scan-quoted-string/mode source start delimiter doubled-delimiter?) - (let ((length (string-length source)) - (delimiter-length (string-length delimiter))) - (and (literal-at? source start delimiter) - (let loop ((offset (+ start delimiter-length)) (escaped? #f)) - (cond - ((>= offset length) #f) - (escaped? (loop (+ offset 1) #f)) - ((char=? (string-ref source offset) #\\) - (loop (+ offset 1) #t)) - ((literal-at? source offset delimiter) - (let (next (+ offset delimiter-length)) - ;; ISO graph-query character sequences escape their delimiter - ;; by doubling it. Backslash escaping remains admitted for - ;; the same source grammars, so both forms stay lossless. - (if (and doubled-delimiter? - (literal-at? source next delimiter)) - (loop (+ next delimiter-length) #f) - next))) - (else (loop (+ offset 1) #f))))))) - -(def (scan-quoted-string source start delimiter) - (scan-quoted-string/mode source start delimiter #t)) - -;; scan-quoted-strings -;; : (-> String Fixnum List Fixnum) -;; | doc m% -;; `scan-quoted-strings` tries delimiters in declaration order. -;; -;; # Examples -;; -;; ```scheme -;; (scan-quoted-strings "\"value\"" 0 '("\"" "'")) -;; ;; => 7 -;; ``` -;; % -(def (scan-quoted-strings source start delimiters) - (ormap (lambda (delimiter) - (scan-quoted-string source start delimiter)) - delimiters)) - -;; Backslash escapes are admitted; adjacent quoted strings remain distinct. -(def (scan-escaped-quoted-strings source start delimiters) - (ormap (lambda (delimiter) - (scan-quoted-string/mode source start delimiter #f)) - delimiters)) - -(def (line-end source start) - (scan-while source start - (lambda (ch) - (and (not (char=? ch #\newline)) - (not (char=? ch #\return)))))) - -(def (skip-horizontal source start end) - (let loop ((offset start)) - (if (and (< offset end) - (horizontal-whitespace? (string-ref source offset))) - (loop (+ offset 1)) - offset))) - -;; scan-heredoc -;; : (-> String Fixnum Fixnum) -;; | doc m% -;; `scan-heredoc` scans a marker-delimited multiline literal. -;; -;; # Examples -;; -;; ```scheme -;; (scan-heredoc "< 15 -;; ``` -;; % -(def (scan-heredoc source start) - (let (length (string-length source)) - (and (literal-at? source start "<<") - (let* ((marker-start0 (+ start 2)) - (marker-start - (if (and (< marker-start0 length) - (char=? (string-ref source marker-start0) #\-)) - (+ marker-start0 1) - marker-start0)) - (marker-end - (and (< marker-start length) - (identifier-start? (string-ref source marker-start)) - (scan-while source marker-start identifier-rest?)))) - (and marker-end - (< marker-end length) - (newline? (string-ref source marker-end)) - (let ((marker (substring source marker-start marker-end)) - (body-start (+ marker-end 1))) - (let loop ((line-start body-start)) - (and (< line-start length) - (let* ((end (line-end source line-start)) - (content-start - (skip-horizontal source line-start end))) - (if (and (literal-at? source content-start marker) - (= (+ content-start - (string-length marker)) - end)) - end - (and (< end length) - (loop (+ end 1))))))))))))) - -(def (scan-line-comment source start prefixes) - (let (prefix (scan-longest-literal source start prefixes)) - (and prefix - (scan-while source (+ start (string-length prefix)) - (lambda (ch) (not (newline? ch))))))) - -;; scan-block-comment -;; : (-> String Fixnum String String Fixnum) -;; | doc m% -;; `scan-block-comment` scans one non-nested delimiter pair. -;; -;; # Examples -;; -;; ```scheme -;; (scan-block-comment "/* value */" 0 "/*" "*/") -;; ;; => 11 -;; ``` -;; % -(def (scan-block-comment source start opening closing) - (let ((length (string-length source)) - (closing-length (string-length closing))) - (and (literal-at? source start opening) - (let loop ((offset (+ start (string-length opening)))) - (cond - ((>= offset length) #f) - ((literal-at? source offset closing) - (+ offset closing-length)) - (else (loop (+ offset 1)))))))) - -;; scan-nested-block-comment -;; : (-> String Fixnum String String Fixnum) -;; | doc m% -;; `scan-nested-block-comment` tracks balanced nested delimiters. -;; -;; # Examples -;; -;; ```scheme -;; (scan-nested-block-comment "/* /* */ */" 0 "/*" "*/") -;; ;; => 11 -;; ``` -;; % -(def (scan-nested-block-comment source start opening closing) - (let ((length (string-length source)) - (opening-length (string-length opening)) - (closing-length (string-length closing))) - (and (literal-at? source start opening) - (let loop ((offset (+ start opening-length)) (depth 1)) - (cond - ((>= offset length) #f) - ((literal-at? source offset opening) - (loop (+ offset opening-length) (+ depth 1))) - ((literal-at? source offset closing) - (let ((next (+ offset closing-length)) - (remaining (- depth 1))) - (if (zero? remaining) next (loop next remaining)))) - (else (loop (+ offset 1) depth))))))) - -(def (literal-at? source start literal) - (let ((source-length (string-length source)) - (literal-length (string-length literal))) - (and (<= (+ start literal-length) source-length) - (let loop ((index 0)) - (or (= index literal-length) - (and (char=? (string-ref source (+ start index)) - (string-ref literal index)) - (loop (+ index 1)))))))) - -;;; Compile a static literal catalog into a character trie once. The returned -;;; scanner follows at most the matching source prefix, independent of catalog -;;; size, and retains the longest terminal seen along that path. -;; : (-> (List String) (-> String Nat (Maybe Nat))) -(def (make-literal-end-scanner literals) - (def (make-node) (vector #f (make-table test: eqv?))) - (def (insert! root literal) - (unless (and (string? literal) (positive? (string-length literal))) - (error "lexer literals must be non-empty strings" literal)) - (let loop ((node root) (index 0)) - (if (= index (string-length literal)) - (vector-set! node 0 #t) - (let* ((children (vector-ref node 1)) - (character (string-ref literal index)) - (child (table-ref children character #f))) - (unless child - (set! child (make-node)) - (table-set! children character child)) - (loop child (+ index 1)))))) - (let (root (make-node)) - (for-each (cut insert! root <>) literals) - (lambda (source start) - (let (source-length (string-length source)) - (let loop ((node root) (offset start) (selected #f)) - (if (= offset source-length) - selected - (let (child - (table-ref (vector-ref node 1) - (string-ref source offset) #f)) - (if child - (let (next (+ offset 1)) - (loop child next - (if (vector-ref child 0) next selected))) - selected)))))))) - -;; scan-longest-literal -;; : (-> String Fixnum List String) -;; | doc m% -;; `scan-longest-literal` selects the longest matching declared literal. -;; -;; # Examples -;; -;; ```scheme -;; (scan-longest-literal "<=" 0 '("<" "<=")) -;; ;; => "<=" -;; ``` -;; % -(def (scan-longest-literal source start literals) - (foldl (lambda (candidate selected) - (if (and (literal-at? source start candidate) - (or (not selected) - (> (string-length candidate) - (string-length selected)))) - candidate - selected)) - #f - literals)) - -(def (scan-emit source kind start end byte-start) - (let* ((lexeme (substring source start end)) - (byte-end (+ byte-start (u8vector-length (string->utf8 lexeme))))) - (make-token kind lexeme byte-start byte-end))) diff --git a/src/runtime/scan/basic.ss b/src/runtime/scan/basic.ss new file mode 100644 index 00000000..43b52deb --- /dev/null +++ b/src/runtime/scan/basic.ss @@ -0,0 +1,136 @@ +;;; -*- Gerbil -*- +;;; Scan basic owner: original pure lexical algorithms. +(import (only-in :std/string/utf8 string-utf8-length) + (only-in ../token make-token)) +(export scan-while identifier-start? identifier-rest? scan-nonempty-while scan-whitespace horizontal-whitespace? scan-horizontal-whitespace newline? scan-newline scan-line scan-character-run scan-until-delimiters scan-identifier line-end skip-horizontal literal-at? scan-emit) + +;; Scanner functions return the exclusive source-character end offset or #f. +;; : (-> String Nat (-> Char Boolean) (Maybe Nat)) +(def (scan-while source start predicate) + (let (length (string-length source)) + (let loop ((offset start)) + (if (and (< offset length) (predicate (string-ref source offset))) + (loop (+ offset 1)) + offset)))) + +;; identifier-start? +;; : (-> Char Boolean) +(def (identifier-start? ch) + (or (char-alphabetic? ch) (char=? ch #\_))) + +;; identifier-rest? +;; : (-> Char Boolean) +(def (identifier-rest? ch) + (or (identifier-start? ch) (char-numeric? ch) (char=? ch #\-))) + +;; scan-nonempty-while +;; : (-> Procedure Procedure String Fixnum Fixnum) +(def (scan-nonempty-while first-predicate rest-predicate source start) + (and (first-predicate (string-ref source start)) + (scan-while source start rest-predicate))) + +;; scan-whitespace +;; : (-> String Fixnum Fixnum) +(def scan-whitespace + (cut scan-nonempty-while char-whitespace? char-whitespace? <> <>)) + +;; horizontal-whitespace? +;; : (-> Char Boolean) +(def (horizontal-whitespace? ch) + (or (char=? ch #\space) (char=? ch #\tab))) + +;; scan-horizontal-whitespace +;; : (-> String Fixnum Fixnum) +(def scan-horizontal-whitespace + (cut scan-nonempty-while + horizontal-whitespace? horizontal-whitespace? <> <>)) + +;; newline? +;; : (-> Char Boolean) +(def (newline? ch) + (or (char=? ch #\newline) (char=? ch #\return))) + +;; scan-newline +;; : (-> String Fixnum Fixnum) +(def scan-newline (cut scan-nonempty-while newline? newline? <> <>)) + +;; One complete line, including its LF, CRLF, or bare CR terminator. +;; This primitive is useful for line-oriented grammars without allocating a +;; substring or emitting one token per source character. +(def (scan-line source start) + (let (length (string-length source)) + (and (< start length) + (let loop ((offset start)) + (cond + ((= offset length) length) + ((char=? (string-ref source offset) #\newline) + (+ offset 1)) + ((char=? (string-ref source offset) #\return) + (if (and (< (+ offset 1) length) + (char=? (string-ref source (+ offset 1)) #\newline)) + (+ offset 2) + (+ offset 1))) + (else (loop (+ offset 1)))))))) + +;; A maximal run of one character, admitted only when it meets the declared +;; minimum. The grammar owns the character and the bound. +(def (scan-character-run source start character minimum) + (and (string? character) + (= (string-length character) 1) + (exact-integer? minimum) + (positive? minimum) + (<= 0 start) + (< start (string-length source)) + (let (end (scan-while + source start + (lambda (ch) (char=? ch (string-ref character 0))))) + (and (>= (- end start) minimum) end)))) + +;; Maximal nonempty atom run, stopping before Unicode whitespace or a delimiter. +;; Delimiters are language-owned; this primitive knows no S-expression policy. +(def (scan-until-delimiters source start delimiters) + (and (< start (string-length source)) + (let (end (scan-while + source start + (lambda (character) + (and (not (char-whitespace? character)) + (not (let loop ((index 0)) + (and (< index (string-length delimiters)) + (or (char=? character + (string-ref delimiters index)) + (loop (+ index 1)))))))))) + (and (> end start) end)))) + +;; scan-identifier +;; : (-> String Fixnum Fixnum) +(def scan-identifier + (cut scan-nonempty-while identifier-start? identifier-rest? <> <>)) + +(def (line-end source start) + (scan-while source start + (lambda (ch) + (and (not (char=? ch #\newline)) + (not (char=? ch #\return)))))) + +(def (skip-horizontal source start end) + (let loop ((offset start)) + (if (and (< offset end) + (horizontal-whitespace? (string-ref source offset))) + (loop (+ offset 1)) + offset))) + +(def (literal-at? source start literal) + (let ((source-length (string-length source)) + (literal-length (string-length literal))) + (and (<= (+ start literal-length) source-length) + (let loop ((index 0)) + (or (= index literal-length) + (and (char=? (string-ref source (+ start index)) + (string-ref literal index)) + (loop (+ index 1)))))))) + +(def (scan-emit source kind start end byte-start) + (let* ((lexeme (substring source start end)) + (byte-end (+ byte-start (string-utf8-length lexeme)))) + (make-token kind lexeme byte-start byte-end))) + diff --git a/src/runtime/scan/comment.ss b/src/runtime/scan/comment.ss new file mode 100644 index 00000000..a5419563 --- /dev/null +++ b/src/runtime/scan/comment.ss @@ -0,0 +1,106 @@ +;;; -*- Gerbil -*- +;;; Scan comment owner: original pure lexical algorithms. +(import (only-in "./basic.ss" scan-while identifier-start? identifier-rest? newline? line-end skip-horizontal literal-at?) + (only-in "./literal.ss" scan-longest-literal)) +(export scan-heredoc scan-line-comment scan-block-comment scan-nested-block-comment) + +;; scan-heredoc +;; : (-> String Fixnum Fixnum) +;; | doc m% +;; `scan-heredoc` scans a marker-delimited multiline literal. +;; +;; # Examples +;; +;; ```scheme +;; (scan-heredoc "< 15 +;; ``` +;; % +(def (scan-heredoc source start) + (let (length (string-length source)) + (and (literal-at? source start "<<") + (let* ((marker-start0 (+ start 2)) + (marker-start + (if (and (< marker-start0 length) + (char=? (string-ref source marker-start0) #\-)) + (+ marker-start0 1) + marker-start0)) + (marker-end + (and (< marker-start length) + (identifier-start? (string-ref source marker-start)) + (scan-while source marker-start identifier-rest?)))) + (and marker-end + (< marker-end length) + (newline? (string-ref source marker-end)) + (let ((marker (substring source marker-start marker-end)) + (body-start (+ marker-end 1))) + (let loop ((line-start body-start)) + (and (< line-start length) + (let* ((end (line-end source line-start)) + (content-start + (skip-horizontal source line-start end))) + (if (and (literal-at? source content-start marker) + (= (+ content-start + (string-length marker)) + end)) + end + (and (< end length) + (loop (+ end 1))))))))))))) + +(def (scan-line-comment source start prefixes) + (let (prefix (scan-longest-literal source start prefixes)) + (and prefix + (scan-while source (+ start (string-length prefix)) + (lambda (ch) (not (newline? ch))))))) + +;; scan-block-comment +;; : (-> String Fixnum String String Fixnum) +;; | doc m% +;; `scan-block-comment` scans one non-nested delimiter pair. +;; +;; # Examples +;; +;; ```scheme +;; (scan-block-comment "/* value */" 0 "/*" "*/") +;; ;; => 11 +;; ``` +;; % +(def (scan-block-comment source start opening closing) + (let ((length (string-length source)) + (closing-length (string-length closing))) + (and (literal-at? source start opening) + (let loop ((offset (+ start (string-length opening)))) + (cond + ((>= offset length) #f) + ((literal-at? source offset closing) + (+ offset closing-length)) + (else (loop (+ offset 1)))))))) + +;; scan-nested-block-comment +;; : (-> String Fixnum String String Fixnum) +;; | doc m% +;; `scan-nested-block-comment` tracks balanced nested delimiters. +;; +;; # Examples +;; +;; ```scheme +;; (scan-nested-block-comment "/* /* */ */" 0 "/*" "*/") +;; ;; => 11 +;; ``` +;; % +(def (scan-nested-block-comment source start opening closing) + (let ((length (string-length source)) + (opening-length (string-length opening)) + (closing-length (string-length closing))) + (and (literal-at? source start opening) + (let loop ((offset (+ start opening-length)) (depth 1)) + (cond + ((>= offset length) #f) + ((literal-at? source offset opening) + (loop (+ offset opening-length) (+ depth 1))) + ((literal-at? source offset closing) + (let ((next (+ offset closing-length)) + (remaining (- depth 1))) + (if (zero? remaining) next (loop next remaining)))) + (else (loop (+ offset 1) depth))))))) + diff --git a/src/runtime/scan/lexical.ss b/src/runtime/scan/lexical.ss new file mode 100644 index 00000000..367c5904 --- /dev/null +++ b/src/runtime/scan/lexical.ss @@ -0,0 +1,80 @@ +;;; -*- Gerbil -*- +;;; Scan lexical owner: original pure lexical algorithms. +(import (only-in :std/vector/vector vector-map/index) + (only-in "./regular.ss" regular-kind-bit regular-mask scan-regular-run make-ranked-regular-scanner) + (only-in "./basic.ss" literal-at?) + (only-in "./literal.ss" literal-trie-child ranked-literal-before? ranked-literal-root-ascii prepare-ranked-literal-root make-ranked-literal-scanner/prepared ranked-core-winner)) +(export make-ranked-lexical-scanner-factory) + +(def (make-ranked-lexical-scanner-factory entries) + (let* ((entries (map (lambda (entry) + (list (string-copy (car entry)) (cadr entry) + (caddr entry) (cadddr entry))) entries)) + (root (prepare-ranked-literal-root entries)) + (ascii (ranked-literal-root-ascii root)) + (literal-only (make-ranked-literal-scanner/prepared root ascii))) + (lambda (regular-entries admissions) + (let* ((eligible (filter (lambda (entry) + (or (not admissions) (vector-ref admissions (cadddr entry)))) entries)) + ;; An unrestricted mode has no per-terminal mask operation. + (admitted (and admissions (not (= (length eligible) (length entries))) + (vector-copy admissions))) + (has-literals? (pair? eligible)) + (regular (and (pair? regular-entries) (make-ranked-regular-scanner regular-entries))) + (available (fold (lambda (entry mask) (bitwise-ior mask (regular-kind-bit (car entry)))) 0 regular-entries)) + (winners + (vector-map/index + (lambda (mask _) + (fold (lambda (entry best) + (let (candidate (cdr entry)) + (if (and (not (zero? (bitwise-and mask (regular-kind-bit (car entry))))) + (or (not best) (ranked-literal-before? candidate best))) candidate best))) + #f regular-entries)) (make-vector 32 #f)))) + (def (publish selected end) + (and selected (list (car selected) end (cadr selected) (caddr selected)))) + (def (prefer left right) + (cond ((not left) right) ((not right) left) + ((or (> (cadr right) (cadr left)) + (and (= (cadr right) (cadr left)) + (or (> (caddr right) (caddr left)) + (and (= (caddr right) (caddr left)) + (<= (cadddr right) (cadddr left)))))) right) + (else left))) + (cond ((not has-literals?) (or regular (lambda (_source _start) #f))) + ((and (not regular) (null? (cdr eligible))) + ;; A closed singleton mode needs neither trie traversal nor + ;; winner competition. Literal bounds still use Scheme chars. + (let* ((entry (car eligible)) (text (car entry)) + (winner (cdr entry)) (width (string-length text))) + (lambda (source start) + (and (literal-at? source start text) + (publish winner (+ start width)))))) + ((not regular) (lambda (source start) (literal-only source start admitted))) + (else + (lambda (source start) + (let (limit (string-length source)) + (and (not (= start limit)) + (let* ((ch (string-ref source start)) + (active (bitwise-and available (regular-mask ch #t)))) + (if (not (zero? (bitwise-and active 32))) + (prefer (regular source start) (literal-only source start admitted)) + (let* ((node (if ascii + (let (code (char->integer ch)) + (if (< code 128) (vector-ref ascii code) (literal-trie-child root ch))) + (literal-trie-child root ch))) + (initial (ranked-core-winner node active admitted winners))) + (let scan ((node node) (at (+ start 1)) (active active) + (selected initial) (selected-end (+ start 1))) + (cond ((= at limit) (publish selected selected-end)) + ((and (not node) (not (zero? active))) + (scan-regular-run source at limit active winners)) + (else + (let* ((ch (string-ref source at)) + (child (and node (literal-trie-child node ch))) + (next (if (zero? active) 0 (bitwise-and active (regular-mask ch #f))))) + (if (and (not child) (zero? next)) + (publish selected selected-end) + (let (candidate (ranked-core-winner child next admitted winners)) + (scan child (+ at 1) next (or candidate selected) + (if candidate (+ at 1) selected-end)))))))))))))))))))) + diff --git a/src/runtime/scan/literal.ss b/src/runtime/scan/literal.ss new file mode 100644 index 00000000..17fcd627 --- /dev/null +++ b/src/runtime/scan/literal.ss @@ -0,0 +1,193 @@ +;;; -*- Gerbil -*- +;;; Scan literal owner: original pure lexical algorithms. +(import (only-in "./basic.ss" literal-at?)) +(export literal-trie-terminal literal-trie-child keep-first-literal collect-literal make-literal-trie checked-lexer-literal make-bounded-literal-end-scanner make-literal-end-scanner ranked-literal-before? prepare-ranked-literal-terminals! ranked-literal-root-ascii prepare-ranked-literal-root make-ranked-literal-scanner/prepared make-ranked-literal-scanner ranked-core-literal ranked-core-winner scan-longest-literal) + +;;; Compile a static literal catalog into a character trie once. The returned +;;; scanner follows at most the matching source prefix, independent of catalog +;;; size, and retains the longest terminal seen along that path. +;; : (-> (List String) (-> String Nat (Maybe Nat))) +(defrule (literal-trie-terminal node) (vector-ref node 0)) + +(defrule (literal-trie-child node-expr character-expr) + (let* ((edges (vector-ref node-expr 1)) (character character-expr)) + (cond ((pair? edges) (and (eqv? character (car edges)) (cdr edges))) + ((not edges) #f) + (else (table-ref edges character #f))))) + +(def (keep-first-literal previous payload) (or previous payload)) + +(def (collect-literal previous payload) (cons payload previous)) + +;;; Common prepared literal IR: accepting payload and character-indexed edges. +;;; Combining duplicate payloads is preparation work, never scanner dispatch. +(def (make-literal-trie entries (initial #f) (combine keep-first-literal)) + (def (make-node) (vector initial (make-table test: eqv?))) + (let (root (make-node)) + (for-each + (lambda (entry) + (let (literal (car entry)) + (unless (string? literal) (error "literal trie requires strings" literal)) + (let insert ((node root) (index 0)) + (if (= index (string-length literal)) + (vector-set! node 0 (combine (literal-trie-terminal node) (cdr entry))) + (let* ((children (vector-ref node 1)) + (character (string-ref literal index)) + (child (table-ref children character #f))) + (unless child + (set! child (make-node)) + (table-set! children character child)) + (insert child (+ index 1))))))) entries) + ;; Preparation owns mutation. Freeze empty and singleton edge sets into + ;; direct representations; branching nodes retain their character index. + ;; An explicit work list avoids growing the stack with literal depth. + (let freeze ((pending (list root))) + (unless (null? pending) + (let* ((node (car pending)) (children (vector-ref node 1)) + (edges (table->list children)) (rest (cdr pending))) + (for-each (lambda (edge) (set! rest (cons (cdr edge) rest))) edges) + (cond ((null? edges) (vector-set! node 1 #f)) + ((null? (cdr edges)) (vector-set! node 1 (car edges)))) + (freeze rest)))) + root)) + +(def (checked-lexer-literal literal) + (unless (and (string? literal) (positive? (string-length literal))) + (error "lexer literals must be non-empty strings" literal)) + literal) + +(def (make-bounded-literal-end-scanner literals) + (let (root (make-literal-trie + (map (lambda (literal) (cons (checked-lexer-literal literal) #t)) literals))) + (lambda (source start source-length) + (and (exact-integer? start) (exact-integer? source-length) + (<= 0 start source-length (string-length source)) + (let loop ((node root) (offset start) (selected #f)) + (if (= offset source-length) + selected + (let (child (literal-trie-child node (string-ref source offset))) + (if child + (let (next (+ offset 1)) + (loop child next (if (literal-trie-terminal child) next selected))) + selected)))))))) + +(def (make-literal-end-scanner literals) + (let (scan (make-bounded-literal-end-scanner literals)) + (lambda (source start) (scan source start (string-length source))))) + +;;; Merge literal-only lexical rules into one trie shared by every LR mode. +;;; Entries carry (literal name precedence declaration-index); the optional +;;; admitted vector filters complete rules without copying the trie per mode. +(def (ranked-literal-before? left right) + (or (> (cadr left) (cadr right)) + (and (= (cadr left) (cadr right)) (< (caddr left) (caddr right))))) + +;;; Order terminal catalogs once; mode filtering then stops at its first winner. +(def (prepare-ranked-literal-terminals! root) + (let visit ((pending (list root))) + (unless (null? pending) + (let* ((node (car pending)) (edges (vector-ref node 1)) + (children (cond ((not edges) '()) + ((pair? edges) (list (cdr edges))) + (else (map cdr (table->list edges)))))) + (vector-set! node 0 (list-sort ranked-literal-before? (vector-ref node 0))) + (visit (append children (cdr pending)))))) + root) + +;;; Dense ASCII fanout gets a request-independent root directory. Sparse and +;;; non-ASCII transitions retain the compact trie; the directory shares nodes. +(def (ranked-literal-root-ascii root) + (let (edges (vector-ref root 1)) + (and edges (not (pair? edges)) + (let* ((entries (table->list edges)) + (ascii (filter (lambda (entry) (< (char->integer (car entry)) 128)) entries))) + (and (>= (length ascii) 8) + (let (directory (make-vector 128 #f)) + (for-each (lambda (entry) + (vector-set! directory (char->integer (car entry)) (cdr entry))) ascii) + directory)))))) + +(def (prepare-ranked-literal-root entries) + (prepare-ranked-literal-terminals! + (make-literal-trie + (map (lambda (entry) + (cons (checked-lexer-literal (car entry)) + (list (cadr entry) (caddr entry) (cadddr entry) + (string-length (car entry))))) entries) + '() collect-literal))) + +(def (make-ranked-literal-scanner/prepared root (ascii (ranked-literal-root-ascii root))) + (def (best-admitted candidates admitted) + (if admitted + (let loop ((remaining candidates)) + (and (pair? remaining) + (if (vector-ref admitted (caddr (car remaining))) + (car remaining) (loop (cdr remaining))))) + (and (pair? candidates) (car candidates)))) + (def (publish terminal start) + (and terminal (list (car terminal) (+ start (cadddr terminal)) + (cadr terminal) (caddr terminal)))) + (lambda (source start (admitted #f)) + (let (source-length (string-length source)) + (and (not (= start source-length)) + (let* ((ch (string-ref source start)) + (child (if ascii + (let (code (char->integer ch)) + (if (< code 128) (vector-ref ascii code) + (literal-trie-child root ch))) + (literal-trie-child root ch)))) + (and child + (let scan ((node child) (offset (+ start 1)) + (selected (best-admitted (vector-ref child 0) admitted))) + (if (= offset source-length) + (publish selected start) + (let (child (literal-trie-child node (string-ref source offset))) + (if child + (let* ((candidates (vector-ref child 0)) + (terminal (and (pair? candidates) (best-admitted candidates admitted)))) + (scan child (+ offset 1) (or terminal selected))) + (publish selected start))))))))))) + +(def (make-ranked-literal-scanner entries) + (make-ranked-literal-scanner/prepared (prepare-ranked-literal-root entries))) + +;;; Product of the literal trie and the closed monotone run DFA. Rule masks +;;; are frozen by mode preparation; source scanning carries only automaton state +;;; and the last accepting winner. Numeric pending/rollback states retain their +;;; existing complete scanner rather than treating them as monotone runs. +(defrule (ranked-core-literal node admitted) + (and node + (let find ((candidates (vector-ref node 0))) + (and (pair? candidates) + (if (or (not admitted) (vector-ref admitted (caddr (car candidates)))) + (car candidates) (find (cdr candidates))))))) + +(defrule (ranked-core-winner node active admitted winners) + (let ((literal (ranked-core-literal node admitted)) (run (vector-ref winners active))) + (if (and run + (or (not literal) (> (cadr run) (cadr literal)) + (and (= (cadr run) (cadr literal)) (< (caddr run) (caddr literal))))) run literal))) + +;; scan-longest-literal +;; : (-> String Fixnum List String) +;; | doc m% +;; `scan-longest-literal` selects the longest matching declared literal. +;; +;; # Examples +;; +;; ```scheme +;; (scan-longest-literal "<=" 0 '("<" "<=")) +;; ;; => "<=" +;; ``` +;; % +(def (scan-longest-literal source start literals) + (foldl (lambda (candidate selected) + (if (and (literal-at? source start candidate) + (or (not selected) + (> (string-length candidate) + (string-length selected)))) + candidate + selected)) + #f + literals)) + diff --git a/src/runtime/scan/number.ss b/src/runtime/scan/number.ss new file mode 100644 index 00000000..db786ea5 --- /dev/null +++ b/src/runtime/scan/number.ss @@ -0,0 +1,159 @@ +;;; -*- Gerbil -*- +;;; Scan number owner: original pure lexical algorithms. +(import (only-in "./basic.ss" scan-while scan-nonempty-while) + (only-in "./literal.ss" scan-longest-literal)) +(export scan-decimal-digits scan-number-fraction-end scan-number-exponent-digits-position scan-number-exponent-end scan-number-literal ascii-digit-value ascii-digit-for-base? scan-separated-digits numeric-prefix-base scan-radix-number scan-profile-decimal-mantissa scan-profile-exponent scan-number-literal/profile) + +;; scan-decimal-digits +;; : (-> String Fixnum Fixnum) +(def scan-decimal-digits + (cut scan-nonempty-while char-numeric? char-numeric? <> <>)) + +;; scan-number-fraction-end +;; : (-> String Fixnum Fixnum Fixnum) +(def (scan-number-fraction-end source whole-end length) + (if (and (< whole-end length) + (char=? (string-ref source whole-end) #\.) + (< (+ whole-end 1) length) + (char-numeric? (string-ref source (+ whole-end 1)))) + (scan-while source (+ whole-end 1) char-numeric?) + whole-end)) + +;; scan-number-exponent-digits-position +;; : (-> String Fixnum Fixnum Fixnum) +(def (scan-number-exponent-digits-position source exponent-end length) + (let (position (+ exponent-end 1)) + (if (and (< position length) + (memv (string-ref source position) '(#\+ #\-))) + (+ position 1) + position))) + +;; scan-number-exponent-end +;; : (-> String Fixnum Fixnum Fixnum) +(def (scan-number-exponent-end source fraction-end length) + (if (and (< fraction-end length) + (memv (string-ref source fraction-end) '(#\e #\E))) + (let (digits-position + (scan-number-exponent-digits-position source fraction-end length)) + (if (and (< digits-position length) + (char-numeric? (string-ref source digits-position))) + (scan-while source digits-position char-numeric?) + fraction-end)) + fraction-end)) + +(def (scan-number-literal source start) + (and (char-numeric? (string-ref source start)) + (let* ((length (string-length source)) + (whole-end (scan-while source start char-numeric?)) + (fraction-end + (scan-number-fraction-end source whole-end length))) + (scan-number-exponent-end source fraction-end length)))) + +;; : (-> Char (Maybe Fixnum)) +(def (ascii-digit-value ch) + (cond + ((and (char>=? ch #\0) (char<=? ch #\9)) + (- (char->integer ch) (char->integer #\0))) + ((and (char>=? ch #\a) (char<=? ch #\f)) + (+ 10 (- (char->integer ch) (char->integer #\a)))) + ((and (char>=? ch #\A) (char<=? ch #\F)) + (+ 10 (- (char->integer ch) (char->integer #\A)))) + (else #f))) + +;; : (-> Char Fixnum Boolean) +(def (ascii-digit-for-base? ch base) + (alet (value (ascii-digit-value ch)) + (< value base))) + +;; : (-> String Fixnum Fixnum Char (Maybe Fixnum)) +(def (scan-separated-digits source start base separator) + (let (length (string-length source)) + (and (< start length) + (ascii-digit-for-base? (string-ref source start) base) + (let loop ((offset (+ start 1))) + (cond + ((>= offset length) offset) + ((ascii-digit-for-base? (string-ref source offset) base) + (loop (+ offset 1))) + ((and (char=? (string-ref source offset) separator) + (< (+ offset 1) length) + (ascii-digit-for-base? + (string-ref source (+ offset 1)) base)) + (loop (+ offset 2))) + (else offset)))))) + +;; : (-> String Fixnum) +(def (numeric-prefix-base prefix) + (let (last (char-downcase + (string-ref prefix (- (string-length prefix) 1)))) + (case last + ((#\b) 2) + ((#\o) 8) + ((#\x) 16) + (else (error "numeric radix prefix must end in b, o, or x" prefix))))) + +;; : (-> String Fixnum [String] Char (Maybe Fixnum)) +(def (scan-radix-number source start prefixes separator) + (alet (prefix (scan-longest-literal source start prefixes)) + (scan-separated-digits + source (+ start (string-length prefix)) + (numeric-prefix-base prefix) separator))) + +;; : (-> String Fixnum Char Boolean Boolean (Maybe Fixnum)) +(def (scan-profile-decimal-mantissa + source start separator leading-period? trailing-period?) + (let (length (string-length source)) + (cond + ((and leading-period? + (< (+ start 1) length) + (char=? (string-ref source start) #\.) + (ascii-digit-for-base? (string-ref source (+ start 1)) 10)) + (scan-separated-digits source (+ start 1) 10 separator)) + ((and (< start length) + (ascii-digit-for-base? (string-ref source start) 10)) + (let* ((whole-end + (scan-separated-digits source start 10 separator)) + (period? (and (< whole-end length) + (char=? (string-ref source whole-end) #\.)))) + (if period? + (or (scan-separated-digits source (+ whole-end 1) 10 separator) + (and trailing-period? (+ whole-end 1)) + whole-end) + whole-end))) + (else #f)))) + +;; : (-> String Fixnum Char Fixnum) +(def (scan-profile-exponent source mantissa-end separator) + (let (length (string-length source)) + (if (and (< mantissa-end length) + (memv (string-ref source mantissa-end) '(#\e #\E))) + (let* ((after-indicator (+ mantissa-end 1)) + (digits-start + (if (and (< after-indicator length) + (memv (string-ref source after-indicator) '(#\+ #\-))) + (+ after-indicator 1) + after-indicator))) + (or (scan-separated-digits source digits-start 10 separator) + mantissa-end)) + mantissa-end))) + +;; scan-number-literal/profile +;; : (-> String Fixnum [String] String [String] Boolean Boolean (Maybe Fixnum)) +;; | doc m% +;; Scans one grammar-declared numeric profile without assigning a +;; language identity in runtime code. +;; % +(def (scan-number-literal/profile + source start prefixes separator suffixes leading-period? trailing-period?) + (let (separator-character (string-ref separator 0)) + (or (scan-radix-number source start prefixes separator-character) + (alet* ((mantissa-end + (scan-profile-decimal-mantissa + source start separator-character + leading-period? trailing-period?)) + (number-end + (scan-profile-exponent source mantissa-end separator-character))) + (or (alet (suffix (scan-longest-literal source number-end suffixes)) + (+ number-end (string-length suffix))) + number-end))))) + diff --git a/src/runtime/scan/profile.ss b/src/runtime/scan/profile.ss new file mode 100644 index 00000000..23d91a6a --- /dev/null +++ b/src/runtime/scan/profile.ss @@ -0,0 +1,96 @@ +;;; -*- Gerbil -*- +;;; Scan profile owner: original pure lexical algorithms. +(import (only-in ../../grammar/lexical-algebra text-profile?)) +(export prepare-text-class prepare-text-profile make-bounded-text-profile-scanner make-text-profile-scanner) + +;;; Compile class and control dispatch once at parser-machine preparation. +;;; Matching carries source/end offsets only; optional rollback and guarded +;;; commitment are explicit data operations rather than language callbacks. +(def (prepare-text-class expression) + (case (car expression) + ((numeric) char-numeric?) + ((alphabetic) char-alphabetic?) + ((ascii-letter) (lambda (ch) (or (and (char>=? ch #\A) (char<=? ch #\Z)) (and (char>=? ch #\a) (char<=? ch #\z))))) + ((characters) (let (characters (string->list (cadr expression))) + (lambda (ch) (and (memv ch characters) #t)))) + ((union) + (let compile ((classes (cdr expression))) + (if (null? classes) + (lambda (_ch) #f) + (let ((accept? (prepare-text-class (car classes))) + (rest? (compile (cdr classes)))) + (lambda (ch) (or (accept? ch) (rest? ch))))))))) + +(def (prepare-text-profile expression) + (case (car expression) + ((literal) + (let* ((text (cadr expression)) (width (string-length text))) + (lambda (source start limit) + (let (end (+ start width)) + (and (<= end limit) + (let loop ((index 0)) + (or (= index width) + (and (char=? (string-ref text index) (string-ref source (+ start index))) + (loop (+ index 1))))) end))))) + ((run) + (let ((accept? (prepare-text-class (cadr expression))) + (minimum (caddr expression)) (maximum (cadddr expression))) + (lambda (source start limit) + (let loop ((end start) (count 0)) + (if (and (< end limit) (or (not maximum) (< count maximum)) + (accept? (string-ref source end))) + (loop (+ end 1) (+ count 1)) + (and (>= count minimum) end)))))) + ((run-containing) + (let ((accept? (prepare-text-class (cadr expression))) + (required? (prepare-text-class (caddr expression))) + (minimum (cadddr expression)) (maximum (car (cddddr expression)))) + (lambda (source start limit) + (let loop ((end start) (count 0) (seen? #f)) + (if (and (< end limit) (or (not maximum) (< count maximum)) + (accept? (string-ref source end))) + (loop (+ end 1) (+ count 1) (or seen? (required? (string-ref source end)))) + (and seen? (>= count minimum) end)))))) + ((ends-in ends-not-in) + (let ((accept? (prepare-text-class (cadr expression))) + (step (prepare-text-profile (caddr expression))) + (positive? (eq? (car expression) 'ends-in))) + (lambda (source start limit) + (let (end (step source start limit)) + (and end (> end start) + (eq? positive? (and (accept? (string-ref source (- end 1))) #t)) end))))) + ((seq) + (let (steps (map prepare-text-profile (cdr expression))) + (lambda (source start limit) + (let loop ((rest steps) (end start)) + (if (null? rest) end + (let (next ((car rest) source end limit)) + (and next (loop (cdr rest) next)))))))) + ((optional) + (let (step (prepare-text-profile (cadr expression))) + (lambda (source start limit) (or (step source start limit) start)))) + ((if-next) + (let ((accept? (prepare-text-class (cadr expression))) + (step (prepare-text-profile (caddr expression))) + (otherwise (if (= (length expression) 4) + (prepare-text-profile (cadddr expression)) + (lambda (_source start _limit) start)))) + (lambda (source start limit) + (if (and (< start limit) (accept? (string-ref source start))) + (step source start limit) (otherwise source start limit))))) + ((not-next) + (let (accept? (prepare-text-class (cadr expression))) + (lambda (source start limit) + (and (or (= start limit) (not (accept? (string-ref source start)))) start)))))) + +(def (make-bounded-text-profile-scanner expression) + (unless (text-profile? expression) (error "invalid or nullable text profile" expression)) + (let (scan (prepare-text-profile expression)) + (lambda (source start limit) + (and (exact-integer? start) (exact-integer? limit) (<= 0 start limit (string-length source)) + (let (end (scan source start limit)) (and end (> end start) end)))))) + +(def (make-text-profile-scanner expression) + (let (scan (make-bounded-text-profile-scanner expression)) + (lambda (source start) (scan source start (string-length source))))) + diff --git a/src/runtime/scan/quoted.ss b/src/runtime/scan/quoted.ss new file mode 100644 index 00000000..12675b20 --- /dev/null +++ b/src/runtime/scan/quoted.ss @@ -0,0 +1,175 @@ +;;; -*- Gerbil -*- +;;; Scan quoted owner: original pure lexical algorithms. +(import (only-in "./basic.ss" literal-at?)) +(export scan-quoted-string/mode scan-quoted-string scan-quoted-strings scan-escaped-quoted-strings scan-balanced-word scan-quoted-string/profile) + +;; scan-quoted-string +;; : (-> String Fixnum String Fixnum) +;; | doc m% +;; `scan-quoted-string` scans one escaped, delimiter-bound string. +;; +;; # Examples +;; +;; ```scheme +;; (scan-quoted-string "'value'" 0 "'") +;; ;; => 7 +;; ``` +;; % +(def (scan-quoted-string/mode source start delimiter doubled-delimiter?) + (let ((length (string-length source)) + (delimiter-length (string-length delimiter))) + (and (literal-at? source start delimiter) + (let loop ((offset (+ start delimiter-length)) (escaped? #f)) + (cond + ((>= offset length) #f) + (escaped? (loop (+ offset 1) #f)) + ((char=? (string-ref source offset) #\\) + (loop (+ offset 1) #t)) + ((literal-at? source offset delimiter) + (let (next (+ offset delimiter-length)) + ;; ISO graph-query character sequences escape their delimiter + ;; by doubling it. Backslash escaping remains admitted for + ;; the same source grammars, so both forms stay lossless. + (if (and doubled-delimiter? + (literal-at? source next delimiter)) + (loop (+ next delimiter-length) #f) + next))) + (else (loop (+ offset 1) #f))))))) + +(def (scan-quoted-string source start delimiter) + (scan-quoted-string/mode source start delimiter #t)) + +;; scan-quoted-strings +;; : (-> String Fixnum List Fixnum) +;; | doc m% +;; `scan-quoted-strings` tries delimiters in declaration order. +;; +;; # Examples +;; +;; ```scheme +;; (scan-quoted-strings "\"value\"" 0 '("\"" "'")) +;; ;; => 7 +;; ``` +;; % +(def (scan-quoted-strings source start delimiters) + (ormap (lambda (delimiter) + (scan-quoted-string source start delimiter)) + delimiters)) + +;; Backslash escapes are admitted; adjacent quoted strings remain distinct. +(def (scan-escaped-quoted-strings source start delimiters) + (ormap (lambda (delimiter) + (scan-quoted-string/mode source start delimiter #f)) + delimiters)) + +;;; A closed scanner primitive for words containing quoted or balanced spans. +;;; The language supplies only delimiter data. The scanner owns recursion, +;;; source bounds, and failure on an unterminated span. +(def (scan-balanced-word source start stops quotes pairs) + (let (length (string-length source)) + (def (at? offset text) + (let (end (+ offset (string-length text))) + (and (<= end length) + (string=? (substring source offset end) text)))) + (def (pair-at offset) + (foldl + (lambda (entry current) + (if (and (at? offset (car entry)) + (or (not current) + (> (string-length (car entry)) + (string-length (car current))))) + entry current)) + #f pairs)) + (def (quote-at offset) + (find (lambda (text) (at? offset text)) quotes)) + (def (stopped? offset) + (or (char-whitespace? (string-ref source offset)) + (ormap (lambda (text) (at? offset text)) stops))) + (def (after-escape offset) + (min length (+ offset 2))) + (def (quoted-end offset quote) + (let (delimiter (string-ref quote 0)) + (let loop ((cursor (+ offset 1))) + (cond + ((= cursor length) + (error "unterminated balanced word quote" offset)) + ((char=? (string-ref source cursor) delimiter) + (+ cursor 1)) + ((and (not (char=? delimiter #\')) + (char=? (string-ref source cursor) #\\)) + (loop (after-escape cursor))) + ((and (char=? delimiter #\") (pair-at cursor)) + (loop (paired-end cursor (pair-at cursor)))) + (else (loop (+ cursor 1))))))) + (def (paired-end offset entry) + (let ((opening (cadr entry)) + (closing (caddr entry)) + (prefix (car entry))) + (let loop ((cursor (+ offset (string-length prefix))) (depth 1)) + (cond + ((= cursor length) + (error "unterminated balanced word pair" offset prefix)) + ((char=? (string-ref source cursor) #\\) + (loop (after-escape cursor) depth)) + ((quote-at cursor) + (loop (quoted-end cursor (quote-at cursor)) depth)) + ((pair-at cursor) + (loop (paired-end cursor (pair-at cursor)) depth)) + ((char=? (string-ref source cursor) opening) + (loop (+ cursor 1) (+ depth 1))) + ((char=? (string-ref source cursor) closing) + (if (= depth 1) (+ cursor 1) + (loop (+ cursor 1) (- depth 1)))) + (else (loop (+ cursor 1) depth)))))) + (and (< start length) + (not (stopped? start)) + (let loop ((cursor start)) + (cond + ((= cursor length) cursor) + ((and (> cursor start) (stopped? cursor)) cursor) + ((char=? (string-ref source cursor) #\\) + (loop (after-escape cursor))) + ((quote-at cursor) + (loop (quoted-end cursor (quote-at cursor)))) + ((pair-at cursor) + (loop (paired-end cursor (pair-at cursor)))) + (else (loop (+ cursor 1)))))))) + +;;; A closed lexical profile validates escaped source without decoding it. +;;; Simple escapes consume one character; \u consumes exactly the configured +;;; number of hex characters. Zero disables that escape; adjacent quotes remain +;;; distinct. Numeric hex characters retain the existing Scheme lexical class. +(def (scan-quoted-string/profile source start delimiter escapes unicode-width) + (let ((length (string-length source)) (delimiter-char (string-ref delimiter 0))) + (def (hex? ch) + (or (char-numeric? ch) + (and (char>=? ch #\a) (char<=? ch #\f)) + (and (char>=? ch #\A) (char<=? ch #\F)))) + (def (simple? ch) + (let loop ((index 0)) + (and (< index (string-length escapes)) + (or (char=? ch (string-ref escapes index)) (loop (+ index 1)))))) + (def (escape-end slash) + (let (next (+ slash 1)) + (and (< next length) + (let (ch (string-ref source next)) + (cond + ((simple? ch) (+ next 1)) + ((and (positive? unicode-width) (char=? ch #\u)) + (let (end (+ next 1 unicode-width)) + (and (<= end length) + (let check ((index (+ next 1))) + (or (= index end) + (and (hex? (string-ref source index)) + (check (+ index 1))))) + end))) + (else #f)))))) + (and (< start length) (char=? (string-ref source start) delimiter-char) + (let loop ((offset (+ start 1))) + (and (< offset length) + (let (ch (string-ref source offset)) + (cond + ((char=? ch delimiter-char) (+ offset 1)) + ((char=? ch #\\) (alet (end (escape-end offset)) (loop end))) + (else (loop (+ offset 1)))))))))) + diff --git a/src/runtime/scan/regular.ss b/src/runtime/scan/regular.ss new file mode 100644 index 00000000..36e685f5 --- /dev/null +++ b/src/runtime/scan/regular.ss @@ -0,0 +1,136 @@ +;;; -*- Gerbil -*- +;;; Scan regular owner: original pure lexical algorithms. +(import (only-in :std/vector/vector vector-map/index) + (only-in "./basic.ss" identifier-start? identifier-rest? horizontal-whitespace? newline?)) +(export regular-kind-bit regular-character-mask regular-ascii-first regular-ascii-rest regular-mask regular-accept-mask regular-number-transition regular-transition scan-regular-run make-ranked-regular-scanner) + +;;; Closed run expressions share one subset DFA. The number rule has pending +;;; fraction/exponent states; the last accepting state is the maximal match. +;;; Mode-specific winners are compiled once. +(def (regular-kind-bit kind) + (case kind + ((whitespace+) 1) + ((horizontal-whitespace+) 2) + ((newline+) 4) + ((decimal-digit+) 8) + ((identifier) 16) + ((number) 32) + (else (error "unsupported regular lexical expression" kind)))) + +(def (regular-character-mask ch first?) + (bitwise-ior + (if (char-whitespace? ch) 1 0) + (if (horizontal-whitespace? ch) 2 0) + (if (newline? ch) 4 0) + (if (char-numeric? ch) 8 0) + (if (if first? (identifier-start? ch) (identifier-rest? ch)) 16 0) + (if (and first? (char-numeric? ch)) 32 0))) + +(def regular-ascii-first + (vector-map/index + (lambda (index _) (regular-character-mask (integer->char index) #t)) + (make-vector 128))) + +(def regular-ascii-rest + (vector-map/index + (lambda (index _) (regular-character-mask (integer->char index) #f)) + (make-vector 128))) + +(def (regular-mask ch first?) + (let (code (char->integer ch)) + (if (< code 128) + (vector-ref (if first? regular-ascii-first regular-ascii-rest) code) + (regular-character-mask ch first?)))) + +;; Number states: whole=32, dot=64, fraction=128, exponent=256, +;; sign=512, exponent-digits=1024. Only whole/fraction/exponent-digits accept. +(def (regular-accept-mask active) + (bitwise-ior + (bitwise-and active 31) + (if (zero? (bitwise-and active 1184)) 0 32))) + +(def (regular-number-transition active ch numeric?) + (cond + (numeric? + (bitwise-ior + (if (zero? (bitwise-and active 32)) 0 32) + (if (zero? (bitwise-and active 192)) 0 128) + (if (zero? (bitwise-and active 1792)) 0 1024))) + ((char=? ch #\.) + (if (zero? (bitwise-and active 32)) 0 64)) + ((or (char=? ch #\e) (char=? ch #\E)) + (if (zero? (bitwise-and active 160)) 0 256)) + ((or (char=? ch #\+) (char=? ch #\-)) + (if (zero? (bitwise-and active 256)) 0 512)) + (else 0))) + +(def (regular-transition active ch) + (let (mask (regular-mask ch #f)) + (bitwise-ior + (bitwise-and active mask) + (regular-number-transition + active ch (not (zero? (bitwise-and mask 8))))))) + +;;; Private hygienic lowering: arguments are stable caller bindings, except +;;; offset, which is evaluated once by the loop initializer. A subset without +;;; a number state is monotone and needs no acceptance rollback machinery. +(defrules scan-regular-run () + ((_ source offset limit initial winners) + (let loop ((at offset) (active initial)) + (let (next (if (< at limit) + (bitwise-and active (regular-mask (string-ref source at) #f)) + 0)) + (if (zero? next) + (let (winner (vector-ref winners active)) + (list (car winner) at (cadr winner) (caddr winner))) + (loop (+ at 1) next)))))) + +;;; entries: (expression-kind token-name precedence declaration-index). +;;; A lexical mode admits whole rules before building this scanner. +(def (make-ranked-regular-scanner entries) + (let* ((available + (fold (lambda (entry mask) + (bitwise-ior mask (regular-kind-bit (car entry)))) + 0 entries)) + (winners + (vector-map/index + (lambda (mask _) + (fold + (lambda (entry best) + (if (zero? (bitwise-and mask (regular-kind-bit (car entry)))) + best + (let (candidate (cdr entry)) + (if (or (not best) + (> (cadr candidate) (cadr best)) + (and (= (cadr candidate) (cadr best)) + (< (caddr candidate) (caddr best)))) + candidate best)))) + #f entries)) + (make-vector (if (zero? (bitwise-and available 32)) 32 64))))) + (if (zero? (bitwise-and available 32)) + (lambda (source start) + (let* ((length (string-length source)) + (initial (and (< start length) + (bitwise-and available (regular-mask (string-ref source start) #t))))) + (and initial (not (zero? initial)) + (scan-regular-run source (+ start 1) length initial winners)))) + (lambda (source start) + (let* ((length (string-length source)) + (initial (and (< start length) + (bitwise-and available (regular-mask (string-ref source start) #t))))) + (and initial (not (zero? initial)) + ;; Initial masks are in 0..63; bit 32 is the number start. + (if (< initial 32) + (scan-regular-run source (+ start 1) length initial winners) + (let loop ((offset (+ start 1)) (active initial) + (accepted initial) (accepted-end (+ start 1))) + (let* ((next (if (< offset length) + (regular-transition active (string-ref source offset)) 0)) + (accept (regular-accept-mask next))) + (cond + ((zero? next) + (let (winner (vector-ref winners accepted)) + (list (car winner) accepted-end (cadr winner) (caddr winner)))) + ((zero? accept) (loop (+ offset 1) next accepted accepted-end)) + (else (loop (+ offset 1) next accept (+ offset 1))))))))))))) + diff --git a/src/runtime/shell-parser.ss b/src/runtime/shell-parser.ss new file mode 100644 index 00000000..0664f73a --- /dev/null +++ b/src/runtime/shell-parser.ss @@ -0,0 +1,426 @@ +;;; -*- Gerbil -*- +;;; Lossless Bash command/word/here-document syntax entry. +;;; Unsupported command forms reject explicitly until their grammar is owned. + +(import (only-in :std/string/utf8 string-utf8-length) + (only-in ./parse-cost with-parser-cost-stage) + (only-in ../language/command-profile command-plan-role-matcher command-plan-text? command-plan-kind command-plan-forms command-plan-match-form command-plan-trigger? command-plan-entry) + (only-in ./funcs recognition-sequence-append recognition-sequence->list recognition-sequence-arity) + (only-in :gerbil-parser/src/runtime/artifact + +diagnostic-schema+ make-failure-parse-artifact + make-success-parse-artifact) + (only-in :gerbil-parser/src/runtime/recognition + make-recognition-child + recognition-child-field recognition-node-start) + (only-in :gerbil-parser/src/runtime/token + make-token token-end token-kind token-lexeme token-start) + (only-in ./contextual-scanner delimiter-obligation-quoted? decode-marker) + (only-in ../language/result-profile result-plan-node result-plan-token) + (only-in ./shell-word make-shell-word-parser)) +(export shell-here-document-link? + shell-here-document-link-marker-start + shell-here-document-link-body-start + make-shell-parser) + +(defstruct shell-here-document-link (marker-start body-start) transparent: #t) + +(def (make-shell-parser regions results parts-plan commands) + (let-values (((shell-word-components shell-assignment-components shell-here-content-components) + (make-shell-word-parser regions results parts-plan))) + + +(def forms-index (make-hash-table-eq)) +(for-each (lambda (row) (hash-put! forms-index (car row) row)) (command-plan-forms commands)) +(def (form id) (or (hash-get forms-index id) (error "undeclared command form" id))) +(def assignment-entry (command-plan-entry commands 'assignment-tail)) +(def pipeline-entry (command-plan-entry commands 'pipeline-head)) +(def assignment-form (form assignment-entry)) +(def pipeline-form (form pipeline-entry)) +(def header-first (cons 'or (map cadr (list-tail pipeline-form 4)))) +(def trivia? (command-plan-role-matcher commands 'trivia)) +(def separator? (command-plan-role-matcher commands 'separator)) +(def redirect? (command-plan-role-matcher commands 'redirect)) +(def here-redirect? (command-plan-role-matcher commands 'here-redirect)) +(def strip-redirect? (command-plan-role-matcher commands 'strip-redirect)) +(def reserved? (command-plan-role-matcher commands 'reserved)) +(def case-end? (command-plan-role-matcher commands 'case-end)) +(def conditional-operator? (command-plan-role-matcher commands 'conditional-operator)) +(def pipeline? (command-plan-role-matcher commands 'pipeline)) +(def and-or? (command-plan-role-matcher commands 'and-or)) +(def (child field value) + (make-recognition-child field value)) + +(def (syntax-node kind start end children) + (result-plan-node results (command-plan-kind commands kind) start end (recognition-sequence->list children))) + +(def (failure-diagnostic condition) + (list (cons 'schema +diagnostic-schema+) + (cons 'code "BASH-SYNTAX-REJECTED") + (cons 'reasonKind 'parse-rejected) + (cons 'message (error-message condition)))) + +(def (fallback-token source) + (if (zero? (string-length source)) '() + (list (result-plan-token results (make-token 'unparsed-source source 0 + (string-utf8-length source)))))) + +;;; Returns the artifact plus source-span links from redirection markers to +;;; their deferred here-document bodies. The links do not change CST order. +(def (parse-shell-core/receipt source scan grammar-digest) + (unless (string? source) (error "Bash source must be a string" source)) + (with-catch + (lambda (condition) + (values + (with-parser-cost-stage 'artifact-publication + (make-failure-parse-artifact + grammar-digest source (fallback-token source) + (failure-diagnostic condition))) + '())) + (lambda () + (let ((remaining (scan source)) + (emitted-reversed '()) + (pending-markers '()) (pending-back '()) + (links '())) + (def (marker-head) + (when (null? pending-markers) + (set! pending-markers (reverse pending-back)) (set! pending-back '())) + (and (pair? pending-markers) (car pending-markers))) + (def (enqueue-marker! marker) (set! pending-back (cons marker pending-back))) + (def (emit! token) + (set! emitted-reversed (cons (result-plan-token results token) emitted-reversed))) + (def (take-raw!) + (unless (pair? remaining) (error "unexpected Bash EOF")) + (let (token (car remaining)) + (set! remaining (cdr remaining)) + (emit! token) + token)) + (def (skip-trivia!) + (let loop () + (when (and (pair? remaining) (trivia? (car remaining))) + (take-raw!) + (loop)))) + (def (peek) + (skip-trivia!) + (and (pair? remaining) (car remaining))) + (def (last-byte) + (if (pair? emitted-reversed) + (token-end (car emitted-reversed)) 0)) + (def (take-word!) + (let (raw (peek)) + (unless (and raw (eq? (token-kind raw) 'word)) + (error "expected Bash word" raw)) + (set! remaining (cdr remaining)) + (let-values (((word pieces) (shell-word-components raw))) + (for-each emit! pieces) + word))) + (def (parse-redirection! descriptor) + (let* ((operator (peek)) + (start (if descriptor + (token-start descriptor) + (token-start operator)))) + (unless (redirect? operator) + (error "expected Bash redirection" operator)) + (take-raw!) + (let (target (peek)) + (unless (and target + (if (here-redirect? operator) + (eq? (token-kind target) 'heredoc-marker) + (eq? (token-kind target) 'word))) + (error "missing Bash redirection target" operator)) + (let* ((target-value + (if (eq? (token-kind target) 'heredoc-marker) + (begin + (enqueue-marker! + (cons (token-start target) + (delimiter-obligation-quoted? + (decode-marker 'shell-quote-removal (token-lexeme target) + (strip-redirect? operator))))) + (take-raw!)) + (take-word!))) + (children + (append + (if descriptor + (list (child 'descriptor descriptor)) '()) + (list (child 'operator operator) + (child 'target target-value))))) + (syntax-node 'Redirection start (last-byte) children))))) + (def (descriptor-before-redirection?) + (and (pair? remaining) + (eq? (token-kind (car remaining)) 'word) + (command-plan-text? commands 'descriptor (token-lexeme (car remaining))) + (pair? (cdr remaining)) + (redirect? (cadr remaining)) + (= (token-end (car remaining)) + (token-start (cadr remaining))))) + (def (array-assignment-open? raw) + (and (pair? remaining) + (command-plan-trigger? commands (cadr assignment-form) remaining) + (= (token-end raw) (token-start (car remaining))))) + (def (parse-simple-command!) + (let* ((first (peek)) + (start (and first (token-start first))) + (named? #f) + (children '())) + (unless first (error "expected Bash command")) + (when (reserved? first) + (error "Bash compound command is not admitted yet" + (token-lexeme first))) + (let loop () + (let (token (peek)) + (cond + ((descriptor-before-redirection?) + (let (descriptor (take-raw!)) + (set! children + (cons (child 'redirect + (parse-redirection! descriptor)) + children)) + (loop))) + ((redirect? token) + (set! children + (cons (child 'redirect (parse-redirection! #f)) + children)) + (loop)) + ((and token (eq? (token-kind token) 'word)) + (let-values (((assignment pieces) + (if named? + (values #f #f) + (shell-assignment-components token)))) + (if assignment + (begin + (set! remaining (cdr remaining)) + (for-each emit! pieces) + (set! children + (cons (child 'assignment + (if (array-assignment-open? token) + (parse-form! assignment-entry (recognition-node-start assignment) + (list (child 'assignment assignment))) + assignment)) + children))) + (let (word (take-word!)) + (set! children + (cons (child (if named? 'argument 'name) word) + children)) + (set! named? #t))) + (loop))) + ((null? children) + (error "expected Bash command or redirection" token)) + (else + (syntax-node 'SimpleCommand start (last-byte) + (reverse children)))))))) + (def (has-command? children) + (ormap (lambda (entry) + (eq? (recognition-child-field entry) 'command)) + children)) + (def (parse-compound-list! start terminators + (allow-empty? #f)) + (let (children (parse-list! terminators)) + (unless (or allow-empty? (has-command? children)) + (error "Bash command list is empty")) + (syntax-node 'CommandList start (last-byte) children))) + (def (matches? trigger) + (peek) + (command-plan-trigger? commands trigger remaining)) + (def (execute-program! program kind (initial '())) + (let ((children initial) (projection kind)) + (def (publish! field value) + (set! children (recognition-sequence-append children (list (child field value))))) + (def (execute! body) + (for-each (lambda (step) + (case (car step) + ((as) (set! projection (cadr step))) + ((raw) (peek) (publish! (cadr step) (take-raw!))) + ((take) + (unless (matches? (caddr step)) (error "expected declared command token" (caddr step) (peek))) + (publish! (cadr step) (take-raw!))) + ((word) (publish! (cadr step) (take-word!))) + ((command) (publish! (cadr step) (parse-command!))) + ((call) (publish! (cadr step) (parse-form! (caddr step)))) + ((node) (publish! (cadr step) (parse-program-node! (caddr step) (cdddr step)))) + ((list) + (publish! (cadr step) (parse-compound-list! (last-byte) (caddr step) (cadddr step)))) + ((optional) (when (matches? (cadr step)) (execute! (cddr step)))) + ((branch) (execute! (if (matches? (cadr step)) (caddr step) (cadddr step)))) + ((choose) + (let choices ((rows (cdr step))) + (cond ((null? rows) (error "no declared command branch accepts token" (peek))) + ((matches? (caar rows)) (execute! (cdar rows))) + (else (choices (cdr rows)))))) + ((many until) + (let loop () + (let (matched? (matches? (cadr step))) + (when (if (eq? (car step) 'many) matched? (not matched?)) + (unless (pair? remaining) (error "unterminated declared command repetition" (cadr step))) + (let (before remaining) + (execute! (cddr step)) + (when (eq? before remaining) (error "command program did not advance input"))) + (loop))))) + ((balance) + (let loop ((depth (list-ref step 3))) + (unless (peek) (error "unterminated declared balanced command body")) + (cond + ((matches? (cadr step)) (publish! (list-ref step 4) (take-raw!)) (loop (+ depth 1))) + ((matches? (caddr step)) + (publish! (list-ref step 5) (take-raw!)) (unless (= depth 1) (loop (- depth 1)))) + (else + (let (before remaining) + (execute! (list-tail step 6)) + (when (eq? before remaining) (error "balanced command program did not advance input"))) + (loop depth))))))) body)) + (execute! program) + (values projection children))) + (def (parse-program-node! kind program (start #f) (initial '())) + (let (beginning (or start (and (peek) (token-start (peek))))) + (unless beginning (error "expected declared compound command" kind)) + (let-values (((projection children) (execute-program! program kind initial))) + (syntax-node projection beginning (last-byte) children)))) + (def (parse-form! id (start #f) (initial '())) + (let (row (form id)) + (parse-program-node! (cadddr row) (list-tail row 4) start initial))) + (def (parse-compound-redirections! command) + (let loop ((children (list (child 'command command)))) + (let (next (peek)) + (cond + ((descriptor-before-redirection?) + (let (descriptor (take-raw!)) + (loop (recognition-sequence-append children + (list (child 'redirect + (parse-redirection! + descriptor))))))) + ((redirect? next) + (loop (recognition-sequence-append children + (list (child 'redirect + (parse-redirection! #f)))))) + ((= (recognition-sequence-arity children) 1) command) + (else + (syntax-node 'RedirectedCommand + (recognition-node-start command) + (last-byte) children)))))) + (def (parse-command!) + (peek) + (let (id (command-plan-match-form commands remaining)) + (if id (parse-compound-redirections! (parse-form! id)) (parse-simple-command!)))) + (def (parse-pipeline!) + (let ((prefixes '()) + (start-token (peek))) + (when (matches? header-first) + (let-values (((_projection children) + (execute-program! (list-tail pipeline-form 4) 'Pipeline))) + (set! prefixes (recognition-sequence->list children)))) + (let* ((first (parse-command!)) + (children + (append prefixes (list (child 'command first))))) + (let loop () + (let (operator (peek)) + (if (pipeline? operator) + (begin + (take-raw!) + (set! children + (recognition-sequence-append children + (list (child 'operator operator) + (child 'command + (parse-command!))))) + (loop)) + (if (and (null? prefixes) (= (recognition-sequence-arity children) 1)) first + (syntax-node 'Pipeline + (if (null? prefixes) + (recognition-node-start first) + (token-start start-token)) + (last-byte) children)))))))) + (def (parse-and-or!) + (let* ((first (parse-pipeline!)) + (children (list (child 'command first)))) + (let loop () + (let (operator (peek)) + (if (and-or? operator) + (begin + (take-raw!) + (set! children + (recognition-sequence-append children + (list (child 'operator operator) + (child 'command (parse-pipeline!))))) + (loop)) + (if (= (recognition-sequence-arity children) 1) first + (syntax-node 'AndOrList + (recognition-node-start first) + (last-byte) children))))))) + (def (parse-here-documents!) + (let loop ((nodes '())) + (let (next (peek)) + (if (and next + (memq (token-kind next) + '(heredoc-content heredoc-end))) + (let ((start (token-start next)) (children '())) + (unless (marker-head) + (error "here-document body has no redirection")) + (let body-loop () + (let (line (peek)) + (unless (and line + (memq (token-kind line) + '(heredoc-content heredoc-end))) + (error "unterminated Bash here-document body")) + (let (value + (if (or (eq? (token-kind line) 'heredoc-end) + (cdr (marker-head))) + (take-raw!) + (begin + (set! remaining (cdr remaining)) + (let-values + (((node pieces) + (shell-here-content-components line))) + (for-each emit! pieces) + node)))) + (set! children + (cons (child + (if (eq? (token-kind line) 'heredoc-end) + 'delimiter 'content) + value) + children))) + (unless (eq? (token-kind line) 'heredoc-end) + (body-loop)))) + (set! links + (cons (make-shell-here-document-link + (car (marker-head)) start) + links)) + (set! pending-markers (cdr pending-markers)) + (loop (cons + (child 'here-document + (syntax-node 'HereDocument start (last-byte) + (reverse children))) + nodes))) + (reverse nodes))))) + (def (parse-list! terminators) + (let loop ((children '())) + (let (next (peek)) + (cond + ((or (not next) (command-plan-trigger? commands terminators remaining)) + (reverse children)) + ((separator? next) + (when (and (eq? (token-kind next) 'operator) + (not (has-command? children))) + (error "Bash list separator has no command" next)) + (take-raw!) + (let (with-separator (cons (child 'separator next) children)) + (if (eq? (token-kind next) 'newline) + (loop (append (reverse (parse-here-documents!)) + with-separator)) + (loop with-separator)))) + (else + (loop (cons (child 'command (parse-and-or!)) children))))))) + (let* ((children (with-parser-cost-stage 'command-recognition (parse-list! '(none)))) + (source-end (string-utf8-length source))) + (when (marker-head) + (error "missing Bash here-document body")) + (let* ((root (syntax-node 'BashFile 0 source-end children)) + (artifact + (with-parser-cost-stage 'artifact-publication + (make-success-parse-artifact + grammar-digest source + (reverse emitted-reversed) root trivia?)))) + (values artifact (reverse links)))))))) + +(def (parse-shell-core source scan grammar-digest) + (let-values (((artifact _links) + (parse-shell-core/receipt source scan grammar-digest))) + artifact)) + + (values parse-shell-core parse-shell-core/receipt))) diff --git a/src/runtime/shell-word.ss b/src/runtime/shell-word.ss new file mode 100644 index 00000000..4f81252e --- /dev/null +++ b/src/runtime/shell-word.ss @@ -0,0 +1,193 @@ +;;; -*- Gerbil -*- +;;; Lossless word decomposition with explicit quote and expansion boundaries. + +(import (only-in :gerbil-parser/src/runtime/recognition + prepare-recognition-source make-recognition-child) + (only-in ../language/part-profile part-plan-recipe part-plan-context part-context-match part-context-literal-end part-plan-literal + part-step-operation part-step-kind part-step-opening part-step-closing part-step-delimiter part-plan-name-end part-plan-binding part-step-binding + binding-plan-name-end binding-plan-prefix-end binding-plan-operator-end binding-plan-subscript-at? binding-plan-subscript-width binding-plan-region-plan binding-plan-region-scopes) + (only-in ../language/result-profile result-plan-projection result-projection-build result-projection-parts-field result-projection-build-parts) + (only-in :gerbil-parser/src/runtime/funcs + recognition-sequence-append recognition-sequence->list) + (only-in :gerbil-parser/src/runtime/token token-lexeme) + (only-in :gerbil-parser/src/runtime/region-scanner + prepare-region-source prepare-scoped-region-source region-source-quote-end region-source-pair-end)) +(export make-shell-word-parser) + +(defstruct shell-word-source (spans text regions subscripts)) + +(def (make-shell-word-parser regions results parts-plan) +(def literal-kind (part-plan-literal parts-plan)) +(def word-context (part-plan-context parts-plan 'word)) +(def here-context (part-plan-context parts-plan 'HereDocument)) +(def assignment-plan (part-plan-binding parts-plan 'assignment)) +(def parameter-plan (part-plan-binding parts-plan 'parameter)) +(def subscript-regions (and parameter-plan (binding-plan-region-plan parameter-plan regions))) +(def subscript-scopes (and parameter-plan (binding-plan-region-scopes parameter-plan regions))) +(def (prepare-word token) + ;; Scanner-created token lexemes are read-only throughout this engine call. + ;; Result publication owns its admitted copy; region preparation is needed + ;; only when an actual quote or substitution is encountered. + (let* ((spans (prepare-recognition-source token)) (text (token-lexeme token))) + (values (make-shell-word-source spans text #f #f) text))) + +(def (word-regions source) + (or (shell-word-source-regions source) + (let (context (prepare-region-source regions (shell-word-source-text source))) + (shell-word-source-regions-set! source context) + context))) + +;;; Bind projection signatures once. The grammar owns node/field/token names; +;;; this executor owns only typed recognition captures and character boundaries. +(def projections (make-hash-table-eq)) +(def (bind! id signature) + (hash-put! projections id (result-plan-projection results id signature))) +(bind! 'Word '((parts parts))) +(bind! 'HereDocumentLine '((parts parts))) +(when assignment-plan (bind! 'Assignment '((name span required) (operator span required) (parts parts)))) +(bind! literal-kind '((text span required))) +(for-each (lambda (row) + (let* ((form (caddr row)) (operation (car form)) + (kind (if (eq? operation 'parameter) 'ParameterExpansion (cadr form)))) + (case operation + ((name escape) (bind! kind '((text span required)))) + ((quote) (bind! kind '((open span required) (parts parts) (close span required)))) + ((pair quoted-body) (bind! kind '((open span required) (body span optional) (close span required)))) + ((parameter) + (bind! kind '((open span required) (prefix span optional) (name span optional) (subscript node optional) (operator span optional) (parts parts) (close span required))) + (when subscript-regions (bind! 'ArraySubscript '((open span required) (parts parts) (close span required)))))))) + (cadr (part-plan-recipe parts-plan))) +(def (project raw id start end captures) + (result-projection-build (hash-get projections id) (shell-word-source-spans raw) start end captures)) +(def (leaf raw kind start end) + (project raw kind start end (vector (cons start end)))) + +(def (word-subscript-regions source) + (or (shell-word-source-subscripts source) + (let (context (prepare-scoped-region-source subscript-regions (shell-word-source-text source) subscript-scopes)) + (shell-word-source-subscripts-set! source context) context))) + +(def (parse-parameter raw text start end step) + (let* ((binding (part-step-binding step)) + (body-start (+ start (part-step-opening step))) + (body-end (- end (part-step-closing step))) + (prefix-end (binding-plan-prefix-end binding text body-start body-end)) + (name-start (or prefix-end body-start)) + (parameter-end (binding-plan-name-end binding text name-start body-end)) + (subscript-start (and (< parameter-end body-end) + (binding-plan-subscript-at? binding text parameter-end body-end) parameter-end)) + (after-subscript (if subscript-start + (region-source-pair-end (word-subscript-regions raw) subscript-start) parameter-end)) + (operator-end (or (binding-plan-operator-end binding text after-subscript body-end) after-subscript))) + (when (> after-subscript body-end) (error "binding subscript crosses parameter boundary" subscript-start)) + (let-values (((operand-children operand-tokens) (parse-parts raw text operator-end body-end #f))) + (let (subscript + (and subscript-start + (let ((inner-start (+ subscript-start (binding-plan-subscript-width binding))) + (inner-end (- after-subscript 1))) + (let-values (((parts tokens) (parse-parts raw text inner-start inner-end #f))) + (let-values (((value produced) + (project raw 'ArraySubscript subscript-start after-subscript + (vector (cons subscript-start inner-start) (cons parts tokens) (cons inner-end after-subscript))))) + (cons value produced)))))) + (project raw (part-step-kind step) start end + (vector (cons start body-start) + (and prefix-end (cons body-start name-start)) + (and (> parameter-end name-start) (cons name-start parameter-end)) + subscript + (and (> operator-end after-subscript) (cons after-subscript operator-end)) + (cons operand-children operand-tokens) (cons body-end end))))))) + +(def (parse-quoted raw text start end kind opening-length closing-length) + (let ((body-start (+ start opening-length)) (body-end (- end closing-length))) + (let-values (((parts interior) (parse-parts raw text body-start body-end kind))) + (project raw kind start end + (vector (cons start body-start) (cons parts interior) (cons body-end end)))))) + +(def (parse-opaque-substitution raw text start end kind opening-length closing-length) + (let ((body-start (+ start opening-length)) (body-end (- end closing-length))) + (project raw kind start end + (vector (cons start body-start) (and (> body-end body-start) (cons body-start body-end)) (cons body-end end))))) + +;;; Token publication shares immutable sequence branches across nested results. +;;; Only the public word/assignment/body boundary materializes the ordered list; +;;; nesting never copies a growing token suffix. No relocated child views enter +;;; this token-only sequence, so materialization retains the original tokens. +;;; Returns syntax parts, or final children when a declared field is supplied, +;;; together with their nonoverlapping source token sequence. +(def (parse-parts raw text start end context (field #f)) + (let (context-plan (cond ((not context) word-context) + ((eq? context 'HereDocument) here-context) + (else (part-plan-context parts-plan context)))) + (let loop ((offset start) (parts '()) (tokens '())) + (if (= offset end) + ;; This spine is private until return; token branches and nested + ;; result nodes are shared values and are never reversed. + (values (reverse! parts) tokens) + (let-values + (((part produced next) + (let* ((step (part-context-match context-plan text offset end)) + (operation (and step (part-step-operation step))) + (name-end (and (eq? operation 'name) + (part-plan-name-end step text (+ offset 1) end)))) + (cond + ((memq operation '(quote quoted-body)) + (let (after (region-source-quote-end (word-regions raw) + (+ offset (- (part-step-opening step) 1)) (part-step-delimiter step))) + (let-values (((part produced) + (if (eq? operation 'quote) + (parse-quoted raw text offset after (part-step-kind step) (part-step-opening step) (part-step-closing step)) + (parse-opaque-substitution raw text offset after (part-step-kind step) + (part-step-opening step) (part-step-closing step))))) + (values part produced after)))) + ((memq operation '(parameter pair)) + (let (after (region-source-pair-end (word-regions raw) offset)) + (let-values (((part produced) + (if (eq? operation 'parameter) + (parse-parameter raw text offset after step) + (parse-opaque-substitution raw text offset after (part-step-kind step) + (part-step-opening step) (part-step-closing step))))) + (values part produced after)))) + ((and name-end (> name-end (+ offset 1))) + (let-values (((part produced) (leaf raw (part-step-kind step) offset name-end))) + (values part produced name-end))) + ((eq? operation 'escape) + (let (after (min end (+ offset 2))) + (let-values (((part produced) (leaf raw (part-step-kind step) offset after))) + (values part produced after)))) + (else + (let (after (part-context-literal-end context-plan text offset end)) + (let-values (((part produced) (leaf raw literal-kind offset after))) + (values part produced after)))))))) + (loop next (cons (if field (make-recognition-child field part) part) parts) + (recognition-sequence-append tokens produced))))))) + +(def (components token context id) + (let-values (((raw text) (prepare-word token))) + (let* ((length (string-length text)) (projection (hash-get projections id)) + (field (result-projection-parts-field projection))) + (let-values (((parts tokens) (parse-parts raw text 0 length context field))) + (let-values (((value produced) (result-projection-build-parts projection + (shell-word-source-spans raw) 0 length parts tokens))) + (values value (recognition-sequence->list produced))))))) +(def (shell-word-components token) (components token #f 'Word)) +(def (shell-here-content-components token) (components token 'HereDocument 'HereDocumentLine)) + +;;; An assignment is recognized only at a command position by the caller. +;;; It returns #f for ordinary words, or a node and ordered source tokens. +(def (shell-assignment-components token) + (let* ((text (token-lexeme token)) + (length (string-length text)) + (name-end (and assignment-plan (binding-plan-name-end assignment-plan text 0 length))) + (operator-end (and name-end (> name-end 0) + (binding-plan-operator-end assignment-plan text name-end length)))) + (if operator-end + (let-values (((raw text) (prepare-word token))) + (let-values (((parts tokens) (parse-parts raw text operator-end length #f))) + (let-values (((value produced) + (project raw 'Assignment 0 length + (vector (cons 0 name-end) (cons name-end operator-end) (cons parts tokens))))) + (values value (recognition-sequence->list produced))))) + (values #f #f)))) + + (values shell-word-components shell-assignment-components shell-here-content-components)) diff --git a/src/runtime/significant.ss b/src/runtime/significant.ss index d7e32b27..6bbb1ae7 100644 --- a/src/runtime/significant.ss +++ b/src/runtime/significant.ss @@ -2,9 +2,21 @@ ;;; Runtime-only projection from lossless tokens to parser-significant tokens. (import (only-in ../compiler/machine parser-machine-trivia)) -(export parser-significant-tokens) +(export parser-significant-tokens parser-significant-joined) ;; : (-> ParserMachine (List Token) (List Token)) (def (parser-significant-tokens machine tokens) (let (trivia? (parser-machine-trivia machine)) (filter (lambda (token) (not (trivia? token))) tokens))) + +;; A fork has a reversed deterministic prefix and a fresh forward suffix. +;; Filter the suffix once, then prepend significant prefix tokens in order; +;; `rest` stays an exact tail of the combined significant stream for GLR. +(def (parser-significant-joined machine prefix-reversed suffix) + (let* ((trivia? (parser-machine-trivia machine)) + (rest (filter (lambda (token) (not (trivia? token))) suffix))) + (values + (foldl (lambda (token combined) + (if (trivia? token) combined (cons token combined))) + rest prefix-reversed) + rest))) diff --git a/src/runtime/source-engines.ss b/src/runtime/source-engines.ss new file mode 100644 index 00000000..32b34f29 --- /dev/null +++ b/src/runtime/source-engines.ss @@ -0,0 +1,92 @@ +;;; Closed source recipes share the source engine protocol; no language imports. +(import (only-in ./parse-cost with-parser-cost-stage) + (only-in ../language/command-profile command-plan-kind) + (only-in :clan/poo/object .o .ref .slot?) + (only-in ../language/source-strategy SourceStrategy. declare-source-strategy-provider make-source-engine) + (only-in ./region-scanner region-plan?) + (only-in ./contextual-scanner prepare-contextual-scanner-plan) + (only-in ./shell-parser make-shell-parser) + (only-in ./source-scanner make-source-scanner make-contextual-source-scanner source-scanner-tokens) + (only-in ./recognition make-recognition-child) + (only-in ../language/result-profile ResultProfile. result-plan-node result-plan-token) + (only-in ../compiler/contextual-program compile-command-contextual-program compile-line-contextual-program + contextual-program-ir contextual-program-regions contextual-program-scanner + contextual-program-results contextual-program-parts contextual-program-commands) + (only-in ./contextual-ir contextual-ir-ref) + (only-in ./token token-start token-end) + (only-in ./artifact make-success-parse-artifact make-failure-parse-artifact)) +(export ShellSourceStrategy. LineSourceStrategy.) +;;; Shape classification is separate from semantic compilation. The compiler +;;; admits each plan once and uses that same owned plan for identity and runtime. +(def (shell-source-admit? candidate) + (and (andmap (lambda (slot) (.slot? candidate slot)) + '(regions scanner parts results commands)) + (region-plan? (.ref candidate 'regions)))) +(def (compile-shell-source candidate) + (let* ((program (compile-command-contextual-program candidate)) + (results (contextual-program-results program)) + (regions (contextual-program-regions program)) + (scanner-ir (contextual-program-scanner program)) + (commands (contextual-program-commands program)) + (parts (contextual-program-parts program)) + (plan (prepare-contextual-scanner-plan scanner-ir))) + (def (factory source) (make-contextual-source-scanner plan source 'source)) + (def (scan source) + (let (worker (with-parser-cost-stage 'source-prepare (factory source))) + (with-parser-cost-stage 'source-scan (source-scanner-tokens worker 'command)))) + (let-values (((parse receipt) (make-shell-parser regions results parts commands))) + (values + (contextual-program-ir program) + (make-source-engine scan factory parse receipt results (command-plan-kind commands 'BashFile) program))))) +(def +shell-source-provider+ + (declare-source-strategy-provider 'shell shell-source-admit? compile-shell-source)) +(def ShellSourceStrategy. + (.o (:: self SourceStrategy.) provider: +shell-source-provider+ regions: #f scanner: #f results: #f parts: #f commands: #f)) +(def (line-source-admit? candidate) + (and (andmap (lambda (slot) (.slot? candidate slot)) '(root-kind token-kind required-prefix)) + (symbol? (.ref candidate 'root-kind)) (symbol? (.ref candidate 'token-kind)) + (string? (.ref candidate 'required-prefix)) + (.slot? candidate 'results))) +(def (compile-line-source candidate) + (let* ((program (compile-line-contextual-program candidate)) + (ir (contextual-program-ir program)) + (recognition (contextual-ir-ref (contextual-ir-ref ir 'recognition) 'program)) + (root-kind (contextual-ir-ref ir 'root-kind)) + (token-kind (contextual-ir-ref recognition 'token-kind)) + (prefix (contextual-ir-ref recognition 'required-prefix)) + (results (contextual-program-results program))) + (def (factory source) + (make-source-scanner source #f + (lambda (text start context _mode) + (let (limit (string-length text)) + (if (= start limit) (values #f start context) + (let loop ((end start)) + (if (or (= end limit) (char=? (string-ref text end) #\newline)) + (values token-kind (if (= end limit) end (+ end 1)) context) + (loop (+ end 1))))))))) + (def (scan source) + (let (worker (with-parser-cost-stage 'source-prepare (factory source))) + (with-parser-cost-stage 'source-scan + (map (lambda (token) (result-plan-token results token)) + (source-scanner-tokens worker 'lines))))) + (def (parse source scanner digest) + (let (tokens (scanner source)) + (if (and (<= (string-length prefix) (string-length source)) + (equal? prefix (substring source 0 (string-length prefix)))) + (make-success-parse-artifact digest source tokens + (result-plan-node results root-kind 0 + (if (null? tokens) 0 (token-end (last tokens))) + (map (lambda (token) (make-recognition-child 'line token)) tokens)) (lambda (_) #f)) + (make-failure-parse-artifact digest source tokens + '((schema . "gerbil-parser.diagnostic.v1") (code . "SOURCE-PREFIX") + (reasonKind . parse-rejected) (failureKind . source-rejected) + (message . "source does not match its declared prefix") (byteOffset . 0)))))) + (values ir (make-source-engine scan factory parse #f results root-kind program)))) +(def +line-source-provider+ + (declare-source-strategy-provider 'lines line-source-admit? compile-line-source)) +(def LineSourceStrategy. + (.o (:: self SourceStrategy.) provider: +line-source-provider+ + root-kind: 'SourceFile token-kind: 'Line required-prefix: "" + (results (.o (:: result ResultProfile.) + nodes: (list (list (.ref self 'root-kind) '(line))) + tokens: (list (.ref self 'token-kind)))))) diff --git a/src/runtime/source-index.ss b/src/runtime/source-index.ss new file mode 100644 index 00000000..08120873 --- /dev/null +++ b/src/runtime/source-index.ss @@ -0,0 +1,300 @@ +;;; -*- Gerbil -*- +;;; Persistent measured token/mode sequence. Only fresh chunks are mutable +;;; while being built; published nodes and chunk backing vectors never mutate. +(import (only-in ./token make-token token-kind token-lexeme token-start token-end)) +(export source-index-count source-index-height source-index-start source-index-end + source-index-shift source-index-concat source-index-slice source-index->lists + source-index-audit source-index-storage + make-source-index-builder source-index-builder-token! source-index-builder-shared! + source-index-builder-finish source-index-builder-fresh source-index-builder-shared + source-index-builder-chunks source-index-builder-mode-chunks + make-source-index-cursor source-index-cursor-copy source-index-cursor-token + source-index-cursor-mode source-index-cursor-rank source-index-cursor-next! + source-index-cursor-seek!) +(def +chunk-size+ 64) +(defstruct source-leaf (tokens modes begin count bytes chars start end)) +(defstruct source-branch (left right count bytes chars start end height)) +(defstruct source-view (base delta)) +(def (source-index-count tree) + (cond ((not tree) 0) ((source-view? tree) (source-index-count (source-view-base tree))) + ((source-leaf? tree) (source-leaf-count tree)) (else (source-branch-count tree)))) +(def (source-index-height tree) + (cond ((not tree) 0) ((source-view? tree) (source-index-height (source-view-base tree))) + ((source-leaf? tree) 1) (else (source-branch-height tree)))) +(def (source-index-start tree) + (cond ((source-view? tree) (+ (source-view-delta tree) (source-index-start (source-view-base tree)))) + ((source-leaf? tree) (source-leaf-start tree)) (else (source-branch-start tree)))) +(def (source-index-end tree) + (cond ((source-view? tree) (+ (source-view-delta tree) (source-index-end (source-view-base tree)))) + ((source-leaf? tree) (source-leaf-end tree)) (else (source-branch-end tree)))) +(def (tree-chars tree) + (cond ((not tree) 0) ((source-view? tree) (tree-chars (source-view-base tree))) + ((source-leaf? tree) (source-leaf-chars tree)) (else (source-branch-chars tree)))) +(def (tree-bytes tree) + (if tree (- (source-index-end tree) (source-index-start tree)) 0)) +(def (source-index-shift tree delta) + (cond ((or (not tree) (zero? delta)) tree) + ((source-view? tree) + (source-index-shift (source-view-base tree) (+ delta (source-view-delta tree)))) + (else (make-source-view tree delta)))) +(def (branch-parts tree) + (if (source-view? tree) + (let-values (((left right) (branch-parts (source-view-base tree)))) + (values (source-index-shift left (source-view-delta tree)) + (source-index-shift right (source-view-delta tree)))) + (values (source-branch-left tree) (source-branch-right tree)))) +(def (branch left right) + (unless (= (source-index-end left) (source-index-start right)) + (error "source index join has a byte gap")) + (make-source-branch left right (+ (source-index-count left) (source-index-count right)) + (+ (tree-bytes left) (tree-bytes right)) (+ (tree-chars left) (tree-chars right)) + (source-index-start left) (source-index-end right) + (+ 1 (max (source-index-height left) (source-index-height right))))) +(def (balance left right) + (cond + ((> (source-index-height left) (+ (source-index-height right) 1)) + (let-values (((a b) (branch-parts left))) + (if (>= (source-index-height a) (source-index-height b)) + (branch a (branch b right)) + (let-values (((c d) (branch-parts b))) (branch (branch a c) (branch d right)))))) + ((> (source-index-height right) (+ (source-index-height left) 1)) + (let-values (((a b) (branch-parts right))) + (if (>= (source-index-height b) (source-index-height a)) + (branch (branch left a) b) + (let-values (((c d) (branch-parts a))) (branch (branch left c) (branch d b)))))) + (else (branch left right)))) +(def (source-index-concat left right) + (cond ((not left) right) ((not right) left) + ((> (source-index-height left) (+ (source-index-height right) 1)) + (let-values (((a b) (branch-parts left))) (balance a (source-index-concat b right)))) + ((> (source-index-height right) (+ (source-index-height left) 1)) + (let-values (((a b) (branch-parts right))) (balance (source-index-concat left a) b))) + (else (branch left right)))) +(def (leaf tokens modes begin count) + (let ((start (token-start (vector-ref tokens begin))) + (end (token-end (vector-ref tokens (+ begin count -1))))) + (make-source-leaf tokens modes begin count (- end start) + (let loop ((at begin) (chars 0)) + (if (= at (+ begin count)) chars + (loop (+ at 1) (+ chars (string-length (token-lexeme (vector-ref tokens at))))))) start end))) +(def (split tree count) + (cond ((zero? count) (values #f tree)) + ((= count (source-index-count tree)) (values tree #f)) + ((source-view? tree) + (let-values (((a b) (split (source-view-base tree) count))) + (values (source-index-shift a (source-view-delta tree)) + (source-index-shift b (source-view-delta tree))))) + ((source-leaf? tree) + (values (leaf (source-leaf-tokens tree) (source-leaf-modes tree) (source-leaf-begin tree) count) + (leaf (source-leaf-tokens tree) (source-leaf-modes tree) + (+ (source-leaf-begin tree) count) (- (source-leaf-count tree) count)))) + (else + (let-values (((a b) (branch-parts tree))) + (let (left-count (source-index-count a)) + (if (< count left-count) + (let-values (((x y) (split a count))) (values x (source-index-concat y b))) + (let-values (((x y) (split b (- count left-count)))) + (values (source-index-concat a x) y)))))))) +(def (source-index-slice tree start count) + (unless (and (integer? start) (integer? count) (>= start 0) (>= count 0) + (<= (+ start count) (source-index-count tree))) + (error "invalid source index slice" start count)) + (let-values (((prefix tail) (split tree start))) + (let-values (((middle suffix) (split tail count))) middle))) + +;;; A cursor descends once, then iterates inside immutable chunks. Its copied +;;; state is a certificate trial; rejecting that trial never consumes the owner. +(defstruct source-cursor (root pending leaf offset limit delta rank cache)) +(def (cursor-install! cursor tree rank delta pending) + (cond ((not tree) (set! (source-cursor-leaf cursor) #f) (set! (source-cursor-pending cursor) pending)) + ((source-view? tree) + (cursor-install! cursor (source-view-base tree) rank (+ delta (source-view-delta tree)) pending)) + ((source-leaf? tree) + (set! (source-cursor-leaf cursor) tree) + (set! (source-cursor-offset cursor) (+ (source-leaf-begin tree) rank)) + (set! (source-cursor-limit cursor) (+ (source-leaf-begin tree) (source-leaf-count tree))) + (set! (source-cursor-delta cursor) delta) (set! (source-cursor-pending cursor) pending)) + (else + (let ((left (source-branch-left tree)) (right (source-branch-right tree))) + (if (< rank (source-index-count left)) + (cursor-install! cursor left rank delta (cons (cons right delta) pending)) + (cursor-install! cursor right (- rank (source-index-count left)) delta pending)))))) +(def (make-source-index-cursor tree (rank 0)) + (unless (<= 0 rank (source-index-count tree)) (error "invalid source cursor rank" rank)) + (let (cursor (make-source-cursor tree '() #f 0 0 0 rank #f)) + (unless (= rank (source-index-count tree)) (cursor-install! cursor tree rank 0 '())) cursor)) +(def source-index-cursor-rank source-cursor-rank) +(def (source-index-cursor-copy cursor) + (make-source-cursor (source-cursor-root cursor) (source-cursor-pending cursor) + (source-cursor-leaf cursor) (source-cursor-offset cursor) (source-cursor-limit cursor) + (source-cursor-delta cursor) (source-cursor-rank cursor) (source-cursor-cache cursor))) +(def (source-index-cursor-token cursor) + (and (source-cursor-leaf cursor) + (or (source-cursor-cache cursor) + (let* ((raw (vector-ref (source-leaf-tokens (source-cursor-leaf cursor)) (source-cursor-offset cursor))) + (delta (source-cursor-delta cursor)) + (token (if (zero? delta) raw + (make-token (token-kind raw) (token-lexeme raw) + (+ delta (token-start raw)) (+ delta (token-end raw)))))) + (set! (source-cursor-cache cursor) token) token)))) +(def (source-index-cursor-mode cursor) + (and (source-cursor-leaf cursor) + (vector-ref (source-leaf-modes (source-cursor-leaf cursor)) (source-cursor-offset cursor)))) +(def (source-index-cursor-next! cursor) + (when (source-cursor-leaf cursor) + (set! (source-cursor-cache cursor) #f) + (set! (source-cursor-rank cursor) (+ (source-cursor-rank cursor) 1)) + (set! (source-cursor-offset cursor) (+ (source-cursor-offset cursor) 1)) + (when (= (source-cursor-offset cursor) (source-cursor-limit cursor)) + (if (null? (source-cursor-pending cursor)) (set! (source-cursor-leaf cursor) #f) + (let (next (car (source-cursor-pending cursor))) + (cursor-install! cursor (car next) 0 (cdr next) (cdr (source-cursor-pending cursor)))))))) +(def (byte-rank tree byte) + (cond ((not tree) 0) + ((<= byte (source-index-start tree)) 0) + ((>= byte (source-index-end tree)) (source-index-count tree)) + ((source-view? tree) (byte-rank (source-view-base tree) (- byte (source-view-delta tree)))) + ((source-leaf? tree) + (let loop ((at 0)) + (if (or (= at (source-leaf-count tree)) + (>= (token-start (vector-ref (source-leaf-tokens tree) (+ at (source-leaf-begin tree)))) byte)) at + (loop (+ at 1))))) + (else + (let ((a (source-branch-left tree)) (b (source-branch-right tree))) + (if (< byte (source-index-end a)) (byte-rank a byte) + (+ (source-index-count a) (byte-rank b byte))))))) +(def (source-index-cursor-seek! cursor byte) + (let (current (source-index-cursor-token cursor)) + (unless (or (not current) (>= (token-start current) byte)) + (if (< byte (+ (source-cursor-delta cursor) (source-leaf-end (source-cursor-leaf cursor)))) + ;; Recognition and boundary scanning stay near the current chunk. + ;; Advancing there preserves locality instead of searching from root + ;; and rescanning the same leaf prefix for every declaration boundary. + (let advance () + (when (< (+ (source-cursor-delta cursor) + (token-start (vector-ref (source-leaf-tokens (source-cursor-leaf cursor)) + (source-cursor-offset cursor)))) byte) + (source-index-cursor-next! cursor) + (when (source-cursor-leaf cursor) (advance)))) + (let (rank (byte-rank (source-cursor-root cursor) byte)) + (set! (source-cursor-rank cursor) rank) (set! (source-cursor-cache cursor) #f) + (if (= rank (source-index-count (source-cursor-root cursor))) + (begin (set! (source-cursor-leaf cursor) #f) (set! (source-cursor-pending cursor) '())) + (cursor-install! cursor (source-cursor-root cursor) rank 0 '()))))))) + +(def (relabel tree patches) + ;; Patches are sorted ranks relative to this tree. Copy one bounded mode + ;; vector per touched leaf; token backing storage is always retained. + (cond ((null? patches) (values tree 0)) + ((source-view? tree) + (let-values (((base chunks) (relabel (source-view-base tree) patches))) + (values (source-index-shift base (source-view-delta tree)) chunks))) + ((source-leaf? tree) + (let (modes (vector-copy (source-leaf-modes tree))) + (for-each (lambda (patch) (vector-set! modes (+ (source-leaf-begin tree) (car patch)) (cdr patch))) patches) + (values (make-source-leaf (source-leaf-tokens tree) modes (source-leaf-begin tree) + (source-leaf-count tree) (source-leaf-bytes tree) (source-leaf-chars tree) + (source-leaf-start tree) (source-leaf-end tree)) 1))) + (else + (let* ((a (source-branch-left tree)) (b (source-branch-right tree)) (count (source-index-count a))) + (let divide ((rest patches) (left '())) + (if (and (pair? rest) (< (caar rest) count)) + (divide (cdr rest) (cons (car rest) left)) + (let-values (((x xc) (relabel a (reverse left))) + ((y yc) (relabel b (map (lambda (patch) (cons (- (car patch) count) (cdr patch))) rest)))) + (values (branch x y) (+ xc yc))))))))) + +;;; A request builder coalesces adjacent origins before slicing old trees. +;;; It never owns a previous snapshot after finish: only reachable tree pieces +;;; survive. Caller observers report scanner results or certified source spans. +(defstruct source-builder (root tokens modes buffered origin fresh shared chunks mode-chunks)) +(def (make-source-index-builder (prefix #f)) + (make-source-builder prefix '() '() 0 #f 0 (source-index-count prefix) 0 0)) +(def source-index-builder-fresh source-builder-fresh) +(def source-index-builder-shared source-builder-shared) +(def source-index-builder-chunks source-builder-chunks) +(def source-index-builder-mode-chunks source-builder-mode-chunks) +(def (flush-fresh! builder) + (when (positive? (source-builder-buffered builder)) + (let ((tokens (list->vector (reverse (source-builder-tokens builder)))) + (modes (list->vector (reverse (source-builder-modes builder))))) + (set! (source-builder-root builder) + (source-index-concat (source-builder-root builder) (leaf tokens modes 0 (vector-length tokens)))) + (set! (source-builder-chunks builder) (+ (source-builder-chunks builder) 1))) + (set! (source-builder-tokens builder) '()) (set! (source-builder-modes builder) '()) + (set! (source-builder-buffered builder) 0))) +(def (flush-origin! builder) + (let (origin (source-builder-origin builder)) + (when origin + (let-values (((piece chunks) + (relabel (source-index-slice (vector-ref origin 0) (vector-ref origin 1) (vector-ref origin 2)) + (reverse (vector-ref origin 4))))) + (set! (source-builder-root builder) + (source-index-concat (source-builder-root builder) (source-index-shift piece (vector-ref origin 3)))) + (set! (source-builder-mode-chunks builder) (+ (source-builder-mode-chunks builder) chunks))) + (set! (source-builder-origin builder) #f)))) +(def (source-index-builder-token! builder token mode) + (unless (and (positive? (- (token-end token) (token-start token))) (integer? mode) (>= mode 0)) + (error "invalid source index token/mode")) + (flush-origin! builder) + (when (pair? (source-builder-tokens builder)) + (unless (= (token-end (car (source-builder-tokens builder))) (token-start token)) + (error "source index fresh chunk has a byte gap"))) + (set! (source-builder-tokens builder) (cons token (source-builder-tokens builder))) + (set! (source-builder-modes builder) (cons mode (source-builder-modes builder))) + (set! (source-builder-buffered builder) (+ (source-builder-buffered builder) 1)) + (set! (source-builder-fresh builder) (+ (source-builder-fresh builder) 1)) + (when (= (source-builder-buffered builder) +chunk-size+) (flush-fresh! builder))) +(def (source-index-builder-shared! builder tree start count delta (mode #f)) + (unless (and (<= 0 start) (<= 0 count) (<= (+ start count) (source-index-count tree))) + (error "invalid shared source span")) + (when mode + (unless (and (= count 1) (integer? mode) (>= mode 0)) (error "invalid shared source mode override"))) + (when (positive? count) + (flush-fresh! builder) + (let (origin (source-builder-origin builder)) + (if (and origin (eq? tree (vector-ref origin 0)) (= delta (vector-ref origin 3)) + (= start (+ (vector-ref origin 1) (vector-ref origin 2)))) + (begin + (when mode (vector-set! origin 4 (cons (cons (vector-ref origin 2) mode) (vector-ref origin 4)))) + (vector-set! origin 2 (+ (vector-ref origin 2) count))) + (begin (flush-origin! builder) (set! (source-builder-origin builder) (vector tree start count delta (if mode (list (cons 0 mode)) '())))))) + (set! (source-builder-shared builder) (+ (source-builder-shared builder) count)))) +(def (source-index-builder-finish builder) + (flush-fresh! builder) (flush-origin! builder) (source-builder-root builder)) +(def (source-index->lists tree) + (let loop ((cursor (make-source-index-cursor tree)) (tokens '()) (modes '())) + (let (token (source-index-cursor-token cursor)) + (if (not token) (values (reverse tokens) (reverse modes)) + (let (mode (source-index-cursor-mode cursor)) + (source-index-cursor-next! cursor) (loop cursor (cons token tokens) (cons mode modes))))))) +;;; Debug/audit operations deliberately traverse the tree; never used to update it. +(def (source-index-storage tree) + (cond ((not tree) '()) ((source-view? tree) (source-index-storage (source-view-base tree))) + ((source-leaf? tree) (list (source-leaf-tokens tree))) + (else (append (source-index-storage (source-branch-left tree)) (source-index-storage (source-branch-right tree)))))) +(def (source-index-audit tree) + (cond ((not tree) #t) + ((source-view? tree) + (and (not (source-view? (source-view-base tree))) (not (zero? (source-view-delta tree))) + (source-index-audit (source-view-base tree)))) + ((source-leaf? tree) + (and (<= 1 (source-leaf-count tree) +chunk-size+) + (= (source-leaf-bytes tree) (- (source-leaf-end tree) (source-leaf-start tree))) + (let loop ((at (source-leaf-begin tree)) (end (source-leaf-start tree)) (chars 0)) + (if (= at (+ (source-leaf-begin tree) (source-leaf-count tree))) + (and (= end (source-leaf-end tree)) (= chars (source-leaf-chars tree))) + (let ((token (vector-ref (source-leaf-tokens tree) at)) + (mode (vector-ref (source-leaf-modes tree) at))) + (and (= end (token-start token)) (< end (token-end token)) (integer? mode) (>= mode 0) + (loop (+ at 1) (token-end token) (+ chars (string-length (token-lexeme token)))))))))) + (else + (let ((a (source-branch-left tree)) (b (source-branch-right tree))) + (and (source-index-audit a) (source-index-audit b) + (<= (abs (- (source-index-height a) (source-index-height b))) 1) + (= (source-branch-height tree) (+ 1 (max (source-index-height a) (source-index-height b)))) + (= (source-branch-count tree) (+ (source-index-count a) (source-index-count b))) + (= (source-index-end a) (source-index-start b)) + (= (source-branch-start tree) (source-index-start a)) (= (source-branch-end tree) (source-index-end b)) + (= (source-branch-bytes tree) (+ (tree-bytes a) (tree-bytes b))) + (= (source-branch-chars tree) (+ (tree-chars a) (tree-chars b)))))))) diff --git a/src/runtime/source-lines.ss b/src/runtime/source-lines.ss new file mode 100644 index 00000000..b5cda54a --- /dev/null +++ b/src/runtime/source-lines.ss @@ -0,0 +1,97 @@ +;;; -*- Gerbil -*- +;;; Request source traversal has no grammar/compiler initialization dependency. +(import (only-in :std/string/utf8 string-utf8-length utf8->string) :std/ffi) +(export fold-source-lines fold-source-byte-lines fold-source-byte-spans + fold-source-frame-boundaries! fold-source-input-bytes source-line-events line-starts-with?) + +;; The visitor still runs once for each original line, in order. Scan only +;; up to its next boundary before invoking it; no speculative scan past a +;; callback and no retained foreign pointer or new line-index allocation. +(C-ffi-macrology) +(C-include "") +(def-C-lambda source-next-line-end (scheme-object unsigned-int64 unsigned-int64) unsigned-int64 + "const uint8_t *bytes = U8_DATA(___arg1); + uint64_t cursor = ___arg2, size = ___arg3; + while (cursor < size) { + uint8_t byte = bytes[cursor++]; + if (byte == 10) break; + if (byte == 13) { + if (cursor < size && bytes[cursor] == 10) ++cursor; + break; + } + } + ___return(cursor);") + +;; A call-scoped UTF-8 buffer can drive the same byte Fold without an +;; encode/decode/encode round trip. Retain official decoder admission: do not +;; reinterpret arbitrary bytes as valid text or weaken scalar checks. +;; The caller must not mutate the owned buffer during synchronous traversal; +;; no body pointer or mutable buffer is retained in published event cells. +(def (fold-source-input-bytes source) + (cond + ((string? source) (string->utf8 source)) + ((u8vector? source) + (utf8->string source) + source) + (else (error "event Fold requires text or UTF-8 byte storage")))) + +;; Generated native products already own the UTF-8 source. Traverse that +;; representation once, retaining each frame's bytes instead of encoding its +;; decoded line again. ASCII CR/LF cannot occur inside a UTF-8 continuation. +(def (fold-source-byte-lines (bytes : :u8vector) initial visit) + (fold-source-byte-spans bytes initial + (lambda (from until state) + (let (line-bytes (subu8vector bytes from until)) + (visit (utf8->string line-bytes) line-bytes from until state))))) + +;; Borrow the request buffer; no line byte slice or decoded string is created. +(def (fold-source-byte-spans (bytes : :u8vector) initial visit) + (let (size (u8vector-length bytes)) + (let loop ((start 0) (state initial)) + (if (fx= start size) state + (let (after (source-next-line-end bytes start size)) + (loop after (visit start after state))))))) + +;; The source is string->utf8 output or checked UTF-8 input. A nonempty slice of that valid +;; buffer is valid UTF-8 iff both ends are character boundaries. Empty slices +;; remain valid even inside a codepoint, matching utf8->string of an empty view. +(def (fold-source-frame-boundaries! bytes from until) + (unless (<= 0 from until (u8vector-length bytes)) + (error "native fold bounds outside source")) + (unless (or (= from until) + (and (not (<= 128 (u8vector-ref bytes from) 191)) + (or (= until (u8vector-length bytes)) + (not (<= 128 (u8vector-ref bytes until) 191))))) + (error "native fold bounds split UTF-8 source" from until))) + +(def (line-starts-with? line prefix) + (and (<= (string-length prefix) (string-length line)) + (string=? (substring line 0 (string-length prefix)) prefix))) + +(def (fold-source-lines source initial visit) + (let (size (string-length source)) + (let loop ((start 0) (cursor 0) (byte-start 0) (state initial)) + (cond + ((= cursor size) + (if (= start size) state + (let* ((line (substring source start size)) + (byte-end (+ byte-start (string-utf8-length line)))) + (visit line byte-start byte-end state)))) + ((or (char=? (string-ref source cursor) #\newline) + (char=? (string-ref source cursor) #\return)) + (let* ((after (+ cursor + (if (and (char=? (string-ref source cursor) #\return) + (< (+ cursor 1) size) + (char=? (string-ref source (+ cursor 1)) #\newline)) + 2 1))) + (line (substring source start after)) + (byte-end (+ byte-start (string-utf8-length line)))) + (loop after after byte-end (visit line byte-start byte-end state)))) + (else (loop start (+ cursor 1) byte-start state)))))) + +(def (source-line-events source root visit) + (reverse + (cons '(finish) + (fold-source-lines source (list (list 'start root)) + (lambda (line start end reversed) + (foldl cons reversed (visit line start end))))))) diff --git a/src/runtime/source-scanner.ss b/src/runtime/source-scanner.ss new file mode 100644 index 00000000..e85b2ec0 --- /dev/null +++ b/src/runtime/source-scanner.ss @@ -0,0 +1,180 @@ +;;; -*- Gerbil -*- +;;; Immutable source-scanner state for languages whose lexical context changes +;;; across token boundaries. A state is a reusable checkpoint for one source. + +(import (only-in :std/string/utf8 string-utf8-length) + (only-in ./token make-token token-kind token-lexeme token-start token-end) + (only-in ./contextual-scanner prepare-contextual-scanner contextual-scanner-initial-state + contextual-scanner-step contextual-scan-state-character-offset contextual-scan-state-byte-offset + contextual-scan-state-source-length contextual-scan-state-common-suffix-length + prepare-contextual-scan-suffix contextual-scan-suffix-state)) +(export source-scanner-contextual? source-scanner-driver? source-scanner-for-source? source-scanner-tokens make-source-scanner + make-contextual-source-scanner source-scanner-initial-state + source-scanner-step + source-scan-state? + source-scan-state-character-offset + source-scan-state-byte-offset + source-scan-state-context + source-scan-state-with-context + make-source-scan-session source-scan-session? source-scan-session-tokens + source-scan-session-scanned-token-count source-scan-session-reused-token-count) + +(defstruct source-scanner-driver (source initial-context scan) transparent: #t) +(defstruct (contextual-source-driver source-scanner-driver) (executor position)) +(def source-scanner-contextual? contextual-source-driver?) +(def (make-contextual-source-scanner plan source position) + (let* ((executor (prepare-contextual-scanner plan source)) + (initial (contextual-scanner-initial-state executor))) + (make-contextual-source-driver source initial #f executor position))) + +(defstruct source-scan-state (owner source character-offset byte-offset context) + transparent: #t) + +(def (make-source-scanner source initial-context step) + (unless (and (string? source) (procedure? step)) + (error "source scanner requires source and scan procedure")) + (make-source-scanner-driver source initial-context step)) + +(def (source-scanner-for-source? scanner source) + (and (source-scanner-driver? scanner) + (eq? source (source-scanner-driver-source scanner)))) + +(def (source-scanner-initial-state scanner) + (make-source-scan-state scanner (source-scanner-driver-source scanner) 0 0 + (source-scanner-driver-initial-context scanner))) + +;;; A language may update its immutable context when a parser reduction +;;; admits a deferred lexical obligation, without changing source position. +(def (source-scan-state-with-context state context) + (make-source-scan-state + (source-scan-state-owner state) (source-scan-state-source state) + (source-scan-state-character-offset state) + (source-scan-state-byte-offset state) + context)) + +(def (step-contextual-source scanner state) + (let (context (source-scan-state-context state)) + (unless (and (= (contextual-scan-state-character-offset context) + (source-scan-state-character-offset state)) + (= (contextual-scan-state-byte-offset context) + (source-scan-state-byte-offset state))) + (error "contextual source checkpoint offsets disagree")) + (let-values (((token next) + (contextual-scanner-step + (contextual-source-driver-executor scanner) context + (contextual-source-driver-position scanner)))) + (values token + (make-source-scan-state scanner (source-scanner-driver-source scanner) + (contextual-scan-state-character-offset next) + (contextual-scan-state-byte-offset next) next))))) + +;;; Engine callbacks return (values kind exclusive-character-end context). +;;; Contextual workers forward the token and offsets owned by the shared IR executor. +;;; Ordinary workers check progress and construct source-backed byte spans. +(def (source-scanner-step scanner state mode) + (unless (and (source-scan-state? state) + (eq? scanner (source-scan-state-owner state))) + (error "scanner checkpoint belongs to another worker")) + (if (contextual-source-driver? scanner) + (step-contextual-source scanner state) + (let* ((source (source-scanner-driver-source scanner)) + (start (source-scan-state-character-offset state)) + (length (string-length source))) + (let-values (((kind end context) + ((source-scanner-driver-scan scanner) + source start (source-scan-state-context state) mode))) + (if kind + (begin + (unless (and (symbol? kind) (integer? end) + (< start end) (<= end length)) + (error "source scanner did not advance" kind start end)) + (let* ((lexeme (substring source start end)) + (byte-end (+ (source-scan-state-byte-offset state) + (string-utf8-length lexeme))) + (token (make-token kind lexeme + (source-scan-state-byte-offset state) + byte-end))) + (values token + (make-source-scan-state scanner source end byte-end context)))) + (begin + (unless (= start length) + (error "source scanner stopped before EOF" start length)) + (values #f (source-scan-state-with-context state context)))))))) + +;;; A scan worker drains its own immutable checkpoints; traversal and token +;;; ownership stay here for both engine callbacks and closed contextual profiles. +(def (source-scanner-tokens scanner mode) + (unless (source-scanner-driver? scanner) (error "invalid source scanner worker")) + (let loop ((state (source-scanner-initial-state scanner)) (tokens '())) + (let-values (((token next) (source-scanner-step scanner state mode))) + (if token (loop next (cons token tokens)) (reverse tokens))))) + +;;; Sessions own token records; callers receive copies so publication or edits +;;; cannot corrupt a future reuse candidate. Checkpoint vectors stay private. +(defstruct source-scan-session-state (worker mode tokens-field states scanned-token-count reused-token-count)) +(def source-scan-session? source-scan-session-state?) +(def source-scan-session-scanned-token-count source-scan-session-state-scanned-token-count) +(def source-scan-session-reused-token-count source-scan-session-state-reused-token-count) +(def (copy-session-token token (delta 0)) + (make-token (token-kind token) (string-copy (token-lexeme token)) + (+ (token-start token) delta) (+ (token-end token) delta))) +(def (source-scan-session-tokens session) + (map copy-session-token (source-scan-session-state-tokens-field session))) + +(def (source-session-state-index states character) + (let loop ((low 0) (high (vector-length states))) + (if (>= low high) #f + (let* ((middle (quotient (+ low high) 2)) + (offset (source-scan-state-character-offset (vector-ref states middle)))) + (cond ((= offset character) middle) + ((< offset character) (loop (+ middle 1) high)) + (else (loop low middle))))))) + +(def (make-source-scan-session worker mode (previous #f)) + (unless (and (source-scanner-driver? worker) + (or (not previous) (source-scan-session? previous))) + (error "invalid source scanning session")) + (let* ((initial (source-scanner-initial-state worker)) + (old-worker (and previous (source-scan-session-state-worker previous))) + (eligible? (and previous (contextual-source-driver? worker) + (contextual-source-driver? old-worker) + (eq? mode (source-scan-session-state-mode previous)) + (eq? (contextual-source-driver-position worker) + (contextual-source-driver-position old-worker)))) + (old-states (and eligible? (source-scan-session-state-states previous))) + (old-context (and eligible? (source-scan-state-context (vector-ref old-states 0)))) + (new-context (source-scan-state-context initial)) + (delta (and eligible? (- (contextual-scan-state-source-length new-context) + (contextual-scan-state-source-length old-context)))) + (suffix-start + (and eligible? + (- (contextual-scan-state-source-length new-context) + (contextual-scan-state-common-suffix-length old-context new-context))))) + (let loop ((state initial) (tokens '()) (states (list initial)) (scanned 0)) + (let* ((index (and eligible? + (>= (source-scan-state-character-offset state) suffix-start) + (source-session-state-index old-states + (- (source-scan-state-character-offset state) delta)))) + (before (and index (vector-ref old-states index))) + (suffix (and before + (prepare-contextual-scan-suffix + (source-scan-state-context before) (source-scan-state-context state))))) + (if suffix + (let* ((byte-delta (- (source-scan-state-byte-offset state) + (source-scan-state-byte-offset before))) + (tail (list-tail (source-scan-session-state-tokens-field previous) index)) + (reused (map (lambda (token) (copy-session-token token byte-delta)) tail)) + (rebound + (map (lambda (old) + (let (context (contextual-scan-suffix-state suffix (source-scan-state-context old))) + (make-source-scan-state worker (source-scanner-driver-source worker) + (contextual-scan-state-character-offset context) + (contextual-scan-state-byte-offset context) context))) + (vector->list old-states (+ index 1))))) + (make-source-scan-session-state worker mode (append-reverse tokens reused) + (list->vector (append-reverse states rebound)) scanned (length reused))) + (let-values (((token next) (source-scanner-step worker state mode))) + (if token + (loop next (cons token tokens) (cons next states) (+ scanned 1)) + (make-source-scan-session-state worker mode (reverse tokens) + (list->vector (reverse states)) scanned 0)))))))) diff --git a/src/runtime/structured.ss b/src/runtime/structured.ss new file mode 100644 index 00000000..cf0f15a3 --- /dev/null +++ b/src/runtime/structured.ss @@ -0,0 +1,137 @@ +;;; Closed structured text scanners and proof policies shared by language recipes. +(import (only-in :clan/poo/object .o .ref .slot? object?) + (only-in :clan/poo/mop define-type validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + :gerbil-parser/src/runtime/cst + (only-in :gerbil-parser/src/runtime/token token? token-lexeme)) +(export StructuredProofPolicy. StructuredProofPolicyContract bind-structured-proof-policy) +(def StructuredProofPolicy. + (.o code: "GERBIL-PARSER-STRUCTURED-PROOF" proof-kind: 'Proof label-kind: 'LabelExpression + name-field: 'name body-field: 'body step-field: 'step operator-field: 'operator argument-field: 'argument + name-kind: 'NameExpression application-kind: 'OperatorApplication close: #\> terminal-command: "QED" + denied-commands: '("USE" "HIDE" "INSTANCE" "DEFINE" "HAVE" "TAKE" "WITNESS") + denied-kinds: '(LocalDefinition LocalFunctionDefinition OperatorDefinition FunctionDefinition InstanceDeclaration))) +(def (proof-policy? value) + (with-catch (lambda (_) #f) (lambda () + (and (object? value) + (andmap (lambda (slot) (symbol? (.ref value slot))) + '(proof-kind label-kind name-field body-field step-field operator-field argument-field name-kind application-kind)) + (char? (.ref value 'close)) (string? (.ref value 'code)) (string? (.ref value 'terminal-command)) + (list? (.ref value 'denied-commands)) (andmap string? (.ref value 'denied-commands)) + (list? (.ref value 'denied-kinds)) (andmap symbol? (.ref value 'denied-kinds)))))) +(define-type (StructuredProofPolicyContract @ PooFlowContract.) + identity: 'gerbil-parser/structured-proof + .classify: (lambda (candidate context) + (let (ok? (proof-policy? candidate)) + (poo-flow-classification-evidence 'gerbil-parser/structured-proof candidate ok? + (if ok? '() '((expected structured-proof))) context)))) +(def (bind-structured-proof-policy profile) + (validate StructuredProofPolicyContract profile) + (let ((code (string-copy (.ref profile 'code))) (proof-kind (.ref profile 'proof-kind)) + (label-kind (.ref profile 'label-kind)) (name-field (.ref profile 'name-field)) + (body-field (.ref profile 'body-field)) (step-field (.ref profile 'step-field)) + (operator-field (.ref profile 'operator-field)) (argument-field (.ref profile 'argument-field)) + (name-kind (.ref profile 'name-kind)) (application-kind (.ref profile 'application-kind)) + (close-char (.ref profile 'close)) (terminal-command (string-copy (.ref profile 'terminal-command))) + (denied-commands (map string-copy (.ref profile 'denied-commands))) (denied-kinds (append (.ref profile 'denied-kinds) '()))) +(def (children value) + (cond ((syntax-node? value) (syntax-node-children value)) + ((syntax-field? value) (syntax-field-children value)) + (else '()))) + +(def (first-token value) + (if (token? value) value + (let loop ((rest (children value))) + (and (pair? rest) (or (first-token (car rest)) (loop (cdr rest))))))) + +(def (fields node name) + (filter (lambda (child) + (and (syntax-field? child) (eq? (syntax-field-name child) name))) + (syntax-node-children node))) + +(def (proof-error step reason) + (list (cons 'schema "gerbil-parser.diagnostic.v1") + (cons 'code code) + (cons 'reasonKind 'parse-rejected) + (cons 'failureKind 'proof-level-rejected) + (cons 'message reason) + (cons 'byteOffset (syntax-node-start step)))) + +(def (field-node value) + (let loop ((rest (children value))) + (and (pair? rest) + (if (syntax-node? (car rest)) (car rest) + (or (field-node (car rest)) (loop (cdr rest))))))) + +(def (proof-step-level step current) + (let* ((name (token-lexeme (first-token (car (fields step name-field))))) + (close (let loop ((at 1)) + (if (char=? (string-ref name at) close-char) at (loop (+ at 1))))) + (level (substring name 1 close))) + (cond ((equal? level "*") (or current 1)) + ((equal? level "+") (+ (or current 0) 1)) + (else (string->number level))))) + +(def (step-command step) + (token-lexeme (first-token (car (fields step body-field))))) + +(def (step-admits-proof? step) + (let* ((body (car (fields step body-field))) (node (field-node body))) + (and (not (member (step-command step) + denied-commands)) + (not (and node (memq (syntax-node-kind node) + denied-kinds)))))) + +;; Each frame is (level . closed?). A greater level attaches a subproof to +;; the previous step; a following sibling can only resume an open frame. +;; QED may itself have a subproof, so closing is delayed until the next level. +(def (validate-proof proof) + (let (steps (map field-node (fields proof step-field))) + (let loop ((rest steps) (stack '()) (previous #f)) + (if (null? rest) + (if (every cdr stack) #f + (proof-error (or previous proof) "structured proof is missing QED")) + (let* ((step (car rest)) + (level (proof-step-level step (and (pair? stack) (caar stack)))) + (higher? (and (pair? stack) (> level (caar stack)))) + (open-stack + (if higher? stack + (let pop ((frames stack)) + (if (and (pair? frames) (cdar frames)) + (pop (cdr frames)) frames))))) + (cond + ((and previous higher? (not (step-admits-proof? previous))) + (proof-error step "this proof step cannot have a subproof")) + ((and previous (not higher?) (null? open-stack)) + (proof-error step "proof step follows the completed outer QED")) + ((and (pair? open-stack) (not higher?) (not (= level (caar open-stack)))) + (proof-error step "proof step has an inconsistent level")) + (else + (let* ((frames (if (or (null? open-stack) higher?) + (cons (cons level #f) open-stack) open-stack)) + (next (cons (cons level (equal? (step-command step) terminal-command)) + (cdr frames)))) + (loop (cdr rest) next step))))))))) + +(def (validate-label node) + (let (name (field-node (car (fields node name-field)))) + (and (not + (or (and name (eq? (syntax-node-kind name) name-kind)) + (and name (eq? (syntax-node-kind name) application-kind) + (not (field-node (car (fields name operator-field)))) + (every (lambda (argument) + (let (value (field-node argument)) + (and value (eq? (syntax-node-kind value) name-kind)))) + (fields name argument-field))))) + (proof-error node "label requires a name and identifier parameters")))) + +(def (structured-proof-diagnostic value) + (or (and (syntax-node? value) + (cond ((eq? (syntax-node-kind value) proof-kind) (validate-proof value)) + ((eq? (syntax-node-kind value) label-kind) (validate-label value)) + (else #f))) + (let loop ((rest (children value))) + (and (pair? rest) + (or (structured-proof-diagnostic (car rest)) (loop (cdr rest))))))) + +structured-proof-diagnostic)) diff --git a/src/runtime/token.ss b/src/runtime/token.ss index 7cc4a93c..8b278108 100644 --- a/src/runtime/token.ss +++ b/src/runtime/token.ss @@ -1,11 +1,13 @@ ;;; Lossless source token representation. +;;; Official inline contracts stage direct calls without changing the plain +;;; vector protocol, first-class procedures or checked vector reads. (export make-token token? token-kind token-lexeme token-start token-end) -(def (make-token kind lexeme start end) +(definline (make-token (kind : :t) (lexeme : :t) (start : :t) (end : :t)) => :t (vector kind lexeme start end)) -(def (token? value) +(definline (token? (value : :t)) => :t (and (vector? value) (= (vector-length value) 4) (symbol? (vector-ref value 0)) @@ -14,7 +16,7 @@ (integer? (vector-ref value 3)) (<= 0 (vector-ref value 2) (vector-ref value 3)))) -(def (token-kind token) (vector-ref token 0)) -(def (token-lexeme token) (vector-ref token 1)) -(def (token-start token) (vector-ref token 2)) -(def (token-end token) (vector-ref token 3)) +(definline (token-kind (token : :t)) => :t (vector-ref token 0)) +(definline (token-lexeme (token : :t)) => :t (vector-ref token 1)) +(definline (token-start (token : :t)) => :t (vector-ref token 2)) +(definline (token-end (token : :t)) => :t (vector-ref token 3)) diff --git a/src/testing/language-parser.ss b/src/testing/language-parser.ss new file mode 100644 index 00000000..060bed82 --- /dev/null +++ b/src/testing/language-parser.ss @@ -0,0 +1,568 @@ +;;; -*- Gerbil -*- +;;; Language test declarations share the admitted POO loader and invariants. +(import (only-in :std/test check test-case test-suite) + (only-in :clan/poo/object .ref .cc) + (only-in :clan/poo/mop validate) + (only-in ../../language-support/entry LanguageLoaderContract) + (only-in ../../language-support/development language-parser-entry-ref run-language-test language-loader-fixtures language-loader-fixture-count + +language-parser-entry-schema+) + (only-in ../runtime/artifact parse-artifact-valid? parse-artifact-success? + parse-artifact-ref parse-artifact-roundtrip parse-artifact-events + token-event? token-event-token-kind event-kind) + (only-in ../runtime/identity sha256-text) + (only-in ../../language-support/fixture syntax-fixture-source syntax-fixture-source-digest syntax-fixture-expected-status) + (only-in ./parser-ast check-parser-ast) + (only-in ./language-product language-model-test-receipt + check-language-abi-entry check-language-portable-rejection) + (only-in ../compiler/normalize grammar-ir-ref) + (only-in ./language-strategy check-language-strategies check-language-installed + check-language-fixture-strategies) + (only-in ../runtime/cst parse-artifact->cst syntax-node? syntax-node-kind + syntax-field-name + syntax-node-children syntax-field? syntax-field-children) + (only-in ../runtime/token token? token-lexeme) + (only-in ../compiler/bound-ir bound-grammar-ir-ref bound-grammar-ir-binding) + (only-in ../compiler/parser-ir parser-ir-ref) + (only-in ../compiler/lr lr-spec-ref production-lhs production-rhs) + (only-in ../compiler/machine parser-machine-ir) + (only-in ../runtime/lexer lex-source) + (only-in ../runtime/significant parser-significant-tokens) + (only-in ../runtime/lr-parser lr-parse/receipt) + (only-in ../language/descriptor language-grammar? language-grammar-machine language-grammar-grammar) + (only-in ../runtime/incremental make-edit parse-source/incremental) + (only-in ../runtime/recovery parse-source/recover) + (only-in ../runtime/region-scanner source-prefix-at?) + (only-in :std/string/utf8 string-utf8-length) + (only-in :std/sync/barrier make-barrier barrier-post! barrier-wait!)) +(export deflanguage-parser-tests) + +(def (checked-test-loader loader) + (validate LanguageLoaderContract loader) + loader) + +(def (checked-language-source loader source accepted?) + (unless (and (equal? (language-parser-entry-ref loader 'schema) + +language-parser-entry-schema+) (string? source)) + (error "language test requires an admitted loader and string source" source)) + (let (artifact ((.ref loader '.parse) source)) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-success? artifact) => accepted?) + (check (parse-artifact-roundtrip artifact) => source) + (check (parse-artifact-ref artifact 'sourceDigest) => (sha256-text source)) + artifact)) + +(def (checked-language-fixtures loader (expected-status #f)) + (let ((artifacts (run-language-test loader 'fixtures)) (fixtures (language-loader-fixtures loader))) + (check (pair? artifacts) => #t) + (check (length artifacts) => (length fixtures)) + (when expected-status + (check (map syntax-fixture-expected-status fixtures) + => (make-list (length fixtures) expected-status))) + (for-each (lambda (artifact fixture) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => (syntax-fixture-source fixture)) + (check (parse-artifact-ref artifact 'sourceDigest) => (syntax-fixture-source-digest fixture))) + artifacts fixtures) + artifacts)) + +(def (checked-fixture-group loader status) + (let (fixtures (language-loader-fixtures loader status)) + (checked-language-fixtures (checked-test-loader (.cc loader 'fixtures fixtures)) status))) + +(def (test-fixture-catalog loader key) + (case key + ((total) (language-loader-fixture-count loader)) + ((accepted rejected) (language-loader-fixture-count loader key)) + ((first-digest) (let (fixtures (language-loader-fixtures loader)) + (and (pair? fixtures) (syntax-fixture-source-digest (car fixtures))))))) + +(def (artifact-field-names artifact kind) + (let loop ((pending (list (parse-artifact->cst artifact))) (names '())) + (if (null? pending) names + (let (value (car pending)) + (cond + ((syntax-node? value) + (loop (append (syntax-node-children value) (cdr pending)) + (if (eq? (syntax-node-kind value) kind) + (append (map syntax-field-name (filter syntax-field? (syntax-node-children value))) names) names))) + ((syntax-field? value) (loop (append (syntax-field-children value) (cdr pending)) names)) + (else (loop (cdr pending) names))))))) + +(def (nonempty-language-sources sources) + (unless (and (list? sources) (pair? sources) (andmap string? sources)) + (error "language test source collection must be nonempty strings" sources)) + sources) + +(def (artifact-node-kinds artifact) + (map (lambda (event) (vector-ref event 2)) + (filter (lambda (event) (eq? (event-kind event) 'start-node)) + (parse-artifact-events artifact)))) + +(def (cst-direct-nodes value) + (cond ((syntax-node? value) (list value)) + ((syntax-field? value) (apply append (map cst-direct-nodes (syntax-field-children value)))) + (else '()))) + +(def (cst-lexemes value) + (cond ((token? value) (list (token-lexeme value))) + ((syntax-node? value) (apply append (map cst-lexemes (syntax-node-children value)))) + ((syntax-field? value) (apply append (map cst-lexemes (syntax-field-children value)))) + (else '()))) + +;;; Child rows select the declared node kinds through transparent fields, then +;;; require exact count/order. Other direct node kinds are not selected. +(def (cst-node-matches? value pattern) + (and (syntax-node? value) (eq? (syntax-node-kind value) (cadr pattern)) + (andmap + (lambda (condition) + (case (car condition) + ((lexemes) (andmap (lambda (text) (member text (cst-lexemes value))) (cdr condition))) + ((children) + (let* ((patterns (cdr condition)) (kinds (map cadr patterns)) + (selected (filter (lambda (child) (memq (syntax-node-kind child) kinds)) + (apply append (map cst-direct-nodes (syntax-node-children value)))))) + (and (= (length selected) (length patterns)) + (andmap values (map cst-node-matches? selected patterns))))) + (else (error "unknown CST test condition" condition)))) + (cddr pattern)))) + +(def (artifact-subtree-matches? artifact pattern) + (def (visit value) + (cond ((syntax-node? value) + (or (cst-node-matches? value pattern) (ormap visit (syntax-node-children value)))) + ((syntax-field? value) (ormap visit (syntax-field-children value))) + (else #f))) + (and (visit (parse-artifact->cst artifact)) #t)) + +(def (artifact-field-counts artifact kind field) + (let loop ((pending (list (parse-artifact->cst artifact))) (counts '())) + (if (null? pending) (reverse counts) + (let (value (car pending)) + (cond + ((syntax-node? value) + (loop (append (syntax-node-children value) (cdr pending)) + (if (eq? (syntax-node-kind value) kind) + (cons (length (filter (lambda (child) + (and (syntax-field? child) (eq? (syntax-field-name child) field))) + (syntax-node-children value))) counts) + counts))) + ((syntax-field? value) (loop (append (syntax-field-children value) (cdr pending)) counts)) + (else (loop (cdr pending) counts))))))) + +(def (test-grammar-machine loader) + (let (descriptor (.ref loader 'descriptor)) + (unless (language-grammar? descriptor) (error "grammar test requires a Grammar IR loader")) + (language-grammar-machine descriptor))) + +(def (check-parallel-language loader workers sources) + (let* ((inputs (nonempty-language-sources sources)) + (expected (map (lambda (source) (checked-language-source loader source #t)) inputs)) + (barrier (make-barrier workers)) + (threads (map (lambda (index) + (spawn (lambda () + (barrier-post! barrier) (barrier-wait! barrier) + (with-catch (lambda (condition) (cons 'failed condition)) + (lambda () (cons 'parsed ((.ref loader '.parse) + (list-ref inputs (modulo index (length inputs)))))))))) + (iota workers))) + (results (map thread-join! threads))) + ;; Join all workers before propagating a parser exception. + (for-each (lambda (result) (when (eq? (car result) 'failed) (raise (cdr result)))) results) + (check (map cdr results) + => (map (lambda (index) (list-ref expected (modulo index (length inputs)))) (iota workers))))) + +(def (test-incremental-receipt loader source old new) + (let* ((offset (let find ((at 0)) + (cond ((source-prefix-at? source at old) at) + ((>= at (string-length source)) (error "incremental test replacement not found" old)) + (else (find (+ at 1)))))) + (prefix (substring source 0 offset)) + (edited (string-append prefix new (substring source (+ offset (string-length old)) (string-length source)))) + (base (checked-language-source loader source #t))) + (let-values (((artifact receipt) + (parse-source/incremental (test-grammar-machine loader) source base + (make-edit (string-utf8-length prefix) (string-utf8-length old) new)))) + (check artifact => (checked-language-source loader edited #t)) + receipt))) + +(def (test-recovery-rejection loader source) + (let-values (((artifact receipt) (parse-source/recover (test-grammar-machine loader) source))) + (check (parse-artifact-success? artifact) => #f) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source) + receipt)) + +(def (test-parser-ir-ref ir key) + (case key + ((rule-count) (length (parser-ir-ref ir 'rules))) + (else (parser-ir-ref ir key)))) + +(def (test-lr-value ir key) + (let (spec (parser-ir-ref ir 'lr-spec)) + (case key + ((state-count) (lr-spec-ref spec 'state-count)) + ((fork-count) + (apply + (map (lambda (row) (length (filter (lambda (entry) (eq? (cadr entry) 'fork)) row))) + (vector->list (lr-spec-ref spec 'actions))))) + (else (error "unknown LR test value" key))))) + +(def (nullable-prefix-productions ir prefixes) + (filter (lambda (production) + (and (null? (production-rhs production)) + (ormap (lambda (prefix) + (string-prefix? prefix (symbol->string (production-lhs production)))) prefixes))) + (lr-spec-ref (parser-ir-ref ir 'lr-spec) 'productions))) + +(def (initial-shifts? ir literals) + (let (actions (vector-ref (lr-spec-ref (parser-ir-ref ir 'lr-spec) 'actions) 0)) + (and (andmap (lambda (literal) + (let (entry (assoc (list 'terminal 'literal literal) actions)) + (and entry (eq? (cadr entry) 'shift)))) literals) #t))) + +(def (test-bound-rule ir name) + (or (bound-grammar-ir-binding ir 'rule name) (error "missing bound test rule" name))) + +(def (test-bound-source binding key) + (bound-grammar-ir-ref (bound-grammar-ir-ref binding 'source) key)) + +(def (test-bound-source-prefix? binding key prefix) + (let (value (test-bound-source binding key)) + (and (string? value) (string-prefix? prefix value)))) + +(def (test-lr-receipt machine source budget) + (let ((spec (parser-ir-ref (parser-machine-ir machine) 'lr-spec)) + (tokens (parser-significant-tokens machine (lex-source machine source)))) + (let-values (((_root rest receipt) (lr-parse/receipt spec tokens budget))) + (cons (cons 'rest rest) receipt)))) + +(defrules language-bound-expectation (source source-prefix references reference-names binding-id) + ((_ binding (source key expected)) (check (test-bound-source binding 'key) => expected)) + ((_ binding (source-prefix key prefix)) (check (test-bound-source-prefix? binding 'key prefix) => #t)) + ((_ binding (references expected)) (check (bound-grammar-ir-ref binding 'references) => expected)) + ((_ binding (reference-names expected)) + (check (map (lambda (row) (bound-grammar-ir-ref row 'name)) + (bound-grammar-ir-ref binding 'references)) => expected)) + ((_ binding (binding-id expected)) (check (bound-grammar-ir-ref binding 'bindingId) => expected))) + +(defrules language-bound-row () + ((_ ir (name expectation ...)) + (let (binding (test-bound-rule ir 'name)) + (language-bound-expectation binding expectation) ...))) + +(defrules language-lr-expectation (state-count fork-count empty-nullable-prefixes initial-shifts) + ((_ ir (state-count expected)) (check (test-lr-value ir 'state-count) => expected)) + ((_ ir (fork-count expected)) (check (test-lr-value ir 'fork-count) => expected)) + ((_ ir (empty-nullable-prefixes prefix ...)) + (check (nullable-prefix-productions ir '(prefix ...)) => '())) + ((_ ir (initial-shifts literal ...)) (check (initial-shifts? ir '(literal ...)) => #t))) + +(defrules language-artifact-expectation (nodes tokens counts without-nodes diagnostics artifact-ref projection root subtree without-subtree field-counts fields token-counts) + ((_ artifact (nodes kind ...)) + (begin (check (and (memq 'kind (artifact-node-kinds artifact)) #t) => #t) ...)) + ((_ artifact (without-nodes kind ...)) + (begin (check (and (memq 'kind (artifact-node-kinds artifact)) #t) => #f) ...)) + ((_ artifact (counts (kind count) ...)) + (begin (check (length (filter (lambda (value) (eq? value 'kind)) + (artifact-node-kinds artifact))) => count) ...)) + ((_ artifact (fields kind name ...)) + (let (names (artifact-field-names artifact 'kind)) + (check (and (memq 'name names) #t) => #t) ...)) + ((_ artifact (token-counts (kind count) ...)) + (let (kinds (map token-event-token-kind (filter token-event? (parse-artifact-events artifact)))) + (check (length (filter (lambda (value) (eq? value 'kind)) kinds)) => count) ...)) + ((_ artifact (tokens kind ...)) + (let (kinds (map token-event-token-kind (filter token-event? (parse-artifact-events artifact)))) + (check (and (memq 'kind kinds) #t) => #t) ...)) + ((_ artifact (root kind)) (check (car (artifact-node-kinds artifact)) => 'kind)) + ((_ artifact (field-counts kind field (count ...))) + (check (artifact-field-counts artifact 'kind 'field) => '(count ...))) + ((_ artifact (without-subtree pattern)) (check (artifact-subtree-matches? artifact 'pattern) => #f)) + ((_ artifact (subtree pattern)) (check (artifact-subtree-matches? artifact 'pattern) => #t)) + ((_ artifact (diagnostics count)) (check (length (parse-artifact-ref artifact 'diagnostics)) => count)) + ((_ artifact (diagnostics)) + (check (pair? (parse-artifact-ref artifact 'diagnostics)) => #t)) + ((_ artifact (artifact-ref key expected)) + (check (parse-artifact-ref artifact 'key) => expected)) + ((_ artifact (projection project (key expected) ...)) + (let (value (project artifact)) + (check (cdr (assq 'key value)) => expected) ...))) + +(defrules language-model-expectation (nonzero) + ((_ receipt (nonzero key)) + (check (let (value (bound-grammar-ir-ref receipt 'key)) + (and (number? value) (not (zero? value)))) => #t)) + ((_ receipt (key expected)) (check (bound-grammar-ir-ref receipt 'key) => expected))) + +(defrules language-test-clause (accepted rejected accepted-many rejected-many identity fixtures ast property bindings equal parser-ir bound-ir bound-rules lr lr-receipt parallel incremental replace recovery-rejected status installed strategy-parity fixture-parity fixture-group accepted-fixture fixture-catalog syntax-kind distinct-contract model-receipt native-entry portable-rejected) + ((_ loader (accepted label source expectation ...)) + (test-case label + (let (artifact (checked-language-source loader source #t)) + (language-artifact-expectation artifact expectation) ... (void)))) + ((_ loader (rejected label source expectation ...)) + (test-case label + (let (artifact (checked-language-source loader source #f)) + (language-artifact-expectation artifact expectation) ... (void)))) + ((_ loader (accepted-many label sources expectation ...)) + (test-case label + (for-each (lambda (source) + (let (artifact (checked-language-source loader source #t)) + (language-artifact-expectation artifact expectation) ... (void))) (nonempty-language-sources sources)))) + ((_ loader (rejected-many label sources expectation ...)) + (test-case label + (for-each (lambda (source) + (let (artifact (checked-language-source loader source #f)) + (language-artifact-expectation artifact expectation) ... (void))) (nonempty-language-sources sources)))) + ((_ loader (identity label (key expected) ...)) + (test-case label (check (language-parser-entry-ref loader 'key) => expected) ...)) + ((_ loader (fixtures label (status expected))) + (test-case label (checked-language-fixtures loader 'expected))) + ((_ loader (fixtures label)) + (test-case label + (checked-language-fixtures loader))) + ((_ loader (ast label source pattern)) + (test-case label + (let* ((text source) (artifact (checked-language-source loader text #t))) + (check-parser-ast artifact text pattern)))) + ((_ loader (parallel label workers sources)) + (test-case label (check-parallel-language loader workers sources))) + ((_ loader (incremental label source (replace old new) (key expected) ...)) + (test-case label (let (receipt (test-incremental-receipt loader source old new)) + (check (bound-grammar-ir-ref receipt 'key) => expected) ...))) + ((_ loader (recovery-rejected label source (key expected) ...)) + (test-case label (let (receipt (test-recovery-rejection loader source)) + (check (bound-grammar-ir-ref receipt 'key) => expected) ...))) + ((_ loader (parser-ir label value (key expected) ...)) + (test-case label (let (ir value) (check (test-parser-ir-ref ir 'key) => expected) ...))) + ((_ loader (bound-ir label value (binding-count expected))) + (test-case label (check (bound-grammar-ir-ref value 'bindingCount) => expected))) + ((_ loader (bound-rules label value row ...)) + (test-case label (let (ir value) (language-bound-row ir row) ...))) + ((_ loader (lr label value expectation ...)) + (test-case label (let (ir value) (language-lr-expectation ir expectation) ...))) + ((_ loader (lr-receipt label machine source budget (key expected) ...)) + (test-case label (let (receipt (test-lr-receipt machine source budget)) + (check (bound-grammar-ir-ref receipt 'key) => expected) ...))) + ((_ loader (installed label route)) + (test-case label (check-language-installed loader 'route))) + ((_ loader (strategy-parity label specification option ...)) + (test-case label (check-language-strategies loader 'specification '(option ...)))) + ((_ loader (fixture-parity label expected-status option ...)) + (test-case label + (checked-language-fixtures loader) + (check-language-fixture-strategies loader 'expected-status '(option ...)))) + ((_ loader (fixture-group label expected-status expectation ...)) + (test-case label + (for-each (lambda (artifact) (language-artifact-expectation artifact expectation) ... (void)) + (checked-fixture-group loader 'expected-status)))) + ((_ loader (accepted-fixture label index expectation ...)) + (test-case label + (let (artifact (checked-language-source loader (syntax-fixture-source (list-ref (.ref loader 'fixtures) index)) #t)) + (language-artifact-expectation artifact expectation) ... (void)))) + ((_ loader (fixture-catalog label (key expected) ...)) + (test-case label (check (test-fixture-catalog loader 'key) => expected) ...)) + ((_ loader (syntax-kind label kind category (name ...))) + (test-case label + (check (assq 'kind (grammar-ir-ref (language-grammar-grammar (.ref loader 'descriptor)) 'syntax-kinds)) + => '(kind category (name ...))))) + ((_ loader (distinct-contract label other)) + (test-case label (check (equal? (.ref loader 'contract) (.ref (checked-test-loader other) 'contract)) => #f))) + ((_ loader (model-receipt label model source config tool workers (key expected) ...)) + (test-case label + (let (receipt (language-model-test-receipt loader model source config 'tool workers)) + (language-model-expectation receipt (key expected)) ...))) + ((_ loader (native-entry label source accepted-status)) + (test-case label + (let* ((text source) (accepted? (eq? 'accepted-status 'accepted)) + (artifact (checked-language-source loader text accepted?))) + (check-language-abi-entry loader text artifact accepted?)))) + ((_ loader (portable-rejected label module-path message)) + (test-case label (check-language-portable-rejection module-path message))) + ((_ loader (property label (bindings (name value) ...) (equal actual expected) ...)) + (test-case label + (let* ((name value) ...) + (check actual => expected) ...)))) + +;;; The loader expression is evaluated once. No free-form test-case body is +;;; accepted: each row and artifact expectation must match the closed syntax. +(defsyntax (deflanguage-parser-tests stx) + (def (nonempty? rows pred) (and (pair? rows) (andmap pred rows))) + (def (symbol-row? row) (and (list? row) (= (length row) 2) (symbol? (car row)))) + (def (node-pattern? pattern) + (and (list? pattern) (>= (length pattern) 2) (eq? (car pattern) 'node) (symbol? (cadr pattern)) + (andmap (lambda (row) + (and (list? row) (pair? row) + (case (car row) + ((lexemes) (nonempty? (cdr row) string?)) + ((children) (nonempty? (cdr row) node-pattern?)) + (else #f)))) (cddr pattern)))) + (def (bound-expectation? row) + (and (list? row) (pair? row) + (case (car row) + ((source source-prefix) (and (= (length row) 3) (symbol? (cadr row)))) + ((references reference-names binding-id) (= (length row) 2)) + (else #f)))) + (def (lr-expectation? row) + (and (list? row) (pair? row) + (case (car row) + ((state-count fork-count) (= (length row) 2)) + ((empty-nullable-prefixes initial-shifts) (nonempty? (cdr row) string?)) + (else #f)))) + (def (expectation? row) + (and (list? row) (pair? row) + (case (car row) + ((nodes tokens without-nodes) (nonempty? (cdr row) symbol?)) + ((counts token-counts) (nonempty? (cdr row) (lambda (item) + (and (symbol-row? item) (exact-integer? (cadr item)) (>= (cadr item) 0))))) + ((diagnostics) (or (= (length row) 1) + (and (= (length row) 2) (exact-integer? (cadr row)) (>= (cadr row) 0)))) + ((fields) (and (>= (length row) 3) (andmap symbol? (cdr row)))) + ((root) (and (= (length row) 2) (symbol? (cadr row)))) + ((subtree without-subtree) (and (= (length row) 2) (node-pattern? (cadr row)))) + ((field-counts) + (and (= (length row) 4) (symbol? (cadr row)) (symbol? (caddr row)) + (list? (cadddr row)) (andmap (lambda (count) (and (exact-integer? count) (>= count 0))) (cadddr row)))) + ((artifact-ref) (and (= (length row) 3) (symbol? (cadr row)))) + ((projection) (and (>= (length row) 3) (nonempty? (cddr row) symbol-row?))) + (else #f)))) + (def (positive-integer? value) (and (exact-integer? value) (> value 0))) + (def (nonnegative-integer? value) (and (exact-integer? value) (>= value 0))) + (def (template? node names) + (cond ((string? node) #t) + ((symbol? node) (or (eq? node 'index) (memq node names))) + ((and (list? node) (pair? node)) + (case (car node) + ((concat) (nonempty? (cdr node) (lambda (part) (template? part names)))) + ((choice) (nonempty? (cdr node) string?)) + ((character) (and (= (length node) 2) (string? (cadr node)) (positive? (string-length (cadr node))))) + ((integer) (and (= (length node) 3) (exact-integer? (cadr node)) (positive-integer? (caddr node)))) + ((if-even) (and (= (length node) 3) (template? (cadr node) names) (template? (caddr node) names))) + ((repeat) (and (= (length node) 4) (nonnegative-integer? (cadr node)) + (positive-integer? (caddr node)) (template? (cadddr node) names))) + ((when-equal) (and (= (length node) 4) (symbol? (cadr node)) (memq (cadr node) names) + (string? (caddr node)) (template? (cadddr node) names))) + ((bind) + (and (= (length node) 3) (list? (cadr node)) (pair? (cadr node)) + (let loop ((rows (cadr node)) (scope names) (seen '())) + (if (null? rows) (template? (caddr node) scope) + (let (row (car rows)) + (and (symbol-row? row) (not (eq? (car row) 'index)) + (not (memq (car row) seen)) (template? (cadr row) scope) + (loop (cdr rows) (cons (car row) scope) (cons (car row) seen)))))))) + (else #f))) + (else #f))) + (def (source-specification? spec) + (and (list? spec) (pair? spec) + (case (car spec) + ((sources) (nonempty? (cdr spec) string?)) + ((indexed) (and (= (length spec) 3) (positive-integer? (cadr spec)) (template? (caddr spec) '()))) + ((seeded) (and (= (length spec) 4) (nonnegative-integer? (cadr spec)) (< (cadr spec) 2147483648) + (positive-integer? (caddr spec)) (template? (cadddr spec) '()))) + ((append) (nonempty? (cdr spec) source-specification?)) + (else #f)))) + (def (strategy-options? options fixture?) + (and (pair? options) + (let loop ((rows options) (seen '())) + (if (null? rows) + (and (memq 'routes seen) + (let (routes (cdr (assq 'routes options))) + (or (>= (length routes) 2) (memq 'accepted seen) (memq 'lexical seen))) + (or (not (memq 'coverage seen)) (memq 'lexical seen))) + (let (row (car rows)) + (and (list? row) (pair? row) (not (memq (car row) seen)) + (case (car row) + ((routes) (let route-loop ((remaining (cdr row)) (used '())) + (or (null? remaining) + (and (memq (car remaining) '(entry indexed source-default source-ranked source-raw source-generic)) + (not (memq (car remaining) used)) + (route-loop (cdr remaining) (cons (car remaining) used)))))) + ((lexical) (and (= (length row) 2) (memq (cadr row) '(fast fallback optional)))) + ((accepted) (and (= (length row) 2) (boolean? (cadr row)) (pair? (cdr (assq 'routes options))))) + ((coverage) (and (not fixture?) (= (length row) 3) (nonnegative-integer? (cadr row)) (nonnegative-integer? (caddr row)))) + ((diagnostics) (and fixture? (= (length row) 2) (nonnegative-integer? (cadr row)))) + (else #f)) + (loop (cdr rows) (cons (car row) seen)))))))) + (def (clause? row) + (and (list? row) (>= (length row) 2) (string? (cadr row)) + (positive? (string-length (cadr row))) + (case (car row) + ((accepted rejected accepted-many rejected-many) + (and (>= (length row) 3) (andmap expectation? (cdddr row)))) + ((fixtures) (or (= (length row) 2) + (and (= (length row) 3) (list? (caddr row)) (= (length (caddr row)) 2) + (eq? (caaddr row) 'status) (memq (cadr (caddr row)) '(accepted rejected))))) + ((identity) (nonempty? (cddr row) (lambda (item) + (and (symbol-row? item) (memq (car item) '(schema language version contract)))))) + ((ast) (= (length row) 4)) + ((parallel) + (and (= (length row) 4) (exact-integer? (caddr row)) (positive? (caddr row)))) + ((incremental) + (and (>= (length row) 5) (list? (cadddr row)) (= (length (cadddr row)) 3) + (eq? (car (cadddr row)) 'replace) (string? (cadr (cadddr row))) + (positive? (string-length (cadr (cadddr row)))) (string? (caddr (cadddr row))) + (nonempty? (list-tail row 4) symbol-row?))) + ((recovery-rejected) (and (>= (length row) 4) (nonempty? (cdddr row) symbol-row?))) + ((parser-ir) + (and (>= (length row) 4) + (nonempty? (cdddr row) (lambda (item) + (and (symbol-row? item) + (memq (car item) '(rule-count root-rule materialization conflict-policy case-insensitive?))))))) + ((bound-ir) + (and (= (length row) 4) (symbol-row? (cadddr row)) (eq? (car (cadddr row)) 'binding-count))) + ((bound-rules) + (and (>= (length row) 4) (nonempty? (cdddr row) (lambda (item) + (and (list? item) (pair? item) (symbol? (car item)) + (nonempty? (cdr item) bound-expectation?)))))) + ((lr) (and (>= (length row) 4) (nonempty? (cdddr row) lr-expectation?))) + ((lr-receipt) + (and (>= (length row) 6) (exact-integer? (list-ref row 4)) (positive? (list-ref row 4)) + (nonempty? (list-tail row 5) (lambda (item) + (and (symbol-row? item) (memq (car item) + '(rest branchesExplored speculativeBranchesExplored maxSpeculativeDepth successfulCompletions))))))) + ((fixture-group) (and (>= (length row) 3) (memq (caddr row) '(accepted rejected)) (andmap expectation? (cdddr row)))) + ((accepted-fixture) (and (>= (length row) 3) (nonnegative-integer? (caddr row)) (andmap expectation? (cdddr row)))) + ((fixture-catalog) (nonempty? (cddr row) (lambda (item) (and (symbol-row? item) (memq (car item) '(total accepted rejected first-digest)))))) + ((syntax-kind) (and (= (length row) 5) (symbol? (caddr row)) (memq (cadddr row) '(node token)) + (list? (list-ref row 4)) (andmap symbol? (list-ref row 4)))) + ((distinct-contract) (= (length row) 3)) + ((native-entry) (and (= (length row) 4) (memq (cadddr row) '(accepted rejected)))) + ((portable-rejected) (and (= (length row) 4) (string? (caddr row)) (string? (cadddr row)))) + ((model-receipt) + (and (>= (length row) 8) (string? (caddr row)) (positive? (string-length (caddr row))) + (andmap (lambda (ch) (or (char-alphabetic? ch) (char-numeric? ch) (memq ch '(#\_ #\-)))) (string->list (caddr row))) + (string? (cadddr row)) (string? (list-ref row 4)) + (let (tool (list-ref row 5)) + (and (list? tool) (pair? tool) + (or (equal? tool '(unresolved)) + (and (= (length tool) 3) (eq? (car tool) 'stdout) (string? (cadr tool)) + (nonnegative-integer? (caddr tool)) (<= (caddr tool) 255))))) + (positive-integer? (list-ref row 6)) + (nonempty? (list-tail row 7) (lambda (item) + (and (symbol-row? item) (or (and (eq? (car item) 'nonzero) (eq? (cadr item) 'exit-status)) (memq (car item) + '(schema syntax-contract model source-digest config-digest syntax-accepted roundtrip tool-path workers + exit-status completed tlc-version states-generated distinct-states states-left graph-depth admitted output-digest)))))))) + ((installed) (and (= (length row) 3) (memq (caddr row) '(step source)))) + ((strategy-parity) (and (>= (length row) 4) (source-specification? (caddr row)) + (strategy-options? (cdddr row) #f))) + ((fixture-parity) (and (>= (length row) 4) (memq (caddr row) '(accepted rejected)) + (strategy-options? (cdddr row) #t))) + ((property) + (and (>= (length row) 4) (list? (caddr row)) (pair? (caddr row)) + (eq? (caaddr row) 'bindings) (andmap symbol-row? (cdaddr row)) + (nonempty? (cdddr row) (lambda (item) + (and (list? item) (= (length item) 3) (eq? (car item) 'equal)))))) + (else #f)))) + (syntax-case stx (loader) + ((_ name title (loader entry) clause ...) + (and (identifier? #'name) (stx-string? #'title)) + (let ((rows (syntax->datum #'(clause ...))) (labels '())) + (unless (nonempty? rows clause?) + (raise-syntax-error #f "invalid language parser test clause or expectation" stx)) + (for-each (lambda (row) + (when (member (cadr row) labels) + (raise-syntax-error #f "duplicate language parser test label" stx)) + (set! labels (cons (cadr row) labels))) rows) + #'(begin + (def name (let (owner (checked-test-loader entry)) + (test-suite title (language-test-clause owner clause) ...))) + (export name)))) + (_ (raise-syntax-error #f "language parser tests require a name, title and loader" stx)))) diff --git a/src/testing/language-product.ss b/src/testing/language-product.ss new file mode 100644 index 00000000..e591a1b3 --- /dev/null +++ b/src/testing/language-product.ss @@ -0,0 +1,58 @@ +;;; -*- Gerbil -*- +;;; Closed tool fixtures and descriptor/native/build admission test execution. +(import (only-in :std/test check check-exception) + (only-in :std/misc/process run-process) + (only-in :std/string/misc string-trim-eol) + (only-in :clan/poo/object .ref) + (only-in ../language/entry parse-language-source) + (only-in ../language/tlc qualify-language-tlc-model tla-plus-model-receipt->alist) + (only-in ../ffi/language-artifact-codec bind-language-abi abi-parse-binary-payload/bytes) + (only-in ../ffi/rust-runtime-aot rust-runtime-source) + (only-in :std/vector/u8vector little u8vector-u32-ref)) +(export language-model-test-receipt check-language-abi-entry check-language-portable-rejection) + +(def (call-with-tool-test-directory procedure) + (let* ((template (path-expand "gerbil-parser-model-test.XXXXXX" (getenv "TMPDIR" "/tmp"))) + (directory (string-trim-eol (run-process ["mktemp" "-d" template])))) + (unwind-protect (procedure directory) + (when (file-exists? directory) (delete-file-or-directory directory #t))))) + +(def (write-test-file path text) + (call-with-output-file path (lambda (port) (write-string text port)))) + +;;; Single quote shell data, including embedded quotes. No fixture text is +;;; evaluated as code; stdout and exit status are the entire tool algebra. +(def (shell-quote-data text) + (call-with-output-string + (lambda (port) + (write-char #\' port) + (string-for-each (lambda (ch) + (if (char=? ch #\') (write-string "'\\''" port) (write-char ch port))) text) + (write-char #\' port)))) + +(def (language-model-test-receipt loader model source config tool workers) + (call-with-tool-test-directory + (lambda (directory) + (let ((spec (path-expand (string-append model ".tla") directory)) + (cfg (path-expand (string-append model ".cfg") directory)) + (executable (path-expand "tool-fixture" directory))) + (write-test-file spec source) (write-test-file cfg config) + (unless (eq? (car tool) 'unresolved) + (write-test-file executable + (string-append "#!/bin/sh\nprintf '%s' " (shell-quote-data (cadr tool)) + "\nexit " (number->string (caddr tool)) "\n")) + (run-process ["chmod" "+x" executable])) + (tla-plus-model-receipt->alist + (qualify-language-tlc-model (.ref loader 'descriptor) spec cfg + tlc: (if (eq? (car tool) 'unresolved) "this-tool-must-not-be-resolved" executable) + workers: workers)))))) + +(def (check-language-abi-entry loader source artifact accepted?) + (let* ((descriptor (.ref loader 'descriptor)) (native (bind-language-abi descriptor))) + (check artifact => (parse-language-source descriptor source)) + (check (u8vector-u32-ref (abi-parse-binary-payload/bytes native (string->utf8 source)) 8 little) + => (if accepted? 0 1)))) + +(def (check-language-portable-rejection module-path message) + (check-exception (rust-runtime-source module-path) + (lambda (exception) (equal? (error-message exception) message)))) diff --git a/src/testing/language-strategy.ss b/src/testing/language-strategy.ss new file mode 100644 index 00000000..dc579f4f --- /dev/null +++ b/src/testing/language-strategy.ss @@ -0,0 +1,166 @@ +;;; -*- Gerbil -*- +;;; Closed source generators and descriptor-bound native strategy qualification. +;;; This engine module has no language imports, callbacks or name dispatch. +(import (only-in :std/test check) + (only-in :clan/poo/object .ref .slot?) + (only-in ../../language-support/development language-loader-fixtures) + (only-in ../language/descriptor language-grammar-machine) + (only-in ../compiler/machine parser-machine-runtime parser-machine-trivia + parser-machine-grammar-digest parser-machine-direct-source) + (only-in ../runtime/lr-parser lr-initial-checkpoint lr-checkpoint-drive lr-runtime-direct-step) + (only-in ../runtime/lexer lex-source scan-source-token) + (only-in ../runtime/token token-end) + (only-in ../runtime/artifact make-success-parse-artifact parse-artifact-valid? + parse-artifact-success? parse-artifact-roundtrip parse-artifact-ref) + (only-in ../../language-support/fixture syntax-fixture-source)) +(export check-language-strategies check-language-installed check-language-fixture-strategies + generate-language-test-sources parse-indexed-reference) + +;;; Indexed reference execution selects no generated reduction or source route. +;;; It retains the same ranked scanner, trivia and canonical artifact builder. +(def (parse-indexed-reference machine source (executor #f)) + (let ((character-offset 0) (byte-offset 0) (pending-character #f) (tokens-reversed '())) + (def (next-input mode) + (if (= character-offset (string-length source)) #f + (let-values (((input-token next-character) + (scan-source-token machine source character-offset byte-offset mode))) + (let (start-character character-offset) + (set! character-offset next-character) + (set! byte-offset (token-end input-token)) + (if ((parser-machine-trivia machine) input-token) + (begin (set! tokens-reversed (cons input-token tokens-reversed)) (next-input mode)) + (begin (set! pending-character start-character) input-token)))))) + (def (after-shift input-token _states _values _actions _shifts) + (set! tokens-reversed (cons input-token tokens-reversed)) + (set! pending-character #f)) + (let-values (((status payload) + (lr-checkpoint-drive (lr-initial-checkpoint (parser-machine-runtime machine) '()) + next-input after-shift #f executor))) + (unless (and (eq? status 'accepted) (= character-offset (string-length source)) + (not pending-character) (null? (cadr payload))) + (error "indexed language test reference did not complete" status)) + (make-success-parse-artifact (parser-machine-grammar-digest machine) source + (reverse tokens-reversed) (car payload) (parser-machine-trivia machine))))) + +;;; Every seeded row starts a fresh exact LCG stream. Nested repetition consumes +;;; draws depth-first, preserving source order and the original case index. +(def (generate-language-test-sources specification) + (def (generated count seed template) + (let (state seed) + (def (draw modulus) + (set! state (modulo (+ (* state 1103515245) 12345) 2147483648)) + (modulo state modulus)) + (def (render node index environment) + (cond + ((string? node) node) + ((eq? node 'index) (number->string index)) + ((symbol? node) (let (entry (assq node environment)) + (if entry (cdr entry) (error "unbound generator value" node)))) + (else + (case (car node) + ((concat) (apply string-append (map (lambda (part) (render part index environment)) (cdr node)))) + ((choice) (list-ref (cdr node) (draw (length (cdr node))))) + ((character) (string (string-ref (cadr node) (draw (string-length (cadr node)))))) + ((integer) (number->string (+ (cadr node) (draw (caddr node))))) + ((if-even) (render (if (even? index) (cadr node) (caddr node)) index environment)) + ((repeat) + (let loop ((remaining (+ (cadr node) (draw (caddr node)))) (parts '())) + (if (= remaining 0) (apply string-append (reverse parts)) + (loop (- remaining 1) (cons (render (cadddr node) index environment) parts))))) + ((bind) + (let loop ((rows (cadr node)) (values environment)) + (if (null? rows) (render (caddr node) index values) + (let (row (car rows)) + (loop (cdr rows) (cons (cons (car row) (render (cadr row) index values)) values)))))) + ((when-equal) + (if (equal? (render (cadr node) index environment) (caddr node)) + (render (cadddr node) index environment) "")) + (else (error "unknown source generator" node)))))) + (map (lambda (index) (render template index '())) (iota count)))) + (case (car specification) + ((sources) (cdr specification)) + ((indexed) (list (apply string-append (generated (cadr specification) 0 (caddr specification))))) + ((seeded) (generated (caddr specification) (cadr specification) (cadddr specification))) + ((append) (apply append (map generate-language-test-sources (cdr specification)))) + (else (error "unknown language source collection" specification)))) + +(def (native-test-profile loader) + (unless (.slot? loader 'native-test-profile) + (error "language strategy requires a declared generated native test profile")) + (.ref loader 'native-test-profile)) + +(def (check-language-installed loader route) + (let (machine (language-grammar-machine (.ref loader 'descriptor))) + (check (procedure? (case route + ((step) (lr-runtime-direct-step (parser-machine-runtime machine))) + ((source) (parser-machine-direct-source machine)) + (else (error "unknown installed test route" route)))) => #t))) + +(def (test-route-artifact loader source route) + (let (machine (language-grammar-machine (.ref loader 'descriptor))) + (case route + ((entry) ((.ref loader '.parse) source)) + ((indexed) (parse-indexed-reference machine source)) + (else + (let (recognize (.ref (native-test-profile loader) 'source)) + (case route + ((source-default) (recognize machine source)) + ((source-ranked) (recognize machine source #t #f)) + ((source-raw) (recognize machine source #t #t #f)) + ((source-generic) (recognize machine source #f)) + (else (error "unknown language parser test route" route)))))))) + +(def (check-strategy-source loader source options) + (let* ((routes (cdr (assq 'routes options))) + (lexical (assq 'lexical options)) (accepted (assq 'accepted options)) + (artifacts (map (lambda (route) (test-route-artifact loader source route)) routes)) + (lexical-result #f)) + (when lexical + (let* ((fast ((.ref (native-test-profile loader) 'lexer) source)) + (expectation (cadr lexical))) + (set! lexical-result fast) + (case expectation + ((fast) (check (and fast #t) => #t)) + ((fallback) (check fast => #f)) + ((optional) (void)) + (else (error "unknown lexical test expectation" expectation))) + (when fast (check fast => (lex-source (language-grammar-machine (.ref loader 'descriptor)) source))) + (void))) + (when (pair? artifacts) + (for-each (lambda (artifact) (check artifact => (car artifacts))) (cdr artifacts))) + (when accepted + (for-each (lambda (artifact) + (if (cadr accepted) + (begin (check (and artifact #t) => #t) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source)) + (check artifact => #f))) artifacts)) + (and lexical-result #t))) + +(def (check-language-strategies loader specification options) + (let* ((sources (generate-language-test-sources specification)) + (coverage (assq 'coverage options)) (fast-count 0) (fallback-count 0)) + (unless (and (pair? sources) (andmap string? sources)) + (error "language strategy sources must be nonempty strings")) + (for-each + (lambda (source) + (let (fast? (check-strategy-source loader source options)) + (when coverage + (if fast? (set! fast-count (+ fast-count 1)) (set! fallback-count (+ fallback-count 1)))))) sources) + (when coverage + (check (> fast-count (cadr coverage)) => #t) + (check (> fallback-count (caddr coverage)) => #t)))) + +(def (check-language-fixture-strategies loader status options) + ;; The existing declared service checks all fixture identities and structure. + ;; Selection below narrows only the native-route parity obligations. + (let (fixtures (language-loader-fixtures loader status)) + (check (pair? fixtures) => #t) + (for-each + (lambda (fixture) + (let ((source (syntax-fixture-source fixture)) (diagnostics (assq 'diagnostics options))) + (check-strategy-source loader source options) + (when diagnostics + (check (length (parse-artifact-ref ((.ref loader '.parse) source) 'diagnostics)) => (cadr diagnostics))))) + fixtures))) diff --git a/src/testing/parser-ast.ss b/src/testing/parser-ast.ss index 49632939..8f00420c 100644 --- a/src/testing/parser-ast.ss +++ b/src/testing/parser-ast.ss @@ -13,9 +13,22 @@ syntax-field-end syntax-field-children) (only-in ../runtime/token token? token-kind token-lexeme token-start token-end)) -(export check-parser-ast parser-ast-pattern parser-ast-diff +(export check-parser-ast check-accepted-parse + parser-ast-pattern parser-ast-diff parser-artifact-ast-diff) +;;; Evaluate the source and candidate once, then assert the complete accepted +;;; artifact against any number of independent parser routes. `defrules` +;;; introduces its local bindings hygienically. +(defrules check-accepted-parse () + ((_ source candidate reference ...) + (let* ((expected-source source) + (result candidate)) + (check (parse-artifact-valid? result) => #t) + (check (parse-artifact-success? result) => #t) + (check (parse-artifact-roundtrip result) => expected-source) + (check result => reference) ...))) + ;; This is an AST grammar for this parser engine, not a general data matcher. ;; Fields and byte spans are mandatory; omitted structure cannot pass by a ;; source-roundtrip-only assertion. diff --git a/t/abi-datum-support.ss b/t/abi-datum-support.ss new file mode 100644 index 00000000..17f53b21 --- /dev/null +++ b/t/abi-datum-support.ss @@ -0,0 +1,11 @@ +(export abi-datum-read) +(def (abi-datum-read text) + (def (decode value) + (cond + ((and (pair? value) (eq? (car value) 'object)) + (let (table (make-hash-table)) + (for-each (lambda (entry) (hash-put! table (car entry) (decode (cadr entry)))) (cdr value)) + table)) + ((and (pair? value) (eq? (car value) 'list)) (map decode (cdr value))) + (else value))) + (call-with-input-string text (lambda (port) (decode (read port))))) diff --git a/t/antlr4-source-test.ss b/t/antlr4-source-test.ss index 2f929ee0..82b04032 100644 --- a/t/antlr4-source-test.ss +++ b/t/antlr4-source-test.ss @@ -3,9 +3,9 @@ (import :std/test :gerbil-parser/language-support - (only-in :gerbil-parser/languages/gql/iso-39075-2024/grammar - gql-iso-parser-ir) - :gerbil-parser/languages/gql/iso-39075-2024/source + (only-in :gerbil-parser/languages/gql/grammar + gql-parser-ir) + :gerbil-parser/languages/gql/parser (only-in :gerbil-parser/src/compiler/parser-ir parser-ir-ref) :gerbil-parser/src/compiler/lr (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec)) @@ -47,12 +47,31 @@ (check (cadr (assq 'regularIdentifier - (antlr4-source-parser-grammar-rules source))) + (antlr4-source-parser-grammar-rules source + '(("REGULAR_IDENTIFIER" token identifier) ("IDENTIFIER" token identifier))))) => '(alias RegularIdentifier (choice (precedence left 2 (token identifier)) (precedence left 1 (literal "MATCH"))))))) + (test-case "token names do not supply implicit language semantics" + (let (source (parse-antlr4-source "tiny" "v1" "commit" tiny-grammar)) + (check-exception (antlr4-source-parser-grammar-rules source) true))) + (test-case "duplicate token bindings fail before lowering" + (let (source (parse-antlr4-source "tiny" "v1" "commit" tiny-grammar)) + (check-exception + (antlr4-source-parser-grammar-rules source + '(("REGULAR_IDENTIFIER" token identifier) + ("REGULAR_IDENTIFIER" token number))) true))) + (test-case "explicit binding determines a nonliteral token independent of its name" + (let* ((source (parse-antlr4-source "tiny" "v1" "commit" tiny-grammar)) + (rules (antlr4-source-parser-grammar-rules source + '(("REGULAR_IDENTIFIER" token number))))) + (check (cadr (assq 'regularIdentifier rules)) + => '(alias RegularIdentifier + (choice (precedence left 2 (token number)) + (precedence left 1 (literal "MATCH"))))))) (test-case "an unresolved parser reference fails closed" + (check-exception (parse-antlr4-source "tiny" "v1" "commit" @@ -66,7 +85,8 @@ (expression (cadr (assq 'expression - (antlr4-source-parser-grammar-rules source))))) + (antlr4-source-parser-grammar-rules source + '(("REGULAR_IDENTIFIER" token identifier) ("IDENTIFIER" token identifier))))))) (check expression => '(alias Expression (choice @@ -83,35 +103,41 @@ (reference expression))) (precedence left 1 (token identifier))))))) (test-case "the complete OpenGQL 1.9.0 grammar catalog is immutable" - (check (parser-ir-ref gql-iso-parser-ir 'schema) + (check (parser-ir-ref gql-parser-ir 'schema) => "gerbil-parser.parser-ir.v1") - (check (antlr4-source-digest gql-iso-antlr4-source) + (check (antlr4-source-digest gql-antlr4-source) => +gql-antlr4-digest+) - (check (antlr4-source-name gql-iso-antlr4-source) => "GQL") - (check (length (antlr4-source-rules gql-iso-antlr4-source)) => 1018) - (check (length (antlr4-source-parser-rules gql-iso-antlr4-source)) + (check (antlr4-source-name gql-antlr4-source) => "GQL") + (check (length (antlr4-source-rules gql-antlr4-source)) => 1018) + (check (length (antlr4-source-parser-rules gql-antlr4-source)) => 574) - (check (length (antlr4-source-lexer-rules gql-iso-antlr4-source)) + (check (antlr4-source-parser-grammar-rules gql-antlr4-source gql-antlr4-token-bindings) + => (cdr (assq 'rules gql-grammar))) + (check (length (antlr4-source-lexer-rules gql-antlr4-source)) => 444) (for-each (lambda (name) - (check (antlr4-source-rule gql-iso-antlr4-source name) + (check (antlr4-source-rule gql-antlr4-source name) ? antlr4-rule?)) '("gqlProgram" "valueExpression" "labelExpression" "REGULAR_IDENTIFIER"))) (test-case "all OpenGQL parser productions lower to canonical GrammarExpr v1" - (let (rules (antlr4-source-parser-grammar-rules gql-iso-antlr4-source)) + (let (rules (antlr4-source-parser-grammar-rules gql-antlr4-source gql-antlr4-token-bindings)) (check (length rules) => 574) (check (caar rules) => 'gqlProgram) (check (> (length - (antlr4-source-parser-literals gql-iso-antlr4-source)) + (antlr4-source-parser-literals gql-antlr4-source gql-antlr4-token-bindings)) 100) => #t))) (test-case "the complete OpenGQL grammar compiles through the sole LR owner" (let* ((rules - (antlr4-source-parser-grammar-rules gql-iso-antlr4-source)) + (antlr4-source-parser-grammar-rules gql-antlr4-source gql-antlr4-token-bindings)) (spec (compile-lr-spec rules 'gqlProgram 'selective-glr))) (check (lr-spec-ref spec 'schema) => "gerbil-parser.lr-spec.v1") (check (> (lr-spec-ref spec 'state-count) 0) => #t))))) (export antlr4-source-tests) + +;; gxtest discovers only exported names ending in -test. +(def antlr4-source-test antlr4-source-tests) +(export antlr4-source-test) diff --git a/t/artifact-publication-cost-test.ss b/t/artifact-publication-cost-test.ss new file mode 100644 index 00000000..9105315d --- /dev/null +++ b/t/artifact-publication-cost-test.ss @@ -0,0 +1,208 @@ +;;; Publication observations describe real work and preserve canonical products. +(import :std/test + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact make-success-parse-artifact/canonical-events + make-success-parse-artifact/raw-event-tape make-failure-parse-artifact + parse-artifact-events parse-artifact-valid-for-source? sha256-text) + (only-in :gerbil-parser/src/runtime/recognition + make-recognition-node make-recognition-child) + (only-in :gerbil-parser/src/runtime/event-program event-program-node-value) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/parse-cost current-parser-cost-observer with-parser-cost-stage) + (only-in ./benchmarks/parser-stage-cost/benchmark measure-parser-stages + measure-artifact-event-storage copy-canonical-event-storage + measure-parser-component measure-parser-batch parser-cpu-milliseconds)) +(export artifact-publication-cost-test) + +(def (observe-publication thunk) + (let (rows '()) + (let (value + (parameterize ((current-parser-cost-observer + (lambda (name receipt) (set! rows (cons (cons name receipt) rows))))) + (thunk))) + (values value (reverse rows))))) +(def (never-trivia? token) #f) +(def +publication-test-grammar+ (sha256-text "publication-cost-contract")) + +(def artifact-publication-cost-test + (test-suite "artifact publication cost contracts" + (test-case "equal CPU clock intervals retain exact endpoint identity" + (let ((before (make-f64vector 8 0.0)) (after (make-f64vector 8 0.0))) + (f64vector-set! before 0 40.1) + (f64vector-set! after 0 40.106735) + (let (first (parser-cpu-milliseconds before after)) + (f64vector-set! before 0 30.1) + (f64vector-set! after 0 30.106735) + (check first => 6.735) + (check (parser-cpu-milliseconds before after) => first) + ;; A genuine one-microsecond separation must remain separated. + (f64vector-set! after 1 0.000001) + (check (parser-cpu-milliseconds before after) => 6.736) + (check (< first (parser-cpu-milliseconds before after)) => #t) + ;; User and system intervals use the same clock representation. + (f64vector-set! after 0 30.106734) + (check (parser-cpu-milliseconds before after) => first)))) + (test-case "wide field-bearing children retain source order and reject improper tails" + (for-each + (lambda (width) + (let* ((source (make-string width #\a)) + (tokens (map (lambda (at) (make-token 'word "a" at (+ at 1))) (iota width))) + (children (map (lambda (token) (make-recognition-child 'value token)) tokens)) + (root (make-recognition-node 'Root 0 width children)) + (artifact (make-success-parse-artifact +publication-test-grammar+ source tokens root never-trivia?))) + (check (parse-artifact-valid-for-source? artifact source) => #t) + (check (length (parse-artifact-events artifact)) => (+ 2 (* 3 width))) + (when (pair? children) + (check-exception + (make-success-parse-artifact +publication-test-grammar+ source tokens + (make-recognition-node 'Root 0 width (cons (car children) #f)) never-trivia?) true)))) + '(0 1 1024))) + (test-case "shared sampler validates dimensions before warmup side effects" + (let* ((calls 0) (thunk (lambda () (set! calls (+ calls 1)) 'complete))) + (for-each + (lambda (sizes) + (check-exception + (measure-parser-component 'invalid (car sizes) (cadr sizes) thunk 'complete) true)) + '((0 1) (-1 1) (1.0 1) (1 0) (1 1.0))) + (check-exception (measure-parser-component 'invalid 1 3 thunk 'complete 2) true) + (check calls => 0))) + (test-case "shared sampler preserves complete batches and original failures" + (let* ((calls 0) + (thunk (lambda () (set! calls (+ calls 1)) '(complete result))) + (summary (parameterize ((current-output-port (open-output-string))) + (measure-parser-component 'shared 2 2 thunk '(complete result))))) + (check calls => 5) + (check (cdr (assq 'sampleCount summary)) => 2) + (check (length (cdr (assq 'samples summary))) => 2) + (check (cdr (assq 'allocationScope summary)) => 'single-caller)) + (check (with-catch (lambda (condition) condition) + (lambda () (measure-parser-batch 'failed 0 1 + (lambda () (raise 'original-failure)) 'complete))) + => 'original-failure)) + (test-case "storage control copies containers and shares canonical payloads" + (let* ((lexeme (string-copy "α🙂")) + (events (list (vector 'start-node 0 'Root 0) + (vector 'start-field 'value 0) + (vector 'token 0 'word lexeme 0 6) + (vector 'finish-field 'value 6) + (vector 'finish-node 0 'Root 6))) + (copy (copy-canonical-event-storage events))) + (check copy => events) + (let loop ((original events) (copied copy)) + (unless (null? original) + (check (eq? original copied) => #f) + (check (eq? (car original) (car copied)) => #f) + (loop (cdr original) (cdr copied)))) + (check (eq? (vector-ref (list-ref copy 2) 3) lexeme) => #t) + (vector-set! (car copy) 2 'Changed) + (check (vector-ref (car events) 2) => 'Root))) + (test-case "storage measurement admits accepted empty Unicode and rejected products" + (for-each + (lambda (source) + (let* ((end (u8vector-length (string->utf8 source))) + (tokens (if (zero? end) '() (list (make-token 'word source 0 end)))) + (root (make-recognition-node 'Root 0 end + (map (lambda (token) (make-recognition-child 'value token)) tokens)))) + (for-each + (lambda (artifact) + (let (summary (parameterize ((current-output-port (open-output-string))) + (measure-artifact-event-storage artifact source 3))) + (check (cdr (assq 'samples summary)) => 3) + (check (cdr (assq 'events summary)) => (length (parse-artifact-events artifact))) + (check (parse-artifact-valid-for-source? artifact source) => #t))) + (list (make-success-parse-artifact +publication-test-grammar+ source tokens root never-trivia?) + (make-failure-parse-artifact +publication-test-grammar+ source tokens + '((message . "expected another token"))))))) + '("" "α🙂")) + (check-exception (measure-artifact-event-storage '() "" 1) true) + (check-exception (measure-artifact-event-storage '() "" 0) true)) + (test-case "recognition and event publication preserve Unicode and empty artifacts" + (for-each + (lambda (source) + (let* ((bytes (string->utf8 source)) (end (u8vector-length bytes)) + (token (and (positive? end) (make-token 'word source 0 end))) + (tokens (if token (list token) '())) + (root (make-recognition-node 'Root 0 end + (if token (list (make-recognition-child #f token)) '()))) + (program (event-program-node-value 'Root 0 end token)) + (reference (make-success-parse-artifact +publication-test-grammar+ + source tokens root never-trivia?))) + (for-each + (lambda (value) + (let-values (((artifact rows) + (observe-publication + (lambda () (make-success-parse-artifact +publication-test-grammar+ + source tokens value never-trivia?))))) + (check artifact => reference) + (check (parse-artifact-valid-for-source? artifact source) => #t) + (check (map car rows) + => '(artifact-source-encoding artifact-event-publication artifact-source-identity)) + (check (map (lambda (row) (cdr (assq 'completed (cdr row)))) rows) => '(#t #t #t)))) + (list root program)))) + '("" "abc" "α🙂")) + (check (current-parser-cost-observer) => #f)) + (test-case "committed tape and supplied canonical events report only owned work" + (let* ((source "α") (bytes (string->utf8 source)) + (token (make-token 'word source 0 2)) (tokens (list token)) + (root (make-recognition-node 'Root 0 2 (list (make-recognition-child #f token)))) + (reference (make-success-parse-artifact +publication-test-grammar+ source tokens root never-trivia?)) + (tape (vector 'open-node 'Root 0 'token token 0 'close-node 'Root 2 + 'invalid-unused-tail #f #f))) + (let-values (((artifact rows) + (observe-publication + (lambda () (make-success-parse-artifact/raw-event-tape + +publication-test-grammar+ source tokens tape 3 never-trivia? bytes))))) + (check artifact => reference) + (check (parse-artifact-valid-for-source? artifact source) => #t) + (check (map car rows) => '(artifact-event-publication artifact-source-identity))) + (let-values (((artifact rows) + (observe-publication + (lambda () (make-success-parse-artifact/canonical-events + +publication-test-grammar+ source (parse-artifact-events reference) bytes))))) + (check artifact => reference) + ;; Already supplied bytes/events are capabilities, not executed stages. + (check (map car rows) => '(artifact-source-identity))))) + (test-case "rejected artifact publication remains complete and lossless" + (let* ((source "α") (tokens (list (make-token 'invalid source 0 2))) + (diagnostic '((message . "expected word"))) + (publish (lambda () (make-failure-parse-artifact +publication-test-grammar+ source tokens diagnostic))) + (reference (publish))) + (let-values (((artifact rows) (observe-publication publish))) + (check artifact => reference) + (check (parse-artifact-valid-for-source? artifact source) => #t) + (check (map car rows) + => '(artifact-event-publication artifact-source-encoding artifact-source-identity))))) + (test-case "failed tape publication retains its error and incomplete observation" + (let ((rows '()) (publish + (lambda () (make-success-parse-artifact/raw-event-tape +publication-test-grammar+ + "" '() (vector 'unknown #f 0) 1 never-trivia? (make-u8vector 0))))) + (def (failure thunk) + (with-catch (lambda (condition) (list (error-message condition) (error-irritants condition))) thunk)) + (let (reference (failure publish)) + (check (parameterize ((current-parser-cost-observer + (lambda (name receipt) (set! rows (cons (cons name receipt) rows))))) + (failure publish)) => reference)) + (check (map car rows) => '(artifact-event-publication)) + (check (cdr (assq 'completed (cdar rows))) => #f) + (check (current-parser-cost-observer) => #f))) + (test-case "stage summaries distinguish failed intervals from complete requests" + (let* ((reference (make-failure-parse-artifact +publication-test-grammar+ "" '() + '((message . "recovered publication failure")))) + (parse (lambda (source) + (with-catch (lambda (condition) reference) + (lambda () (with-parser-cost-stage 'recoverable-publication + (raise 'original-condition)))))) + (summary (parameterize ((current-output-port (open-output-string))) + (measure-parser-stages parse "" 3)))) + (for-each + (lambda (row) + (check (cdr (assq 'request-samples row)) => 3) + (check (cdr (assq 'samples row)) => 3) + (if (eq? (cdr (assq 'stage row)) 'recoverable-publication) + (begin (check (cdr (assq 'completed-samples row)) => 0) + (check (cdr (assq 'incomplete-samples row)) => 3)) + (begin (check (cdr (assq 'completed-samples row)) => 3) + (check (cdr (assq 'incomplete-samples row)) => 0)))) + summary) + (check (length summary) => 2) + (check (current-parser-cost-observer) => #f))))) diff --git a/t/artifact-window-publication-test.ss b/t/artifact-window-publication-test.ss new file mode 100644 index 00000000..7354301c --- /dev/null +++ b/t/artifact-window-publication-test.ss @@ -0,0 +1,57 @@ +(import :std/test + (only-in ./fixtures/artifact-window window-tokens window-artifact) + (only-in :gerbil-parser/src/runtime/artifact + make-certified-window-artifact make-failure-parse-artifact + parse-artifact-events parse-artifact-valid-for-source? sha256-text)) +(export artifact-window-publication-test) + +(def artifact-window-publication-test + (test-suite "shared window publication" + (test-case "expanded contracted and Unicode windows equal fresh products" + (for-each + (lambda (entry) + (let* ((old-source (car entry)) (source (cadr entry)) + (old (window-tokens old-source)) (new (window-tokens source)) + (old-window (list (cadr old))) + (new-window (take (cdr new) (- (length new) 2))) + (base (window-artifact old-source old)) + (delta (- (u8vector-length (string->utf8 source)) + (u8vector-length (string->utf8 old-source))))) + (let-values (((product shared) + (make-certified-window-artifact base source 1 old-window new-window delta))) + (check product => (window-artifact source new)) + (check (parse-artifact-valid-for-source? product source) => #t) + (check (> shared 0) => #t) + (check (eq? (car (parse-artifact-events base)) + (car (parse-artifact-events product))) => #t) + (check (parse-artifact-events base) + => (parse-artifact-events (window-artifact old-source old)))))) + '(("abc" "axyc") ("abc" "ac") ("abc" "aλ🙂c") ("abc" "axc")))) + (test-case "empty groups preserve explicit certified assignment order" + (let* ((old (window-tokens "abcd")) (new (window-tokens "aλ🙂d")) + (base (window-artifact "abcd" old)) + (old-window (take (cdr old) 2)) (new-window (take (cdr new) 2))) + (let-values (((product shared) + (make-certified-window-artifact base "aλ🙂d" 1 old-window new-window 4 + (list '() new-window) '((1 . 1) (3 . 7))))) + (check product => (window-artifact "aλ🙂d" new)) + (check (parse-artifact-valid-for-source? product "aλ🙂d") => #t)) + (check-exception + (make-certified-window-artifact base "aλ🙂d" 1 old-window new-window 4 + (list new-window new-window)) true))) + (test-case "large complete event streams retain IDs and independent list ownership" + (let* ((source (make-string 20000 #\a)) (tokens (window-tokens source)) + (base (window-artifact source tokens))) + (let-values (((product shared) + (make-certified-window-artifact base source 0 tokens tokens 0 + (map list tokens)))) + (check product => base) + (check (parse-artifact-valid-for-source? product source) => #t) + (check shared => 2) + (check (eq? (parse-artifact-events base) (parse-artifact-events product)) => #f))) + (let* ((source (make-string 20000 #\a)) (tokens (window-tokens source)) + (product (make-failure-parse-artifact (sha256-text "window-failure") source tokens + '((message . "expected another token"))))) + (check (parse-artifact-valid-for-source? product source) => #t) + (check (vector-ref (car (parse-artifact-events product)) 1) => 0) + (check (vector-ref (list-ref (parse-artifact-events product) 19999) 1) => 19999))))) diff --git a/t/audit/opencypher-tck-syntax.ss b/t/audit/opencypher-tck-syntax.ss index 3ee12908..4d591de7 100644 --- a/t/audit/opencypher-tck-syntax.ss +++ b/t/audit/opencypher-tck-syntax.ss @@ -2,8 +2,8 @@ ;;; -*- Gerbil -*- ;;; Observational syntax audit over a commit-bound extracted TCK manifest. -(import (only-in :gerbil-parser/languages/cypher/opencypher-2024-1/parser - parse-opencypher-2024-1) +(import (only-in :gerbil-parser/languages/cypher/parser + parse-opencypher) (only-in :gerbil-parser/src/runtime/artifact parse-artifact-ref parse-artifact-roundtrip @@ -141,7 +141,7 @@ (unresolved? (record-unresolved? record)) (expected? (expected-syntax-acceptance? outcome skip? unresolved?)) - (artifact (parse-opencypher-2024-1 source)) + (artifact (parse-opencypher source)) (ok? (parse-artifact-success? artifact)) (diagnostic (and (not ok?) diff --git a/t/benchmark-test.ss b/t/benchmark-test.ss index 1f05e98f..eae9a969 100644 --- a/t/benchmark-test.ss +++ b/t/benchmark-test.ss @@ -5,7 +5,9 @@ :asp-gerbil-scheme/src/benchmark/framework :gerbil-parser/src/runtime/artifact :gerbil-parser/src/runtime/parser - :gerbil-parser/languages/arithmetic/v1/parser) + :gerbil-parser/languages/arithmetic/parser + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark cpu-pair-request-plan measure-parser-cpu-pairs) + (only-in :gerbil-parser/t/fixtures/progress report-parser-batch!)) (def benchmark-path "t/benchmarks/parser-hot-path/benchmark.ss") @@ -16,10 +18,44 @@ (parse-artifact-success? (parse-source arithmetic-parser "1 + 2 * value - 3 / 4")) (error "benchmark parse failed")) - (loop (- remaining 1))))) + (loop (- remaining 1)))) + ;; Emit only after a real batch completes. The measured thunk includes this + ;; output cost, so the unchanged wall-time ceiling remains conservative. + (report-parser-batch! 1000)) (def parser-benchmark-tests (test-suite "standard parser benchmark" + (test-case "paired observations conserve requests and balance quadratic leg position" + (for-each + (lambda (calls) + (for-each + (lambda (group) + (let (plan (cpu-pair-request-plan calls group)) + (check (length plan) => 8) + (check (andmap (lambda (work) (and (exact-integer? (cdr work)) (positive? (cdr work)))) plan) => #t) + (for-each + (lambda (variant) + (check (apply + (map cdr (filter (lambda (work) (eq? (car work) variant)) plan))) => (* 4 calls))) + '(left right)) + (for-each + (lambda (degree) + (check + (apply + + (map (lambda (work index) + (* (if (eq? (car work) 'left) 1 -1) (cdr work) (expt index degree))) + plan (iota (length plan)))) => 0)) + '(0 1 2)))) '(0 1))) + '(1 2 3 25 256 768))) + (test-case "paired sampler executes every complete request in its declared budget" + (let* ((source "1 + 2 * value - 3 / 4") + (expected (parse-source arithmetic-parser source)) + (left-count 0) (right-count 0)) + (measure-parser-cpu-pairs 'request-budget 'control 20 256 expected + (lambda () (set! left-count (+ left-count 1)) (parse-source arithmetic-parser source)) + (lambda () (set! right-count (+ right-count 1)) (parse-source arithmetic-parser source))) + ;; Each side owns one validated warmup plus twenty complete groups. + (check left-count => (+ 1 (* 20 4 256))) + (check right-count => (+ 1 (* 20 4 256))))) (test-case "generated hot path satisfies ASP Gerbil benchmark contract" (check (benchmark-contract-valid? benchmark-path) => #t) ;; Calibrate the Gambit nursery before measurement, then collect outside @@ -32,6 +68,12 @@ parse-batch)) (write receipt) (newline) + (force-output) (check (benchmark-contract-receipt-pass? receipt) => #t))))) -(export parser-benchmark-tests) +(def benchmark-test parser-benchmark-tests) +(export parser-benchmark-tests benchmark-test) + +;; Admit wall/CPU contracts after all shared-runtime semantic owners join. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/benchmarks/arithmetic-scale/aot-benchmark.ss b/t/benchmarks/arithmetic-scale/aot-benchmark.ss new file mode 100644 index 00000000..e65c0056 --- /dev/null +++ b/t/benchmarks/arithmetic-scale/aot-benchmark.ss @@ -0,0 +1,107 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Measure a preloaded arithmetic source in an optimized native executable. +;;; Recognition modes stop before ParseArtifact event materialization. They use +;;; the full-source lexer; the preflight compares its product with directed lexing. + +(import (only-in :gerbil-parser/languages/arithmetic/parser + arithmetic-parser parse-arithmetic) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-grammar-digest parser-machine-runtime + parser-machine-trivia) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact parse-artifact-success? + parse-artifact-roundtrip) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse/prepared) + (only-in :gerbil-parser/src/runtime/significant + parser-significant-tokens)) + +(def (addition-source terms shape) + (let* ((width (if (eq? shape 'lines) 3 2)) + (source (make-string (- (* width terms) (fx- width 1)) #\1))) + (let loop ((offset 1)) + (when (< offset (string-length source)) + (string-set! source offset #\+) + (when (eq? shape 'lines) + (string-set! source (fx+ offset 1) #\newline)) + (loop (fx+ offset width)))) + source)) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((terms (string->number (argument 0 "1024"))) + (shape (string->symbol (argument 1 "lines"))) + (mode (string->symbol (argument 2 "artifact"))) + (samples (string->number (argument 3 "20"))) + (iterations (string->number (argument 4 "10")))) + (unless (and (integer? terms) (positive? terms) + (memq shape '(lines terms)) + (memq mode '(artifact recognition source-recognition)) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations)) + (error "expected size, lines/terms, artifact/recognition/source-recognition, samples, iterations" + args)) + (let* ((source (addition-source terms shape)) + (runtime (parser-machine-runtime arithmetic-parser)) + (significant + (and (eq? mode 'recognition) + (parser-significant-tokens + arithmetic-parser (lex-source arithmetic-parser source))))) + (when (memq mode '(recognition source-recognition)) + (let* ((tokens (lex-source arithmetic-parser source)) + (input (parser-significant-tokens arithmetic-parser tokens))) + (let-values (((root rest) (lr-parse/prepared runtime input))) + (unless (and (null? rest) + (equal? + (parse-arithmetic source) + (make-success-parse-artifact + (parser-machine-grammar-digest arithmetic-parser) + source tokens root + (parser-machine-trivia arithmetic-parser)))) + (error "recognition does not match directed artifact" terms))))) + (def (parse-one) + (if (eq? mode 'artifact) + (parse-arithmetic source) + (let* ((input + (if significant + significant + (parser-significant-tokens + arithmetic-parser (lex-source arithmetic-parser source))))) + (let-values (((root rest) (lr-parse/prepared runtime input))) + (unless (null? rest) + (error "prepared LR left unconsumed tokens" terms)) + root)))) + (def (check result) + (when (eq? mode 'artifact) + (unless (and (parse-artifact-success? result) + (string=? (parse-artifact-roundtrip result) source)) + (error "arithmetic ParseArtifact did not roundtrip" terms)))) + (write (list (cons 'input-shape shape) (cons 'terms terms) + (cons 'source-characters (string-length source)) + (cons 'mode mode) (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let parse-loop ((remaining iterations) (result #f)) + (if (zero? remaining) + (let ((elapsed-ms + (/ (* 1000.0 (- (##current-time-point) started)) + iterations)) + (cpu-ms + (/ (* 1000.0 (- (cpu-time) cpu-started)) + iterations))) + (check result) + (when (>= sample 0) + (write (list (cons 'sample sample) + (cons 'elapsed-ms elapsed-ms) + (cons 'cpu-ms cpu-ms))) + (newline))) + (parse-loop (fx- remaining 1) (parse-one))))) + (sample-loop (fx+ sample 1))))))) + +(export main) diff --git a/t/benchmarks/arithmetic-scale/benchmark.ss b/t/benchmarks/arithmetic-scale/benchmark.ss new file mode 100644 index 00000000..6d43b25f --- /dev/null +++ b/t/benchmarks/arithmetic-scale/benchmark.ss @@ -0,0 +1,120 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Pure Scheme end-to-end parser scaling benchmark. + +(import (only-in :gerbil-parser/languages/arithmetic/parser + arithmetic-parser parse-arithmetic) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-grammar-digest parser-machine-runtime + parser-machine-trivia) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact + parse-artifact-success? parse-artifact-roundtrip) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse/prepared) + (only-in :gerbil-parser/src/runtime/significant + parser-significant-tokens)) + +(def (addition-source terms) + (let (source (make-string (- (* 2 terms) 1) #\1)) + (let loop ((offset 1)) + (when (< offset (string-length source)) + (string-set! source offset #\+) + (loop (+ offset 2)))) + source)) + +(def (measure terms) + (let* ((source (addition-source terms)) + (samples + (map (lambda (_) + (let* ((started (##current-time-point)) + (artifact (parse-arithmetic source)) + (elapsed-ms + (* 1000.0 (- (##current-time-point) started)))) + (unless (and (parse-artifact-success? artifact) + (string=? source + (parse-artifact-roundtrip artifact))) + (error "arithmetic scaling fixture did not roundtrip" + terms)) + elapsed-ms)) + (iota 5)))) + (write (list (cons 'terms terms) + (cons 'bytes (string-length source)) + (list 'samples-ms samples) + (cons 'best-ms (apply min samples)))) + (newline))) + +(def (measure-phase name thunk) + (let (samples + (map (lambda (_) + (##gc) + (let* ((started (##current-time-point)) + (cpu-started (cpu-time)) + (result (thunk)) + (cpu-ms (* 1000.0 (- (cpu-time) cpu-started))) + (elapsed-ms + (* 1000.0 (- (##current-time-point) started)))) + (unless result + (error "phase returned no result" name)) + (cons elapsed-ms cpu-ms))) + (iota 5))) + (list name + (cons 'best-elapsed-ms (apply min (map car samples))) + (cons 'best-cpu-ms (apply min (map cdr samples))) + (cons 'samples-ms samples)))) + +(def (measure-stages terms) + ;; Prepared tokens make these phase costs comparable across revisions, but + ;; they are not a decomposition of the directed streaming parser path. + (let* ((source (addition-source terms)) + (tokens (lex-source arithmetic-parser source)) + (significant (parser-significant-tokens arithmetic-parser tokens)) + (runtime (parser-machine-runtime arithmetic-parser))) + (let-values (((root rest) (lr-parse/prepared runtime significant))) + (unless (null? rest) + (error "prepared LR left unconsumed tokens" terms)) + (unless + (equal? + (parse-arithmetic source) + (make-success-parse-artifact + (parser-machine-grammar-digest arithmetic-parser) + source tokens root (parser-machine-trivia arithmetic-parser))) + (error "directed and prepared parsers disagree" terms)) + (write + (list (cons 'terms terms) + (measure-phase 'lexical-analysis + (lambda () (lex-source arithmetic-parser source))) + (measure-phase + 'significant-token-filter + (lambda () + (parser-significant-tokens arithmetic-parser tokens))) + (measure-phase + 'lr-execution + (lambda () + (let-values + (((parsed remaining) + (lr-parse/prepared runtime significant))) + (unless (null? remaining) + (error "prepared LR left unconsumed tokens" terms)) + parsed))) + (measure-phase + 'artifact-materialization + (lambda () + (make-success-parse-artifact + (parser-machine-grammar-digest arithmetic-parser) + source tokens root + (parser-machine-trivia arithmetic-parser)))))) + (newline)))) + +(def (main . args) + (parse-arithmetic "1+1") + (for-each measure + (if (null? args) + '(100 200 400 800 1600) + (map string->number args))) + (for-each measure-stages + (if (null? args) + '(1600) + (map string->number args)))) + +(export main) diff --git a/t/benchmarks/arithmetic-scale/fused-lr-benchmark-body.ss b/t/benchmarks/arithmetic-scale/fused-lr-benchmark-body.ss new file mode 100644 index 00000000..ed02f943 --- /dev/null +++ b/t/benchmarks/arithmetic-scale/fused-lr-benchmark-body.ss @@ -0,0 +1,167 @@ +;;; -*- Gerbil -*- +;;; Appended to generated LR source to make a standalone matched AOT benchmark. + +(import (only-in :gerbil-parser/src/runtime/lexer + lex-source scan-source-token) + (only-in :gerbil-parser/src/runtime/significant + parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-grammar-digest parser-machine-trivia) + (only-in :gerbil-parser/languages/arithmetic/parser + parse-arithmetic)) + +(def (addition-source terms shape) + (let* ((width (if (eq? shape 'lines) 3 2)) + (source (make-string (- (* width terms) (fx- width 1)) #\1))) + (let loop ((offset 1)) + (when (< offset (string-length source)) + (string-set! source offset #\+) + (when (eq? shape 'lines) + (string-set! source (fx+ offset 1) #\newline)) + (loop (fx+ offset width)))) + source)) + +(def validation-sources + '("1" "123" "a" "a+b" "1+2*3" "(1+2)*3" "-1+2" + "-(1+2)" "+a" "1-2-3" "1/(2+3)" "(-1)" "a*b/c" + "1 +\n 2 +\n 3" "(a + b) * (c - d)" "1+2*3-4/5" + "((((1))))" "a + -b" "-a*-b" "1\n+\n2" + "π+1" "α + β" "1 + 中" "é+2")) + +(def (parse-source/fused source) + (let ((source-length (string-length source)) + (trivia? (parser-machine-trivia arithmetic-parser)) + (runtime (parser-machine-runtime arithmetic-parser)) + (character-offset 0) + (byte-offset 0) + (pending-character #f) + (tokens-reversed '())) + (def (next-input mode) + (if (= character-offset source-length) + #f + (let-values (((input-token next-character) + (scan-source-token arithmetic-parser source + character-offset byte-offset mode))) + (let (start-character character-offset) + (set! character-offset next-character) + (set! byte-offset (token-end input-token)) + (if (trivia? input-token) + (begin + (set! tokens-reversed (cons input-token tokens-reversed)) + (next-input mode)) + (begin + (set! pending-character start-character) + input-token)))))) + (def (after-shift input-token _states _values _actions _shifts) + (set! tokens-reversed (cons input-token tokens-reversed)) + (set! pending-character #f)) + (let-values (((status payload) + (direct-drive runtime next-input after-shift))) + (unless (and (eq? status 'accepted) + (= character-offset source-length) + (not pending-character) + (null? (cadr payload))) + (error "generated streaming LR left its fast path" status source)) + (make-success-parse-artifact + (parser-machine-grammar-digest arithmetic-parser) + source (reverse tokens-reversed) (car payload) trivia?)))) + +(def (assert-same-artifact source) + (let* ((tokens (lex-source arithmetic-parser source)) + (input (parser-significant-tokens arithmetic-parser tokens)) + (runtime (parser-machine-runtime arithmetic-parser))) + (let-values (((reference reference-rest) + (lr-parse/prepared runtime input)) + ((generated generated-rest) + (direct-parse input #t))) + (unless (and (null? reference-rest) + (null? generated-rest) + (equal? reference generated) + (equal? + (parse-source/fused source) + (parse-arithmetic source)) + (equal? + (parse-arithmetic source) + (make-success-parse-artifact + (parser-machine-grammar-digest arithmetic-parser) + source tokens generated + (parser-machine-trivia arithmetic-parser)))) + (error "generated LR differs from the complete artifact" source))))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((terms (string->number (argument 0 "1024"))) + (shape (string->symbol (argument 1 "lines"))) + (mode (string->symbol (argument 2 "prepared"))) + (samples (string->number (argument 3 "20"))) + (iterations (string->number (argument 4 "10")))) + (unless (and (integer? terms) (positive? terms) + (memq shape '(lines terms)) + (memq mode '(prepared source artifact)) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations)) + (error "expected size, lines/terms, prepared/source/artifact, samples, iterations" + args)) + (let* ((source (addition-source terms shape)) + (input (parser-significant-tokens + arithmetic-parser (lex-source arithmetic-parser source))) + (runtime (parser-machine-runtime arithmetic-parser))) + (for-each assert-same-artifact validation-sources) + (assert-same-artifact source) + (def (parse-one generated?) + (if (eq? mode 'artifact) + (values (if generated? + (parse-source/fused source) + (parse-arithmetic source)) + '()) + (let (prepared + (if (eq? mode 'prepared) + input + (parser-significant-tokens + arithmetic-parser (lex-source arithmetic-parser source)))) + (if generated? + (direct-parse prepared #t) + (lr-parse/prepared runtime prepared))))) + (def (measure generated?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (let-values (((_root rest) (parse-one generated?))) + (unless (null? rest) + (error "LR left unconsumed tokens" mode))) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'terms terms) (cons 'shape shape) + (cons 'source-characters (string-length source)) + (cons 'mode mode) (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (let ((baseline-cpu (if (odd? sample) second-cpu first-cpu)) + (baseline-elapsed + (if (odd? sample) second-elapsed first-elapsed)) + (generated-cpu (if (odd? sample) first-cpu second-cpu)) + (generated-elapsed + (if (odd? sample) first-elapsed second-elapsed))) + (write + (list (cons 'sample sample) + (cons 'baseline-cpu-ms baseline-cpu) + (cons 'generated-cpu-ms generated-cpu) + (cons 'baseline-elapsed-ms baseline-elapsed) + (cons 'generated-elapsed-ms generated-elapsed))) + (newline))))) + (sample-loop (fx+ sample 1))))))) + +(export main) diff --git a/t/benchmarks/arithmetic-scale/generate-fused-lr.ss b/t/benchmarks/arithmetic-scale/generate-fused-lr.ss new file mode 100644 index 00000000..58b8e9b8 --- /dev/null +++ b/t/benchmarks/arithmetic-scale/generate-fused-lr.ss @@ -0,0 +1,287 @@ +;;; -*- Gerbil -*- +;;; Experimental fused Scheme LR generator for the arithmetic benchmark. +;;; This consumes the compiled Parser IR; the grammar remains the sole author. +(import (only-in :gerbil-parser/languages/arithmetic/grammar + arithmetic-parser-ir arithmetic-parser) + (only-in :std/misc/ports read-all-as-string) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-grammar-digest) + (only-in :gerbil-parser/src/compiler/lr + lr-spec-ref production-table production-lhs production-rhs + compute-nullable) + (only-in :gerbil-parser/src/compiler/fused-reduction fused-production-expression) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-runtime-lexical-mode-catalog + lr-lexical-mode-terminals)) + +(def spec (cdr (assq 'lr-spec arithmetic-parser-ir))) +(def actions (lr-spec-ref spec 'actions)) +(def gotos (lr-spec-ref spec 'gotos)) +(def table (production-table (lr-spec-ref spec 'productions))) +(def nullable-index + (let-values (((_names index) (compute-nullable (lr-spec-ref spec 'productions)))) index)) +(def mode-catalog + (lr-runtime-lexical-mode-catalog (lr-prepare spec))) + +(def (state-mode-id state) + (let (terminals (map car (vector-ref actions state))) + (let loop ((index 0)) + (if (= index (vector-length mode-catalog)) + (error "LR lexical mode missing from prepared catalog" state) + (if (equal? terminals + (lr-lexical-mode-terminals + (vector-ref mode-catalog index))) + index + (loop (fx+ index 1))))))) + +(def (nth-tail name count) + (let loop ((n count) (value name)) + (if (zero? n) value (loop (- n 1) `(cdr ,value))))) + +(def (goto-expression lhs) + (let ((clauses '())) + (let loop ((i 0)) + (when (< i (vector-length gotos)) + (let (entry (assoc lhs (vector-ref gotos i))) + (when entry + (set! clauses (cons `((,i) ,(cdr entry)) clauses)))) + (loop (+ i 1)))) + `(case (car remaining-states) + ,@(reverse clauses) + (else #f)))) + +(def (reduce-expression production-id (stream? #f)) + (let* ((production (vector-ref table production-id)) + (count (length (production-rhs production))) + (bindings + (let loop ((i 0) (acc '())) + (if (= i count) + (reverse acc) + (loop (+ i 1) + (cons (list (string->symbol + (string-append "v" (number->string i))) + `(car ,(nth-tail 'semantic-values (- count i 1)))) + acc))))) + (remaining-states (nth-tail 'states count)) + (remaining-values (nth-tail 'semantic-values count))) + `(let* ((remaining-states ,remaining-states) + (remaining-values ,remaining-values) + ,@bindings + (offset (if (pair? rest) (token-start (car rest)) + input-end-offset)) + (value ,(fused-production-expression production nullable-index)) + (target ,(goto-expression (production-lhs production)))) + (if target + (loop (cons target remaining-states) + (cons value remaining-values) rest + ,@(if stream? '((fx+ actions 1) shifts) '())) + (fallback))))) + +(def (action-expression entry) + (case (cadr entry) + ((shift) + `(if (pair? rest) + (loop (cons ,(caddr entry) states) + (cons (list (make-recognition-child #f (car rest))) + semantic-values) + (cdr rest)) + (fallback))) + ((reduce) (reduce-expression (caddr entry))) + ((accept) + '(let (children + (and (pair? semantic-values) + (recognition-sequence->list (car semantic-values)))) + (if (and (pair? children) + (null? (cdr children)) + (not (recognition-child-field (car children)))) + (values (recognition-child-value (car children)) rest) + (fallback)))) + (else '(fallback)))) + +(def (terminal-predicate terminal) + (case (cadr terminal) + ((eof) '(null? rest)) + ((literal) + `(and (pair? rest) + (string? (token-lexeme (car rest))) + (string=? (token-lexeme (car rest)) ,(caddr terminal)))) + ((token) + `(and (pair? rest) (eq? (token-kind (car rest)) ',(caddr terminal)))) + (else (error "unsupported terminal" terminal)))) + +(def (state-expression state) + (let* ((row (vector-ref actions state)) + (literals (filter (lambda (entry) + (eq? (cadr (car entry)) 'literal)) row)) + (tokens (filter (lambda (entry) + (eq? (cadr (car entry)) 'token)) row)) + (eof (filter (lambda (entry) + (eq? (cadr (car entry)) 'eof)) row))) + `(cond + ,@(map (lambda (entry) + `(,(terminal-predicate (car entry)) + ,(action-expression entry))) + (append literals eof tokens)) + (else (fallback))))) + +(def (stream-action-expression entry) + (case (cadr entry) + ((shift) + `(if (pair? rest) + (let ((next-states (cons ,(caddr entry) states)) + (next-values + (cons (list (make-recognition-child #f (car rest))) + semantic-values)) + (next-actions (fx+ actions 1)) + (next-shifts (fx+ shifts 1))) + (after-shift (car rest) next-states next-values + next-actions next-shifts) + (loop next-states next-values (cdr rest) + next-actions next-shifts)) + (fallback))) + ((reduce) (reduce-expression (caddr entry) #t)) + ((accept) + '(let (children + (and (pair? semantic-values) + (recognition-sequence->list (car semantic-values)))) + (if (and (pair? children) + (null? (cdr children)) + (not (recognition-child-field (car children)))) + (values 'accepted + (list (recognition-child-value (car children)) rest)) + (fallback)))) + (else '(fallback)))) + +(def (stream-state-expression state) + (let* ((row (vector-ref actions state)) + (literals (filter (lambda (entry) + (eq? (cadr (car entry)) 'literal)) row)) + (tokens (filter (lambda (entry) + (eq? (cadr (car entry)) 'token)) row)) + (eof (filter (lambda (entry) + (eq? (cadr (car entry)) 'eof)) row))) + `(let (rest + (if (null? rest) + (let (input-token + (next-input (vector-ref modes ,(state-mode-id state)))) + (if input-token + (begin + (set! input-end-offset + (max input-end-offset (token-end input-token))) + (list input-token)) + '())) + rest)) + (cond + ,@(map (lambda (entry) + `(,(terminal-predicate (car entry)) + ,(stream-action-expression entry))) + (append literals eof tokens)) + (else (fallback)))))) + +(def (stream-drive-expression) + `(def (direct-drive runtime next-input after-shift) + (let ((modes (lr-runtime-lexical-mode-catalog runtime)) + (input-end-offset 0)) + (def (fallback) (values 'fallback #f)) + (let loop ((states '(0)) (semantic-values '()) (rest '()) + (actions 0) (shifts 0)) + (case (car states) + ,@(let loop ((i 0) (acc '())) + (if (= i (vector-length actions)) + (reverse acc) + (loop (fx+ i 1) + (cons `((,i) ,(stream-state-expression i)) acc)))) + (else (fallback))))))) + +(def (module-expression) + `(begin + (import (only-in :gerbil-parser/languages/arithmetic/grammar + arithmetic-parser) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-runtime) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-parse/prepared lr-runtime-lexical-mode-catalog) + (only-in :gerbil-parser/src/runtime/recognition + make-recognition-child make-recognition-fragment + recognition-child-field recognition-child-value) + (only-in :gerbil-parser/src/runtime/reduce + recognition-children-field recognition-children-alias) + (only-in :gerbil-parser/src/runtime/funcs + recognition-sequence->list recognition-sequence-for-action recognition-sequence-append + recognition-sequence-start) + (only-in :gerbil-parser/src/runtime/token + token-kind token-lexeme token-start token-end)) + (export direct-parse direct-drive) + (def (direct-parse tokens (strict? #f)) + (let* ((runtime (parser-machine-runtime arithmetic-parser)) + (input-end-offset + (fold (lambda (input-token offset) + (max offset (token-end input-token))) + 0 tokens))) + (def (fallback) + (if strict? + (error "generated LR left the deterministic fast path") + (lr-parse/prepared runtime tokens))) + (let loop ((states '(0)) (semantic-values '()) (rest tokens)) + (case (car states) + ,@(let loop ((i 0) (acc '())) + (if (= i (vector-length actions)) + (reverse acc) + (loop (+ i 1) + (cons `((,i) ,(state-expression i)) acc)))) + (else (fallback)))))) + ,(stream-drive-expression))) + +(def (production-module-expression) + `(begin + (import (only-in :gerbil-parser/src/runtime/lr-parser + lr-runtime-lexical-mode-catalog) + (only-in :gerbil-parser/src/runtime/recognition + make-recognition-child make-recognition-fragment + recognition-child-field recognition-child-value) + (only-in :gerbil-parser/src/runtime/reduce + recognition-children-field recognition-children-alias) + (only-in :gerbil-parser/src/runtime/funcs + recognition-sequence->list recognition-sequence-for-action recognition-sequence-append + recognition-sequence-start) + (only-in :gerbil-parser/src/runtime/token + token-kind token-lexeme token-start token-end)) + (export direct-drive direct-grammar-digest) + (def direct-grammar-digest + ,(parser-machine-grammar-digest arithmetic-parser)) + ,(stream-drive-expression))) + +(def (emit-generated port module?) + (display ";;; Generated from arithmetic Parser IR; regenerate with generate-fused-lr.ss.\n" port) + (write + (if module? + (production-module-expression) + (module-expression)) port) + (newline port) + (unless module? + (call-with-input-file + "t/benchmarks/arithmetic-scale/fused-lr-benchmark-body.ss" + (lambda (source) + (let loop ((character (read-char source))) + (unless (eof-object? character) + (write-char character port) + (loop (read-char source)))))))) + +(def (main . args) + (cond + ((= (length args) 1) + (call-with-output-file (car args) + (lambda (port) (emit-generated port #f)))) + ((and (= (length args) 2) (equal? (car args) "module")) + (call-with-output-file (cadr args) + (lambda (port) (emit-generated port #t)))) + ((and (= (length args) 2) (equal? (car args) "check")) + (let ((expected + (call-with-output-string + (lambda (port) (emit-generated port #t)))) + (actual (call-with-input-file (cadr args) read-all-as-string))) + (unless (string=? expected actual) + (error "generated arithmetic LR source is stale" (cadr args))) + (displayln "GENERATED-LR-OK"))) + (else (error "expected [module|check] generated source path" args)))) +(export main) diff --git a/t/benchmarks/arithmetic-scale/matched-stream.ss b/t/benchmarks/arithmetic-scale/matched-stream.ss new file mode 100644 index 00000000..6094825a --- /dev/null +++ b/t/benchmarks/arithmetic-scale/matched-stream.ss @@ -0,0 +1,66 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Repeatable long-input ParseArtifact comparison across package revisions. + +(import (only-in :gerbil-parser/languages/arithmetic/parser + parse-arithmetic) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-roundtrip)) + +(def (addition-source terms) + (let (source (make-string (- (* 2 terms) 1) #\1)) + (let loop ((offset 1)) + (when (< offset (string-length source)) + (string-set! source offset #\+) + (loop (+ offset 2)))) + source)) + +(def (addition-lines-source lines) + (let (source (make-string (- (* 3 lines) 2) #\1)) + (let loop ((offset 0)) + (when (< (+ offset 2) (string-length source)) + (string-set! source (+ offset 1) #\+) + (string-set! source (+ offset 2) #\newline) + (loop (+ offset 3)))) + source)) + +(def (main . args) + (let* ((terms (if (null? args) 3200 (string->number (car args)))) + (samples (if (or (null? args) (null? (cdr args))) + 30 (string->number (cadr args)))) + (shape (if (and (pair? args) (pair? (cdr args)) + (pair? (cddr args))) + (string->symbol (caddr args)) 'terms))) + (unless (and (integer? terms) (positive? terms) + (integer? samples) (positive? samples) + (memq shape '(terms lines))) + (error "expected positive size, sample count, and terms/lines shape" + args)) + (let (source (if (eq? shape 'lines) + (addition-lines-source terms) + (addition-source terms))) + (write (list (cons 'input-shape shape) + (cons (if (eq? shape 'lines) 'lines 'terms) terms) + (cons 'source-characters (string-length source)))) + (newline) + (let loop ((n 0)) + (when (< n (+ samples 3)) + (##gc) + ;; Keep process CPU beside elapsed time: a stalled host must not + ;; be mistaken for extra parser work. + (let* ((started (##current-time-point)) + (cpu-started (cpu-time)) + (artifact (parse-arithmetic source)) + (cpu-ms (* 1000.0 (- (cpu-time) cpu-started))) + (ms (* 1000.0 (- (##current-time-point) started)))) + (unless (and (parse-artifact-success? artifact) + (equal? (parse-artifact-roundtrip artifact) source)) + (error "long arithmetic parse failed" terms)) + (when (>= n 3) + (write (list (cons 'sample (- n 3)) + (cons 'elapsed-ms ms) + (cons 'cpu-ms cpu-ms))) + (newline))) + (loop (+ n 1))))))) + +(export main) diff --git a/t/benchmarks/artifact-window/benchmark.ss b/t/benchmarks/artifact-window/benchmark.ss new file mode 100644 index 00000000..115ec332 --- /dev/null +++ b/t/benchmarks/artifact-window/benchmark.ss @@ -0,0 +1,26 @@ +;;; Measure complete publication with unchanged source admission and digest work. +(import (only-in ../../fixtures/artifact-window window-tokens window-artifact) + (only-in ../parser-stage-cost/benchmark measure-parser-component) + (only-in :gerbil-parser/src/runtime/artifact + make-certified-window-artifact make-failure-parse-artifact + parse-artifact-valid-for-source? sha256-text)) +(export benchmark-artifact-window) + +(def (benchmark-artifact-window (samples 20) (batch-count 5) (width 20000)) + (let* ((source (make-string width #\a)) (tokens (window-tokens source)) + (base (window-artifact source tokens)) (assigned (map list tokens)) + (grammar (sha256-text "window-failure")) + (diagnostic '((message . "expected another token"))) + (rejected (make-failure-parse-artifact grammar source tokens diagnostic))) + (unless (and (parse-artifact-valid-for-source? base source) + (parse-artifact-valid-for-source? rejected source)) + (error "window benchmark requires admitted complete products")) + (measure-parser-component 'certified-window-publication samples batch-count + (lambda () + (let-values (((product shared) + (make-certified-window-artifact base source 0 tokens tokens 0 assigned))) + (unless (= shared 2) (error "window benchmark changed shared event accounting")) + product)) base) + (measure-parser-component 'rejected-token-publication samples batch-count + (lambda () (make-failure-parse-artifact grammar source tokens diagnostic)) rejected) + (displayln "ARTIFACT-WINDOW-BENCHMARK-OK"))) diff --git a/t/benchmarks/bound-source-catalog/benchmark.ss b/t/benchmarks/bound-source-catalog/benchmark.ss new file mode 100644 index 00000000..9fe5909b --- /dev/null +++ b/t/benchmarks/bound-source-catalog/benchmark.ss @@ -0,0 +1,47 @@ +;;; Complete binding, reference closure and canonical digest publication. +(import (only-in ../../fixtures/bound-source-catalog bound-source-catalog) + (only-in ../parser-stage-cost/benchmark measure-parser-component measure-parser-cpu-pairs) + (only-in :gerbil-parser/src/compiler/bound-ir bind-grammar-ir bound-grammar-ir-ref) + (only-in :gerbil-parser/src/runtime/identity sha256-bytes)) +(export benchmark-bound-source-catalog compare-bound-source-catalog + benchmark-bound-identity-input) + +;;; Isolated final-section controls attribute costs, not whole-publication speed. +;;; Construct the complete sidecar before timing and retain its admitted identity. +(def (benchmark-bound-identity-input (samples 20) (batch-count 5) (width 512)) + (let-values (((grammar sources) (bound-source-catalog width))) + (let* ((sidecar (bind-grammar-ir grammar "bound-source-benchmark" '(author) sources)) + (sections (bound-grammar-ir-ref sidecar 'sections)) + (text (call-with-output-string (lambda (port) (write sections port)))) + (bytes (string->utf8 text)) + (identity (bound-grammar-ir-ref sidecar 'identityDigest))) + (measure-parser-component (list 'bound-identity-text width) samples batch-count + (lambda () (call-with-output-string (lambda (port) (write sections port)))) text) + (measure-parser-component (list 'bound-identity-encode width) samples batch-count + (lambda () (string->utf8 text)) bytes) + (measure-parser-component (list 'bound-identity-byte-writer width) samples batch-count + (lambda () (call-with-output-u8vector '(char-encoding: UTF-8 eol-encoding: lf) + (lambda (port) (write sections port)))) bytes) + (measure-parser-component (list 'bound-identity-sha256 width) samples batch-count + (lambda () (sha256-bytes bytes)) identity) + (displayln "BOUND-IDENTITY-INPUT-BENCHMARK-OK")))) + +(def (benchmark-bound-source-catalog (samples 20) (batch-count 5) (width 512) + (bind bind-grammar-ir)) + (let-values (((grammar sources) (bound-source-catalog width))) + (let (expected (bind-grammar-ir grammar "bound-source-benchmark" '(author) sources)) + ;; Every baseline/candidate result must equal the entire production sidecar. + (measure-parser-component (list 'bound-source-catalog width) samples batch-count + (lambda () (bind grammar "bound-source-benchmark" '(author) sources)) expected) + (displayln "BOUND-SOURCE-CATALOG-BENCHMARK-OK")))) + +;;; The original constructor is an experiment input, never an engine fallback. +(def (compare-bound-source-catalog original (groups 20) (calls 2) (width 512)) + (let-values (((grammar sources) (bound-source-catalog width))) + (let* ((left (lambda () (original grammar "bound-source-benchmark" '(author) sources))) + (right (lambda () (bind-grammar-ir grammar "bound-source-benchmark" '(author) sources))) + (expected (right))) + (let (ratios (measure-parser-cpu-pairs 'bound-source-catalog width groups calls expected left right)) + (write (list 'BOUND-SOURCE-CPU-RATIOS width ratios)) (newline) (force-output) + (displayln "BOUND-SOURCE-COMPARISON-OK") + ratios)))) diff --git a/t/benchmarks/build.ss b/t/benchmarks/build.ss new file mode 100644 index 00000000..1d197b95 --- /dev/null +++ b/t/benchmarks/build.ss @@ -0,0 +1,21 @@ +#!/usr/bin/env gxi +;;; One standard build owner for the engine evidence workload closure. +(import :std/build-script) +(defbuild-script + '("../../languages/arithmetic/parser-test" + "../../languages/hcl/parser-test" + "parser-stage-cost/benchmark" + "../scenarios/performance/selective-glr/scenario" + "lr-request-stack/derivation" + "lr-request-stack/lookahead" + "gql/runtime/execution-counts" + "gql/runtime/reduction-counts" + "gql/runtime/matched-stages" + "lr-request-stack/benchmark" + "lr-request-stack/sdk-control" + "lr-request-stack/record-access" + "lr-request-stack/record-repeatability" + "lr-request-stack/request-cost" + "lr-request-stack/request-profile" + "lr-request-stack/record-evidence" + "parser-evidence")) diff --git a/t/benchmarks/capture-prefix/benchmark.ss b/t/benchmarks/capture-prefix/benchmark.ss new file mode 100644 index 00000000..a54ad451 --- /dev/null +++ b/t/benchmarks/capture-prefix/benchmark.ss @@ -0,0 +1,53 @@ +;;; Two persistent SDK processes admit two real native images; request CPU is +;;; measured inside each worker, excluding startup and protocol exchange. +(displayln "CAPTURE-CPU-START") (force-output) +(def (worker image scope) + (open-process (list path: "gxi" arguments: (list "t/benchmarks/capture-prefix/worker.ss" image scope) + stdin-redirection: #t stdout-redirection: #t stderr-redirection: #f))) +(def (batch process calls) + (write calls process) (newline process) (force-output process) + (let (row (read process)) + (unless (and (list? row) (= (length row) 3) (eq? (car row) 'CAPTURE-BATCH) + (= (cadr row) calls) (> (caddr row) .001)) + (error "invalid or unresolved complete-request CPU batch" row)) + (write row) (newline) (force-output) + (caddr row))) +(def (percentile values rank) + (list-ref (list-sort < values) (- (ceiling (/ (* rank (length values)) 100)) 1))) +(def (phase name reference candidate calls) + (let loop ((group 0) (ratios '())) + (if (= group 5) + (let (summary (map (lambda (rank) (percentile ratios rank)) '(10 50 90))) + (write (list 'CAPTURE-CPU-SUMMARY name summary)) (newline) (force-output) summary) + (let* ((order (apply append (make-list 4 (if (even? group) '(a b b a b a a b) '(b a a b a b b a))))) + (rows (map (lambda (side) + (cons side (batch (if (eq? side 'a) reference candidate) calls))) order)) + (total (lambda (side) (foldl (lambda (row sum) (if (eq? (car row) side) (+ sum (cdr row)) sum)) 0 rows))) + (ratio (/ (total 'b) (total 'a)))) + (write (list 'CAPTURE-CPU-GROUP name group ratio)) (newline) (force-output) + (loop (+ group 1) (cons ratio ratios)))))) +(def (main reference-image (scope "selector-input")) + (let ((reference (worker reference-image scope)) (candidate (worker "candidate" scope))) + (try + (let ((left (read reference)) (right (read candidate))) + (unless (and (list? left) (list? right) (= (length left) 5) (= (length right) 5) + (eq? (car left) 'CAPTURE-READY) (eq? (car right) 'CAPTURE-READY) + (equal? (cadr left) (cadr right)) (equal? (caddr left) (caddr right)) + (not (equal? (cadddr left) (cadddr right))) + (equal? (car (cddddr left)) (car (cddddr right)))) + (error "native images changed complete artifact" left right)) + (write left) (newline) + (write right) (newline) (force-output)) + (batch reference 256) (batch candidate 256) + (let* ((before (phase 'before reference reference 256)) + (production (phase 'production reference candidate 256)) + (after (phase 'after reference reference 256))) + (unless (and (>= (car before) .90) (<= (caddr before) 1.10) + (>= (car after) .90) (<= (caddr after) 1.10) (< (caddr production) .95)) + (error "complete capture CPU admission failed" before production after)) + (displayln "CAPTURE-COMPLETE-CPU-PROOF-OK") (force-output)) + (finally + (for-each (lambda (process) + (with-catch (lambda (_) (void)) + (lambda () (write 'stop process) (newline process) (force-output process))) + (close-port process)) (list reference candidate)))))) diff --git a/t/benchmarks/capture-prefix/build-reference.ss b/t/benchmarks/capture-prefix/build-reference.ss new file mode 100644 index 00000000..728be61c --- /dev/null +++ b/t/benchmarks/capture-prefix/build-reference.ss @@ -0,0 +1,17 @@ +;;; Normal SDK compiler for an exact Git snapshot and its selected owner set. +(import (only-in :gerbil/compiler compile-module)) +(def (main source output (scope "selector-input")) + (unless (member scope '("selector-input")) + (error "unknown native reference scope" scope)) + (let ((output (path-expand output)) + (runtime (path-normalize (path-expand "../" (path-directory (path-expand source)))))) + ;; Earlier modules' frozen interfaces must precede shared current inputs. + (add-load-path! output) + (for-each + (lambda (path) + (displayln "REFERENCE-COMPILE " path) (force-output) + (compile-module path + (list output-dir: output optimize: #t generate-ssxi: #t invoke-gsc: #t))) + (map (lambda (name) (path-expand (string-append name ".ss") runtime)) + '("lr-parser/state" "layout" "lr-parser/prepare" "lr-parser/recognition" "lr-parser/batch" + "lr-parser/checkpoint" "lr-parser/transfer" "parser" "lr-parser" "reuse" "incremental/session"))))) diff --git a/t/benchmarks/capture-prefix/controls.ss b/t/benchmarks/capture-prefix/controls.ss new file mode 100644 index 00000000..7dc45068 --- /dev/null +++ b/t/benchmarks/capture-prefix/controls.ss @@ -0,0 +1,76 @@ +;;; Direct kernel parity plus nullable, Unicode, relocation and exact tail checks. +(import (only-in :gerbil-parser/src/runtime/lr-parser/state + make-lr-capture-plan lr-recognition-view? lr-recognition-view-base lr-recognition-fragment-lookahead lr-runtime-reduction-widths lr-recognition-fragment? lr-recognition-fragment-children make-lr-recognition-view) + (only-in :gerbil-parser/src/runtime/lr-parser/recognition capture-reduction prepare-capture-plans lr-recognition-relocate lr-recognition-fragment-terminal-yield) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar records-parser) + (only-in :gerbil-parser/t/benchmarks/lr-request-stack/capture-reference reference-capture-reduction)) +(def (require! value message) (unless value (error message))) +(export verify-capture-prefix!) +(def (verify-capture-prefix!) + (let* ((runtime (parser-machine-runtime records-parser)) + (widths (lr-runtime-reduction-widths runtime)) + (plans (prepare-capture-plans runtime)) + (ids (iota (vector-length widths))) + (unary (find (lambda (id) (= (vector-ref widths id) 1)) ids)) + (nullable (find (lambda (id) (= (vector-ref widths id) 0)) ids)) + (combinations 0) + (executor (lambda _ #f))) + (def (observe-children! piece) + (cond ((lr-recognition-view? piece) (observe-children! (lr-recognition-view-base piece))) + ((lr-recognition-fragment? piece) + (lr-recognition-fragment-lookahead piece) + (for-each observe-children! (lr-recognition-fragment-children piece))))) + (def (make-piece kind i) + (let* ((token (make-token 'identifier "α" (* i 2) (+ 2 (* i 2)))) + (empty? (memq kind '(empty-fragment moved-empty nested-empty))) + (fragment? (or empty? (memq kind '(fragment moved-fragment nested-fragment)))) + (piece (if fragment? + (car (reference-capture-reduction runtime executor (if empty? nullable unary) + (if empty? '(0) '(0 0)) (if empty? '() (list token)) + '() (* i 2) '(value) 1)) token))) + (cond ((memq kind '(moved-token moved-fragment moved-empty)) + (lr-recognition-relocate piece 3)) + ((memq kind '(nested-token nested-fragment nested-empty)) + (make-lr-recognition-view (make-lr-recognition-view piece 3) -1)) + (else piece)))) + (for-each + (lambda (production) + (let (width (vector-ref widths production)) + (for-each + (lambda (kind) + (for-each + (lambda (rest) + (set! combinations (+ combinations 1)) + (let* ((prefix (map (lambda (i) (make-piece kind i)) (iota width))) + (suffix (list (make-token 'identifier "tail" 100 104))) + (stack (append (reverse prefix) suffix)) + (states (iota (+ width 1))) + (args (list runtime executor production states stack rest 7 '(value) 1)) + (reference (apply reference-capture-reduction args)) + (candidate (capture-reduction (vector-ref plans production) executor (list-ref states width) stack + (and (pair? rest) (car rest)) 7 '(value) 1))) + (let* ((uncached (make-lr-capture-plan runtime production width #f)) + (fast (capture-reduction uncached executor (list-ref states width) stack + (and (pair? rest) (car rest)) 7 '(value) 1))) + (lr-recognition-fragment-terminal-yield (car fast)) + (observe-children! (car reference)) (observe-children! (car fast)) + (require! (equal? reference fast) "uncached plan changed capture fields") + (require! (eq? (cdr fast) suffix) "uncached plan copied remaining stack")) + (lr-recognition-fragment-terminal-yield (car candidate)) + (observe-children! (car reference)) (observe-children! (car candidate)) + (require! (equal? reference candidate) "capture fields or yield association changed") + (require! (eq? (cdr candidate) suffix) "capture copied or changed the remaining stack") + (require! (equal? (lr-recognition-fragment-children (car candidate)) prefix) + "source order changed"))) + (list '() (list (make-token 'identifier "β" 120 122))))) + '(token moved-token nested-token fragment moved-fragment nested-fragment + empty-fragment moved-empty nested-empty)))) + ids) + (require! (and unary nullable) "nullable and unary productions not exercised") + (require! (= combinations 198) "capture representation inventory incomplete") + (displayln "CAPTURE-METADATA-VARIANTS-OK combinations=" combinations) (force-output) + (displayln "CAPTURE-PREFIX-CONTROLS-OK productions=" (vector-length widths)) (force-output))) + +(def (main . _) (verify-capture-prefix!)) diff --git a/t/benchmarks/capture-prefix/phase-cost.ss b/t/benchmarks/capture-prefix/phase-cost.ss new file mode 100644 index 00000000..e0ad4524 --- /dev/null +++ b/t/benchmarks/capture-prefix/phase-cost.ss @@ -0,0 +1,162 @@ +;;; Recorded-query cost diagnosis. This is not a matched speedup gate. +(import (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar records-parser) + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark measure-parser-batch sample-at-percentile) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime parser-machine-trivia parser-machine-grammar-digest call-with-parser-machine-source) + (only-in :gerbil-parser/src/runtime/incremental make-incremental-session incremental-session-artifact incremental-session-recognition-root) + (only-in :gerbil-parser/src/runtime/incremental/state incremental-session-state-tokens incremental-session-state-modes) + (only-in :gerbil-parser/src/runtime/lexer scan-source-token) + (only-in :gerbil-parser/src/runtime/token token-start token-lexeme) + (only-in :gerbil-parser/src/runtime/lr-parser lr-initial-checkpoint lr-checkpoint-drive current-lr-recognition-observer lr-runtime-lexical-mode-catalog lr-lexical-mode-id lr-recognition-project) + (only-in :gerbil-parser/src/runtime/recognition recognition-child-value) + (only-in :gerbil-parser/src/runtime/artifact make-success-parse-artifact parse-artifact-valid-for-source?)) +(import (only-in :gerbil-parser/src/runtime/lr-parser/state + lr-recognition-fragment-lookahead lr-recognition-fragment? lr-recognition-fragment-children + lr-checkpoint-states lr-checkpoint-semantic-values) + (only-in :gerbil-parser/src/runtime/lr-parser/transfer fragment-yield-matches?)) +(export main) +(defstruct replay-step (offset byte mode token end) final: #t) +(def (main) + (let* ((source (apply string-append (make-list 64 "a=1\n"))) + (session (make-incremental-session records-parser source #t)) + (expected (incremental-session-artifact session)) + (tokens (incremental-session-state-tokens session)) + (modes (incremental-session-state-modes session)) + (runtime (parser-machine-runtime records-parser)) + (catalog (lr-runtime-lexical-mode-catalog runtime)) + (trivia? (parser-machine-trivia records-parser)) + (steps + (let loop ((tokens tokens) (modes modes) (offset 0) (rows '())) + (if (null? tokens) (reverse rows) + (let* ((token (car tokens)) (end (+ offset (string-length (token-lexeme token))))) + (loop (cdr tokens) (cdr modes) end + (cons (make-replay-step offset (token-start token) (vector-ref catalog (car modes)) token end) rows)))))) + (significant (filter (lambda (row) (not (trivia? (replay-step-token row)))) steps))) + (unless (and (= (length steps) 256) (= (length significant) 256) + (parse-artifact-valid-for-source? expected source)) + (error "recorded query scope changed" (length steps) (length significant) (parse-artifact-valid-for-source? expected source))) + (def (scan-replay) + (call-with-parser-machine-source records-parser source + (lambda () + (map (lambda (row) + (let-values (((token _end) + (scan-source-token records-parser source (replay-step-offset row) + (replay-step-byte row) (replay-step-mode row)))) token)) steps)))) + ;; Bind character and byte positions independently before timing replay. + (call-with-parser-machine-source records-parser source + (lambda () + (for-each (lambda (row) + (let-values (((token end) + (scan-source-token records-parser source (replay-step-offset row) + (replay-step-byte row) (replay-step-mode row)))) + (unless (and (equal? token (replay-step-token row)) (= end (replay-step-end row))) + (error "recorded lexical query changed token or character boundary")))) steps))) + (def (lr-replay capture? (validate-modes? #f)) + (let ((pending significant) (root #f) (snapshots '())) + (def (next mode) + (and (pair? pending) + (let (row (car pending)) + (when (and validate-modes? (not (= (lr-lexical-mode-id mode) (lr-lexical-mode-id (replay-step-mode row))))) + (error "LR replay changed an actual lexical mode")) + (set! pending (cdr pending)) (replay-step-token row)))) + (parameterize ((current-lr-recognition-observer (and capture? (lambda (value) (set! root value))))) + (let-values (((status payload) + (lr-checkpoint-drive (lr-initial-checkpoint runtime '()) next #f #f 'installed + (and capture? 32) (and capture? (lambda (frame) (set! snapshots (cons frame snapshots))))))) + (unless (and (eq? status 'accepted) (null? pending) (null? (cadr payload))) + (error "LR replay did not finish" status)) + (vector (car payload) root snapshots))))) + (def full-lr (lr-replay #t #t)) + (def bare-lr (lr-replay #f #t)) + ;; Count the actual retained tree and snapshot depths outside every clock. + ;; These are logical objects, not a native allocation-byte estimate. + (let loop ((pending (list (vector-ref full-lr 1))) (fragments 0) (leaves 0)) + (if (null? pending) + (begin + (unless (and (= fragments 322) (= leaves 256) + (= (length (vector-ref full-lr 2)) 8)) + (error "recorded capture topology changed" fragments leaves)) + (write (list 'CAPTURE-COST-SCOPE 'fragments fragments 'terminals leaves + 'snapshots + (map (lambda (frame) + (list (length (lr-checkpoint-states frame)) + (length (lr-checkpoint-semantic-values frame)))) + (reverse (vector-ref full-lr 2))))) + (newline) (force-output)) + (let (piece (car pending)) + (if (lr-recognition-fragment? piece) + (loop (append (lr-recognition-fragment-children piece) (cdr pending)) + (+ fragments 1) leaves) + (loop (cdr pending) fragments (+ leaves 1)))))) + (def (publish) + (make-success-parse-artifact (parser-machine-grammar-digest records-parser) source tokens + (vector-ref full-lr 0) trivia?)) + (unless (and (equal? (scan-replay) tokens) (equal? (publish) expected) + (equal? (vector-ref full-lr 0) (vector-ref bare-lr 0)) + (equal? (recognition-child-value (car (lr-recognition-project (vector-ref full-lr 1) tokens))) + (vector-ref full-lr 0))) + (error "independent recorded replay changed the complete product")) + (def (whole) + (let* ((current (make-incremental-session records-parser source #t)) + (artifact (incremental-session-artifact current))) + (unless (and (incremental-session-recognition-root current) + (equal? artifact expected) (parse-artifact-valid-for-source? artifact source)) + (error "complete cost request changed")) artifact)) + ;; A separate scope pays first-demand construction and terminal validation. + ;; It does not silently move that work outside the forced-request clock. + (def (force-boundaries! root) + (let loop ((pending (list root))) + (unless (null? pending) + (let (piece (car pending)) + (if (lr-recognition-fragment? piece) + (begin (lr-recognition-fragment-lookahead piece) + (loop (append (lr-recognition-fragment-children piece) (cdr pending)))) + (loop (cdr pending))))))) + (def (whole-forced) + (let* ((current (make-incremental-session records-parser source #t)) + (artifact (incremental-session-artifact current))) + (force-boundaries! (incremental-session-recognition-root current)) + (unless (and (fragment-yield-matches? (incremental-session-recognition-root current) 0 + (incremental-session-state-tokens current)) + (equal? artifact expected) (parse-artifact-valid-for-source? artifact source)) + (error "forced-yield request changed")) artifact)) + ;; Topology/projection preflight observes and warms full-lr's private memos. + ;; Compare timed fresh captures with an independently fresh representation; + ;; do not force derived lists or indexes inside the captured-LR scope. + (def cold-full-lr (lr-replay #t)) + (unless (equal? (vector-ref cold-full-lr 0) (vector-ref full-lr 0)) + (error "fresh LR reference changed semantic output")) + (def stages + (list (list 'whole whole expected) + (list 'whole-forced-yield whole-forced expected) + (list 'recorded-scan scan-replay tokens) + (list 'captured-lr (lambda () (lr-replay #t)) cold-full-lr) + (list 'uncaptured-lr (lambda () (lr-replay #f)) bare-lr) + (list 'publication publish expected) + (list 'validation (lambda () (parse-artifact-valid-for-source? expected source)) #t))) + (for-each (lambda (stage) (for-each (lambda (_) ((cadr stage))) (iota 256))) stages) + (let (rows (make-table test: eq?)) + (for-each + (lambda (sample) + (for-each + (lambda (stage) + (let ((name (car stage)) + (row (measure-parser-batch (car stage) sample 256 (cadr stage) (caddr stage)))) + (table-set! rows name (cons row (table-ref rows name '()))) + (write (list 'CAPTURE-COST-SAMPLE name row)) (newline) (force-output))) + (if (even? sample) stages (reverse stages)))) (iota 5)) + (for-each + (lambda (stage) + (let* ((name (car stage)) (samples (table-ref rows name)) + (median (cdr (assq 'cpu-ms (sample-at-percentile samples 'cpu-ms 50))))) + (write (list 'CAPTURE-COST-MEDIAN name 'cpu-ms-per-request (/ median 256))) (newline))) stages)) + ;; Allocation-only controls have an explicit preceding collection. Their CPU + ;; is never admitted as a speedup; the separate complete CPU gate uses natural + ;; collection throughout. A collection inside this window still rejects bytes. + (for-each + (lambda (sample) + (##gc) + (let (row (measure-parser-batch 'allocation-only sample 256 whole expected)) + (write (list 'CAPTURE-ALLOCATION-ONLY (assq 'sample row) '(calls . 256) + (assq 'gc-count row) (assq 'allocated-bytes row))) + (newline) (force-output))) (iota 3)) + (displayln "CAPTURE-COST-REPLAY-OK tokens=256 significant=256") (force-output))) diff --git a/t/benchmarks/capture-prefix/profile.ss b/t/benchmarks/capture-prefix/profile.ss new file mode 100644 index 00000000..fed5a21a --- /dev/null +++ b/t/benchmarks/capture-prefix/profile.ss @@ -0,0 +1,30 @@ +;;; Inclusive stage diagnosis only; the matched CPU gate remains independent. +(import (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar records-parser) + (only-in :gerbil-parser/src/runtime/incremental make-incremental-session incremental-session-artifact incremental-session-recognition-root) + (only-in :gerbil-parser/src/runtime/parse-cost current-parser-cost-observer with-parser-cost-stage) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid-for-source?) + (only-in :gerbil-parser/src/runtime/parser parse-source)) +(export main) +(def (main) + (let ((source (apply string-append (make-list 64 "a=1\n"))) (totals (make-table test: eq?))) + (def expected (parse-source records-parser source)) + (def (request) + (let* ((session (make-incremental-session records-parser source #t)) + (artifact (incremental-session-artifact session))) + (with-parser-cost-stage 'benchmark-only-verification + (unless (and (incremental-session-recognition-root session) + (equal? artifact expected) + (parse-artifact-valid-for-source? artifact source)) + (error "stage diagnosis request rejected"))))) + (for-each (lambda (_) (request)) (iota 256)) + (parameterize ((current-parser-cost-observer + (lambda (stage row) + (unless (cdr (assq 'completed row)) (error "stage did not complete" stage)) + (let (old (table-ref totals stage '(0 0 0))) + (table-set! totals stage + (list (+ 1 (car old)) (+ (cadr old) (cdr (assq 'cpu-ms row))) + (+ (caddr old) (cdr (assq 'gc-count row))))))))) + (for-each (lambda (_) (with-parser-cost-stage 'complete-captured-request (request))) (iota 256))) + (for-each (lambda (row) (write (cons 'SELECTOR-INCLUSIVE-STAGE row)) (newline)) + (list-sort (lambda (a b) (stringstring (car a)) (symbol->string (car b)))) (table->list totals))) + (displayln "SELECTOR-STAGE-DIAGNOSIS-OK") (force-output))) diff --git a/t/benchmarks/capture-prefix/worker.ss b/t/benchmarks/capture-prefix/worker.ss new file mode 100644 index 00000000..08f13f46 --- /dev/null +++ b/t/benchmarks/capture-prefix/worker.ss @@ -0,0 +1,129 @@ +;;; One prepared runtime per actual native image; no runtime binding replacement. +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(def (main image (scope "selector-input")) + (unless (member scope '("selector-input")) (error "unknown native experiment scope" scope)) + (unless (equal? image "candidate") (add-load-path! image)) + (parameterize ((current-output-port (current-error-port))) + (preload-module "gerbil-parser/t/support/qualification/benchmark-qos") + (preload-module "gerbil-parser/src/runtime/lr-parser/recognition") + (preload-module "gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar") + (preload-module "gerbil-parser/src/runtime/incremental") + (preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") + (eval '(import + (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar records-parser) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/lr-parser/state lr-runtime-reduction-widths) + (only-in :gerbil-parser/src/runtime/token token?) + (only-in :gerbil-parser/src/runtime/lr-parser lr-recognition-view? lr-recognition-fragment? + lr-recognition-fragment-runtime lr-recognition-fragment-production-id lr-recognition-fragment-children) + (only-in :gerbil-parser/src/runtime/incremental/state incremental-session-state-checkpoints incremental-session-state-modes) + (only-in :gerbil-parser/src/runtime/lr-parser/state lr-prefix-snapshot-states lr-prefix-snapshot-deterministic-actions lr-prefix-snapshot-deterministic-shifts) + (only-in :gerbil-parser/src/runtime/incremental make-incremental-session incremental-session-artifact + incremental-session-project-artifact incremental-session-recognition-root) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid-for-source? parse-artifact-ref sha256-text) + (only-in :gerbil-parser/src/runtime/identity sha256-bytes) + (only-in :std/misc/ports read-all-as-u8vector) + (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!))) + (eval '(import (only-in :gerbil-parser/t/support/qualification/benchmark-qos admit-benchmark-qos!))) + (eval '(admit-benchmark-qos!)) + (displayln "CAPTURE-HOST-QOS-OK") (force-output) + (eval '(def capture-source (apply string-append (make-list 64 "a=1\n")))) + (eval '(def capture-expected (parse-source records-parser capture-source))) + (eval '(def (capture-request) + (let* ((session (make-incremental-session records-parser capture-source #t)) + (artifact (incremental-session-artifact session))) + (unless (and (incremental-session-recognition-root session) + (equal? artifact capture-expected) + (parse-artifact-valid-for-source? artifact capture-source)) + (error "complete capture request changed")) + session))) + (eval '(unless (equal? (incremental-session-project-artifact (capture-request)) capture-expected) + (error "independent capture projection differs")))) + (eval '(def capture-width-counts + (let ((counts (make-table test: eqv?))) + (let loop ((pending (list (incremental-session-recognition-root (capture-request))))) + (unless (null? pending) + (let (piece (car pending)) + (if (lr-recognition-fragment? piece) + (let (width (vector-ref (lr-runtime-reduction-widths (lr-recognition-fragment-runtime piece)) + (lr-recognition-fragment-production-id piece))) + (table-set! counts width (+ 1 (table-ref counts width 0))) + (loop (append (lr-recognition-fragment-children piece) (cdr pending)))) + (loop (cdr pending)))))) + (list-sort (lambda (a b) (< (car a) (car b))) (table->list counts))))) + (eval '(def capture-metadata-counts + (let ((counts (make-table test: eq?))) + (let loop ((pending (list (incremental-session-recognition-root (capture-request))))) + (unless (null? pending) + (let (piece (car pending)) + (if (lr-recognition-fragment? piece) + (let (children (lr-recognition-fragment-children piece)) + (when (and (pair? children) (null? (cdr children))) + (let (kind (cond ((lr-recognition-view? (car children)) 'view) + ((token? (car children)) 'token) (else 'fragment))) + (table-set! counts kind (+ 1 (table-ref counts kind 0))))) + (loop (append children (cdr pending)))) + (loop (cdr pending)))))) + (list-sort (lambda (a b) (stringstring (car a)) (symbol->string (car b)))) + (table->list counts))))) + (def (image-file module) + (let (entry (assoc module (list-modules))) + (unless (and entry (string? (cdr entry))) (error "missing native image" module)) + (cdr entry))) + (def selected-modules + '("gerbil-parser/src/runtime/lr-parser/state~0" "gerbil-parser/src/runtime/layout~0" "gerbil-parser/src/runtime/lr-parser/prepare~0" + "gerbil-parser/src/runtime/lr-parser/recognition~0" "gerbil-parser/src/runtime/lr-parser/batch~0" + "gerbil-parser/src/runtime/lr-parser/checkpoint~0" "gerbil-parser/src/runtime/lr-parser/transfer~0" + "gerbil-parser/src/runtime/parser~0" + "gerbil-parser/src/runtime/reuse~0" "gerbil-parser/src/runtime/incremental/session~0")) + (for-each + (lambda (module) + (let* ((file (image-file module)) + (root (path-normalize (path-expand (if (equal? image "candidate") ".gerbil/lib" image))))) + (unless (and (string-prefix? root (path-normalize (path-expand file))) + (not (equal? (path-extension file) ".scm"))) + (error "selected runtime is not the requested native image" module file root)))) selected-modules) + (def (image-digest module) + ((eval 'sha256-bytes) (call-with-input-file (image-file module) (eval 'read-all-as-u8vector)))) + (def shared-images + (map (lambda (module) (list module (image-digest module))) + (filter (lambda (module) (not (member module selected-modules))) + '("gerbil-parser/src/runtime/lr-stack-storage~0" + "gerbil-parser/src/runtime/lr-parser/state~0" + "gerbil-parser/src/runtime/lr-parser/recognition~0" + "gerbil-parser/src/runtime/lr-parser/checkpoint~0" + "gerbil-parser/src/runtime/lr-parser/transfer~0" + "gerbil-parser/src/runtime/parser~0" + "gerbil-parser/src/runtime/incremental/session~0" + "gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar~0")))) + (eval '(def capture-frontier-fingerprint + (let (session (capture-request)) + (sha256-text + (object->string + (list (incremental-session-state-modes session) + (map (lambda (row) + (let (snapshot (vector-ref row 1)) + (list (vector-ref row 0) (lr-prefix-snapshot-states snapshot) + (lr-prefix-snapshot-deterministic-actions snapshot) + (lr-prefix-snapshot-deterministic-shifts snapshot) + (vector-ref row 2) (vector-ref row 3)))) + (vector->list (incremental-session-state-checkpoints session))))))))) + (write (list 'CAPTURE-READY (eval '(sha256-text (object->string capture-expected))) (list (eval 'capture-width-counts) (eval 'capture-metadata-counts) (eval 'capture-frontier-fingerprint)) + (map (lambda (module) (list module (image-digest module))) selected-modules) shared-images)) + (newline) (force-output) + (let (request (eval 'capture-request)) + (let loop () + (let (calls (read)) + (if (eq? calls 'stop) + (eval '(test-child-process-exit! 0)) + (begin + (unless (and (exact-integer? calls) (positive? calls)) (error "invalid CPU batch" calls)) + (let* ((before (##process-statistics)) + (_ (let repeat ((left calls)) + (unless (zero? left) (request) (repeat (- left 1))))) + (after (##process-statistics)) + (cpu (+ (- (f64vector-ref after 0) (f64vector-ref before 0)) + (- (f64vector-ref after 1) (f64vector-ref before 1))))) + (write (list 'CAPTURE-BATCH calls cpu)) (newline) (force-output)) + (loop))))))) diff --git a/t/benchmarks/contextual-scanner/deferred-scale.ss b/t/benchmarks/contextual-scanner/deferred-scale.ss new file mode 100644 index 00000000..8ae6eee4 --- /dev/null +++ b/t/benchmarks/contextual-scanner/deferred-scale.ss @@ -0,0 +1,84 @@ +#!/usr/bin/env gxi +;;; Deferred queue scale dimension; IR compilation and preparation are untimed. +(import + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-role make-contextual-method make-contextual-scan-rule) + (only-in :gerbil-parser/src/compiler/contextual-dispatch compile-contextual-dispatch) + (only-in :gerbil-parser/src/compiler/contextual-scanner-ir compile-contextual-scanner) + (only-in :gerbil-parser/src/runtime/contextual-scanner + prepare-contextual-scanner contextual-scanner-initial-state + contextual-scanner-step contextual-scan-state-canonical) + (only-in :gerbil-parser/src/runtime/token token-kind token-lexeme token-start token-end)) +(export deferred-scale-ir deferred-scale-source deferred-scale-scan main) + +(def (deferred-scale-ir) + (let* ((forms '(open word space newline body end)) + (role (make-contextual-role + 'deferred-scale + (map (lambda (form) + (make-contextual-method form 'any 'any form form)) forms))) + (dispatch (compile-contextual-dispatch + (list role) '(command body) '(command) forms)) + (rules + (list + (make-contextual-scan-rule 'open 'command 'open '(literal "<<") 20 + '(expect-marker #f)) + (make-contextual-scan-rule 'word 'command 'word '(identifier) 0 + '(enqueue-if-expecting raw)) + (make-contextual-scan-rule 'space 'command 'space + '(horizontal-whitespace+) 0 'keep) + (make-contextual-scan-rule 'newline 'command 'newline '(newline-one) 0 + '(activate-next body)) + (make-contextual-scan-rule 'end 'body 'end '(marker-line) 10 + '(finish-marker command body)) + (make-contextual-scan-rule 'body 'body 'body '(body-line) 0 'keep)))) + (compile-contextual-scanner rules dispatch 'command))) + +(def (deferred-scale-source count) + (let (markers (map (lambda (n) (string-append "M" (number->string n))) + (iota count))) + (string-append + (string-join (map (lambda (marker) (string-append "<<" marker)) markers) " ") + "\r\n" + (string-join + (map (lambda (marker) (string-append "α payload\r\n" marker "\r\n")) + markers) "")))) + +;;; Returns a complete public token trace plus canonical final checkpoint. +(def (deferred-scale-scan scanner initial step canonical) + (let loop ((state (initial scanner)) (tokens '())) + (let-values (((token next) (step scanner state 'command))) + (if token + (loop next (cons (list (token-kind token) (token-lexeme token) + (token-start token) (token-end token)) tokens)) + (list (reverse tokens) (canonical next)))))) + +(def (main . args) + (let* ((count (if (pair? args) (string->number (car args)) 512)) + (samples (if (> (length args) 1) (string->number (cadr args)) 5))) + (unless (and (integer? count) (positive? count) + (integer? samples) (positive? samples)) + (error "expected positive delimiter and sample counts")) + (let* ((source (deferred-scale-source count)) + (scanner (prepare-contextual-scanner (deferred-scale-ir) source))) + (let loop ((sample 0)) + (when (< sample samples) + (##gc) + (let* ((cpu (cpu-time)) (wall (##current-time-point)) + (result (deferred-scale-scan + scanner contextual-scanner-initial-state + contextual-scanner-step contextual-scan-state-canonical)) + (cpu-ms (* 1000 (- (cpu-time) cpu))) + (wall-ms (* 1000 (- (##current-time-point) wall)))) + (unless (and (= (length (car result)) (* 5 count)) + (equal? (string-join (map cadr (car result)) "") source) + (= (cdr (assq 'byteOffset (cadr result))) + (u8vector-length (string->utf8 source))) + (null? (cdr (assq 'pending (cadr result)))) + (not (cdr (assq 'active (cadr result))))) + (error "deferred scale lost tokens, order, or byte coverage")) + (write (list 'delimiters count 'sample sample 'cpu-ms cpu-ms + 'elapsed-ms wall-ms)) + (newline) (force-output)) + (loop (+ sample 1)))) + (displayln "DEFERRED-SCALE-OK") (force-output)))) diff --git a/t/benchmarks/contextual-scanner/marker-comparison.ss b/t/benchmarks/contextual-scanner/marker-comparison.ss new file mode 100644 index 00000000..0a2ee62a --- /dev/null +++ b/t/benchmarks/contextual-scanner/marker-comparison.ss @@ -0,0 +1,44 @@ +#!/usr/bin/env gxi +;;; Matched SDK component controls; only copied/guarded use ABBA order. +;;; Does not admit a complete-parser or separately compiled native speedup. +(import (only-in :std/string/misc string=)) +(export main) +(def (copied source marker start end) + (string=? (substring source start end) marker)) +(def (ranged source marker start end) + (and (string= source marker start end) #t)) +(def (guarded source marker start end) + (and (= (- end start) (string-length marker)) + (string=? (substring source start end) marker))) +(def (observe label compare source marker start end expected) + (##gc) + (let* ((before (##process-statistics)) (cpu (cpu-time)) + (matches (let loop ((n 1000) (count 0)) + (if (zero? n) count + (loop (- n 1) (+ count (if (compare source marker start end) 1 0)))))) + (cpu-ms (* 1000 (- (cpu-time) cpu))) + (after (##process-statistics))) + (unless (= matches (if expected 1000 0)) (error "marker comparison mismatch" label matches)) + (write (list label 'cpu-ms cpu-ms + 'gc-count (- (f64vector-ref after 6) (f64vector-ref before 6)) + 'allocation-counter-delta (- (f64vector-ref after 7) (f64vector-ref before 7)))) + (newline) (force-output))) +(def (main . _args) +(for-each + (lambda (entry) + (let* ((name (car entry)) (content (cadr entry)) (marker (caddr entry)) + (expected (cadddr entry)) (source (string-append "prefix" content "suffix"))) + (displayln "SDK-MARKER-COMPONENT " name) + (for-each + (lambda (method) (observe (car method) (cdr method) source marker 6 + (+ 6 (string-length content)) expected)) + (list (cons 'copied copied) (cons 'guarded guarded) + (cons 'guarded guarded) (cons 'copied copied) + (cons 'ranged ranged) (cons 'ranged ranged))))) + (list (list 'long-body (make-string 1024 (integer->char 955)) "EOF" #f) + (list 'equal-long (make-string 1024 (integer->char 955)) + (make-string 1024 (integer->char 955)) #t) + (list 'equal-short "EOF" "EOF" #t) + (list 'empty "" "" #t) + (list 'same-size-mismatch "EOX" "EOF" #f))) +(displayln "SDK-MARKER-COMPONENT-OK")) diff --git a/t/benchmarks/contextual-scanner/matched-parser.ss b/t/benchmarks/contextual-scanner/matched-parser.ss new file mode 100644 index 00000000..759f9e59 --- /dev/null +++ b/t/benchmarks/contextual-scanner/matched-parser.ss @@ -0,0 +1,48 @@ +#!/usr/bin/env gxi +;;; Complete ParseArtifact batches; includes one preparation per plan batch. +(import (only-in :gerbil-parser/languages/hcl/grammar hcl-parser) + (only-in :gerbil-parser/src/compiler/machine parser-machine-grammar-digest) + (only-in :gerbil-parser/src/runtime/parser + parse-source/contextual prepare-contextual-parser parse-source/contextual/prepared) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid? parse-artifact-roundtrip) + (only-in :gerbil-parser/t/benchmarks/contextual-scanner/parser-fixture contextual-product)) +(export main) +(def (main . args) + (let ((requests (if (pair? args) (string->number (car args)) 16)) + (samples (if (> (length args) 1) (string->number (cadr args)) 5))) + (unless (and (integer? requests) (positive? requests) + (integer? samples) (positive? samples)) + (error "expected positive requests and samples" args)) + (displayln "PARSER-FIXTURE requests=" requests " samples=" samples) (force-output) + (let* ((machine hcl-parser) + (product (contextual-product machine (parser-machine-grammar-digest machine))) + (sources (list "x = 1\n" (apply string-append (make-list 32 "名字 = 1\n")) + "x = " "x = @你好\n"))) + (def (run prepared?) + (##gc) + (let* ((cpu (cpu-time)) (wall (##current-time-point)) + (plan (and prepared? (prepare-contextual-parser machine product))) + (results (map (lambda (n) + (let (source (list-ref sources (modulo n 4))) + (if prepared? (parse-source/contextual/prepared plan source) + (parse-source/contextual machine product source)))) (iota requests)))) + (let ((cpu-ms (* 1000 (- (cpu-time) cpu))) + (wall-ms (* 1000 (- (##current-time-point) wall)))) + (displayln "PARSER-MEASURED prepared=" prepared? " cpu-ms=" cpu-ms) (force-output) + (values results cpu-ms wall-ms)))) + (let loop ((sample 0)) + (when (< sample (+ samples 3)) + (let-values (((left lc lw) (run (odd? sample)))) + (let-values (((right rc rw) (run (even? sample)))) + (unless (equal? left right) (error "prepared parser changed full ParseArtifact")) + (for-each (lambda (artifact n) + (unless (and (parse-artifact-valid? artifact) + (string=? (parse-artifact-roundtrip artifact) + (list-ref sources (modulo n 4)))) + (error "prepared parser broke lossless artifact admission"))) left (iota requests)) + (write (list 'requests requests 'sample sample 'warmup (< sample 3) + 'before-cpu (if (even? sample) lc rc) 'after-cpu (if (even? sample) rc lc) + 'before-wall (if (even? sample) lw rw) 'after-wall (if (even? sample) rw lw))) + (newline) (force-output))) + (loop (+ sample 1)))) + (displayln "CONTEXTUAL-PARSER-SCALE-OK") (force-output)))) diff --git a/t/benchmarks/contextual-scanner/matched-scale.ss b/t/benchmarks/contextual-scanner/matched-scale.ss new file mode 100644 index 00000000..3b0a6775 --- /dev/null +++ b/t/benchmarks/contextual-scanner/matched-scale.ss @@ -0,0 +1,159 @@ +#!/usr/bin/env gxi +;;; Source/native scanner scale receipt; optional preparation is timed, IR compilation is not. +(import + (only-in :gerbil-parser/src/runtime/contextual-scanner + prepare-contextual-scanner prepare-contextual-scanner-plan contextual-scanner-initial-state + contextual-scanner-step contextual-scan-state-byte-offset) + (only-in :gerbil-parser/src/compiler/contextual-dispatch compile-contextual-dispatch) + (only-in :gerbil-parser/src/compiler/contextual-scanner-ir compile-contextual-scanner) + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-role make-contextual-method make-contextual-scan-rule) + (only-in :gerbil-parser/src/runtime/token token-lexeme token-kind)) +(import (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation)) +(export main fixture-ir benchmark-matcher-preparation) + +(def (catalog prefix size) + (map (lambda (n) (string->symbol (string-append prefix (number->string n)))) + (iota size))) + +(def (fixture-ir axes (literals '()) (word-matcher '(identifier))) + (let* ((modes (catalog "m" axes)) + (positions (catalog "p" axes)) + (role (make-contextual-role + 'scanner-scale + (list (make-contextual-method 'word 'any 'any 'word 'word) + (make-contextual-method 'space 'any 'any 'space 'space)))) + (dispatch + (compile-contextual-dispatch (list role) modes positions '(word space))) + (rules + (apply append + (map + (lambda (mode) + (append (if (null? literals) '() + (list (make-contextual-scan-rule + (string->symbol (string-append (symbol->string mode) "-catalog")) + mode 'word (list 'literals literals) 10 'keep))) + (list + (make-contextual-scan-rule + (string->symbol (string-append (symbol->string mode) "-word")) + mode 'word word-matcher 0 'keep) + (make-contextual-scan-rule + (string->symbol (string-append (symbol->string mode) "-space")) + mode 'space '(horizontal-whitespace+) 0 'keep)))) + modes))) + (ir (compile-contextual-scanner rules dispatch (last modes)))) + (values ir (last positions)))) + +(def (fixture axes source (literals '())) + (let-values (((ir position) (fixture-ir axes literals))) + (values (prepare-contextual-scanner ir source) position))) + +(def (scan-all scanner position) + (let loop ((state (contextual-scanner-initial-state scanner)) (tokens '())) + (let-values (((token next) (contextual-scanner-step scanner state position))) + (if token (loop next (cons token tokens)) + (values (reverse tokens) (contextual-scan-state-byte-offset next)))))) + +;;; Compare revisions through this same caller; no baseline executor ships here. +;;; IR construction and product checks remain outside the preparation interval. +(def (benchmark-matcher-preparation (samples 11) (iterations 3)) + (unless (and (exact-integer? samples) (positive? samples) + (exact-integer? iterations) (positive? iterations)) + (error "matcher preparation requires positive exact sample/iteration counts")) + (let* ((literals (map (lambda (n) (string-append "word" (number->string n))) (iota 128))) + (stops (append '(" " ";") + (map (lambda (n) (string-append "stop" (number->string n))) (iota 128)))) + (region (list 'region-word + (list stops '((#\" #t ("${"))) '(("${" #\{ #\} 1)) #f))) + (plain-words '("word0" "word7" "word127" "wording")) + (region-words '("${α}" "\"β\"" "word0" "wording"))) + (for-each + (match <> + ([name axes catalog matcher words] + (let* ((expected-words (apply append (make-list 32 words))) + (source (string-join expected-words " ")) + (source-bytes (u8vector-length (string->utf8 source)))) + (let-values (((ir position) (fixture-ir axes catalog matcher))) + (let sample ((index 0)) + (when (< index samples) + (##gc) + (let* ((before (##process-statistics)) + (plan (let repeat ((left iterations) (plan #f)) + (if (zero? left) plan + (repeat (- left 1) (prepare-contextual-scanner-plan ir))))) + (after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) (f64vector-ref before slot))))) + (let-values (((tokens byte-end) (scan-all (prepare-contextual-scanner plan source) position))) + (unless (and (= byte-end source-bytes) + (= (length tokens) (- (* 2 (length expected-words)) 1)) + (equal? (map token-lexeme (filter (lambda (token) (eq? (token-kind token) 'word)) tokens)) + expected-words) + (equal? (string-join (map token-lexeme tokens) "") source)) + (error "prepared matcher product differs from declared words" name))) + (write (list 'matcher-preparation name index 'iterations iterations + 'cpu-ms-per-call (/ (* 1000 (+ (delta 0) (delta 1))) iterations) + 'allocated-bytes-per-call + (let (bytes (admit-parser-allocation (delta 7) (delta 6))) + (and bytes (/ bytes iterations))) + 'gc-count (delta 6))) + (newline) (force-output)) + (sample (+ index 1)))))))) + (list (list 'identifier-one 1 '() '(identifier) plain-words) + (list 'identifier-sixteen 16 '() '(identifier) plain-words) + (list 'literal-sixteen 16 literals '(identifier) plain-words) + (list 'region-sixteen 16 literals region region-words) + (list 'guarded-region-sixteen 16 literals + (list 'unless-prefix '("<<<") '() region) region-words)))) + (displayln "MATCHER-PREPARATION-OK") (force-output)) + +(def (main . args) + (let* ((axes (if (pair? args) (string->number (car args)) 16)) + (samples (if (> (length args) 1) (string->number (cadr args)) 20)) + (words (if (> (length args) 2) (string->number (caddr args)) 128)) + (literal-count (if (> (length args) 3) (string->number (list-ref args 3)) 0)) + (phase (if (> (length args) 4) (list-ref args 4) "scan")) + (requests (if (> (length args) 5) (string->number (list-ref args 5)) 1))) + (unless (and (integer? axes) (<= 1 axes 64) + (integer? samples) (positive? samples) + (integer? words) (positive? words) + (integer? literal-count) (<= 0 literal-count 512) + (integer? requests) (positive? requests) + (member phase '("scan" "prepare-scan" "plan-batch"))) + (error "expected axes in 1..64, positive samples/words/requests, catalog in 0..512, phase scan, prepare-scan or plan-batch" args)) + (let* ((literals (map (lambda (n) (string-append "word" (number->string n))) + (iota literal-count))) + (source (string-join (if (zero? literal-count) (make-list words "word") + (map (lambda (n) (list-ref '("word0" "word7" "word127" "wording") + (modulo n 4))) (iota words))) " "))) + (let-values (((ir position) (fixture-ir axes literals))) + (let (scanner (prepare-contextual-scanner ir source)) + (write (list (cons 'axes axes) (cons 'phase phase) (cons 'literals literal-count) (cons 'cells (* 2 axes axes)) + (cons 'tokens (- (* 2 words) 1)) (cons 'samples samples) (cons 'requests requests))) + (newline) (force-output) + (let loop ((sample 0)) + (when (< sample (+ samples 3)) + (##gc) + (let ((started (##current-time-point)) (cpu-started (cpu-time))) + (let* ((plan (and (string=? phase "plan-batch") (prepare-contextual-scanner-plan ir))) + (result + (let batch ((left requests) (result #f)) + (if (zero? left) result + (let-values (((tokens byte-end) + (scan-all (cond (plan (prepare-contextual-scanner plan source)) + ((string=? phase "prepare-scan") (prepare-contextual-scanner ir source)) + (else scanner)) position))) + (batch (- left 1) (cons tokens byte-end))))))) + (let ((tokens (car result)) (byte-end (cdr result))) + (let ((cpu-ms (* 1000.0 (- (cpu-time) cpu-started))) + (wall-ms (* 1000.0 (- (##current-time-point) started)))) + (unless (and (= (length tokens) (- (* 2 words) 1)) + (= byte-end (string-length source)) + (equal? (string-join (map token-lexeme tokens) "") source) + (= (length (filter (lambda (token) (eq? (token-kind token) 'word)) tokens)) words)) + (error "scanner scale receipt failed semantic or byte coverage")) + (when (>= sample 3) + (write (list (cons 'sample (- sample 3)) + (cons 'cpu-ms cpu-ms) (cons 'elapsed-ms wall-ms))) + (newline) (force-output)))))) + (loop (+ sample 1)))) + (display "SCANNER-SCALE-OK\n") (force-output)))))) diff --git a/t/benchmarks/contextual-scanner/parser-fixture.ss b/t/benchmarks/contextual-scanner/parser-fixture.ss new file mode 100644 index 00000000..3fd470c8 --- /dev/null +++ b/t/benchmarks/contextual-scanner/parser-fixture.ss @@ -0,0 +1,35 @@ +;;; HCL contextual product shared by integration tests and matched parse batches. +(import (only-in :gerbil-parser/src/compiler/machine parser-machine-ir) + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-method make-contextual-role make-contextual-scan-rule) + (only-in :gerbil-parser/src/compiler/contextual-parser-ir compile-contextual-parser)) +(export contextual-product) + +(def (method name form result) + (make-contextual-method name 'any 'any form result)) + +(def (rule name form matcher rank) + (make-contextual-scan-rule name 'normal form matcher rank 'keep)) + +(def (contextual-product machine grammar-digest + (positions '(line)) + (clauses '((line () ()))) + (number-literal "1")) + (let (role (make-contextual-role + 'hcl-parser-fixture + (list (method 'identifier 'identifier 'identifier) + (method 'number 'number 'number) + (method 'punctuation 'punctuation 'punctuation) + (method 'space 'space 'horizontal-whitespace) + (method 'newline 'newline 'newline)))) + (compile-contextual-parser + (parser-machine-ir machine) grammar-digest (list role) + '(normal) positions + '(identifier number punctuation space newline) + (list (rule 'identifier 'identifier '(identifier) 0) + (rule 'number 'number (list 'literal number-literal) 0) + (rule 'punctuation 'punctuation '(literal "=") 0) + (rule 'space 'space '(horizontal-whitespace+) 0) + (rule 'newline 'newline '(newline-one) 0)) + 'normal clauses))) + diff --git a/t/benchmarks/edited-publication/benchmark.ss b/t/benchmarks/edited-publication/benchmark.ss new file mode 100644 index 00000000..ca8387d8 --- /dev/null +++ b/t/benchmarks/edited-publication/benchmark.ss @@ -0,0 +1,96 @@ +#!/usr/bin/env gxi +;;; Isolate publication from actual edited roots with moved reused token leaves. +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser parse-hcl) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-event-program-enabled?) + (only-in :gerbil-parser/src/runtime/incremental make-incremental-session + parse-incremental-session make-edit apply-edit incremental-session-artifact + incremental-session-project-artifact incremental-session-publication-comparison + incremental-session-recognition-root) + (only-in :gerbil-parser/src/runtime/event-program event-program-walk) + (only-in :gerbil-parser/src/runtime/lr-parser lr-recognition-view? + lr-recognition-view-base lr-recognition-fragment? lr-recognition-fragment-children) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-events parse-artifact-valid?)) +(def (median values) + (let* ((sorted (list-sort < values)) (mid (quotient (length sorted) 2))) + (/ (+ (list-ref sorted (- mid 1)) (list-ref sorted mid)) 2.0))) +(def (measure publish family units mode) + (let (times + (map (lambda (sample) + (##gc) + (let* ((start (cpu-time)) (artifact (publish)) + (elapsed (* 1000.0 (- (cpu-time) start)))) + (unless artifact (error "publication returned no artifact")) + (write (list 'publication-sample family units mode sample 'cpu-ms elapsed)) + (newline) (force-output) + elapsed)) (iota 20))) + (list (cons 'mode mode) (cons 'samples 20) + (cons 'cpu-samples-ms times) (cons 'cpu-median-ms (median times))))) + +(def (receipt-count receipt key) + (let (entry (assq key receipt)) (if entry (cdr entry) 0))) +(def (shared-fragments old new) + (let ((original (make-hash-table-eq)) (seen (make-hash-table-eq)) (shared 0)) + (def (visit root table count?) + (let loop ((pending (list root))) + (unless (null? pending) + (let unwrap ((piece (car pending))) + (if (lr-recognition-view? piece) + (unwrap (lr-recognition-view-base piece)) + (if (or (not (lr-recognition-fragment? piece)) (hash-get table piece)) + (loop (cdr pending)) + (begin + (hash-put! table piece #t) + (when (and count? (hash-get original piece)) (set! shared (+ shared 1))) + (loop (append (lr-recognition-fragment-children piece) (cdr pending)))))))))) + (visit old original #f) (visit new seen #t) shared)) +(def (measure-edit units location operation) + (let* ((source (string-join (make-list units "a = 1\n") "")) + (index (case location ((first) 0) ((middle) (quotient units 2)) (else (- units 1)))) + (edit (make-edit (* index 6) (if (eq? operation 'delete) 6 0) + (if (eq? operation 'insert) "b = 2\n" ""))) + (session (make-incremental-session hcl-parser source #t))) + (let-values (((next receipt) (parse-incremental-session session edit))) + (let* ((artifact (incremental-session-artifact next)) + (fresh (parse-hcl (apply-edit source edit))) + (moved 0) (unmoved 0) + (shared (shared-fragments (incremental-session-recognition-root session) + (incremental-session-recognition-root next)))) + (unless (and (parse-artifact-valid? artifact) (equal? artifact fresh) + (equal? artifact (incremental-session-project-artifact next)) + (> shared 0) + (or (eq? location 'last) + (> (receipt-count receipt 'reusedRecognitionFragmentCount) 0))) + (error "edited publication lacks physical sharing, certified transfer for moved cases, or complete equivalence")) + (let-values (((control candidate code) (incremental-session-publication-comparison next))) + (event-program-walk + (lambda (op value offset delta moved?) + (when (eq? op 'token) + (if moved? (set! moved (+ moved 1)) (set! unmoved (+ unmoved 1))))) code) + (when (and (memq location '(first middle)) (= moved 0)) + (error "edited publication must contain moved reused tokens")) + (unless (and (equal? (control) artifact) (equal? (candidate) artifact)) + (error "edited publication roots differ")) + (for-each + (lambda (order) + (for-each + (lambda (mode) + (write (append + (list (cons 'workload 'edited-semantic-publication) + (cons 'family 'hcl-siblings) (cons 'input-units units) + (cons 'location location) (cons 'operation operation) + (cons 'order order) (cons 'complete-artifact-equal? #t) + (cons 'moved-program-tokens moved) (cons 'unmoved-program-tokens unmoved) + (cons 'reused-fragments (receipt-count receipt 'reusedRecognitionFragmentCount)) + (cons 'shared-fragments shared) + (cons 'events (length (parse-artifact-events artifact)))) + (measure (if (eq? mode 'canonical) control candidate) location units mode))) + (newline) (force-output)) + (if (= order 1) '(canonical unified-event) '(unified-event canonical)))) '(1 2))))))) +(def (main . args) + (parameterize ((current-lr-event-program-enabled? #t)) + (for-each (lambda (units) + (for-each (lambda (location) + (for-each (lambda (op) (measure-edit units location op)) '(insert delete))) + '(first middle last))) '(400 800))) + (displayln "EDITED-PUBLICATION-ALL-OK")) +(export main) diff --git a/t/benchmarks/event-fold-frames/README.org b/t/benchmarks/event-fold-frames/README.org new file mode 100644 index 00000000..bb798b7b --- /dev/null +++ b/t/benchmarks/event-fold-frames/README.org @@ -0,0 +1,118 @@ +#+title: Native EventFold frame benchmark + +* Boundary + +This engine-owned native executable compares dynamic string frames, explicit +string frames, and explicit byte frames using the same prefix primitives and +request-owned event publication. It includes whole-source encoding, source +traversal, frame creation, 32 prefix tests per line, and complete event output. +It is not a full Org grammar, FFI scheduler, filesystem query, or end-to-end +100ms admission gate. The dynamic route uses the current primitive defaults, +not an executable from a historical revision. A fourth, benchmark-only +generic byte walker freezes the pre-fixnum algorithm, allowing the cursor +change to be compared inside the same native process. It is not a runtime +fallback and cannot be selected by parser consumers. A fifth route borrows +absolute byte spans without creating a per-line byte vector or decoded string. + +Each batch contains 1,000, 2,000, or 10,000 distinct documents with mixed UTF-8, +CR/LF/CRLF, prefix hits, rejected boundaries, blank lines, and an unterminated +last line. Every document's complete output is compared before timing. Ten +samples balance all five route positions; every sample verifies the full event count. +Compilation and conformance checks are excluded from measured request time. + +* Native entry points + +Use the package's compiled library as =output= and supply its dependency load +path. No Rust executable, JSON exchange, or per-document process is needed. + +#+begin_src sh +just build-event-fold-frame-benchmark OUTPUT LOAD_PATH +just benchmark-event-fold-frames OUTPUT/event-fold-frame-benchmark 10 +#+end_src + +CPU and wall time are distinct. GC count and the raw allocation-counter delta +are retained; =allocated-bytes= is admitted only for a nonnegative delta with +no collection. GC-dependent raw deltas are not proven allocation totals. +Host contention and heap growth prevent a portable wall-time or GC ranking. + +* Initial observation + +Before the fixnum-cursor change, the 2,000-document CPU medians were 45.808ms +(dynamic), 21.452ms (explicit string), and 22.940ms (explicit byte). This +supports removing dynamic frame/prefix allocation, but shows that byte +traversal was not automatically the fastest route. The subsequent cursor +optimization borrows Gerbil's typed buffered-I/O and fixnum arithmetic style: +one guarded byte read per cursor, a cached next position, and fixnum cursor +operations. Bounds checks remain in place; no kernel or heap changes. + +* Same-process cursor control + +A subsequent eight-sample run rotates all four routes. Complete parity passes +for every document at every scale. CPU medians (milliseconds per batch): + +| Documents | Dynamic string | Explicit string | Generic byte reference | Fixnum byte | +|-----------+----------------+-----------------+------------------------+-------------| +| 1000 | 19.877 | 10.639 | 11.368 | 10.354 | +| 2000 | 41.654 | 21.453 | 23.271 | 20.965 | +| 10000 | 192.046 | 109.844 | 116.198 | 106.596 | + +The cursor-only CPU differences are approximately 8-10% in this diagnostic +workload. Dynamic-to-explicit differences also include prefix/view allocation +removal; neither comparison is a whole-parser throughput claim. At 10,000 +documents the generic/fixnum wall medians are 131.927/172.467ms, respectively: +the wall-time regression is retained, not promoted as a latency win. All rows +had collections, so allocation totals are not admitted from raw deltas. +Original stdout is retained as =event-fold-frame-benchmark-matched.log= next +to the local executable, with all 96 timed rows, not selected best samples. + +* Borrowed spans and functional traversal + +Generated folds now pass the immutable source buffer and absolute bounds to +helpers and predicates instead of decoding, slicing and re-encoding each +frame. Nonempty helper/slice bounds still reject UTF-8 continuation endpoints; +empty spans remain valid at any offset, matching the independent decoder. + +The engine also shares a pure =byte-span-skip= combinator: callers supply a +predicate instead of reimplementing cursor loops for whitespace, keys, ordered +list markers and list content starts. This follows the official LL(1) pattern +of named higher-order procedures around a shared traversal mechanism, not a +replacement grammar or a new macro language. Original checked reads remain. + +A ten-sample, same-process run compares all five routes, with complete output +parity at every scale. CPU medians (milliseconds per batch): + +| Documents | Dynamic string | Explicit string | Generic byte | Copied byte | Borrowed span | +|-----------+----------------+-----------------+--------------+-------------+---------------| +| 1000 | 22.414 | 12.069 | 12.373 | 11.449 | 10.949 | +| 2000 | 45.464 | 23.713 | 24.858 | 23.756 | 22.076 | +| 10000 | 215.509 | 122.228 | 132.829 | 114.004 | 106.170 | + +Borrowed spans reduce CPU approximately 4-7% against copied byte frames in this +workload; this is not a large whole-parser gain. The prefix workload does not +isolate the new skip combinator's cost, so it cannot establish that higher-order +scanning is faster than hand-written loops. All 150 rows collected garbage; +raw allocation deltas are not admitted allocation totals. Local stdout is +retained as =event-fold-frame-benchmark-borrowed.log= beside the executable. +The 100ms complete consumer gate remains a separate, unpassed qualification. + +* Native engine control caller + +=t/engine-controls.ss= compiles the existing Fold-source, regular-DFA +and parser-runtime suites into one =std/test/base= harness. This is an +alternative qualification caller, not an additional default test matrix or +runtime process framework. =just build-engine-controls= prepares and +links the caller against an already built SDK; =just test-engine-controls= +requires all three modules, all 18 cases, final =OK= and the native exit status. + +After rebasing onto =bb53704=, the generated native Fold passed 100 controls, +including decoder-oracle checks for split UTF-8 bounds and empty mid-character +spans. The native suite caller passed all 18 source-owned cases; the complete +local Just invocation took 0.789 seconds, including receipt checks. This is not +a parser-throughput measurement and excludes the cold AOT build. + +Source GXI attempts hit the unchanged five-second idle gate before harness +output, including an LR attempt using the compiled interface. Gerbil's +=tools/gxtest.ss= prepares/evaluates modules before running the harness. +The native caller removes this interpretation/loading stage, but does not +prove that the source GXI startup issue itself is repaired. Failed attempts +are excluded from successful qualification; no timeout or heap increase is used. diff --git a/t/benchmarks/event-fold-frames/benchmark.ss b/t/benchmarks/event-fold-frames/benchmark.ss new file mode 100644 index 00000000..7fb17ed7 --- /dev/null +++ b/t/benchmarks/event-fold-frames/benchmark.ss @@ -0,0 +1,119 @@ +;;; Native engine frame costs, not an Org grammar or end-to-end latency gate. +(import (only-in :gerbil-parser/src/runtime/source-lines + fold-source-lines fold-source-byte-lines fold-source-byte-spans) + (only-in :gerbil-parser/src/runtime/event-fold-lines + current-fold-line-view fold-line-prefix-boundary?) + (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation)) +(import (only-in :std/string/utf8 utf8->string)) +(export main) + +(def prefixes + (map (lambda (index) (string-append "#+key" (number->string index) ":")) (iota 32))) +(def encoded-prefixes (map string->utf8 prefixes)) + +;; Frozen pre-fixnum algorithm, benchmark-only; never a production fallback. +(def (reference-byte-lines bytes initial visit) + (let (size (u8vector-length bytes)) + (def (emit from until state) + (let (view (subu8vector bytes from until)) + (visit (utf8->string view) view from until state))) + (let loop ((start 0) (cursor 0) (state initial)) + (cond + ((= cursor size) (if (= start size) state (emit start size state))) + ((memv (u8vector-ref bytes cursor) '(10 13)) + (let (after (+ cursor + (if (and (= (u8vector-ref bytes cursor) 13) + (< (+ cursor 1) size) + (= (u8vector-ref bytes (+ cursor 1)) 10)) + 2 1))) + (loop after after (emit start after state)))) + (else (loop start (+ cursor 1) state)))))) + +(def (mask line bytes explicit? (from 0) (until (and bytes (u8vector-length bytes)))) + (let loop ((names prefixes) (encoded encoded-prefixes) (bit 1) (flags 0)) + (if (null? names) flags + (loop (cdr names) (cdr encoded) (* bit 2) + (if (if explicit? + (fold-line-prefix-boundary? line (car names) #f bytes (car encoded) from until) + (fold-line-prefix-boundary? line (car names) #f)) + (+ flags bit) flags))))) + +(def (parse text mode) + ;; Both generated routes encode the whole source for source-bounded helpers. + (let (source (string->utf8 text)) + (unless (>= (u8vector-length source) (string-length text)) + (error "invalid native frame source extent")) + (def (visit line bytes start end events) + (cons (list 'line start end (mask line bytes #t)) events)) + (reverse! + (case mode + ((borrowed-byte-spans) + (fold-source-byte-spans source '() + (lambda (start end events) + (cons (list 'line start end (mask #f source #t start end)) events)))) + ((byte-frames) (fold-source-byte-lines source '() visit)) + ((generic-byte-frames) (reference-byte-lines source '() visit)) + (else (fold-source-lines text '() + (lambda (line start end events) + (let (bytes (string->utf8 line)) + (if (eq? mode 'explicit-string-frames) + (visit line bytes start end events) + (parameterize ((current-fold-line-view (cons line bytes))) + (cons (list 'line start end (mask line #f #f)) events))))))))))) + +(def (document index) + (string-append "#+KEY1: value " (number->string index) "\r\n" + "#+key31: λ中🦀\n#+key10:not-a-boundary\r" + "ordinary UTF-8 é\r\n\nunterminated")) + +(def (positive-count text) + (let (number (string->number text)) + (unless (and (exact-integer? number) (positive? number)) + (error "benchmark count must be a positive exact integer" text)) + number)) + +(def (measure documents mode expected) + (let* ((before (##process-statistics)) (start (##current-time-point)) + (count (foldl (lambda (text sum) (+ sum (length (parse text mode)))) 0 documents)) + (wall (* 1000 (- (##current-time-point) start))) + (after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) (f64vector-ref before slot))))) + (unless (= count expected) (error "native frame benchmark lost events" mode count expected)) + (list 'wall-ms wall 'cpu-ms (* 1000 (+ (delta 0) (delta 1))) + 'gc-count (delta 6) 'allocation-counter-delta (delta 7) + 'allocated-bytes (admit-parser-allocation (delta 7) (delta 6))))) + +(def (main (samples-text "7") . sizes-text) + (let ((samples (positive-count samples-text)) + (sizes (map positive-count (if (null? sizes-text) '("1000" "2000" "10000") sizes-text)))) + (for-each + (lambda (size) + (let* ((documents (map document (iota size))) + (expected (* size 6))) + ;; Check every document outside timing. No shrinking or hit-only view. + (for-each + (lambda (text) + (let (reference (parse text 'dynamic-string-frames)) + (for-each + (lambda (mode) + (unless (equal? reference (parse text mode)) + (error "native frame benchmark semantic mismatch" mode text))) + '(explicit-string-frames generic-byte-frames byte-frames borrowed-byte-spans)))) + documents) + (displayln "FRAME-PARITY-OK documents=" size) (force-output) + (let sample ((index 0)) + (when (< index samples) + ;; Rotate order so one route is not always the first cold sample. + (let* ((modes '(dynamic-string-frames explicit-string-frames generic-byte-frames + byte-frames borrowed-byte-spans)) + (offset (modulo index (length modes)))) + (for-each + (lambda (mode) + (write (cons 'event-fold-frames + (append (list 'documents size 'sample index 'mode mode) + (measure documents mode expected)))) + (newline) (force-output)) + (append (drop modes offset) (take modes offset)))) + (sample (+ index 1)))))) + sizes)) + (displayln "FRAME-BENCHMARK-OK") (force-output)) diff --git a/t/benchmarks/gql/README.org b/t/benchmarks/gql/README.org new file mode 100644 index 00000000..93b3ee19 --- /dev/null +++ b/t/benchmarks/gql/README.org @@ -0,0 +1,564 @@ +#+title: ISO GQL performance + +Run from the repository root after the native package build: + +#+begin_src shell +just test-local gql .gerbil +just benchmark-gql .gerbil +#+end_src + +The language owns =runtime-benchmark-test.ss=, the unchanged ASP runtime +contract at =runtime/benchmark.ss=, and semantic diagnostic tests at +=benchmark-profile-test.ss=. CI runs them through test-suite with the rest +of the language tests. Both native CI jobs also execute the 40 x 100 component +and actor diagnostics, preserving sample/summary receipts in the job logs. +Benchmark support is compiled explicitly, outside the +production library catalog. + +* Read the receipts + +=runtime/matched-stages.ss= reports 40 samples of 100 calls by default: + +| Stage | Work measured | Boundary | +|---+---+---| +| global-lexing | Full lexical catalog | A control, not the directed scanner path | +| prepared-lr | LR over significant tokens captured from the public parser | Token admission is identical; no source scanning | +| artifact-publication | Source identity, lossless events and publication | Reuses the admitted recognition root | +| full-source | Actual public language entry | Includes streaming scanner, LR and publication | + +Every prepared artifact is checked against the full public result. Each +component warms up once and performs one initial GC, followed by natural GC +during the samples. Receipts include wall and CPU P50/P95 *per 100-call batch*, wall P50/P95 per parse, bytes allocated per parse, and each sample's +GC count/time. =wallP50Sample=, =wallP95Sample= and =maxWallSample= retain +CPU, GC and allocation counters from the *same wall-ranked observation*. +The signed =allocation-counter-delta= is always retained. =allocated-bytes= +is admitted only for a nonnegative delta in a no-GC interval. Allocation P50 +and P95 per parse use those admitted samples; =allocationSampleCount= reports +their count. If none qualify, both percentiles are =#f= rather than zero. +Actor batches retain process counters but do not publish caller-attributed +allocation percentiles. This is a measurement contract, not a claim that GC +is defective or that a language engine should replace Gambit's allocator. +Independent wall/CPU percentiles may select different observations and must +not be subtracted to attribute tail latency. Each sample also reports signed +=wall-minus-cpu-ms=; this can include elapsed waiting and measurement noise +(and can be negative), so it does not identify the responsible scheduler or +OS mechanism. GC time is already included in wall time and is not an extra +additive stage. Setup, output and semantic comparisons are outside the timed +batch. These component controls *must not be summed or used as exact shares of the streaming full-source time*. They do identify which components warrant +further profiling without silently switching the production parse algorithm. + +The existing admission contract remains 150ms wall P95 per 100 calls. A batch +P95 of 137ms is approximately 1.37ms per parse; it is not a single-query P95. +Neither CPU percentiles nor an average across a batch replace that gate or +measure the distribution of individual query latency. + +For unexplained full-source outliers, compile =runtime/native-timing.ss= +with =gxc -V= (Darwin needs =-ld-options "-Wl,-undefined,dynamic_lookup"=) and run its =main= through the native preload +helper. It records paired monotonic/wall clocks, CPU, GC, allocation, page +faults and voluntary/involuntary context switches for the same 40 x 100 work. +Wall minus CPU is elapsed time without process CPU consumption, not proof of +any specific scheduler cause; GC wall is not an independent additive phase. + +* Coverage and limitations + +=parser-test.ss= owns the pinned official corpus, malformed inputs and lossless +CST conformance. =query-syntax-test.ss= owns the language query surface. +=benchmark-profile-test.ss= checks exact stage equivalence with Unicode and +trivia, and rejects invalid measurement sizes and rejected source inputs. +The 40 x 100 performance workload remains the representative query from +=grammar.ss=; it does not claim a percentile across the whole official corpus. + +* Local measurement, 2026-10-04 + +Before rebasing onto the concurrent runtime changes in =6757550=, the first +native 40 x 100 diagnostic run on =638e544= produced these batch medians: + +| Component control | Wall P50 | Wall P95 | CPU P50 | Bytes/parse | +|---+----+----+----+----| +| global lexing | 1.195ms | 2.323ms | 1.157ms | 52,356 | +| prepared LR | 8.553ms | 18.916ms | 8.400ms | 136,180 | +| publication | 1.640ms | 2.820ms | 1.641ms | 44,708 | +| public full source | 12.136ms | 54.503ms | 11.983ms | about 228,647 | + +The full-source P95 sample included one GC taking 42.449ms of its 54.503ms +wall time. This identifies elapsed time inside collection in that sample; +it does not establish a Gambit defect or identify the objects being collected. +The isolated LR control has the largest CPU/allocation cost among the three +prepared controls, making recognition allocation the next profiling target. +It does not establish an additive LR share of the streamed public entry. +The maximum full-source sample was 95.279ms, with 58.364ms CPU and one +41.271ms GC: it also includes elapsed time without CPU consumption. These +are historical local receipts, not universal latency guarantees or an explanation +of the different historical 137ms/593ms outliers. + +* Paired tail observations on =74f104c= + +The Ubuntu native CI run [[https://github.com/tao3k/gerbil-parser/actions/runs/37174656471][37174656471]] +completed the unchanged admission gate at 81.257ms wall P95. Its separate +40 x 100 full-source diagnostic selected these actual observations: + +| Wall rank | Sample | Wall | CPU | GC wall | GC count | +|---+----+----+----+----+----| +| P50 | 14 | 24.873ms | 24.878ms | 0ms | 0 | +| P95 | 19 | 79.411ms | 79.387ms | 54.006ms | 1 | +| Maximum | 9 | 80.609ms | 80.595ms | 55.386ms | 1 | + +The P95 sample spends about 68% of its elapsed time in GC. This is a paired +observation, not the difference between independent percentiles. The ordinary +no-GC P50 batch allocates 22,887,376 bytes (about 228,874 per parse). Allocation +in the prepared LR control therefore remains a concrete investigation target; +these measurements do not locate an individual allocation site or establish +that changing the heap policy improves parsing. + +On the same source revision, the local diagnostic's wall P95 observation was +sample 29: 212.683ms wall, 58.673ms CPU, one GC taking 199.528ms wall. Its +maximum, sample 32, had 439.664ms wall, 29.318ms CPU and *no GC*. The latter +rules out GC as the cause of that particular maximum and demonstrates elapsed +time without comparable CPU consumption. It does not identify the waiting +mechanism. The unchanged local admission gate failed at 214.611ms P95; the +Linux pass does not replace that failure. These historical samples are tied +to =74f104c=, not measurements of subsequent diagnostic changes. + +* Standard library review: prepared index dispatch + +[[https://srfi.schemers.org/srfi-1/srfi-1.html#Rationale][SRFI-1's reference implementation guidance]] +prefers common-case fast paths, constant-space iteration and avoiding temporary +structures. The installed Gerbil v0.19 compiler's =generate-runtime-begin%= +and =generate-runtime-define-values%= similarly use explicit pair matching, +tail recursion and reverse accumulators. Existing recognition materialization +already uses Gerbil's =:std/list/list-builder=, retaining its implementation +rather than duplicating a list builder. + +The concrete change from this review is narrower: prepared LR action and goto +indexes contain only proper association lists or their prepared tables. +Lookup now dispatches with =pair?=/=null?=, instead of traversing the complete +list with =list?= before =assoc=. The association scan, equality behavior, +first-duplicate rule and original entry identity remain unchanged. This removes +one redundant traversal on list-backed queries; it does not reduce recognition +allocation or claim to solve the observed GC/waiting tail. =t/lr-index-test.ss= +covers empty, singleton, threshold and wide rows, misses, equal string keys, +and duplicate entry identity in both index representations. End-to-end cost +must still be judged from native prepared-LR and full-source samples. + +Native comparison must retain the same compiler configuration. Gerbil +=:std/make= defaults to =optimize: #t=; a plain =gxc= rebuild of a production +module does not match that configuration. Rebuild production modules with the +package graph or matching =gxc -O= settings before comparing allocation/CPU +receipts. Diagnostic support compilation alone is not production qualification. +The rebased change passed 11 native cases across the index, GQL profile and +recognition-sequence/GLR modules after compiling the affected GQL runtime +modules with optimization enabled. The separate complete local package build +was stopped by the unchanged 30-second output-idle fence after its import +closure projection; these focused passes do not establish a full package pass. + +* Empty operand-action allocation experiment + +The optimized LR module still constructed a callback capturing the byte offset +and fragment constructor before calling =foldl= on empty operand actions. +Both semantic backends now return the original value for an empty action list; +nonempty actions keep the original fold and field/alias semantics. + +The Scheme/ASP runtime owners emit matched observations directly into job logs. +Use =runtime/matched-stages.ss= with the existing benchmark contracts to compare +prepared LR and full-source costs. Result snapshots are not stored in the +language package. Generated Scheme places the null check before constructing +the fold callback; allocation savings do not imply a wall/CPU or GC guarantee. + +=lr-empty-action-test.ss= checks nullable, identity and decorated operands, +rejection equality across materialized/event backends, and edits through an +empty source against independent production replay. Together with the GQL +profile, event-program and layout modules, 27 native cases passed. The original +150ms runtime admission gate is unchanged; these diagnostics do not turn the +known local wall-time failure into admission success. + +* Optional actor library + +The package remains a library. Importing =parser-actor-support= starts no worker, +listener or daemon. The application explicitly creates and closes local actors: + +#+begin_src scheme +(import :gerbil-parser/languages/gql/parser + :gerbil-parser/parser-actor-support) +(let (worker (spawn-parser-actor gql-parser)) + (try + (parser-actor-parse worker "RETURN 1" 5) + (finally (parser-actor-stop! worker)))) +#+end_src + +This uses the installed Gerbil =:std/actor= LocalActor, envelope continuations, +request expiry and shutdown protocol, with a spawned green thread. Each actor +serializes its parse requests; applications can own multiple actors. Generated +drivers must be installed before sharing the prepared machine, which must not +be mutated while actors use it. Each synchronous submission snapshots the +caller's mutable source string. Results use the same complete ParseArtifact +as the direct API, including rejected artifacts. + +A timeout bounds the caller's wait; Gerbil discards expired queued envelopes. +It does not preempt an already running parser. Shutdown waits behind earlier +work and joins the owned thread. This API does not implement a global pool, +mailbox capacity limit, remote actor server or hard cancellation. Applications +own admission and resource limits. Green thread concurrency is not a promise +of multicore speedup or lower individual parse CPU cost. + +#+begin_src shell +just benchmark-gql-actors .gerbil +#+end_src + +=runtime/actors.ss= reports 40 batches of 100 complete requests for 1 client / +1 actor, 4 clients / 1 actor, and 4 clients / 4 actors. Actors are created before +measurement and joined afterwards. Timed work includes client spawning, +message/reply scheduling, source snapshots and parsing. Clients share one +prepared language machine; the last artifact from every client is checked +against the direct public parser. Compare these batch costs with full-source +controls; batch-normalized P95 is not an individual queued-request percentile. +=actor-test.ss= verifies concurrent reply isolation, Unicode/rejected artifact +identity, invalid-request recovery and actual shutdown. + +In the first local actor run, batch P50/P95 was 12.573/58.333ms (1 client, +1 actor), 13.216/58.192ms (4 clients, 1 actor), and 15.263/63.209ms (4 clients, +4 actors). These runs do not establish a speedup over direct parsing. + +The process allocation counter showed large isolated jumps during thread +switching (one batch reported 921,816,080 bytes while normal batches reported +about 23.6MB). Their cause is unqualified. Actor receipts preserve those raw +process deltas and label their scope =process-with-thread-switches=; they report +=allocatedBytesPerParse= as unavailable rather than assigning the jumps to +requests or presenting a misleading median as per-request memory cost. + +* Rebased runtime measurement + +After rebasing onto =1cdab80= and rebuilding with ASP v0.1.2.2, the unchanged +local runtime gate failed at 214.611ms batch P95. The diagnostic completed +40 samples and reported full-source wall P50/P95 19.782/212.683ms versus CPU +P50/P95 14.614/58.673ms. One no-GC sample took 439.664ms wall but 29.318ms CPU; +a later GC sample took 327.971ms wall, 66.641ms CPU and 287.830ms GC wall. +These are paired measurements of substantial non-CPU delay, not an identified +OS cause. The failed gate remains failed; these observations do not authorize +raising its ceiling. The earlier clean CPU/allocation controls remain useful +historical comparisons, with the runtime revision explicitly recorded. + +* Multi-operand reduction callbacks + +The generic materialized and event reducers now consume operand and semantic +value lists through private tail-recursive helpers. Offset and constructor are +explicit parameters, so each concatenating reduction avoids a captured fold +callback. Operand order, nullable reductions and =foldl2= termination behavior +are preserved; token admission and artifact publication keep their existing +owners. Generated direct reducers and selective GLR admission are unchanged. + +Use the existing 40 x 100 component diagnostics to compare allocated bytes and +paired CPU/GC observations. =lr-empty-action-test.ss= also asserts exact field +and token order for three operands, trivia and a nullable middle operand, in +addition to backend equality and incremental production replay. An allocation +change alone does not satisfy the 150ms wall P95 contract. + +* Decorated operand actions and derivation counts + +Field and alias action chains now use explicit-parameter tail recursion in both +semantic backends. This preserves =foldl1= action order and terminal identity +while removing the callback that captured offset and constructor for every +nonempty action list. The nested field/alias regression in +=t/lr-empty-action-test.ss= checks independent node and field order, trivia, +lossless publication, and equality between the materialized and event paths. + +The existing matched-stage command also emits =GQL-REDUCTION-COUNTS= before any +measured batch. For the representative query its selected derivation contains +288 reductions, 75 concatenating reductions, 192 identity operands and 132 +decorated operands. =profile-gql-reductions= verifies that enabling the observer +leaves the public artifact identical; rejected inputs produce no counts report. +Counts describe the selected derivation, not discarded GLR work or actor +requests. The observer, tree traversal, validation and logging are untimed. + +Compare no-GC allocation observations across baseline, candidate, reversal and +candidate repeat using the same native compiler, input, heap and sample counts. +Keep wall-ranked CPU/GC observations paired. Reduced allocation does not by +itself prove a latency improvement or satisfy the unchanged 150ms wall gate. + +* Investigating GC with Gambit's own counters + +GC wall time alone cannot distinguish collector CPU work from elapsed time +without CPU consumption, or identify which application objects survived. +=matched-stages.ss= therefore reports =gc-cpu-ms= alongside cumulative GC wall +and count deltas in each paired observation. After the existing initial GC it +also emits =PARSER-GC-BASELINE=, including heap, live, movable and still bytes. +The shared =t/benchmarks/parser-stage-cost/benchmark.ss= module owns component +sampling, allocation admission and GC snapshots. GQL, actor workloads and the +cross-language LR request benchmark import it directly. Component receipts use +=PARSER-COMPONENT-SAMPLE= and =PARSER-COMPONENT-SUMMARY=; GQL workload receipts +retain their language identity. No GQL sampling aliases are retained. +Each summary retains that baseline as =gcBaseline=. + +=latest-gc= is =#f= when the batch did not collect. Otherwise it reports the +*latest collection in the whole VM*, including its CPU/wall time, heap size, +allocation counter, and live/movable/still bytes. If several collections occur +in one batch, this snapshot describes only the last; the GC time/count deltas +still cover all of them. Heap and live bytes are snapshots, not differences, +and cannot be attributed solely to GQL. Loaded modules, parser tables, +prepared inputs and reference artifacts also belong to this VM. In particular, +compiled permanent objects are not equivalent to dynamically allocated live +objects, so module size alone does not measure collection work. + +The slot meanings were checked against the installed Gambit revision's +[[https://github.com/gambit/gambit/blob/dcd677cd3e40860bdd27dfbdbf5e3ce46ab03813/lib/_kernel.scm#L3999][process-statistics implementation]] +and its [[https://github.com/gambit/gambit/blob/dcd677cd3e40860bdd27dfbdbf5e3ce46ab03813/lib/mem.c][memory implementation]]. +These diagnostics leave the 1 GiB heap cap, natural timed collections, and +existing admission deadlines unchanged. Compare allocation sites and live +heap under identical input/compiler/process conditions before drawing a +runtime conclusion. Validate exact artifacts when changing representation. + +Gerbil's official guide recommends +[[https://gerbil.scheme.org/guide/getting-started.html][GCC for compiled code]] +and describes [[https://gerbil.scheme.org/guide/intro.html][full-program optimization]] +as a distinct executable build option. Record the actual compiler and build +mode for each comparison: optimized separately compiled modules (=gxc -O=) +do not by themselves establish that an executable was built with FPO. Neither +compiler advice nor these counters establish a speedup without a matched run. + +A fresh-process startup control on the same optimized module products, input, +1 GiB cap and 40 x 100 batches reproduced these full-source baseline live bytes +in two pairs: 201,709,952 with compiled-interface admission, and 117,666,952 +with runtime-module loading alone. The 84,043,000-byte difference is startup +footprint in this harness, not per-parse allocation. The runtime-only control +loads the generated native runtime wrappers and calls the same compiled entry; +it omits interface admission rather than substituting a different parser. +This identifies a concrete measurement-path contribution. It does not identify +the remaining live objects, measure an FPO executable, qualify latency under +host contention, or establish a Gambit defect. Functional source tests still +need their macro interfaces; those and runtime-only deployment are distinct +measurement contexts. + +* Counting UTF-8 bytes without encoding temporary buffers + +The scanner and incremental runtime use Gerbil's =:std/string/utf8= +=string-utf8-length= for eight length-only calls. The installed standard +implementation delegates to Gambit's =##string->utf8-length=; the +[[https://gerbil.scheme.org/reference/std/text/utf8.html][official UTF-8 documentation]] +also describes its optional start/end interval. Checkpoint validation counts +that interval directly instead of first copying the source prefix. Hashing, +byte slicing and boundary validation still use actual UTF-8 buffers where +those bytes are needed. + +A controlled native comparison against revision +=ceab4bee9e6a14f9bc85053750b1b6d06466889d= on the representative 138-byte GQL query +(51 tokens, 37 significant), with 40 batches of 100 calls per stage and a +1 GiB heap cap, produced the following median bytes per parse among batches +with no collection: + +| Implementation | Global lexing | Prepared LR | Artifact publication | Full source | +|---+----+----+----+----| +| Original | 52,356 | 111,140 | 44,324 | 203,340.32 | +| Direct length | 50,724 | 111,140 | 44,324 | 201,708.32 | +| Original restored | 52,356 | 111,140 | 44,324 | 203,340.32 | +| Direct length repeated | 50,724 | 111,140 | 44,324 | 201,708.32 | + +Each row retains 40, 39, 40 and 37 no-GC observations respectively. The four +runs used the same Gerbil 0.19 / Gambit +=dcd677cd3e40860bdd27dfbdbf5e3ce46ab03813= toolchain, optimized native module +build, input and harness; compiler optimization settings were unchanged. +The 1,632-byte reduction is about 3.1% of lexical allocation and 0.8% of +full-source allocation on this input. Prepared LR and publication serve as +unchanged stage controls. This result measures allocation, not retained heap +or a latency improvement; host contention prevented a wall-time conclusion, +and the 150ms admission gate remains unchanged. + +After rebasing onto =df43581dba835a2b044056a39e6b7c96ab83ea50=, a fresh +original/candidate pair with the same 40 x 100 configuration reproduced every +allocation median in the first two rows, including both unchanged controls. +This pair preserves the newer scanner index and native publication changes. + +Regression coverage exercises 1–4-byte scalars, NUL and combining marks, +Unicode checkpoint restoration, contextual scanning, Bash, incremental edits +and matched GQL artifacts. Reproduce the comparison with the matched-stage +command above, rebuilding each implementation before its fresh-process run. + +* Borrowing the immutable stack cell for unary LR reductions + +The private =pop-reduction= helper used to allocate a one-element source-value +list when the production width was one. Both consumers (=reduce-value= and +=reduce-value/events=) stop at the production's RHS width, so this case can +borrow the immutable semantic stack's first cell. The remaining semantic and +state suffixes are unchanged. Zero-width reductions retain their empty value +list; wider reductions still construct the required source-order list. + +Against =8535fb4ba5478e6786b06aec7d4a4efec796c888=, the same representative +138-byte input, toolchain, heap and 40 x 100 configuration used above produced +these median bytes per parse in no-GC batches: + +| Implementation | Global lexing | Prepared LR | Artifact publication | Full source | +|---+----+----+----+----| +| Original | 50,724 | 111,140 | 44,324 | 201,708.32 | +| Borrow unary cell | 50,724 | 100,916 | 44,324 | 191,484.32 | +| Original restored | 50,724 | 111,140 | 44,324 | 201,708.32 | +| Borrow unary cell repeated | 50,724 | 100,916 | 44,324 | 191,484.32 | + +Each row retains 40, 39, 40 and 37 no-GC observations respectively. This is +10,224 fewer bytes per parse: about 9.2% of prepared LR allocation and 5.1% +of full-source allocation. Lexing and publication are unchanged controls. +The native preprocessed C products confirm removal of the pair allocation +in the unary branch; compiler settings and the three-value return protocol +remain unchanged. The measured byte saving is specific to this input, execution +path and toolchain, not a portable per-pair size or a GC/latency guarantee. + +=GQL-REDUCTION-COUNTS= now also partitions the selected derivation into +=emptyReductions=, =unaryReductions= and =multiOperandReductions=. Their sum +is =reductions=. These untimed counts describe committed syntax, not every +executed reduction in failed/speculative branches; do not multiply them by +an assumed object size to infer runtime allocation. + +=t/lr-empty-action-test.ss= exercises nested unary field/alias reductions +above retained left siblings, both semantic backends, Unicode, source order +and incremental edits against independent full replay. Existing empty and +multi-operand cases remain covered. Host contention still prevents a latency +conclusion; the existing admission deadlines remain unchanged. + +* Binding reduction stacks into the consumer + +The next source-composition experiment replaces the private =pop-reduction= +procedure and its two immediately unpacking =let-values= callers with +=with-pop-reduction=. This uses Gerbil's ordinary +[[https://gerbil.scheme.org/guide/intro.html#additional-syntactic-sugar][=defrule= syntax]] +and a named =let=: the loop binds the source-order operands and remaining +immutable stack suffixes directly into the reducer body. Arguments are +evaluated once. Empty, borrowed unary and wider reductions retain their +previous values, ordering and underflow check. No compiler or GC settings +change. + +The first native A/B pair on =ba072f486e64d464764431a4a8546bc77754593f= +used the same 138-byte input, toolchain, heap and 40 x 100 configuration. +A second pair rebuilt the LR module against +=617fb7ce783d031f29571de210208df7059720c0=, retaining the same native +dependency cache for the matched controls. Both pairs produced: + +| Implementation | Global lexing | Prepared LR | Artifact publication | Full source | +|---+----+----+----+----| +| Three-value helper | 50,724 | 100,916 | 44,324 | 191,484.32 | +| Bind into consumer | 50,724 | 82,484 | 44,324 | 173,052.32 | + +These are median allocated bytes per parse in no-GC batches; each row has +40, 39, 40 and 37 observations respectively. The saving is 18,432 bytes per +parse, about 18.3% of prepared LR allocation and 9.6% of full-source +allocation. The native preprocessed C for the original helper contains +the three-value return-container allocation; that helper and its return +protocol are absent from the candidate. This is measured execution-path +evidence, not a portable object-size or selected-reduction-count formula. + +Wall and CPU samples remain paired in the raw benchmark output. Host +contention prevents attributing P50/P95 changes to this allocation reduction; +collection frequency and latency are separate questions. The existing +bounded test and benchmark deadlines remain unchanged. + +* Executed prepared LR actions outside the timed batches + +=runtime/execution-counts.ss= now emits =GQL-LR-EXECUTION= before the first +measurement. It advances the existing prepared executor by one action per +immutable checkpoint, verifies every action/shift delta, and requires the +same explicit EOF acceptance and prepared result. Publishing that result +must reproduce the directed parser's public artifact. A call that finishes +a fallback suffix cannot be admitted as a complete checkpoint trace. +Unused fork rows in the grammar do not invalidate an otherwise deterministic +visited path. No production-runtime observer or counter is added. + +On the representative 138-byte input, the receipt reports: + +| Counter | Observed value | +|---+----| +| Shift/reduce actions | 325 | +| Shifts | 37 | +| Reductions | 288 | +| Accepts | 1 | +| Non-EOF observations | 279 | +| EOF observations | 47 | +| Literal-free non-EOF observations | 0 | + +The observations include the final accept; their total is actions plus one. +For this input, executed reductions match the 288 selected-derivation +reductions. That equality is checked evidence for this path, not a general +conversion from selected syntax counts to executed or speculative work. This +is a prepared-token control, not a trace of the streaming full-source lane. +Checkpoint allocations and validation remain outside all timed batches. + +This trace also explains a withdrawn experiment against +=d5f5ea99575f0999ed9cd77a246f55e3cfc2341c=: skipping =string-upcase= on +rows without literals did not reduce allocation for this input. Native +baseline, candidate and restored-baseline phases, each 40 x 100 with the same +input/toolchain/heap, all retained these no-GC median bytes per parse: + +| Global lexing | Prepared LR | Artifact publication | Full source | +|----+----+----+----| +| 50,724 | 82,484 | 44,324 | 173,052.32 | + +Each phase retained 40, 39, 40 and 37 no-GC observations respectively. There +were no visited non-EOF literal-free rows for the guard to eliminate. The +candidate was withdrawn; no timing improvement is claimed. The diagnostic-enabled 40 x 100 control retained the same medians and +no-GC observation counts, confirming that the added trace is outside the +measured batches. + +* Withdrawn unary stack identity experiment + +The installed Gerbil 0.19 runtime's =util.ss= returns existing list tails in +=memf= and returns the original list in =remf= when no element is removed. +These structural-sharing examples motivated a parser-owned experiment using +[[https://gerbil.scheme.org/reference/gerbil/runtime/lists.html][core list primitives]] and +[[https://gerbil.scheme.org/guide/intro.html#additional-syntactic-sugar][ordinary defrule syntax]]. +Scheme pairs are mutable; this parser's semantic stack spines are immutable. + +After a successful unary pop, =remaining-values= is the original stack's +=cdr=. If the reducer returns the original =car= by =eq?=, the pushed stack +can be the entire original stack. Distinct decorated values, empty and wider +reductions retain a new head. The state stack still advances to the new goto. +The experiment replaced the three generic semantic pushes in the deterministic +and selective GLR consumers with this private macro: + +#+begin_src scheme +(defrule (push-reduction-value value count semantic-values remaining-values) + (let* ((result value) (width count) + (original semantic-values) (suffix remaining-values)) + (if (and (eqv? width 1) (eq? result (car original))) + original + (cons result suffix)))) +#+end_src + +Each argument is bound once. Native generated Scheme contains the three +expanded identity checks and no runtime helper procedure. A preceding ordinary +function variant used the same guard. Both variants were tested against +=f923edcd089034cccf0f4339d680a277ae1ed4a8=, each in baseline, candidate, +restored-baseline and repeated-candidate phases. Each phase used the same +138-byte input, native toolchain, heap and 40 x 100 component configuration. +The subsequent =4dfccd2= rebase changed documentation and Cargo README metadata; +the LR source remained byte-identical to the restored baseline. + +Both four-phase series produced these median allocated bytes per parse in +no-GC batches: + +| Implementation | Global lexing | Prepared LR | Publication | Full source | No-GC observations by stage | +|----------------+---------------+-------------+-------------+-------------+----------------------------| +| Original / restored | 50,724 | 82,484 | 44,324 | 173,052.32 | 40 / 39 / 40 / 37 | +| Candidate / repeat | 50,724 | 78,596 | 44,324 | 169,164.32 | 40 / 39 / 40 / 38 | + +The repeatable saving is 3,888 bytes per parse: 4.7% of prepared LR allocation +and 2.2% of full-source allocation. This is an execution measurement, not a +portable pair-size or selected-production-count formula. + +CPU measurements did not establish a consistent improvement. The following +are no-GC median CPU milliseconds per 100-call batch, in execution order: + +| Phase | Function prepared LR | Function full source | Macro prepared LR | Macro full source | +|-------+----------------------+----------------------+-------------------+-------------------| +| Baseline | 10.143 | 13.052 | 11.118 | 14.824 | +| Candidate | 9.086 | 14.633 | 12.290 | 16.331 | +| Restored baseline | 11.253 | 13.624 | 12.765 | 16.971 | +| Repeated candidate | 11.270 | 14.830 | 11.724 | 16.676 | + +The changing original controls prevent attributing all differences to the +candidate. These results qualify neither a CPU/wall speedup nor the cause of +a regression. Both candidates were withdrawn. The published runtime retains +the original stack push; the lower allocation row describes the experiment. +Diagnostic completion does not satisfy the unchanged 150ms wall admission +contract, and no compiler, GC or watchdog settings were changed. + +Both candidate forms passed the five native LR operand/action cases, including +Unicode, fields, aliases and both semantic backends. The restored production +implementation was rebuilt and checked again. The retained regression extends +=t/lr-empty-action-test.ss=: two edits derived from the same incremental +session must each match an independent full parse, while the first successor +and the original published artifact remain unchanged. diff --git a/t/benchmarks/gql/runtime/actors.ss b/t/benchmarks/gql/runtime/actors.ss new file mode 100644 index 00000000..c102f803 --- /dev/null +++ b/t/benchmarks/gql/runtime/actors.ss @@ -0,0 +1,47 @@ +;;; -*- Gerbil -*- +;;; In-process actor scheduling costs; uses the same prepared language machine. +(import :gerbil-parser/languages/gql/parser + (only-in :gerbil-parser/parser-actor-support + spawn-parser-actor parser-actor-parse parser-actor-stop!) + (only-in ../../parser-stage-cost/benchmark measure-parser-component)) +(export main profile-gql-actors) + +(def (profile-gql-actors source samples batch-count clients workers) + (unless (and (integer? samples) (> samples 0) + (integer? batch-count) (> batch-count 0) + (integer? clients) (<= 1 clients batch-count) + (integer? workers) (<= 1 workers clients)) + (error "invalid actor benchmark dimensions" samples batch-count clients workers)) + (let* ((expected (parse-gql source)) + (actors (list->vector + (map (lambda (_) (spawn-parser-actor gql-parser)) (iota workers))))) + (try + (measure-parser-component + (list 'actor-requests (cons 'clients clients) (cons 'workers workers)) + samples batch-count + (lambda () + (let (threads + (map + (lambda (index) + (spawn + (lambda () + (let loop ((remaining (+ (quotient batch-count clients) + (if (< index (modulo batch-count clients)) 1 0))) + (last #f)) + (if (zero? remaining) last + (loop (- remaining 1) + (parser-actor-parse + (vector-ref actors (modulo index workers)) source))))))) + (iota clients))) + (map (lambda (thread) (thread-join! thread 5)) threads))) + (make-list clients expected) batch-count 'process-with-thread-switches) + (finally (for-each parser-actor-stop! (vector->list actors)))))) + +(def (main . args) + (let ((samples (if (pair? args) (string->number (car args)) 40)) + (batch-count (if (> (length args) 1) (string->number (cadr args)) 100))) + (for-each (lambda (dimensions) + (profile-gql-actors +gql-representative-query+ + samples batch-count (car dimensions) (cadr dimensions))) + '((1 1) (4 1) (4 4))) + (displayln "GQL-ACTORS-OK") (force-output))) diff --git a/t/benchmarks/gql-runtime/benchmark.ss b/t/benchmarks/gql/runtime/benchmark.ss similarity index 100% rename from t/benchmarks/gql-runtime/benchmark.ss rename to t/benchmarks/gql/runtime/benchmark.ss diff --git a/t/benchmarks/gql/runtime/compiled-timing.ss b/t/benchmarks/gql/runtime/compiled-timing.ss new file mode 100644 index 00000000..23121e20 --- /dev/null +++ b/t/benchmarks/gql/runtime/compiled-timing.ss @@ -0,0 +1,67 @@ +;;; Native timing evidence for the unchanged 100-parse/40-sample workload. +;;; Diagnostic only: does not replace the benchmark's wall-clock gate. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process + test-child-process-exit!) + :std/ffi + (only-in :gerbil-parser/languages/gql/parser + parse-gql) + (only-in :gerbil-parser/languages/gql/parser + +gql-representative-query+) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success?)) +(export main) +(C-include "" "") +(def-C-lambda (monotonic-seconds) => double + "struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) ___result = -1.0; + else ___result = ts.tv_sec + ts.tv_nsec * 1e-9;") + +(def-C-lambda (voluntary-switches) => long + "struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage) != 0) ___result = -1; + else ___result = usage.ru_nvcsw;") +(def-C-lambda (involuntary-switches) => long + "struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage) != 0) ___result = -1; + else ___result = usage.ru_nivcsw;") + +(def (parse-batch) + (let loop ((remaining 100)) + (unless (zero? remaining) + (unless (parse-artifact-success? + (parse-gql +gql-representative-query+)) + (error "native timing query rejected")) + (loop (- remaining 1))))) + +(def (main . _) + (when (< (monotonic-seconds) 0) (error "monotonic clock unavailable")) + (parse-batch) + (##gc) + (let loop ((sample 0) (elapsed '())) + (if (= sample 40) + (begin + (write (list 'GQL-NATIVE-TIMING-OK 'samples 40 'parses-per-sample 100 + 'monotonic-max-ms (apply max elapsed))) + (newline) (force-output) + (test-child-process-exit! 0)) + (let* ((before (##process-statistics)) + (voluntary (voluntary-switches)) + (involuntary (involuntary-switches)) + (wall (time->seconds (current-time))) + (mono (monotonic-seconds))) + (parse-batch) + (let* ((mono-ms (* 1000 (- (monotonic-seconds) mono))) + (wall-ms (* 1000 (- (time->seconds (current-time)) wall))) + (after (##process-statistics)) + (delta (lambda (index) + (- (f64vector-ref after index) + (f64vector-ref before index))))) + ;; Gambit process-statistics indices are runtime cumulative counters. + (write (list 'GQL-NATIVE-SAMPLE sample 'monotonic-ms mono-ms + 'wall-ms wall-ms 'cpu-ms (* 1000 (+ (delta 0) (delta 1))) + 'gc-count (delta 6) 'gc-wall-ms (* 1000 (delta 5)) + 'allocated-bytes (delta 7) + 'minor-faults (delta 10) 'major-faults (delta 11) + 'voluntary-switches (- (voluntary-switches) voluntary) + 'involuntary-switches (- (involuntary-switches) involuntary))) + (newline) (force-output) + (loop (+ sample 1) (cons mono-ms elapsed))))))) diff --git a/t/benchmarks/gql/runtime/execution-counts.ss b/t/benchmarks/gql/runtime/execution-counts.ss new file mode 100644 index 00000000..6e6173e1 --- /dev/null +++ b/t/benchmarks/gql/runtime/execution-counts.ss @@ -0,0 +1,95 @@ +;;; Untimed diagnostics from the same prepared LR executor, one action at a time. +(import :gerbil-parser/languages/gql/parser + (only-in :gerbil-parser/src/compiler/machine parser-machine-ir parser-machine-runtime + parser-machine-grammar-digest parser-machine-trivia) + (only-in :gerbil-parser/src/compiler/lr lr-spec-ref) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-valid? parse-artifact-events + make-success-parse-artifact + token-event? token-event-token-kind token-event-lexeme event-start event-end) + (only-in :gerbil-parser/src/runtime/token make-token token-lexeme) + (only-in :gerbil-parser/src/runtime/significant parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-parse/prepared lr-initial-checkpoint lr-checkpoint-advance + lr-checkpoint-state lr-checkpoint-deterministic-actions + lr-checkpoint-deterministic-shifts lr-checkpoint-remaining-token-count)) +(export profile-gql-prepared-execution) + +(def (profile-gql-prepared-execution source) + (let* ((reference (parse-gql source)) + (runtime (parser-machine-runtime gql-parser)) + (spec (cdr (assq 'lr-spec (parser-machine-ir gql-parser)))) + (rows (lr-spec-ref spec 'actions))) + (unless (and (parse-artifact-success? reference) (parse-artifact-valid? reference)) + (error "execution diagnostics require an accepted GQL artifact")) + (let* ((tokens + (map (lambda (event) + (make-token (token-event-token-kind event) (token-event-lexeme event) + (event-start event) (event-end event))) + (filter token-event? (parse-artifact-events reference)))) + (significant (parser-significant-tokens gql-parser tokens)) + (input (list->vector significant)) + (expected (call-with-values (lambda () (lr-parse/prepared runtime significant)) list))) + (unless (and (null? (cadr expected)) + (equal? (make-success-parse-artifact + (parser-machine-grammar-digest gql-parser) + source tokens (car expected) (parser-machine-trivia gql-parser)) + reference)) + (error "prepared execution changed the directed artifact")) + (let loop ((checkpoint (lr-initial-checkpoint runtime significant)) + (non-eof 0) (eof 0) (literal-free 0) + (folds 0) (folded-tokens 0) (last-fold-remaining #f)) + (let* ((remaining (lr-checkpoint-remaining-token-count checkpoint)) + (row (vector-ref rows (lr-checkpoint-state checkpoint))) + (next-non-eof (+ non-eof (if (zero? remaining) 0 1))) + (next-eof (+ eof (if (zero? remaining) 1 0))) + (next-literal-free + (+ literal-free + (if (and (positive? remaining) + (not (any (lambda (entry) (eq? (cadar entry) 'literal)) row))) + 1 0))) + ;; Count the executor's short-circuit conversion condition using + ;; the canonical row, without performing an extra conversion. + ;; This diagnostic runs outside the timed benchmark intervals. + (fold? + (and (positive? remaining) (lr-spec-ref spec 'case-insensitive?) + (let (lexeme (token-lexeme + (vector-ref input (- (vector-length input) remaining)))) + (and (string? lexeme) + (not (any (lambda (entry) + (and (eq? (cadar entry) 'literal) + (equal? (caddar entry) lexeme))) row)))))) + (next-folds (+ folds (if fold? 1 0))) + (next-folded-tokens + (+ folded-tokens (if (and fold? (not (eqv? remaining last-fold-remaining))) 1 0))) + (next-last-fold-remaining (if fold? remaining last-fold-remaining))) + (let-values (((status next) (lr-checkpoint-advance checkpoint 1))) + (case status + ((checkpoint) + (let ((actions (- (lr-checkpoint-deterministic-actions next) + (lr-checkpoint-deterministic-actions checkpoint))) + (shifts (- (lr-checkpoint-deterministic-shifts next) + (lr-checkpoint-deterministic-shifts checkpoint)))) + (unless (and (= actions 1) (memv shifts '(0 1))) + (error "checkpoint did not execute exactly one LR action")) + (loop next next-non-eof next-eof next-literal-free + next-folds next-folded-tokens next-last-fold-remaining))) + ((accepted) + ;; Fallback may execute a whole suffix within one call. Only a + ;; checkpoint already at the explicit EOF accept entry qualifies. + (let (accept-entry (find (lambda (entry) (eq? (cadar entry) 'eof)) row)) + (unless (and (zero? remaining) accept-entry + (eq? (cadr accept-entry) 'accept) (equal? next expected)) + (error "single-action execution did not reach the same EOF acceptance"))) + (let ((actions (lr-checkpoint-deterministic-actions checkpoint)) + (shifts (lr-checkpoint-deterministic-shifts checkpoint))) + (list (cons 'scope 'prepared-lr-single-action-checkpoints) + (cons 'actions actions) (cons 'shifts shifts) + (cons 'reductions (- actions shifts)) (cons 'accepts 1) + (cons 'nonEofObservations next-non-eof) + (cons 'eofObservations next-eof) + (cons 'literalFreeNonEofObservations next-literal-free) + (cons 'caseFoldObservations next-folds) + (cons 'caseFoldTokens next-folded-tokens) + (cons 'repeatedCaseFoldObservations (- next-folds next-folded-tokens))))) + (else (error "unexpected single-action execution status" status))))))))) diff --git a/t/benchmarks/gql/runtime/matched-stages.ss b/t/benchmarks/gql/runtime/matched-stages.ss new file mode 100644 index 00000000..3af3a551 --- /dev/null +++ b/t/benchmarks/gql/runtime/matched-stages.ss @@ -0,0 +1,68 @@ +#!/usr/bin/env gxi +;;; GQL workload selection; sampling and admission belong to the shared owner. +;;; Prepared directed-token controls are not an additive source-cost breakdown. +(import (only-in ./execution-counts profile-gql-prepared-execution) + (only-in ./reduction-counts profile-gql-reductions) + (only-in ../../parser-stage-cost/benchmark measure-parser-component) + :gerbil-parser/languages/gql/parser + :gerbil-parser/src/compiler/machine + :gerbil-parser/src/runtime/lexer + :gerbil-parser/src/runtime/lr-parser + :gerbil-parser/src/runtime/significant + :gerbil-parser/src/runtime/artifact + :gerbil-parser/src/runtime/token) +(export main profile-gql-stages) + +(def (main . args) + (let ((samples (if (pair? args) (string->number (car args)) 40)) + (batch-count (if (> (length args) 1) (string->number (cadr args)) 100))) + (profile-gql-stages +gql-representative-query+ samples batch-count) + (display "GQL-STAGES-OK\n") (force-output))) + +(def (profile-gql-stages source samples batch-count) + (unless (and (integer? samples) (positive? samples) + (integer? batch-count) (positive? batch-count)) + (error "expected positive sample and batch counts" samples batch-count)) + (let* ((reference (parse-gql source)) + (tokens + (map (lambda (event) + (make-token (token-event-token-kind event) + (token-event-lexeme event) + (event-start event) (event-end event))) + (filter token-event? (parse-artifact-events reference)))) + (significant (parser-significant-tokens gql-parser tokens)) + (runtime (parser-machine-runtime gql-parser)) + (lexed (lex-source gql-parser source)) + (parsed (call-with-values + (lambda () (lr-parse/prepared runtime significant)) list))) + (unless (and (parse-artifact-success? reference) + (parse-artifact-valid? reference) (null? (cadr parsed)) + (equal? (make-success-parse-artifact + (parser-machine-grammar-digest gql-parser) + source tokens (car parsed) + (parser-machine-trivia gql-parser)) reference)) + (error "prepared GQL stages do not reproduce the directed artifact")) + ;; Observer work stays outside measurement and leaves public artifacts intact. + (write (list 'GQL-LR-EXECUTION (profile-gql-prepared-execution source))) + (newline) (force-output) + (write (list 'GQL-REDUCTION-COUNTS (profile-gql-reductions source))) + (newline) (force-output) + (write (list (cons 'sourceBytes (parse-artifact-ref reference 'sourceByteLength)) + (cons 'tokens (length tokens)) + (cons 'significantTokens (length significant)))) + (newline) (force-output) + (list + (measure-parser-component 'global-lexing samples batch-count + (lambda () (lex-source gql-parser source)) lexed) + (measure-parser-component 'prepared-lr samples batch-count + (lambda () (call-with-values + (lambda () (lr-parse/prepared runtime significant)) list)) + parsed) + (measure-parser-component 'artifact-publication samples batch-count + (lambda () (make-success-parse-artifact + (parser-machine-grammar-digest gql-parser) + source tokens (car parsed) + (parser-machine-trivia gql-parser))) reference) + (measure-parser-component 'full-source samples batch-count + (lambda () (parse-gql source)) reference) + ))) diff --git a/t/benchmarks/gql/runtime/reduction-counts.ss b/t/benchmarks/gql/runtime/reduction-counts.ss new file mode 100644 index 00000000..29220144 --- /dev/null +++ b/t/benchmarks/gql/runtime/reduction-counts.ss @@ -0,0 +1,49 @@ +;;; Untimed selected-derivation diagnostics; never part of a timed batch. +(import :gerbil-parser/languages/gql/parser + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success? parse-artifact-valid?) + (only-in :gerbil-parser/src/compiler/machine parser-machine-ir) + (only-in :gerbil-parser/src/compiler/lr lr-spec-ref production-table production-rhs production-action operand-actions) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-recognition-observer lr-recognition-fragment? lr-recognition-fragment-production-id lr-recognition-fragment-children)) +(export profile-gql-reductions) + +(def (profile-gql-reductions source) + (let* ((reference (parse-gql source)) + (root #f) + (observed (parameterize ((current-lr-recognition-observer (lambda (value) (set! root value)))) + (parse-gql source))) + (spec (cdr (assq 'lr-spec (parser-machine-ir gql-parser)))) + (table (production-table (lr-spec-ref spec 'productions))) + (reductions 0) (concatenating-reductions 0) (empty-actions 0) (decorated-operands 0) + (empty-reductions 0) (unary-reductions 0) (multi-reductions 0)) + (unless (and root (parse-artifact-success? reference) + (parse-artifact-valid? reference) (equal? reference observed)) (error "observer changed the public artifact")) + (let loop ((pending (list root))) + (when (pair? pending) + (let* ((piece (car pending)) + (fragment? (lr-recognition-fragment? piece)) + (children (if fragment? (lr-recognition-fragment-children piece) '()))) + (when fragment? + (let* ((production (vector-ref table (lr-recognition-fragment-production-id piece))) + (rhs (production-rhs production)) + (single? (and (eq? (production-action production) 'pass) + (pair? rhs) (null? (cdr rhs))))) + (unless (memq (production-action production) '(pass concat)) + (error "unexpected GQL semantic action" production)) + (set! reductions (+ reductions 1)) + (cond ((null? rhs) (set! empty-reductions (+ empty-reductions 1))) + ((null? (cdr rhs)) (set! unary-reductions (+ unary-reductions 1))) + (else (set! multi-reductions (+ multi-reductions 1)))) + (unless single? (set! concatenating-reductions (+ concatenating-reductions 1))) + (set! decorated-operands + (+ decorated-operands (length (filter (lambda (operand) (pair? (operand-actions operand))) rhs)))) + (set! empty-actions + (+ empty-actions (length (filter (lambda (operand) (null? (operand-actions operand))) rhs)))))) + (loop (append children (cdr pending)))))) + (list (cons 'scope 'selected-derivation) + (cons 'reductions reductions) + (cons 'concatenatingReductions concatenating-reductions) + (cons 'emptyReductions empty-reductions) + (cons 'unaryReductions unary-reductions) + (cons 'multiOperandReductions multi-reductions) + (cons 'identityOperands empty-actions) + (cons 'decoratedOperands decorated-operands)))) diff --git a/t/benchmarks/hcl-scale/canonical-event-benchmark.ss b/t/benchmarks/hcl-scale/canonical-event-benchmark.ss new file mode 100644 index 00000000..0576ea9c --- /dev/null +++ b/t/benchmarks/hcl-scale/canonical-event-benchmark.ss @@ -0,0 +1,81 @@ +;;; -*- Gerbil -*- +;;; Compare committed canonical HCL events with the raw event tape. +;;; Compile with gxc -O before invoking main from gxi. + +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/hcl-source + direct-parse-hcl) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success?)) + +(def (benchmark-source lines shape) + (let (basic + (call-with-output-string + (lambda (port) + (let loop ((i 0)) + (when (< i lines) + (display "key" port) + (display i port) + (display " = 1\n" port) + (loop (fx+ i 1))))))) + (if (eq? shape 'late-complex) + (string-append basic "array = [1, 2]\n") + basic))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((lines (string->number (argument 0 "1024"))) + (samples (string->number (argument 1 "20"))) + (iterations (string->number (argument 2 "10"))) + (shape (string->symbol (argument 3 "basic")))) + (unless (and (integer? lines) (positive? lines) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations) + (memq shape '(basic late-complex))) + (error "expected lines, samples, iterations, basic|late-complex" + args)) + (let* ((source (benchmark-source lines shape)) + (raw (direct-parse-hcl hcl-parser source #t #t #f)) + (canonical (direct-parse-hcl hcl-parser source #t #t #t))) + (unless (and (parse-artifact-success? raw) + (equal? raw canonical)) + (error "canonical path changed complete ParseArtifact" shape)) + (set! raw #f) + (set! canonical #f) + (def (measure canonical?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (direct-parse-hcl hcl-parser source #t #t canonical?) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'lines lines) + (cons 'shape shape) + (cons 'source-characters (string-length source)) + (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (write + (list (cons 'sample sample) + (cons 'raw-cpu-ms + (if (odd? sample) second-cpu first-cpu)) + (cons 'canonical-cpu-ms + (if (odd? sample) first-cpu second-cpu)) + (cons 'raw-elapsed-ms + (if (odd? sample) second-elapsed first-elapsed)) + (cons 'canonical-elapsed-ms + (if (odd? sample) first-elapsed second-elapsed)))) + (newline)))) + (sample-loop (fx+ sample 1))))))) +(export main) diff --git a/t/benchmarks/hcl-scale/comment-benchmark.ss b/t/benchmarks/hcl-scale/comment-benchmark.ss new file mode 100644 index 00000000..42116a4f --- /dev/null +++ b/t/benchmarks/hcl-scale/comment-benchmark.ss @@ -0,0 +1,92 @@ +;;; -*- Gerbil -*- +;;; Compare generated ASCII HCL comment lexing with ranked lexing. +;;; Compile with gxc -O before invoking main from gxi. + +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/hcl-source + direct-parse-hcl direct-lex-hcl) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success?)) + +(def (comment-source lines shape) + (call-with-output-string + (lambda (port) + (let loop ((i 0)) + (when (< i lines) + (display "key" port) + (display i port) + (case shape + ((hash) (display " = 1 # note\n" port)) + ((slash) (display " = 1 // note\n" port)) + ((block) (display " = 1 /* note */\n" port)) + ((late-unicode) + (if (= i (fx- lines 1)) + (display " = 1 # 雪\n" port) + (display " = 1 # note\n" port)))) + (loop (fx+ i 1))))))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((lines (string->number (argument 0 "1024"))) + (samples (string->number (argument 1 "20"))) + (iterations (string->number (argument 2 "10"))) + (shape (string->symbol (argument 3 "hash")))) + (unless (and (integer? lines) (positive? lines) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations) + (memq shape '(hash slash block late-unicode))) + (error "expected lines, samples, iterations, hash|slash|block|late-unicode" + args)) + (let* ((source (comment-source lines shape)) + (fast-tokens (direct-lex-hcl source)) + (ranked (direct-parse-hcl hcl-parser source #t #f)) + (candidate (direct-parse-hcl hcl-parser source #t #t))) + (unless (and (parse-artifact-success? ranked) + (equal? ranked candidate) + (if (eq? shape 'late-unicode) + (not fast-tokens) + (and fast-tokens + (equal? fast-tokens + (lex-source hcl-parser source))))) + (error "HCL comment path changed tokens or ParseArtifact")) + (set! fast-tokens #f) + (set! ranked #f) + (set! candidate #f) + (def (measure fast?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (direct-parse-hcl hcl-parser source #t fast?) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'lines lines) + (cons 'shape shape) + (cons 'source-characters (string-length source)) + (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (write + (list (cons 'sample sample) + (cons 'ranked-cpu-ms + (if (odd? sample) second-cpu first-cpu)) + (cons 'candidate-cpu-ms + (if (odd? sample) first-cpu second-cpu)) + (cons 'ranked-elapsed-ms + (if (odd? sample) second-elapsed first-elapsed)) + (cons 'candidate-elapsed-ms + (if (odd? sample) first-elapsed second-elapsed)))) + (newline)))) + (sample-loop (fx+ sample 1))))))) +(export main) diff --git a/t/benchmarks/hcl-scale/flat-block-benchmark.ss b/t/benchmarks/hcl-scale/flat-block-benchmark.ss new file mode 100644 index 00000000..290c1117 --- /dev/null +++ b/t/benchmarks/hcl-scale/flat-block-benchmark.ss @@ -0,0 +1,89 @@ +;;; -*- Gerbil -*- +;;; Compare the closed flat-block event path with generated recursive events. +;;; Compile with gxc -O before invoking main from gxi. + +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/hcl-source + direct-parse-hcl direct-lex-hcl) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success?)) + +(def (benchmark-source blocks shape) + (call-with-output-string + (lambda (port) + (let loop ((i 0)) + (when (< i blocks) + (when (eq? shape 'nested) + (display "outer" port) + (display i port) + (display " \"env\" {\n" port)) + (display "srv" port) + (display i port) + (when (memq shape '(labeled nested)) + (display " \"web\" internal" port)) + (if (eq? shape 'nested) + (display " {\nkey = 1\n}}\n" port) + (display " {\nkey = 1\nport = 2\n}\n" port)) + (loop (fx+ i 1))))))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((blocks (string->number (argument 0 "256"))) + (samples (string->number (argument 1 "20"))) + (iterations (string->number (argument 2 "10"))) + (shape (string->symbol (argument 3 "unlabeled")))) + (unless (and (integer? blocks) (positive? blocks) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations) + (memq shape '(unlabeled labeled nested))) + (error "expected blocks, samples, iterations, unlabeled|labeled|nested" + args)) + (let* ((source (benchmark-source blocks shape)) + (generic (direct-parse-hcl hcl-parser source #f)) + (specialized (direct-parse-hcl hcl-parser source #t))) + (unless (and (parse-artifact-success? generic) + (equal? generic specialized) + (equal? (direct-lex-hcl source) + (lex-source hcl-parser source))) + (error "flat-block path changed tokens or complete ParseArtifact")) + (set! generic #f) + (set! specialized #f) + (def (measure specialized?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (direct-parse-hcl hcl-parser source specialized?) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'blocks blocks) + (cons 'shape shape) + (cons 'lines (* 4 blocks)) + (cons 'source-characters (string-length source)) + (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (write + (list (cons 'sample sample) + (cons 'generic-cpu-ms + (if (odd? sample) second-cpu first-cpu)) + (cons 'specialized-cpu-ms + (if (odd? sample) first-cpu second-cpu)) + (cons 'generic-elapsed-ms + (if (odd? sample) second-elapsed first-elapsed)) + (cons 'specialized-elapsed-ms + (if (odd? sample) first-elapsed second-elapsed)))) + (newline)))) + (sample-loop (fx+ sample 1))))))) +(export main) diff --git a/t/benchmarks/hcl-scale/lexical-subset-benchmark.ss b/t/benchmarks/hcl-scale/lexical-subset-benchmark.ss new file mode 100644 index 00000000..fedc8241 --- /dev/null +++ b/t/benchmarks/hcl-scale/lexical-subset-benchmark.ss @@ -0,0 +1,71 @@ +;;; -*- Gerbil -*- +;;; Compare the generated closed HCL lexical subset with the ranked lexer. +;;; Compile this module with gxc -O before invoking main from gxi. + +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/hcl-source + direct-lex-hcl) + (only-in :gerbil-parser/src/runtime/lexer lex-source)) + +(def (basic-source lines) + (call-with-output-string + (lambda (port) + (let loop ((i 0)) + (when (< i lines) + (display "key" port) + (display i port) + (display " = 1\n" port) + (loop (fx+ i 1))))))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((lines (string->number (argument 0 "1024"))) + (samples (string->number (argument 1 "20"))) + (iterations (string->number (argument 2 "10")))) + (unless (and (integer? lines) (positive? lines) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations)) + (error "expected lines, samples, iterations" args)) + (let (source (basic-source lines)) + (let ((ranked (lex-source hcl-parser source)) + (generated (direct-lex-hcl source))) + (unless (equal? ranked generated) + (error "generated HCL lexer changed tokens"))) + (def (measure generated?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (if generated? + (direct-lex-hcl source) + (lex-source hcl-parser source)) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'lines lines) + (cons 'source-characters (string-length source)) + (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (write + (list (cons 'sample sample) + (cons 'ranked-cpu-ms + (if (odd? sample) second-cpu first-cpu)) + (cons 'generated-cpu-ms + (if (odd? sample) first-cpu second-cpu)) + (cons 'ranked-elapsed-ms + (if (odd? sample) second-elapsed first-elapsed)) + (cons 'generated-elapsed-ms + (if (odd? sample) first-elapsed second-elapsed)))) + (newline)))) + (sample-loop (fx+ sample 1))))))) +(export main) diff --git a/t/benchmarks/hcl-scale/number-benchmark.ss b/t/benchmarks/hcl-scale/number-benchmark.ss new file mode 100644 index 00000000..ab9a7d73 --- /dev/null +++ b/t/benchmarks/hcl-scale/number-benchmark.ss @@ -0,0 +1,93 @@ +;;; -*- Gerbil -*- +;;; Compare ranked and generated HCL numeric scanning in full parses. +;;; Compile with gxc -O before invoking main from gxi. + +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/hcl-source + direct-parse-hcl direct-lex-hcl) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success?)) + +(def (number-source lines shape) + (call-with-output-string + (lambda (port) + (let loop ((i 0)) + (when (< i lines) + (display "key" port) + (display i port) + (display " = " port) + (case shape + ((integer) (display "125" port)) + ((decimal) (display "1.25" port)) + ((exponent) (display "1.25e+3" port)) + ((mixed) + (case (modulo i 3) + ((0) (display "125" port)) + ((1) (display "1.25" port)) + (else (display "1.25e+3" port))))) + (newline port) + (loop (fx+ i 1))))))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((lines (string->number (argument 0 "1024"))) + (samples (string->number (argument 1 "20"))) + (iterations (string->number (argument 2 "10"))) + (shape (string->symbol (argument 3 "decimal")))) + (unless (and (integer? lines) (positive? lines) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations) + (memq shape '(integer decimal exponent mixed))) + (error "expected lines, samples, iterations, integer|decimal|exponent|mixed" + args)) + (let* ((source (number-source lines shape)) + (fast-tokens (direct-lex-hcl source)) + (ranked (direct-parse-hcl hcl-parser source #t #f)) + (candidate (direct-parse-hcl hcl-parser source #t #t))) + (unless (and fast-tokens + (equal? fast-tokens + (lex-source hcl-parser source)) + (parse-artifact-success? ranked) + (equal? ranked candidate)) + (error "HCL numeric path changed tokens or ParseArtifact" shape)) + (set! fast-tokens #f) + (set! ranked #f) + (set! candidate #f) + (def (measure fast?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (direct-parse-hcl hcl-parser source #t fast?) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'lines lines) + (cons 'shape shape) + (cons 'source-characters (string-length source)) + (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (write + (list (cons 'sample sample) + (cons 'ranked-cpu-ms + (if (odd? sample) second-cpu first-cpu)) + (cons 'candidate-cpu-ms + (if (odd? sample) first-cpu second-cpu)) + (cons 'ranked-elapsed-ms + (if (odd? sample) second-elapsed first-elapsed)) + (cons 'candidate-elapsed-ms + (if (odd? sample) first-elapsed second-elapsed)))) + (newline)))) + (sample-loop (fx+ sample 1))))))) +(export main) diff --git a/t/benchmarks/hcl-scale/quoted-string-benchmark.ss b/t/benchmarks/hcl-scale/quoted-string-benchmark.ss new file mode 100644 index 00000000..fb90d5a1 --- /dev/null +++ b/t/benchmarks/hcl-scale/quoted-string-benchmark.ss @@ -0,0 +1,86 @@ +;;; -*- Gerbil -*- +;;; Compare generated ASCII quoted-string lexing with ranked HCL lexing. +;;; Compile with gxc -O before invoking main from gxi. + +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/hcl-source + direct-parse-hcl direct-lex-hcl) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success?)) + +(def (string-source lines late-unicode?) + (call-with-output-string + (lambda (port) + (let loop ((i 0)) + (when (< i lines) + (display "key" port) + (display i port) + (if (and late-unicode? (= i (fx- lines 1))) + (display " = \"雪\"\n" port) + (display " = \"value\"\n" port)) + (loop (fx+ i 1))))))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((lines (string->number (argument 0 "1024"))) + (samples (string->number (argument 1 "20"))) + (iterations (string->number (argument 2 "10"))) + (shape (argument 3 "ascii"))) + (unless (and (integer? lines) (positive? lines) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations) + (member shape '("ascii" "late-unicode"))) + (error "expected lines, samples, iterations, ascii|late-unicode" args)) + (let* ((source (string-source lines (equal? shape "late-unicode"))) + (fast-tokens (direct-lex-hcl source)) + (ranked (direct-parse-hcl hcl-parser source #t #f)) + (candidate (direct-parse-hcl hcl-parser source #t #t))) + (unless (and (parse-artifact-success? ranked) + (equal? ranked candidate) + (if (equal? shape "ascii") + (and fast-tokens + (equal? fast-tokens + (lex-source hcl-parser source))) + (not fast-tokens))) + (error "HCL quoted-string path changed tokens or ParseArtifact")) + (set! fast-tokens #f) + (set! ranked #f) + (set! candidate #f) + (def (measure fast?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (direct-parse-hcl hcl-parser source #t fast?) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'lines lines) + (cons 'shape shape) + (cons 'source-characters (string-length source)) + (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (write + (list (cons 'sample sample) + (cons 'ranked-cpu-ms + (if (odd? sample) second-cpu first-cpu)) + (cons 'candidate-cpu-ms + (if (odd? sample) first-cpu second-cpu)) + (cons 'ranked-elapsed-ms + (if (odd? sample) second-elapsed first-elapsed)) + (cons 'candidate-elapsed-ms + (if (odd? sample) first-elapsed second-elapsed)))) + (newline)))) + (sample-loop (fx+ sample 1))))))) +(export main) diff --git a/t/benchmarks/hcl-scale/simple-attribute-benchmark.ss b/t/benchmarks/hcl-scale/simple-attribute-benchmark.ss new file mode 100644 index 00000000..636c0e35 --- /dev/null +++ b/t/benchmarks/hcl-scale/simple-attribute-benchmark.ss @@ -0,0 +1,77 @@ +;;; -*- Gerbil -*- +;;; Compare the generated HCL simple-attribute corridor with its generic route. +;;; Compile with gxc -O before invoking main from gxi. + +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/hcl-source + direct-parse-hcl) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success?)) + +(def (basic-source lines) + (call-with-output-string + (lambda (port) + (let loop ((i 0)) + (when (< i lines) + (display "key" port) + (display i port) + (display " = 1\n" port) + (loop (fx+ i 1))))))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((lines (string->number (argument 0 "1024"))) + (samples (string->number (argument 1 "20"))) + (iterations (string->number (argument 2 "10"))) + (shape (argument 3 "simple"))) + (unless (and (integer? lines) (positive? lines) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations) + (member shape '("simple" "late-complex"))) + (error "expected lines, samples, iterations, simple|late-complex" args)) + (let (source + (if (equal? shape "simple") + (basic-source lines) + (string-append (basic-source lines) "complex = [1, 2]\n"))) + (let ((generic (direct-parse-hcl hcl-parser source #f)) + (candidate (direct-parse-hcl hcl-parser source))) + (unless (and (parse-artifact-success? generic) + (equal? generic candidate)) + (error "HCL simple corridor changed complete ParseArtifact"))) + (def (measure simple?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (direct-parse-hcl hcl-parser source simple?) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'lines lines) + (cons 'shape shape) + (cons 'source-characters (string-length source)) + (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (write + (list (cons 'sample sample) + (cons 'generic-cpu-ms + (if (odd? sample) second-cpu first-cpu)) + (cons 'corridor-cpu-ms + (if (odd? sample) first-cpu second-cpu)) + (cons 'generic-elapsed-ms + (if (odd? sample) second-elapsed first-elapsed)) + (cons 'corridor-elapsed-ms + (if (odd? sample) first-elapsed second-elapsed)))) + (newline)))) + (sample-loop (fx+ sample 1))))))) +(export main) diff --git a/t/benchmarks/hcl-scale/step-benchmark.ss b/t/benchmarks/hcl-scale/step-benchmark.ss new file mode 100644 index 00000000..b88e1e0c --- /dev/null +++ b/t/benchmarks/hcl-scale/step-benchmark.ss @@ -0,0 +1,119 @@ +;;; -*- Gerbil -*- +;;; Matched AOT full-artifact HCL benchmark: indexed versus fused reductions. + +(import (only-in :gerbil-parser/languages/hcl/parser + hcl-parser parse-hcl) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-runtime parser-machine-trivia + parser-machine-grammar-digest) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-initial-checkpoint lr-checkpoint-drive) + (only-in :gerbil-parser/src/runtime/lexer scan-source-token) + (only-in :gerbil-parser/src/runtime/token token-end) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact parse-artifact-success?)) + +(def (basic-source lines) + (call-with-output-string + (lambda (port) + (let loop ((i 0)) + (when (< i lines) + (display "key" port) + (display i port) + (display " = 1\n" port) + (loop (fx+ i 1))))))) + +(def (parse-source source generated?) + (let ((machine hcl-parser) + (character-offset 0) + (byte-offset 0) + (pending-character #f) + (tokens-reversed '())) + (def (next-input mode) + (if (= character-offset (string-length source)) + #f + (let-values (((input-token next-character) + (scan-source-token machine source + character-offset byte-offset mode))) + (let (start-character character-offset) + (set! character-offset next-character) + (set! byte-offset (token-end input-token)) + (if ((parser-machine-trivia machine) input-token) + (begin + (set! tokens-reversed (cons input-token tokens-reversed)) + (next-input mode)) + (begin + (set! pending-character start-character) + input-token)))))) + (def (after-shift input-token _states _values _actions _shifts) + (set! tokens-reversed (cons input-token tokens-reversed)) + (set! pending-character #f)) + (let-values (((status payload) + (lr-checkpoint-drive + (lr-initial-checkpoint + (parser-machine-runtime machine) '()) + next-input after-shift #f + (if generated? 'installed #f)))) + (unless (and (eq? status 'accepted) + (= character-offset (string-length source)) + (not pending-character) + (null? (cadr payload))) + (error "HCL benchmark left deterministic path" status)) + (make-success-parse-artifact + (parser-machine-grammar-digest machine) + source (reverse tokens-reversed) + (car payload) (parser-machine-trivia machine))))) + +(def (main . args) + (def (argument index default) + (if (> (length args) index) (list-ref args index) default)) + (let* ((lines (string->number (argument 0 "1024"))) + (samples (string->number (argument 1 "20"))) + (iterations (string->number (argument 2 "10")))) + (unless (and (integer? lines) (positive? lines) + (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations)) + (error "expected lines, samples, iterations" args)) + (let* ((source (basic-source lines)) + (baseline (parse-source source #f)) + (generated (parse-source source #t)) + (public (parse-hcl source))) + (unless (and (parse-artifact-success? baseline) + (equal? baseline generated) + (equal? generated public)) + (error "HCL benchmark changed ParseArtifact")) + (def (measure generated?) + (##gc) + (let ((started (##current-time-point)) + (cpu-started (cpu-time))) + (let loop ((remaining iterations)) + (when (> remaining 0) + (parse-source source generated?) + (loop (fx- remaining 1)))) + (values + (/ (* 1000.0 (- (cpu-time) cpu-started)) iterations) + (/ (* 1000.0 (- (##current-time-point) started)) iterations)))) + (write (list (cons 'lines lines) + (cons 'source-characters (string-length source)) + (cons 'iterations iterations))) + (newline) + (let sample-loop ((sample -3)) + (when (< sample samples) + (let-values (((first-cpu first-elapsed) + (measure (odd? sample)))) + (let-values (((second-cpu second-elapsed) + (measure (even? sample)))) + (when (>= sample 0) + (write + (list (cons 'sample sample) + (cons 'baseline-cpu-ms + (if (odd? sample) second-cpu first-cpu)) + (cons 'generated-cpu-ms + (if (odd? sample) first-cpu second-cpu)) + (cons 'baseline-elapsed-ms + (if (odd? sample) second-elapsed first-elapsed)) + (cons 'generated-elapsed-ms + (if (odd? sample) first-elapsed second-elapsed)))) + (newline)))) + (sample-loop (fx+ sample 1))))))) +(export main) diff --git a/t/benchmarks/incremental-session/benchmark.ss b/t/benchmarks/incremental-session/benchmark.ss new file mode 100644 index 00000000..2d5e80d1 --- /dev/null +++ b/t/benchmarks/incremental-session/benchmark.ss @@ -0,0 +1,537 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Pure Scheme comparison of fresh, one-shot, and cached-LR edits. + +(import (only-in :gerbil-parser/src/runtime/observability call-with-parser-observed-phase) + (only-in :gerbil-parser/languages/arithmetic/parser + arithmetic-parser parse-arithmetic) + (only-in :gerbil-parser/languages/hcl/parser + hcl-parser parse-hcl) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-events parse-artifact-valid?) + (only-in :gerbil-parser/src/runtime/incremental + current-lr-probe-reuse-enabled? current-lr-fragment-reuse-enabled? current-lr-source-index-enabled? current-lr-lexical-plan-reuse-enabled? incremental-session-source-index incremental-session-source-modes apply-edit incremental-session-artifact + make-edit make-incremental-session incremental-session-project-artifact incremental-session-recognition-root + parse-incremental-session parse-source/incremental)) + +(import (only-in :gerbil-parser/src/runtime/lr-parser + lr-recognition-fragment? lr-recognition-fragment-children + lr-recognition-view? lr-recognition-view-base)) + +(import (only-in :gerbil-parser/src/runtime/source-index source-index-count source-index-height source-index-audit source-index->lists)) + +(import (only-in :gerbil-parser/src/compiler/machine + current-lexical-plan-sharing-enabled? parser-machine-prepare-lexer parser-machine-lexical-plans)) +(def (plan-count plans) + (length (foldl (lambda (plan found) (if (memq plan found) found (cons plan found))) '() (vector->list plans)))) +(def (measure-lexical-preparation (include-certificates? #f)) + (for-each + (lambda (row) + (let* ((machine (cdr row)) (plans (parser-machine-lexical-plans machine)) + (times (topology-samples + (lambda () + (let-values (((lexer prepared certificates) (parser-machine-prepare-lexer machine))) + (when include-certificates? (force certificates)) prepared)) 'lexical-preparation))) + (let-values (((lexer prepared certificates) (parser-machine-prepare-lexer machine))) + (write (list (cons 'workload 'lexical-plan-preparation) + (cons 'family (car row)) (cons 'samples +topology-sample-count+) + (cons 'plan-sharing-enabled? (current-lexical-plan-sharing-enabled?)) + (cons 'certificate-preparation? include-certificates?) + (cons 'lr-modes (vector-length plans)) (cons 'prepared-plans (plan-count prepared)) + (cons 'cpu-samples-ms times) (cons 'cpu-median-ms (median times)))) + (newline) (force-output)))) + (list (cons 'arithmetic arithmetic-parser) (cons 'hcl hcl-parser)))) + +(def (audit-source-index session) + (when (and (current-lr-source-index-enabled?) (incremental-session-source-modes session)) + (let (tree (incremental-session-source-index session)) + (unless (source-index-audit tree) (error "source index measures differ")) + (let-values (((tokens modes) (source-index->lists tree))) + (unless (equal? modes (incremental-session-source-modes session)) + (error "source index modes differ from captured parser stream")))))) + +(def (grammar-fragment-count session) + (let loop ((pending (list (incremental-session-recognition-root session))) (count 0)) + (if (null? pending) count + (let (piece (car pending)) + (cond ((lr-recognition-view? piece) + (loop (cons (lr-recognition-view-base piece) (cdr pending)) count)) + ((lr-recognition-fragment? piece) + (loop (append (lr-recognition-fragment-children piece) (cdr pending)) (+ count 1))) + (else (loop (cdr pending) count))))))) + +(def (samples thunk) + (map (lambda (_) + (let* ((started (##current-time-point)) + (result (thunk)) + (elapsed-ms (* 1000.0 (- (##current-time-point) started)))) + (unless result (error "benchmark returned no result")) + elapsed-ms)) + (iota 5))) + +(def (report label measurements) + (list label (cons 'best-ms (apply min measurements)) + (list 'samples-ms measurements))) + +(def (measure terms) + (let* ((source (string-join (make-list terms "001") " + ")) + (source-edit (make-edit (* (quotient terms 2) 6) 3 "0002")) + (changed-source (apply-edit source source-edit)) + (base (parse-arithmetic source)) + (session (make-incremental-session arithmetic-parser source)) + (fresh (parse-arithmetic changed-source))) + (let-values (((one-shot _one-shot-receipt) + (parse-source/incremental + arithmetic-parser source base source-edit)) + ((next cached-receipt) + (parse-incremental-session session source-edit))) + (unless (and (equal? one-shot fresh) + (equal? (incremental-session-artifact next) fresh)) + (error "incremental benchmark products differ" terms)) + (write + (list (cons 'terms terms) + (cons 'relexed-bytes + (cdr (assq 'relexedByteCount cached-receipt))) + (cons 'converged-suffix-tokens + (cdr (assq 'convergedSuffixTokenCount cached-receipt))))) + (newline)) + (write + (list (cons 'terms terms) + (cons 'bytes (string-length source)) + (report 'session-build + (samples + (lambda () + (make-incremental-session arithmetic-parser source)))) + (report 'fresh-edit + (samples (lambda () + (parse-arithmetic changed-source)))) + (report 'one-shot-edit + (samples + (lambda () + (let-values (((artifact _receipt) + (parse-source/incremental + arithmetic-parser source base source-edit))) + artifact)))) + (report 'cached-shifted-event-edit + (samples + (lambda () + (let-values (((next _receipt) + (parse-incremental-session + session source-edit))) + next)))))) + (newline))) + +(def (measure-event-reuse terms workload source-edit) + (let* ((source (string-join (make-list terms "001") " + ")) + (changed-source (apply-edit source source-edit)) + (session (make-incremental-session arithmetic-parser source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (unless (equal? (incremental-session-artifact next) + (parse-arithmetic changed-source)) + (error "event reuse benchmark products differ" workload terms)) + (write + (list (cons 'workload workload) + (cons 'terms terms) + (cons 'reused-recognition-events + (cdr (assq 'reusedRecognitionEventCount receipt))) + (report 'fresh-edit + (samples + (lambda () (parse-arithmetic changed-source)))) + (report 'cached-event-edit + (samples + (lambda () + (let-values (((edited _receipt) + (parse-incremental-session + session source-edit))) + edited)))))) + (newline)))) + +(def (measure-window terms) + (let* ((source (string-join (make-list terms "001") " + ")) + (source-edit + (make-edit (* (quotient terms 2) 6) 9 "0002 + 0003")) + (changed-source (apply-edit source source-edit)) + (session (make-incremental-session arithmetic-parser source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (unless (equal? (incremental-session-artifact next) + (parse-arithmetic changed-source)) + (error "certified window product differs" terms)) + (write + (list (cons 'workload 'multi-token-window) + (cons 'terms terms) + (cons 'reused-recognition-events + (cdr (assq 'reusedRecognitionEventCount receipt))) + (report 'fresh-edit + (samples + (lambda () (parse-arithmetic changed-source)))) + (report 'cached-window-edit + (samples + (lambda () + (let-values (((edited _receipt) + (parse-incremental-session + session source-edit))) + edited)))))) + (newline)))) + +(def (measure-hcl-window lines workload source-edit) + (let* ((source (apply string-append + (make-list lines "x = 1 /*a*/\n"))) + (changed-source (apply-edit source source-edit)) + (session (make-incremental-session hcl-parser source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (unless (and (equal? (incremental-session-artifact next) + (parse-hcl changed-source)) + (assq 'reusedRecognitionEventCount receipt)) + (error "certified HCL window product differs" workload lines)) + (write + (list (cons 'workload workload) + (cons 'lines lines) + (cons 'reused-recognition-events + (cdr (assq 'reusedRecognitionEventCount receipt))) + (report 'fresh-edit + (samples (lambda () + (parse-hcl changed-source)))) + (report 'cached-window-edit + (samples + (lambda () + (let-values (((edited _receipt) + (parse-incremental-session + session source-edit))) + edited)))))) + (newline)))) + +(def (measure-trivia-window lines) + (measure-hcl-window + lines 'hcl-trivia-token-count-change + (make-edit (+ (* (quotient lines 2) 12) 6) + 5 "/*a*/ /*b*/"))) + +(def (measure-mixed-window lines) + (measure-hcl-window + lines 'hcl-mixed-token-count-change + (make-edit (+ (* (quotient lines 2) 12) 4) + 7 "3 /*a*/ /*b*/"))) + +;; Topology-changing edits deliberately cross the event-window admission +;; boundary. Each location measures a complete fresh artifact against a cached +;; session update, retaining exact equality and inverse-edit receipts. +(def (median values) + (let* ((sorted (list-sort < values)) (n (length sorted)) (mid (quotient n 2))) + (if (odd? n) (list-ref sorted mid) + (/ (+ (list-ref sorted (- mid 1)) (list-ref sorted mid)) 2.0)))) + +(def +topology-sample-count+ + (let (count (string->number (or (getenv "GERBIL_PARSER_BENCHMARK_SAMPLES" #f) "5"))) + (unless (and (integer? count) (<= 1 count 100)) + (error "topology sample count must be an integer from 1 to 100")) + count)) + +(def (topology-samples thunk (phase 'unspecified)) + (map (lambda (index) + (##gc) + (let* ((started (cpu-time)) (result (thunk)) + (elapsed (* 1000.0 (- (cpu-time) started)))) + (unless result (error "topology benchmark returned no result")) + ;; Real sample completion, outside the measured call; no heartbeat. + (write (list 'topology-sample phase (+ index 1) 'cpu-ms elapsed)) + (newline) (force-output) + elapsed)) + (iota +topology-sample-count+))) + +(def (measure-topology terms location operation (family 'arithmetic) (capture? #f)) + (let* ((hcl? (memq family '(hcl-siblings hcl-nested-siblings))) + (nested? (eq? family 'hcl-nested-siblings)) + (line (if nested? + (string-append "group {\n" + (apply string-append (make-list 80 "value = 001\n")) "}\n") + "value = 001\n")) + (machine (if hcl? hcl-parser arithmetic-parser)) + (parse (if hcl? parse-hcl parse-arithmetic)) + (source (if hcl? (apply string-append (make-list terms line)) + (string-join (make-list terms "001") " + "))) + (index (case location ((first) 0) ((middle) (quotient terms 2)) + (else (- terms 1)))) + (start (if hcl? (* index (string-length line)) + (if (and (eq? operation 'delete) (= index (- terms 1))) + (- (* index 6) 3) (* index 6)))) + (deleted (if (eq? operation 'delete) (if hcl? (string-length line) 6) 0)) + (inserted (if (eq? operation 'insert) (if hcl? (if nested? line "other = 002\n") "002 + ") "")) + (edit (make-edit start deleted inserted)) + (changed (apply-edit source edit)) + (session (make-incremental-session machine source capture?)) + (forest-count (grammar-fragment-count session)) + (fresh (parse changed))) + (let-values (((next receipt) (parse-incremental-session session edit))) + (unless (and (parse-artifact-valid? fresh) + (equal? fresh (incremental-session-artifact next))) + (error "topology edit differs from fresh parse" terms location operation)) + (when capture? + (audit-source-index next) + (unless (equal? (begin (assert-event-backend next) (incremental-session-project-artifact next)) fresh) + (error "captured topology projection differs" terms location operation))) + (let* ((restored-text (substring source start (+ start deleted))) + (inverse (make-edit start (string-length inserted) restored-text))) + (let-values (((restored ignored) (parse-incremental-session next inverse))) + (unless (equal? (incremental-session-artifact restored) + (incremental-session-artifact session)) + (error "inverse topology edit differs from original" terms location operation)) + (when capture? + (audit-source-index restored) + (unless (equal? (begin (assert-event-backend restored) (incremental-session-project-artifact restored)) + (incremental-session-artifact session)) + (error "inverse captured projection differs" terms location operation))))) + (let* ((fresh-session (make-incremental-session machine changed capture?)) + (_ (begin + (unless (equal? fresh (incremental-session-artifact fresh-session)) + (error "fresh session changed artifact" terms location operation)) + (when capture? + (unless (equal? fresh + (incremental-session-artifact + (make-incremental-session machine changed #f))) + (error "checkpoint-only session changed artifact" terms location operation)) + (audit-source-index fresh-session) + (unless (equal? fresh (incremental-session-project-artifact fresh-session)) + (error "fresh session changed recognition projection" terms location operation))))) + (fresh-times (topology-samples (lambda () (parse changed)) 'fresh)) + (fresh-checkpoint-times + (and capture? + (topology-samples + (lambda () + (incremental-session-artifact + (make-incremental-session machine changed #f))) 'fresh-checkpoint))) + (fresh-session-times + (topology-samples + (lambda () + (incremental-session-artifact + (make-incremental-session machine changed capture?))) 'fresh-session)) + (cached-times + (topology-samples + (lambda () + (let-values (((result ignored) (parse-incremental-session session edit))) + (incremental-session-artifact result))) 'cached))) + (write + (list (cons 'workload 'significant-token-topology-change) + (cons 'family family) (cons 'input-units terms) (cons 'location location) (cons 'operation operation) + (cons 'samples +topology-sample-count+) (cons 'recognition-capture? capture?) + (cons 'complete-artifact-equal? #t) (cons 'inverse-edit-equal? #t) + (cons 'events (length (parse-artifact-events fresh))) + (cons 'initial-grammar-fragments forest-count) + (cons 'resumed-shifts (cdr (assq 'resumedSignificantTokenCount receipt))) + (cons 'remaining-significant-tokens + (cdr (assq 'remainingSignificantTokenCount receipt))) + (cons 'relexed-bytes (cdr (assq 'relexedByteCount receipt))) + (cons 'converged-suffix-tokens + (cdr (assq 'convergedSuffixTokenCount receipt))) + (cons 'reused-recognition-events + (let (entry (assq 'reusedRecognitionEventCount receipt)) + (if entry (cdr entry) 0))) + (cons 'fragment-reuse-enabled? (current-lr-fragment-reuse-enabled?)) + (cons 'probe-reuse-enabled? (current-lr-probe-reuse-enabled?)) + (cons 'certified-lexical-probe-tokens (let (entry (assq 'certifiedLexicalProbeTokenCount receipt)) (if entry (cdr entry) 0))) + (cons 'lexical-plan-reuse-enabled? (current-lr-lexical-plan-reuse-enabled?)) + (cons 'sequence-fusion-enabled? (current-recognition-sequence-fusion-enabled?)) + (cons 'event-program-enabled? (current-lr-event-program-enabled?)) + (cons 'source-index-enabled? (current-lr-source-index-enabled?)) + (cons 'source-index-token-count (source-index-count (incremental-session-source-index session))) + (cons 'source-index-height (source-index-height (incremental-session-source-index next))) + (cons 'source-index-fresh-tokens (let (entry (assq 'sourceIndexFreshTokenCount receipt)) (and entry (cdr entry)))) + (cons 'source-index-shared-tokens (let (entry (assq 'sourceIndexSharedTokenCount receipt)) (and entry (cdr entry)))) + (cons 'source-index-new-mode-chunks (let (entry (assq 'sourceIndexNewModeChunkCount receipt)) (and entry (cdr entry)))) + (cons 'source-index-new-chunks (let (entry (assq 'sourceIndexNewChunkCount receipt)) (and entry (cdr entry)))) + (cons 'reused-recognition-fragments + (let (entry (assq 'reusedRecognitionFragmentCount receipt)) + (if entry (cdr entry) 0))) + (cons 'reused-significant-tokens + (let (entry (assq 'reusedSignificantTokenCount receipt)) + (if entry (cdr entry) 0))) + (cons 'certificate-probe-bytes + (let (entry (assq 'fragmentCertificateProbeByteCount receipt)) + (if entry (cdr entry) 0))) + (cons 'reused-probe-tokens + (let (entry (assq 'fragmentProbeReuseTokenCount receipt)) + (if entry (cdr entry) 0))) + (cons 'reused-probe-bytes + (let (entry (assq 'fragmentProbeReuseByteCount receipt)) + (if entry (cdr entry) 0))) + (cons 'cursor-visited-frames + (let (entry (assq 'fragmentCursorVisitCount receipt)) + (if entry (cdr entry) 0))) + (cons 'fresh-session-artifact-equal? #t) + (cons 'fresh-checkpoint-cpu-samples-ms fresh-checkpoint-times) + (cons 'fresh-checkpoint-cpu-median-ms + (and fresh-checkpoint-times (median fresh-checkpoint-times))) + (cons 'fresh-session-cpu-samples-ms fresh-session-times) + (cons 'fresh-session-cpu-median-ms (median fresh-session-times)) + (cons 'fresh-cpu-samples-ms fresh-times) + (cons 'cached-cpu-samples-ms cached-times) + (cons 'fresh-cpu-median-ms (median fresh-times)) + (cons 'cached-cpu-median-ms (median cached-times)))) + (newline) (force-output))))) + +(def (measure-topology-size terms (family 'arithmetic) (capture? #f)) + (write (list 'topology-size terms 'phase 'semantic-preflight)) + (newline) (force-output) + (for-each + (lambda (location) + (measure-topology terms location 'insert family capture?) + (measure-topology terms location 'delete family capture?)) + '(first middle last))) + +(def (measure-history units nested?) + (let* ((line (if nested? (string-append "group {\n" (apply string-append (make-list 80 "value = 001\n")) "}\n") "value = 001\n")) + (inserted (if nested? line "other = 002\n")) + (width (string-length inserted)) (mid (* (quotient units 2) (string-length line))) + (source (apply string-append (make-list units line))) + (session (make-incremental-session hcl-parser source #t)) + (certified-probes 0) (scanner-probe-bytes 0) + (edits (list (make-edit 0 0 inserted) (make-edit (+ mid width) 0 inserted) + (make-edit 0 width "") (make-edit mid width "") + (make-edit (string-length source) 0 inserted) + (make-edit (string-length source) width "")))) + (write (list 'history-size units 'nested? nested? 'phase 'semantic-preflight)) (newline) (force-output) + (let validate ((rest edits) (current session) (text source)) + (unless (null? rest) + (let-values (((next receipt) (parse-incremental-session current (car rest)))) + (let ((certified (assq 'certifiedLexicalProbeTokenCount receipt)) + (scanned (assq 'fragmentCertificateProbeByteCount receipt))) + (when certified (set! certified-probes (+ certified-probes (cdr certified)))) + (when scanned (set! scanner-probe-bytes (+ scanner-probe-bytes (cdr scanned))))) + (let* ((changed (apply-edit text (car rest))) (fresh (parse-hcl changed))) + (unless (and (equal? fresh (incremental-session-artifact next)) + (equal? fresh (begin (assert-event-backend next) (incremental-session-project-artifact next)))) + (error "history artifact/projector differs" units (length rest))) + (audit-source-index next) + (validate (cdr rest) next changed))))) + (let (times (topology-samples + (lambda () + (let loop ((rest edits) (current session)) + (if (null? rest) (incremental-session-artifact current) + (let-values (((next receipt) (parse-incremental-session current (car rest)))) + (loop (cdr rest) next))))) 'history)) + (write (list (cons 'workload 'sequential-source-history) + (cons 'family (if nested? 'hcl-nested-siblings 'hcl-siblings)) + (cons 'input-units units) (cons 'edits 6) + (cons 'samples +topology-sample-count+) + (cons 'certified-lexical-probe-tokens certified-probes) + (cons 'certificate-probe-bytes scanner-probe-bytes) + (cons 'lexical-plan-reuse-enabled? (current-lr-lexical-plan-reuse-enabled?)) + (cons 'sequence-fusion-enabled? (current-recognition-sequence-fusion-enabled?)) + (cons 'event-program-enabled? (current-lr-event-program-enabled?)) + (cons 'source-index-enabled? (current-lr-source-index-enabled?)) + (cons 'complete-artifact-equal? #t) (cons 'inverse-edit-equal? #t) + (cons 'cached-cpu-samples-ms times) (cons 'cached-cpu-median-ms (median times)))) + (newline) (force-output)))) + +(def (main-default args) + (for-each measure + (if (null? args) '(400 800 1600) + (map string->number args))) + (for-each + (lambda (terms) + (measure-event-reuse + terms 'same-width-trivia-edit + (make-edit (- (* (quotient terms 2) 6) 1) 1 "\t")) + (measure-event-reuse + terms 'same-width-generic-token-edit + (make-edit (* (quotient terms 2) 6) 3 "002")) + (measure-window terms)) + (if (null? args) '(400 800 1600) + (map string->number args))) + (measure-trivia-window 100) + (measure-mixed-window 100)) + + +(import (only-in :gerbil-parser/src/runtime/funcs current-recognition-sequence-fusion-enabled? recognition-sequence->list) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-event-program-enabled? + lr-runtime-event-program? lr-recognition-fragment-runtime lr-recognition-fragment-value lr-recognition-fragment-executor) + (only-in :gerbil-parser/src/compiler/hcl-reductions direct-event-step) + (only-in :gerbil-parser/src/runtime/event-program event-program-value?) + (only-in :gerbil-parser/src/runtime/recognition recognition-child-value)) +(def (assert-event-backend session) + (when (current-lr-event-program-enabled?) + (let (root (let base ((piece (incremental-session-recognition-root session))) + (if (lr-recognition-view? piece) (base (lr-recognition-view-base piece)) piece))) + (unless (and root (lr-runtime-event-program? (lr-recognition-fragment-runtime root)) + (eq? (lr-recognition-fragment-executor root) direct-event-step) + (event-program-value? (recognition-child-value + (car (recognition-sequence->list (lr-recognition-fragment-value root)))))) + (error "event benchmark did not execute the event backend"))))) +(def (phase-direct n) + (call-with-parser-observed-phase #f 'component (lambda () (+ n 1)))) +(def (phase-procedure n) + (let (invoke call-with-parser-observed-phase) + (invoke #f 'component (lambda () (+ n 1))))) +(def (measure-phase-dispatch) + (for-each + (lambda (method) + (##gc) + (let* ((before (##process-statistics)) (cpu (cpu-time)) + (total (let loop ((n 0) (total 0)) + (if (= n 10000) total + (loop (+ n 1) (+ total ((cdr method) n)))))) + (cpu-ms (* 1000 (- (cpu-time) cpu))) + (after (##process-statistics))) + (unless (= total 50005000) (error "phase dispatch changed result" (car method))) + (write (list 'phase-dispatch (car method) 'calls 10000 'cpu-ms cpu-ms + 'gc-count (- (f64vector-ref after 6) (f64vector-ref before 6)) + 'allocation-counter-delta (- (f64vector-ref after 7) (f64vector-ref before 7)))) + (newline) (force-output))) + (list (cons 'procedure phase-procedure) (cons 'direct phase-direct) + (cons 'direct phase-direct) (cons 'procedure phase-procedure))) + (displayln "PHASE-DISPATCH-OK") (force-output)) + +(def (main . args) + (when (and (pair? args) (equal? (car args) "phase-dispatch")) + (measure-phase-dispatch) + (exit 0)) + (when (and (pair? args) (member (car args) '("event-program" "no-event-program"))) + (current-lr-event-program-enabled? (equal? (car args) "event-program")) + (set! args (cdr args))) + (when (and (pair? args) (member (car args) '("sequence-fusion" "no-sequence-fusion"))) + (current-recognition-sequence-fusion-enabled? (equal? (car args) "sequence-fusion")) + (set! args (cdr args))) + (when (and (pair? args) (equal? (car args) "no-plan-sharing")) + (current-lexical-plan-sharing-enabled? #f) + (set! args (cdr args))) + (when (and (pair? args) (equal? (car args) "plan-reuse")) + (current-lr-lexical-plan-reuse-enabled? #t) + (set! args (cdr args))) + (when (and (pair? args) (equal? (car args) "no-plan-reuse")) + (current-lr-lexical-plan-reuse-enabled? #f) + (set! args (cdr args))) + (when (and (pair? args) (member (car args) '("lexical-plan-prepare" "lexical-certificate-prepare"))) + (measure-lexical-preparation (equal? (car args) "lexical-certificate-prepare")) + (exit 0)) + (when (and (pair? args) (equal? (car args) "source-index")) + (current-lr-source-index-enabled? #t) + (set! args (cdr args))) + (when (and (pair? args) (equal? (car args) "no-source-index")) + (current-lr-source-index-enabled? #f) + (set! args (cdr args))) + (when (and (pair? args) (equal? (car args) "no-probe-reuse")) + (current-lr-probe-reuse-enabled? #f) + (set! args (cdr args))) + (when (and (pair? args) (equal? (car args) "no-reuse")) + (current-lr-fragment-reuse-enabled? #f) + (set! args (cdr args))) + (if (and (pair? args) (member (car args) '("history-hcl-capture" "history-hcl-nested-capture"))) + (for-each (lambda (size) (measure-history size (equal? (car args) "history-hcl-nested-capture"))) + (map string->number (cdr args))) + (if (and (pair? args) (member (car args) '("topology" "topology-hcl" "topology-capture" "topology-hcl-capture" "topology-hcl-nested-capture"))) + (let (sizes (if (null? (cdr args)) '(400 800 1600 3200) + (map string->number (cdr args)))) + (unless (every (lambda (size) (and (integer? size) (>= size 2))) sizes) + (error "topology sizes must be integers of at least two" args)) + (for-each (lambda (size) + (measure-topology-size size + (cond ((equal? (car args) "topology-hcl-nested-capture") 'hcl-nested-siblings) + ((member (car args) '("topology-hcl" "topology-hcl-capture")) 'hcl-siblings) + (else 'arithmetic)) + (if (member (car args) '("topology-capture" "topology-hcl-capture" "topology-hcl-nested-capture")) #t #f))) + sizes)) + (main-default args)))) + +(export main) diff --git a/t/benchmarks/language-abi/benchmark.ss b/t/benchmarks/language-abi/benchmark.ss new file mode 100644 index 00000000..870a150d --- /dev/null +++ b/t/benchmarks/language-abi/benchmark.ss @@ -0,0 +1,14 @@ +((benchmarkKind . scenario-e2e) + (max_total . 100ms) + (target_total . 20ms) + (regression_budget . 80ms) + (expected_over_input_budget . 5ms) + (targetRationale . "one hundred warm length-bearing native C parses, including callback, UTF-8 admission, lossless publication and owned payload replacement") + (sampleCount . 40) + (rule . GERBIL-PARSER-NATIVE-LANGUAGE) + (feature . shared-scanner-language-abi-context) + (optimizationFocus . "prepared language/scanner admission outside repeated calls; no per-call runtime, grammar or process isolation") + (inputShape . "one hundred UTF-8 record assignments per sample") + (expectedOutcome . "a 100-call warm batch stays below 100ms; startup and language registration are separately reported") + (measurementPhases collect-before collect-after policy-before policy-after assert-time-gate observe-runtime-memory) + (tags performance native ffi scanner)) diff --git a/t/benchmarks/language-abi/engine-proof.ss b/t/benchmarks/language-abi/engine-proof.ss new file mode 100644 index 00000000..929fd674 --- /dev/null +++ b/t/benchmarks/language-abi/engine-proof.ss @@ -0,0 +1,24 @@ +#!/usr/bin/env gxi +;;; Complete requests compare independent scanner/LR preparation kernels. +(port-settings-set! (current-output-port) '(buffering: #f)) +(displayln "ENGINE-CPU-BOOTSTRAP") (force-output) +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(call-with-compiled-interface-trace + (lambda () + (preload-runtime-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") + (preload-runtime-module "gerbil-parser/t/benchmarks/language-abi/utf8-cpu"))) +(def leave (eval 'gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process#test-child-process-exit!)) +(def (main (mode "combined") (unicode "false")) + (with-catch + (lambda (condition) + (write (list 'ENGINE-CPU-REJECTED (error-message condition) (error-irritants condition))) + (newline) (force-output) (leave 1)) + (lambda () + (unless (and (member mode '("scanner" "lr" "combined")) + (member unicode '("false" "true"))) (error "invalid engine CPU workload")) + (let (summary ((eval 'gerbil-parser/t/benchmarks/language-abi/utf8-cpu#benchmark-abi-utf8-cpu) + 20 256 64 (equal? unicode "true") 'records #f (string->symbol mode))) + (unless (cdr (assq 'cpu-benefit (cdr summary))) + (error "complete engine request CPU gate failed" summary))) + (displayln "ENGINE-CPU-PROOF-OK") (force-output) (leave 0)))) diff --git a/t/benchmarks/language-abi/scaled-benchmark.ss b/t/benchmarks/language-abi/scaled-benchmark.ss new file mode 100644 index 00000000..1bb69905 --- /dev/null +++ b/t/benchmarks/language-abi/scaled-benchmark.ss @@ -0,0 +1,14 @@ +((benchmarkKind . scenario-e2e) + (max_total . 1000ms) + (target_total . 100ms) + (regression_budget . 900ms) + (expected_over_input_budget . 5ms) + (targetRationale . "one warm native C parse at a declared source scale, including UTF-8 admission, lossless publication and owned payload replacement") + (sampleCount . 10) + (rule . GERBIL-PARSER-NATIVE-LANGUAGE-SCALE) + (feature . shared-scanner-language-abi-context) + (optimizationFocus . "source admission and event materialization allocation growth across complete native calls") + (inputShape . "256 or 4096 record assignments; ASCII LF and Unicode CRLF; rows and byte length accompany every sample") + (expectedOutcome . "scaled complete calls retain accepted lossless payloads with recorded CPU, GC and allocated bytes") + (measurementPhases collect-before collect-after policy-before policy-after assert-time-gate observe-runtime-memory) + (tags performance native ffi scanner scale)) diff --git a/t/benchmarks/language-abi/utf8-cpu.ss b/t/benchmarks/language-abi/utf8-cpu.ss new file mode 100644 index 00000000..d8741804 --- /dev/null +++ b/t/benchmarks/language-abi/utf8-cpu.ss @@ -0,0 +1,133 @@ +;;; Complete C ABI calls: input ownership, strict admission, parsing, GPA1 +;;; publication, C result ownership and readback all remain inside timing. +(import (only-in :gerbil-parser/src/runtime/lr-parser/prepare make-lr-action-selector)) +(import (only-in :std/vector/u8vector little u8vector-u32-ref u8vector-u32-set! u8vector-u64-set!) + (only-in :gerbil-parser/t/fixtures/shared-scanner/records records-language-grammar records-contextual-product) + (only-in :gerbil-parser/src/ffi/language-artifact-codec bind-language-abi abi-parse-binary-payload) + (only-in :gerbil-parser/src/ffi/language-abi-context language-abi-context-source-admission-set! language-abi-context-event-row-writer-set!) + (only-in :gerbil-parser/src/ffi/language-handles register-language-context! release-language!) + (only-in :gerbil-parser/t/benchmarks/source-edits/abi-call parse-source/abi!) + (only-in :gerbil-parser/t/benchmarks/ranked-literal/benchmark ranked-cpu-admission)) +(import (only-in :gerbil-parser/t/fixtures/contextual-result-selection install-reference-contextual-results!) + (only-in :gerbil-parser/t/fixtures/lr-action-selection reference-lr-action-selector) + (only-in :gerbil-parser/src/compiler/machine parser-machine-with-lr-action-selector parser-machine-runtime + parser-machine-grammar-digest install-parser-machine-direct-step!) + (only-in :gerbil-parser/src/runtime/lr-parser lr-runtime-direct-step) + (only-in :gerbil-parser/src/language/descriptor make-language-grammar + +language-grammar-schema+ language-grammar-language language-grammar-version + language-grammar-contract language-grammar-grammar language-grammar-ir + language-grammar-machine language-grammar-observability)) +(export benchmark-abi-utf8-cpu qualify-abi-utf8-cpu qualify-abi-utf8-regression qualify-abi-event-cpu) +(def (records-with-selector factory) + (let* ((base (language-grammar-machine records-language-grammar)) + (machine (parser-machine-with-lr-action-selector base factory)) + (step (lr-runtime-direct-step (parser-machine-runtime base)))) + ;; Both controls retain the identical generated reduction step, when present. + (when step + (install-parser-machine-direct-step! machine (parser-machine-grammar-digest base) step)) + (make-language-grammar +language-grammar-schema+ + (language-grammar-language records-language-grammar) + (language-grammar-version records-language-grammar) + (language-grammar-contract records-language-grammar) + (language-grammar-grammar records-language-grammar) + (language-grammar-ir records-language-grammar) machine + (language-grammar-observability records-language-grammar)))) + +;;; The former admission primitive is an independent control, not a runtime mode. +(def (reference-source-admission bytes) + (let (source (utf8->string bytes)) + (unless (equal? bytes (string->utf8 source)) + (error "native source is not canonical UTF-8")) + source)) + +;;; Independent former complete-row encoding, retained only by the benchmark. +(def (reference-event-row! payload offset tag index id start end) + (u8vector-u32-set! payload offset tag little) + (u8vector-u32-set! payload (+ offset 4) index little) + (u8vector-u64-set! payload (+ offset 8) id little) + (u8vector-u32-set! payload (+ offset 16) start little) + (u8vector-u32-set! payload (+ offset 20) end little)) +(def (benchmark-abi-utf8-cpu (groups 20) (calls 128) (lines 32) (unicode? #f) (shape 'records) (events? #f) (engine 'none)) + (unless (and (exact-integer? lines) (positive? lines)) + (error "ABI workload requires positive exact record count")) + (let* ((source (case shape + ((records) (apply string-append (make-list lines (if unicode? "α=1\n" "a=1\n")))) + ((identifier) (string-append (make-string (* 32 lines) (if unicode? #\α #\a)) "=1\n")) + ((padding) (string-append "a=" (make-string (* 32 lines) #\space) "1\n")) + (else (error "unknown complete ABI source shape" shape)))) + (bytes (string->utf8 source)) + (control-grammar (if (memq engine '(lr combined)) + (records-with-selector reference-lr-action-selector) + records-language-grammar)) + (candidate-grammar (if (memq engine '(lr combined)) + (records-with-selector + make-lr-action-selector) + records-language-grammar)) + (original (bind-language-abi control-grammar records-contextual-product)) + (candidate (bind-language-abi candidate-grammar records-contextual-product)) + (expected (abi-parse-binary-payload candidate source)) + (left-output (make-u8vector (u8vector-length expected))) + (right-output (make-u8vector (u8vector-length expected)))) + (unless (zero? (u8vector-u32-ref expected 8 little)) + (error "complete ABI CPU workload must be accepted" shape)) + (when (memq engine '(scanner combined)) (install-reference-contextual-results! original)) + (if events? + (language-abi-context-event-row-writer-set! original reference-event-row!) + (when (eq? engine 'none) + (language-abi-context-source-admission-set! original reference-source-admission))) + (let ((left (register-language-context! original)) + (right (register-language-context! candidate))) + (dynamic-wind void + (lambda () + ;; Validate accepted, rejected, empty and multibyte complete products + ;; outside timing, through both independent public C ABI handles. + (when (or events? (not (eq? engine 'none))) + (for-each + (lambda (text) + (let* ((input (string->utf8 text)) + (wanted (abi-parse-binary-payload candidate text)) + (a (make-u8vector (u8vector-length wanted))) + (b (make-u8vector (u8vector-length wanted)))) + (unless (and (zero? (parse-source/abi! left input a)) + (zero? (parse-source/abi! right input b)) + (equal? a wanted) (equal? b wanted)) + (error "complete event-row reference parity failed" text)))) + '("" "a=1\n" "α=1\r\n" "name=\"猫\"\n" "a=1 b=1\n" "a=\n" "?"))) + (def (call handle output) + (unless (zero? (parse-source/abi! handle bytes output)) + (error "complete ABI request failed")) + output) + (write (list 'ABI-UTF8-WORKLOAD 'source-bytes (u8vector-length bytes) + 'payload-bytes (u8vector-length expected) 'records (if (eq? shape 'records) lines 1) 'units lines 'unicode unicode? 'shape shape)) + (newline) (force-output) + (ranked-cpu-admission (list (cond ((not (eq? engine 'none)) (list 'complete-c-abi-engine engine)) (events? 'complete-c-abi-event-rows) (else 'complete-c-abi-utf8)) shape lines unicode?) groups calls expected + (lambda () (call left left-output)) (lambda () (call right right-output)))) + (lambda () (release-language! left) (release-language! right)))))) +(def (qualify-abi-utf8-cpu (groups 20) (calls 128) (lines 32) (unicode? #f) (shape 'records)) + (let (summary (benchmark-abi-utf8-cpu groups calls lines unicode? shape)) + (unless (cdr (assq 'cpu-benefit (cdr summary))) + (error "complete ABI UTF-8 CPU controls or benefit gate failed" summary)) + (displayln "ABI-UTF8-CPU-PROOF-OK") (force-output) + summary)) + +;;; A many-record request is a separate no-regression obligation. It must not +;;; be relabeled a speedup when strict CPU benefit admission rejects it. +(def (qualify-abi-utf8-regression (groups 20) (calls 256) (lines 64) (unicode? #f) (shape 'records)) + (let* ((summary (benchmark-abi-utf8-cpu groups calls lines unicode? shape)) + (rows (cdr summary)) + (stable? (lambda (values) (and (>= (car values) 0.90) (<= (caddr values) 1.10)))) + (before (cdr (assq 'before rows))) (after (cdr (assq 'after rows))) + (production (cdr (assq 'production rows)))) + (unless (and (stable? before) (stable? after) (<= (caddr production) 1.05)) + (error "complete ABI request regression or control gate failed" summary)) + (displayln "ABI-UTF8-CPU-REGRESSION-OK") (force-output) + summary)) + +;;; Both contexts use the same upstream decoder and full parser; only the +;;; publication primitive differs. Every timed call produces the entire GPA1. +(def (qualify-abi-event-cpu (groups 20) (calls 256) (lines 64) (unicode? #f) (shape 'records)) + (let (summary (benchmark-abi-utf8-cpu groups calls lines unicode? shape #t)) + (unless (cdr (assq 'cpu-benefit (cdr summary))) + (error "complete ABI event CPU controls or benefit gate failed" summary)) + (displayln "ABI-EVENT-CPU-PROOF-OK") (force-output) + summary)) diff --git a/t/benchmarks/language-abi/utf8-proof.ss b/t/benchmarks/language-abi/utf8-proof.ss new file mode 100644 index 00000000..445b910c --- /dev/null +++ b/t/benchmarks/language-abi/utf8-proof.ss @@ -0,0 +1,29 @@ +#!/usr/bin/env gxi +;;; Fresh Scheme VM, compiled complete ABI requests and strict CPU admission. +(port-settings-set! (current-output-port) '(buffering: #f)) +(displayln "ABI-UTF8-PROOF-BOOTSTRAP") (force-output) +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(call-with-compiled-interface-trace + (lambda () + (preload-runtime-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") + (preload-runtime-module "gerbil-parser/t/benchmarks/language-abi/utf8-cpu"))) +(def leave (eval 'gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process#test-child-process-exit!)) +(def (count text) + (let (value (string->number text)) + (unless (and (exact-integer? value) (positive? value)) + (error "ABI CPU proof requires positive exact counts" text)) + value)) +(def (main (groups "20") (calls "128") (lines "128") (shape "identifier") (unicode "false") (admission "benefit")) + (with-catch + (lambda (condition) + (write (list 'ABI-UTF8-CPU-REJECTED (error-message condition) (error-irritants condition))) + (newline) (force-output) (leave 1)) + (lambda () + (unless (member unicode '("true" "false")) (error "expected true or false" unicode)) + (unless (member admission '("benefit" "regression" "events")) (error "unknown CPU admission" admission)) + ((eval (cond ((equal? admission "events") 'gerbil-parser/t/benchmarks/language-abi/utf8-cpu#qualify-abi-event-cpu) + ((equal? admission "benefit") 'gerbil-parser/t/benchmarks/language-abi/utf8-cpu#qualify-abi-utf8-cpu) + (else 'gerbil-parser/t/benchmarks/language-abi/utf8-cpu#qualify-abi-utf8-regression))) + (count groups) (count calls) (count lines) (equal? unicode "true") (string->symbol shape)) + (leave 0)))) diff --git a/t/benchmarks/language-artifact/benchmark.ss b/t/benchmarks/language-artifact/benchmark.ss new file mode 100644 index 00000000..d2a63e7a --- /dev/null +++ b/t/benchmarks/language-artifact/benchmark.ss @@ -0,0 +1,89 @@ +;;; Complete immutable publication and authenticated artifact admission. +(import (only-in :std/io/tempfile make-temporary-file-name) + (only-in :std/list/list-builder with-list-builder) + (only-in :std/misc/ports read-all-as-u8vector) + (only-in :gerbil-parser/src/compiler/language-artifact + materialize-compiled-language-artifact/output-dirs) + (only-in :gerbil-parser/src/runtime/language-artifact + compiled-language-artifact-relative-path + load-compiled-language-artifact/roots + load-compiled-language-artifact/embedded) + (only-in :gerbil-parser/src/runtime/embedded-image pack-language-artifact-image) + (only-in :gerbil-parser/src/runtime/identity sha256-text) + (only-in :std/encoding/zlib compress) + (only-in ../parser-stage-cost/benchmark + measure-parser-component measure-parser-cpu-pairs)) +(export benchmark-language-artifact compare-language-artifact) + +(def +schema+ "gerbil-parser.language-artifact-benchmark.v1") + +;; An independent ordered wire datum; no language parser or compiler cache +;; participates in fixture construction or semantic result admission. +(def (artifact-value width) + (let (payload + (with-list-builder (push!) + (let loop ((index 0)) + (unless (= index width) + (push! (list index '中文 'λ (make-string 64 #\λ) "line\r\nvalue")) + (loop (fx+ index 1)))))) + `((schema . ,+schema+) (payload . ,payload)))) + +;; Setup creates the content using the original text specification. Timed warm +;; publication verifies that file; both loaders admit the complete expected datum. +;; File and image equality are checked again after all timed work. +(def (call-with-artifact-control width proc) + (let* ((value (artifact-value width)) + (text (call-with-output-string (lambda (port) (write value port)))) + (digest (sha256-text text)) + (locator (list (compiled-language-artifact-relative-path digest) digest)) + (compressed (compress (string->utf8 text) compression: 9)) + (image (pack-language-artifact-image compressed)) + (root (make-temporary-file-name "gerbil-parser-artifact-benchmark")) + (path (path-expand (car locator) root))) + (try + (create-directory* (path-directory path)) + (call-with-output-file path + (lambda (port) (write-subu8vector compressed 0 (u8vector-length compressed) port))) + (let (result (proc value locator image root)) + (unless (equal? (call-with-input-file path read-all-as-u8vector) compressed) + (error "artifact benchmark modified immutable input")) + result) + (finally (when (file-exists? root) (delete-file-or-directory root #t)))))) + +(def (warm-roundtrip value root publish load) + (let (locator (publish value (list root))) + (list locator (load +schema+ locator (list root))))) + +(def (benchmark-language-artifact (samples 20) (calls 5) (width 512) + (publish materialize-compiled-language-artifact/output-dirs) + (load load-compiled-language-artifact/roots) + (embedded load-compiled-language-artifact/embedded)) + (call-with-artifact-control width + (lambda (value locator image root) + (let ((roundtrip (lambda () (warm-roundtrip value root publish load))) + (admit (lambda () (embedded +schema+ locator image)))) + (measure-parser-component (list 'artifact-warm-roundtrip width) samples calls + roundtrip (list locator value)) + (measure-parser-component (list 'artifact-embedded-admission width) samples calls + admit value) + (displayln "LANGUAGE-ARTIFACT-BENCHMARK-OK"))))) + +;;; Original owners are experiment inputs, never production fallback routes. +(def (compare-language-artifact original-publish original-load original-embedded + (groups 20) (calls 20) (width 512)) + (call-with-artifact-control width + (lambda (value locator image root) + (let ((roundtrip-ratios + (measure-parser-cpu-pairs 'language-artifact (list 'warm-roundtrip width) + groups calls (list locator value) + (lambda () (warm-roundtrip value root original-publish original-load)) + (lambda () (warm-roundtrip value root materialize-compiled-language-artifact/output-dirs + load-compiled-language-artifact/roots)))) + (admission-ratios + (measure-parser-cpu-pairs 'language-artifact (list 'embedded-admission width) + groups calls value + (lambda () (original-embedded +schema+ locator image)) + (lambda () (load-compiled-language-artifact/embedded +schema+ locator image))))) + (write (list 'LANGUAGE-ARTIFACT-CPU-RATIOS width roundtrip-ratios admission-ratios)) + (newline) (force-output) + (displayln "LANGUAGE-ARTIFACT-COMPARISON-OK"))))) diff --git a/t/benchmarks/layout-columns/benchmark.ss b/t/benchmarks/layout-columns/benchmark.ss new file mode 100644 index 00000000..a1913ee3 --- /dev/null +++ b/t/benchmarks/layout-columns/benchmark.ss @@ -0,0 +1,100 @@ +;;; Source preparation and complete GLR products use the shared batch sampler. +(import (only-in ../../fixtures/layout-columns reference-layout-columns reference-packed-layout-columns) + (only-in ../parser-stage-cost/benchmark measure-parser-component measure-parser-cpu-pairs) + (only-in :gerbil-parser/src/runtime/layout + make-layout-columns current-layout-columns current-layout-frames layout-token-column) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/significant parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse/prepared/receipt) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-runtime parser-machine-grammar-digest parser-machine-trivia) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact parse-artifact-valid-for-source?) + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-layout-language-grammar) + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine)) +(export benchmark-layout-columns benchmark-layout-request-cpu qualify-layout-request-cpu) + +(def (prepared-layout-product machine source columns) + (let* ((tokens (lex-source machine source)) + (significant (parser-significant-tokens machine tokens))) + (parameterize ((current-layout-columns (columns source)) (current-layout-frames '())) + (let-values (((root rest _receipt) + (lr-parse/prepared/receipt (parser-machine-runtime machine) significant))) + (unless (null? rest) (error "layout benchmark left trailing tokens")) + (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens root (parser-machine-trivia machine)))))) + +(def (benchmark-layout-columns (samples 20) (batch-count 10) (padding 32768)) + (let* ((machine (language-grammar-machine tla-plus-layout-language-grammar)) + (source (string-append "---- MODULE J ----\nInit ==\n /\\ TRUE" + (make-string padding #\space) "\n /\\ FALSE\n====\n")) + (expected (prepared-layout-product machine source reference-layout-columns))) + (unless (parse-artifact-valid-for-source? expected source) + (error "layout benchmark requires a complete admitted product")) + (for-each + (lambda (variant) + (let* ((name (car variant)) (constructor (cdr variant)) + (end (- (vector-length (reference-layout-columns source)) 1)) + (probe (make-token 'probe "" end end)) + (column (vector-ref (reference-layout-columns source) end))) + (measure-parser-component (list name 'columns) samples batch-count + (lambda () (parameterize ((current-layout-columns (constructor source))) + (layout-token-column probe))) column) + (measure-parser-component (list name 'prepared-glr-product) samples batch-count + (lambda () (prepared-layout-product machine source constructor)) expected))) + (list (cons 'reference reference-layout-columns) (cons 'production make-layout-columns))) + (displayln "LAYOUT-COLUMNS-BENCHMARK-OK"))) + +;;; Matched complete requests: lexical work, layout construction, GLR and +;;; artifact publication run inside each timed operation. Neither variant +;;; substitutes a token count or root summary for the complete ParseArtifact. +(def (layout-cpu-field row key) (cdr (assq key row))) +(def (benchmark-layout-request-cpu (groups 20) (batch-count 10) (padding 32768) + (stages '(columns complete-product)) + (reference-constructor reference-packed-layout-columns)) + (unless (and (exact-integer? groups) (>= groups 20) + (exact-integer? batch-count) (positive? batch-count) + (exact-integer? padding) (>= padding 0) + (list? stages) (pair? stages) + (andmap (lambda (stage) (memq stage '(columns complete-product))) stages)) + (error "layout CPU benchmark requires declared stages and exact positive batches")) + (let* ((machine (language-grammar-machine tla-plus-layout-language-grammar)) + (source (string-append "---- MODULE J ----\nInit ==\n /\\ TRUE" + (make-string padding #\space) "\n /\\ FALSE\n====\n")) + (columns (reference-layout-columns source)) + (probe (make-token 'probe "" (- (vector-length columns) 1) (- (vector-length columns) 1))) + (expected-product (prepared-layout-product machine source reference-constructor))) + (unless (and (parse-artifact-valid-for-source? expected-product source) + (equal? expected-product (prepared-layout-product machine source make-layout-columns))) + (error "layout CPU variants do not produce the same source-admitted artifact")) + (map (lambda (stage) + (let* ((operation + (if (eq? stage 'columns) + (lambda (constructor) + (lambda () (parameterize ((current-layout-columns (constructor source))) + (layout-token-column probe)))) + (lambda (constructor) + (lambda () (prepared-layout-product machine source constructor))))) + (reference (operation reference-constructor)) + (candidate (operation make-layout-columns)) + (expected (if (eq? stage 'columns) + (vector-ref columns (- (vector-length columns) 1)) expected-product)) + (name (list 'layout-request stage padding)) + (before (measure-parser-cpu-pairs name 'control-before groups batch-count expected reference reference)) + (result (measure-parser-cpu-pairs name 'production groups batch-count expected reference candidate)) + (after (measure-parser-cpu-pairs name 'control-after groups batch-count expected reference reference)) + (stable? (lambda (ratios) (and (>= (car ratios) 0.90) (<= (caddr ratios) 1.10)))) + (summary (list (cons 'before before) (cons 'production result) (cons 'after after) + (cons 'cpu-benefit (and (stable? before) (stable? after) (< (caddr result) 0.95)))))) + (write (list 'LAYOUT-REQUEST-CPU-SUMMARY stage 'reference (if (eq? reference-constructor reference-packed-layout-columns) 'former-packed 'boxed-oracle) + 'padding padding 'groups groups 'batch-count batch-count summary)) + (newline) (force-output) + (cons stage summary))) stages))) + +(def (qualify-layout-request-cpu (groups 20) (batch-count 10) (padding 32768)) + (let (summaries (benchmark-layout-request-cpu groups batch-count padding '(complete-product))) + (unless (andmap (lambda (entry) (layout-cpu-field (cdr entry) 'cpu-benefit)) summaries) + (error "complete layout request CPU controls or benefit gate failed" summaries)) + (displayln "LAYOUT-REQUEST-CPU-PROOF-OK") (force-output) + summaries)) diff --git a/t/benchmarks/literal-mode/benchmark.ss b/t/benchmarks/literal-mode/benchmark.ss new file mode 100644 index 00000000..2083abde --- /dev/null +++ b/t/benchmarks/literal-mode/benchmark.ss @@ -0,0 +1,67 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Pure Scheme scan-cost comparison for a fixed 64-rule literal mode. + +(import (only-in :gerbil-parser/src/runtime/scan + make-literal-end-scanner make-ranked-literal-scanner)) + +(def entries + (map (lambda (index) + (list (string-append "keyword-" (number->string index)) + (string->symbol + (string-append "Keyword" (number->string index))) + 0 index)) + (iota 64))) + +(def separate-scanners + (map (lambda (entry) + (cons entry (make-literal-end-scanner (list (car entry))))) + entries)) + +(def combined-scanner (make-ranked-literal-scanner entries)) + +(def (scan-separately source) + (let loop ((remaining separate-scanners) (selected #f)) + (if (null? remaining) + selected + (let* ((rule (car remaining)) + (entry (car rule)) + (end ((cdr rule) source 0))) + (loop (cdr remaining) + (if (and end + (or (not selected) (> end (cadr selected)))) + (list (cadr entry) end (caddr entry) (cadddr entry)) + selected)))))) + +(def (measure label source scanner repetitions) + (let ((started (##current-time-point)) + (last #f)) + (let loop ((remaining repetitions)) + (unless (zero? remaining) + (set! last (scanner source)) + (loop (- remaining 1)))) + (write + (list (cons 'scanner label) + (cons 'source source) + (cons 'repetitions repetitions) + (cons 'last last) + (cons 'elapsed-total-ms + (* 1000.0 (- (##current-time-point) started))))) + (newline))) + +(def (main . args) + (let (repetitions + (if (null? args) 10000 (string->number (car args)))) + (unless (and (integer? repetitions) (> repetitions 0)) + (error "expected a positive repetition count" args)) + (for-each + (lambda (source) + (unless (equal? (scan-separately source) + (combined-scanner source 0)) + (error "literal scanners disagree" source)) + (measure 'separate source scan-separately repetitions) + (measure 'combined source + (lambda (text) (combined-scanner text 0)) repetitions)) + '("keyword-63!" "not-a-keyword")))) + +(export main) diff --git a/t/benchmarks/lr-request-stack/benchmark.ss b/t/benchmarks/lr-request-stack/benchmark.ss new file mode 100644 index 00000000..4d6baa80 --- /dev/null +++ b/t/benchmarks/lr-request-stack/benchmark.ss @@ -0,0 +1,433 @@ +;;; Complete requests and prepared recognition at matched source scales. +(import (only-in ./derivation profile-lr-request-reductions + prepare-lr-semantic-replay replay-lr-semantics replay-lr-semantic-control + lr-semantic-replay-count profile-lr-semantic-actions) + (only-in ./lookahead benchmark-lr-lookahead) + (only-in :gerbil-parser/t/scenarios/performance/selective-glr/scenario + selective-glr-scenario selective-glr-scenario-pass?) + (only-in :gerbil-parser/languages/gql/parser gql-parser parse-gql) + (only-in :gerbil-parser/languages/fhirpath/parser fhirpath-parser parse-fhirpath) + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-parser parse-arithmetic) + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark + measure-parser-component measure-parser-batch measure-parser-cpu-pairs sample-at-percentile) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime parser-machine-ir + parser-machine-grammar-digest parser-machine-trivia + parser-machine-direct-drive parser-machine-direct-source + parser-machine-backend-representation parser-machine-for-current-semantic-backend) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse/prepared lr-parse/prepared/receipt + lr-prepare lr-runtime-for-current-semantic-backend + current-lr-event-program-enabled? lr-runtime-event-program?) + (only-in :gerbil-parser/src/runtime/parse-cost current-parser-cost-observer) + (only-in :gerbil-parser/src/runtime/significant parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/artifact make-success-parse-artifact + parse-artifact-events parse-artifact-valid-for-source? + token-event? token-event-token-kind token-event-lexeme event-start event-end)) +(export benchmark-lr-request-stack benchmark-lr-request-backends + benchmark-lr-completions benchmark-lr-semantic-preparation + benchmark-lr-casefold-requests benchmark-lr-text-class-requests + compare-lr-staged-actions compare-lr-staged-case) + +;;; Original runtime procedures are independent experiment inputs. Both sides +;;; publish the entire artifact, with the same source, tokens and grammar ID. +(def (compare-lr-staged-family family units machine parse source original-prepare original-parse groups calls + (original-publish make-success-parse-artifact)) + (let* ((expected (parse source)) + (tokens (map (lambda (event) + (make-token (token-event-token-kind event) (token-event-lexeme event) + (event-start event) (event-end event))) + (filter token-event? (parse-artifact-events expected)))) + (significant (parser-significant-tokens machine tokens)) + (spec (cdr (assq 'lr-spec (parser-machine-ir machine)))) + (original (original-prepare spec)) + (current (lr-prepare spec)) + (publish + (lambda (execute runtime publisher) + (let-values (((root rest) (execute runtime significant))) + (unless (null? rest) (error "staged reduction leaves unconsumed input" family units)) + (publisher (parser-machine-grammar-digest machine) + source tokens root (parser-machine-trivia machine))))) + (left (lambda () (publish original-parse original original-publish))) + (right (lambda () (publish lr-parse/prepared current make-success-parse-artifact)))) + (unless (parse-artifact-valid-for-source? expected source) + (error "invalid complete staged reduction reference" family units)) + (measure-parser-component (list family units 'original-preparation-and-request) 20 1 + (lambda () (publish original-parse (original-prepare spec) original-publish)) expected) + (measure-parser-component (list family units 'staged-preparation-and-request) 20 1 + (lambda () (publish lr-parse/prepared (lr-prepare spec) make-success-parse-artifact)) expected) + (measure-parser-component (list family units 'original-prepared-artifact) 20 calls left expected) + (measure-parser-component (list family units 'staged-prepared-artifact) 20 calls right expected) + (let (ratios (measure-parser-cpu-pairs 'staged-lr-actions (list family units) + groups calls expected left right)) + (write (list 'LR-STAGED-ACTION-CPU-RATIOS family units ratios)) + (newline) (force-output)))) + +(def (compare-lr-staged-case family units original-prepare original-parse (groups 20) (calls 200) + (original-publish make-success-parse-artifact)) + (parameterize ((current-lr-event-program-enabled? #f)) + (case family + ((gql-return) + (compare-lr-staged-family family units gql-parser parse-gql + (string-append "RETURN " + (string-join (map (lambda (i) (string-append "v" (number->string i))) (iota units)) ", ") + "\n") original-prepare original-parse groups calls original-publish)) + ((fhirpath-addition) + (compare-lr-staged-family family units fhirpath-parser parse-fhirpath + (string-join (make-list units "1") " + ") original-prepare original-parse groups calls original-publish)) + ((arithmetic-addition) + (compare-lr-staged-family family units arithmetic-parser parse-arithmetic + (string-join (make-list units "1") " + ") original-prepare original-parse groups calls original-publish)) + (else (error "unknown staged LR benchmark family" family)))) + (displayln "LR-STAGED-ACTION-CASE-OK " family " " units) (force-output)) + +(def (compare-lr-staged-actions original-prepare original-parse (groups 20) (calls 200) + (original-publish make-success-parse-artifact)) + (for-each + (lambda (units) + (for-each + (lambda (family) + (compare-lr-staged-case family units original-prepare original-parse groups calls original-publish)) + '(gql-return fhirpath-addition arithmetic-addition))) + '(64 512)) + (displayln "LR-STAGED-ACTIONS-COMPARISON-OK") (force-output)) + +;;; Preparation includes eligibility and generic selection. Repeated selection +;;; uses the prepared runtime, including negative admission. Neither stage is a +;;; decomposition of full parsing or a before/after performance claim. +(def (benchmark-lr-semantic-preparation (samples 20) (preparations 10) (selections 1000)) + (parameterize ((current-lr-event-program-enabled? #t)) + (for-each + (lambda (entry) + (let* ((family (car entry)) + (spec (cdr (assq 'lr-spec (parser-machine-ir (cdr entry))))) + (runtime (lr-prepare spec)) + (select (lambda (runtime) + (lr-runtime-event-program? + (lr-runtime-for-current-semantic-backend runtime)))) + (expected (select runtime))) + (displayln "LR-SEMANTIC-ADMISSION " family " event=" expected) (force-output) + (measure-parser-component (list family 'semantic-preparation) samples preparations + (lambda () (select (lr-prepare spec))) expected) + (measure-parser-component (list family 'prepared-backend-selection) samples selections + (lambda () (select runtime)) expected))) + (list (cons 'gql gql-parser) (cons 'fhirpath fhirpath-parser) + (cons 'arithmetic arithmetic-parser)))) + (displayln "LR-SEMANTIC-PREPARATION-OK") (force-output)) + +;;; A complete group contains equivalent merge, dynamic ranking, fragment +;;; interning and typed ambiguity rejection. Machine preparation is excluded. +(def (benchmark-lr-completions (samples 20) (iterations 20)) + (let (reference (selective-glr-scenario)) + (unless (selective-glr-scenario-pass? reference) + (error "invalid GLR completion benchmark product")) + (measure-parser-component 'selective-glr-completion samples iterations + selective-glr-scenario reference) + (displayln "LR-COMPLETION-OK") (force-output))) +(def (benchmark-family family units machine parse source samples iterations + (measure measure-parser-component) (require-derivation? #t)) + (let* ((reference (parameterize ((current-lr-event-program-enabled? #f)) (parse source))) + (tokens (map (lambda (event) + (make-token (token-event-token-kind event) (token-event-lexeme event) + (event-start event) (event-end event))) + (filter token-event? (parse-artifact-events reference)))) + (significant (parser-significant-tokens machine tokens)) + (materialized-runtime (parser-machine-runtime machine)) + (runtime (lr-runtime-for-current-semantic-backend materialized-runtime)) + (parsed (call-with-values (lambda () (lr-parse/prepared runtime significant)) list))) + (unless (and (parse-artifact-valid-for-source? reference source) (null? (cadr parsed))) + (error "invalid complete stack benchmark product" family units)) + (let-values (((root rest receipt) (lr-parse/prepared/receipt materialized-runtime significant))) + (unless (and (null? rest) + (equal? reference + (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens (car parsed) (parser-machine-trivia machine))) + (equal? reference + (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens root (parser-machine-trivia machine)))) + (error "independent GLR product mismatch" family units))) + ;; Cost observation preserves the fresh public route. This untimed witness + ;; records actual source stages; prepared admission alone cannot establish + ;; which executor a generated source/drive path used. + (let (stages '()) + (let (witness + (parameterize + ((current-parser-cost-observer + (lambda (name receipt) + (set! stages (cons (cons name (cdr (assq 'completed receipt))) stages))))) + (parse source))) + (unless (equal? witness reference) + (error "source route witness changed complete product" family units))) + (let* ((expected-route (if (parser-machine-direct-drive machine) + 'generated-drive-execution 'prepared-checkpoint-execution)) + (completed (assq expected-route stages))) + (unless (and completed (cdr completed)) + (error "complete request did not execute its expected engine route" family units expected-route))) + (write (list 'LR-REQUEST-BACKEND family units + 'requested-event (current-lr-event-program-enabled?) + 'prepared-event (lr-runtime-event-program? runtime) + 'generated-drive (and (parser-machine-direct-drive machine) #t) + 'generated-source (and (parser-machine-direct-source machine) #t) + 'prepared-representation + (parser-machine-backend-representation + (parser-machine-for-current-semantic-backend machine) 'prepared) + 'generated-drive-representation (parser-machine-backend-representation machine 'drive) + 'generated-source-representation (parser-machine-backend-representation machine 'source) + 'source-stages (reverse stages)))) + (newline) (force-output) + (write (list 'LR-STACK-WORKLOAD family units 'source-characters (string-length source) + 'tokens (length significant))) (newline) (force-output) + (let (work (parameterize ((current-lr-event-program-enabled? #f)) + (profile-lr-request-reductions machine significant require-derivation?))) + (write (list 'LR-REQUEST-REDUCTIONS family units work)) + (newline) (force-output) + (measure (list family units 'prepared-lr) samples iterations + (lambda () (call-with-values (lambda () (lr-parse/prepared runtime significant)) list)) parsed) + (when (and (not (lr-runtime-event-program? runtime)) + (eq? (cdr (assq 'scope work)) 'selected-fresh-derivation)) + (let ((replay (prepare-lr-semantic-replay materialized-runtime + (cdr (assq 'lr-spec (parser-machine-ir machine))) significant)) + (calls (max iterations (if (< units 512) 100 20)))) + (unless (equal? (replay-lr-semantics replay) (car parsed)) + (error "semantic replay changed the complete recognition root" family units)) + (write (list 'LR-SEMANTIC-REPLAY-WORK family units + (cons 'reductions (lr-semantic-replay-count replay)))) + (newline) (force-output) + (write (list 'LR-SEMANTIC-ACTION-WORK family units (profile-lr-semantic-actions replay))) + (newline) (force-output) + (measure (list family units 'semantic-replay-control) samples + (if (< units 512) 10000 1250) + (lambda () (replay-lr-semantic-control replay)) (car parsed)) + (measure (list family units 'semantic-replay) samples calls + (lambda () (replay-lr-semantics replay)) (car parsed))))) + (measure (list family units 'artifact-publication) samples + (max iterations (if (< units 512) 100 20)) + (lambda () (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens (car parsed) (parser-machine-trivia machine))) reference) + (measure (list family units 'full-source) samples iterations + (lambda () (parse source)) reference))) +(def (benchmark-request-families samples iterations label + (measure measure-parser-component)) + (unless (and (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations)) + (error "stack benchmark requires positive sample and iteration counts")) + (for-each + (lambda (units) + (benchmark-family (label 'gql-return) units gql-parser parse-gql + (string-append "RETURN " + (string-join (map (lambda (i) (string-append "v" (number->string i))) (iota units)) ", ") + "\n") samples iterations measure) + (let (source (string-join (make-list units "1") " + ")) + (benchmark-family (label 'fhirpath-addition) units fhirpath-parser parse-fhirpath source samples iterations measure) + (benchmark-family (label 'arithmetic-addition) units arithmetic-parser parse-arithmetic source samples iterations measure))) + '(64 512))) + +(def (benchmark-lr-request-stack (samples 20) (iterations 20)) + (benchmark-request-families samples iterations identity) + ;; Retain actual clause/property and invocation forms beside scalable lists. + ;; They use the same full-product, executor and independent GLR admission. + (benchmark-family 'gql-match-filter 1 gql-parser parse-gql + "MATCH (n) WHERE n.score > 2 AND n.active = TRUE RETURN n" + samples (max iterations 100) measure-parser-component #f) + (benchmark-family 'fhirpath-property-invocation 1 fhirpath-parser parse-fhirpath + "Patient.name.where(use = 'official').given" samples (max iterations 100)) + ;; Keep one existing optional request gate. Complete paired controls include + ;; the small workload and the unchanged generated Arithmetic executor. + (for-each + (lambda (units) + (let ((calls (if (= units 1) 900 64)) + (addition (string-join (make-list units "1") " + "))) + (benchmark-lr-lookahead (list 'gql-return units) gql-parser + (string-append "RETURN " + (string-join (map (lambda (i) (string-append "v" (number->string i))) (iota units)) ", ") + "\n") 20 calls) + (benchmark-lr-lookahead (list 'fhirpath-addition units) fhirpath-parser addition 20 calls) + (benchmark-lr-lookahead (list 'arithmetic-addition units) arithmetic-parser addition + 20 (if (= units 1) 4096 calls)))) + '(1 64)) + (displayln "LR-REQUEST-STACK-OK") (force-output)) + +;;; Complete request controls for the shared casefold algorithm. Source and +;;; prepared products use the same independent admission as the general matrix. +;;; Short identifiers and the generated Arithmetic entry retain negative controls. +(def (benchmark-lr-casefold-requests (samples 20) (iterations 20)) + (unless (and (exact-integer? samples) (positive? samples) + (exact-integer? iterations) (positive? iterations)) + (error "casefold requests require positive integral counts")) + (for-each + (lambda (width) + (for-each + (lambda (units) + (benchmark-family + (if (= width 32) 'gql-identifiers-32 'gql-identifiers-256) + units gql-parser parse-gql + (string-append "RETURN " + (string-join + (map (lambda (index) + (string-append "identifier_" (make-string width #\x) "_" (number->string index))) + (iota units)) ", ") "\n") + samples iterations)) + '(64 256))) + '(32 256)) + (benchmark-family 'gql-short-control 64 gql-parser parse-gql + (string-append "RETURN " + (string-join (map (lambda (index) (string-append "v" (number->string index))) (iota 64)) ", ") "\n") + samples iterations) + (benchmark-family 'arithmetic-unchanged-control 64 arithmetic-parser parse-arithmetic + (string-join (make-list 64 "1") " + ") samples iterations) + (displayln "LR-CASEFOLD-REQUESTS-OK") (force-output)) + +;;; Shared character-class dispatch measured through complete FHIRPath requests. +(def (benchmark-lr-text-class-requests (samples 20) (iterations 100)) + (for-each + (lambda (width) + (benchmark-family + (if (= width 32) 'fhirpath-classes-32 'fhirpath-classes-256) + 64 fhirpath-parser parse-fhirpath + (string-join + (map (lambda (index) + (string-append "item_" (make-string width #\x) "_" (number->string index))) + (iota 64)) " + ") samples iterations)) '(32 256)) + (for-each + (lambda (units) + (benchmark-family 'fhirpath-unicode-control units fhirpath-parser parse-fhirpath + (string-join + (map (lambda (index) + (string-append "item_" (make-string 256 #\٣) "_" (number->string index))) + (iota units)) " + ") samples iterations)) '(16 64)) + (benchmark-family 'fhirpath-short-control 64 fhirpath-parser parse-fhirpath + (string-join (make-list 64 "1") " + ") samples iterations) + (benchmark-family 'arithmetic-unchanged-control 64 arithmetic-parser parse-arithmetic + (string-join (make-list 64 "1") " + ") samples iterations) + (displayln "LR-TEXT-CLASS-REQUESTS-OK") (force-output)) + +;;; Compare semantic representations across complete public requests. Every +;;; route publishes the recognition reference, including grammar rejection of +;;; the requested event backend and independent generated source/drive routes. +;;; Prepared roots may differ in representation; +;;; their canonical artifacts must agree before any timed batch is admitted. +(defstruct request-variant (events? thunk expected)) + +(def (batch-value row key) (cdr (assq key row))) +(def (paired-percentile rows key rank) + (batch-value (sample-at-percentile rows key rank) key)) + +;;; Empirical observation band, not a confidence interval or causal proof. +;;; A practical 5% margin is fixed before sampling; natural GC stays in CPU. +(def (paired-decision rows) + (cond ((< (paired-percentile rows 'cpu-ratio 90) 0.95) 'observed-benefit) + ((> (paired-percentile rows 'cpu-ratio 10) 1.05) 'observed-regression) + (else 'not-admitted))) + +(def (measure-backend-pair name variants groups iterations) + (def (run variant sample) + (parameterize ((current-lr-event-program-enabled? (request-variant-events? variant))) + (measure-parser-batch name sample iterations + (request-variant-thunk variant) (request-variant-expected variant)))) + ;; Warm both admitted representations before one initial collection. Variant + ;; construction, derivation, source witnesses and validation are untimed. + (run (car variants) -1) (run (cadr variants) -1) (##gc) + (let loop ((group 0) (observations '())) + (if (= group groups) + (let* ((rows (reverse observations)) + (allocated (filter (lambda (row) (number? (batch-value row 'allocation-ratio))) rows)) + (summary + (list (cons 'workload name) (cons 'groups groups) + (cons 'calls-per-batch iterations) + (cons 'decision (paired-decision rows)) + (cons 'cpu-ratio-p10 (paired-percentile rows 'cpu-ratio 10)) + (cons 'cpu-ratio-p50 (paired-percentile rows 'cpu-ratio 50)) + (cons 'cpu-ratio-p90 (paired-percentile rows 'cpu-ratio 90)) + (cons 'recognition-cpu-ms-per-call + (paired-percentile rows 'recognition-cpu-ms-per-call 50)) + (cons 'event-cpu-ms-per-call + (paired-percentile rows 'event-cpu-ms-per-call 50)) + (cons 'allocation-admitted-groups + (length allocated)) + (cons 'allocation-ratio-p50 + (and (pair? allocated) (paired-percentile allocated 'allocation-ratio 50))) + (cons 'gc-collections + (apply + (map (lambda (row) + (apply + (map (lambda (entry) (batch-value (cdr entry) 'gc-count)) + (batch-value row 'batches)))) rows))) + (cons 'observations rows)))) + (write (list 'LR-BACKEND-PAIR-SUMMARY + (filter (lambda (entry) (not (eq? (car entry) 'observations))) summary))) + (newline) (force-output) + summary) + (let* ((order (if (even? group) '(0 1 1 0) '(1 0 0 1))) + (batches (map (lambda (index) + (cons index (run (list-ref variants index) group))) order)) + (selected (lambda (index key) + (map (lambda (entry) (batch-value (cdr entry) key)) + (filter (lambda (entry) (= (car entry) index)) batches))))) + (def (sum index key) (apply + (selected index key))) + (let* ((recognition (sum 0 'cpu-ms)) (events (sum 1 'cpu-ms)) + (allocations (map (lambda (entry) (batch-value (cdr entry) 'allocated-bytes)) batches))) + (unless (and (positive? recognition) (positive? events)) + (error "paired benchmark CPU resolution is insufficient" name group)) + (let (row + (list (cons 'sample group) (cons 'order order) + (cons 'cpu-ratio (/ events recognition)) + (cons 'recognition-cpu-ms-per-call (/ recognition (* 2 iterations))) + (cons 'event-cpu-ms-per-call (/ events (* 2 iterations))) + (cons 'allocation-ratio + (and (every number? allocations) + (positive? (sum 0 'allocated-bytes)) + (/ (sum 1 'allocated-bytes) (sum 0 'allocated-bytes)))) + (cons 'batches batches))) + (write (list 'LR-BACKEND-PAIR-SAMPLE name row)) (newline) (force-output) + (loop (+ group 1) (cons row observations)))))))) + +(def (benchmark-lr-request-backends (groups 20) (iterations 20)) + (unless (and (integer? groups) (exact? groups) (>= groups 20) + (integer? iterations) (exact? iterations) (positive? iterations)) + (error "matched backend qualification requires at least 20 groups and positive integral calls")) + (let ((catalog (make-hash-table)) (keys '())) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (benchmark-request-families groups iterations identity + (lambda (key _samples _iterations thunk expected) + (let (variant (make-request-variant events? thunk expected)) + (if events? + (hash-put! catalog key (list (car (hash-ref catalog key)) variant)) + (begin (set! keys (cons key keys)) + (hash-put! catalog key (list variant))))))))) + '(#f #t)) + ;; Recognition-only diagnostic phases have no event-program counterpart. + ;; Report their absence; complete-source requests must still be paired. + (let* ((paired + (filter (lambda (key) + (let (variants (hash-ref catalog key)) + (if (= (length variants) 2) #t + (begin + (when (eq? (caddr key) 'full-source) + (error "complete request lacks its backend pair" key)) + (write (list 'LR-BACKEND-PAIR-UNSUPPORTED key + 'available-variants (length variants))) + (newline) (force-output) #f)))) + (reverse keys))) + (summaries (map (lambda (key) + (measure-backend-pair key (hash-ref catalog key) groups iterations)) + paired)) + (controls (filter (lambda (row) + (eq? (car (batch-value row 'workload)) 'gql-return)) summaries)) + (stable? (every (lambda (row) + (and (<= (batch-value row 'cpu-ratio-p10) 1) + (>= (batch-value row 'cpu-ratio-p90) 1) + (<= 0.95 (batch-value row 'cpu-ratio-p50) 1.05))) controls))) + (write (list 'LR-BACKEND-CONTROL-ADMISSION 'gql-fallback-stable stable?)) + (newline) (force-output) + (write (list 'LR-BACKEND-PERFORMANCE-ADMISSION + 'control-stable stable? + 'complete-request-benefit-admitted + (and stable? + (every (lambda (row) + (eq? (batch-value row 'decision) 'observed-benefit)) + (filter (lambda (row) + (let (key (batch-value row 'workload)) + (and (not (eq? (car key) 'gql-return)) + (eq? (caddr key) 'full-source)))) summaries))))) + (newline) (force-output) + (displayln "LR-REQUEST-BACKENDS-OK") (force-output) + summaries))) diff --git a/t/benchmarks/lr-request-stack/capture-reference.ss b/t/benchmarks/lr-request-stack/capture-reference.ss new file mode 100644 index 00000000..d23a11dd --- /dev/null +++ b/t/benchmarks/lr-request-stack/capture-reference.ss @@ -0,0 +1,38 @@ +(import (only-in :gerbil-parser/src/runtime/lr-parser/state + lr-runtime-reduction-widths make-lr-capture-context) + (rename-in (only-in :gerbil-parser/src/runtime/lr-parser/state make-lr-recognition-fragment) + (make-lr-recognition-fragment make-current-capture-fragment)) + (only-in :gerbil-parser/src/runtime/lr-parser/recognition + recognition-piece-start recognition-piece-end recognition-piece-token-count + recognition-piece-terminal-yield terminal-yield-append) + (only-in :gerbil-parser/src/runtime/token token-kind token-lexeme)) +(export reference-capture-reduction) +;;; Preserve the historical oracle body while constructing the current private +;;; representation. This adapter exists only in the frozen test oracle. +(def (make-lr-recognition-fragment runtime executor production entry target children + lookahead value start end count offset yield) + (make-current-capture-fragment (make-lr-capture-context runtime executor production entry target (and lookahead (car lookahead))) + (if (and (pair? children) (null? (cdr children))) + (car children) children) + (and lookahead (cdr lookahead)) value start end count offset yield)) +;;; Frozen capture control; it does not select or replace a runtime binding. +(def (reference-capture-reduction runtime executor production-id states stack rest offset value target) + (if (not stack) #f + (let* ((count (vector-ref (lr-runtime-reduction-widths runtime) production-id)) + (children (reverse (take stack count))) + (remaining (drop stack count)) + (entry-state (list-ref states count)) + (lookahead (if (pair? rest) + (cons (token-kind (car rest)) (token-lexeme (car rest))) + #f)) + (node (make-lr-recognition-fragment + runtime executor production-id entry-state target children lookahead value + (if (pair? children) (recognition-piece-start (car children)) offset) + (if (pair? children) (recognition-piece-end (last children)) offset) + (foldl (lambda (piece count) + (+ count (recognition-piece-token-count piece))) + 0 children) offset + (foldl (lambda (piece yield) + (terminal-yield-append yield (recognition-piece-terminal-yield piece))) + #f children)))) + (cons node remaining)))) diff --git a/t/benchmarks/lr-request-stack/casefold-evidence.ss b/t/benchmarks/lr-request-stack/casefold-evidence.ss new file mode 100644 index 00000000..95a4db79 --- /dev/null +++ b/t/benchmarks/lr-request-stack/casefold-evidence.ss @@ -0,0 +1,66 @@ +;;; Exact allocation evidence and conservative CPU admission from native receipts. +(import (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/receipt read-parser-component-receipt)) +(export main compare-casefold-request-receipts compare-casefold-request-cpu-receipts) +(def (field row key) (cdr (assq key row))) +(def (require! condition . detail) + (unless condition (apply error "casefold request evidence rejected" detail))) +(def (read-request-receipt path) + (read-parser-component-receipt path "LR-CASEFOLD-REQUESTS-OK" 12 20 20)) + +(def (compare-casefold-request-receipts candidate-a reference-a reference-b candidate-b) + (compare-request-receipts candidate-a reference-a reference-b candidate-b 'reduced)) + +;;; A dispatch-only change must preserve every complete-request allocation +;;; counter. CPU admission uses the same controls and repeated-median gate. +(def (compare-casefold-request-cpu-receipts candidate-a reference-a reference-b candidate-b) + (compare-request-receipts candidate-a reference-a reference-b candidate-b 'unchanged)) + +(def (compare-request-receipts candidate-a reference-a reference-b candidate-b allocation-policy) + (let* ((ca (read-request-receipt candidate-a)) (ra (read-request-receipt reference-a)) + (rb (read-request-receipt reference-b)) (cb (read-request-receipt candidate-b)) + (savings (make-hash-table)) (rows '())) + (def (cpu receipt key) (field (hash-get (car receipt) key) 'cpuP50Ms)) + (def (stable-control? key) + (let* ((c [(cpu ca key) (cpu cb key)]) (r [(cpu ra key) (cpu rb key)]) + (ratio (/ (apply + c) (apply + r)))) + (and (<= 0.95 ratio 1.05) + (<= (/ (apply min c) (apply max r)) 1 (/ (apply max c) (apply min r)))))) + (let (controls-stable? + (and (stable-control? '(arithmetic-unchanged-control 64 prepared-lr)) + (stable-control? '(arithmetic-unchanged-control 64 full-source)))) + (for-each + (lambda (key) + (let* ((baseline (car (hash-get (cadr ra) key))) + (candidate (car (hash-get (cadr ca) key))) + (delta (/ (- baseline candidate) 20)) + (cpu-ratio (/ (+ (cpu ca key) (cpu cb key)) (+ (cpu ra key) (cpu rb key)))) + (repeated-benefit? (< (max (cpu ca key) (cpu cb key)) + (* 0.95 (min (cpu ra key) (cpu rb key)))))) + (require! (= baseline (car (hash-get (cadr rb) key))) key 'reference-drift) + (require! (= candidate (car (hash-get (cadr cb) key))) key 'candidate-drift) + (require! (integer? delta) key 'fractional-saving) + (require! (if (or (eq? allocation-policy 'unchanged) + (eq? (car key) 'arithmetic-unchanged-control)) (= delta 0) (> delta 0)) key 'allocation) + (hash-put! savings key delta) + (let (row (list (cons 'stage key) (cons 'saved-bytes delta) + (cons 'allocation-reduction (/ (- baseline candidate) baseline)) + (cons 'cpu-ratio-of-child-medians cpu-ratio) + (cons 'repeated-median-benefit? repeated-benefit?) + (cons 'cpu-benefit-admitted? (and controls-stable? repeated-benefit?)))) + (set! rows (cons row rows)) + (write (list 'CASEFOLD-REQUEST-EVIDENCE row)) (newline)))) + (hash-keys (car ra))) + (for-each + (lambda (key) + (when (eq? (caddr key) 'prepared-lr) + (require! (= (hash-get savings key) + (hash-get savings (list (car key) (cadr key) 'full-source))) key 'boundary-drift))) + (hash-keys savings)) + (write (list 'CASEFOLD-REQUEST-ALLOCATION-OK + 'allocation-policy allocation-policy + 'admitted-intervals (+ (caddr ca) (caddr ra) (caddr rb) (caddr cb)) + 'unchanged-controls-stable? controls-stable?)) (newline) + (reverse rows)))) +(def (main candidate-a reference-a reference-b candidate-b) + (compare-casefold-request-receipts candidate-a reference-a reference-b candidate-b) + (displayln "CASEFOLD-REQUEST-EVIDENCE-OK") (force-output)) diff --git a/t/benchmarks/lr-request-stack/casefold.ss b/t/benchmarks/lr-request-stack/casefold.ss new file mode 100644 index 00000000..a17bcaa7 --- /dev/null +++ b/t/benchmarks/lr-request-stack/casefold.ss @@ -0,0 +1,105 @@ +;;; Guarded CPU pairs with complete entries and identical-operation controls. +(import (only-in :gerbil-parser/languages/gql/parser gql-parser) + (only-in :gerbil-parser/src/compiler/machine parser-machine-ir) + (only-in :gerbil-parser/src/runtime/lr-action-index + index-action-rows lookup-literal-action-entry lookup-casefolded-literal-action-entry + lr-action-row-has-literals?) + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark + measure-parser-batch sample-at-percentile)) +(export benchmark-lr-casefold) +(def (field row key) (cdr (assq key row))) +;;; CPU qualification of the production mechanism against a guarded reference. +;;; Return every original entry, not only a hit count. This remains a lookup +;;; component: its uniform state sweep is not a complete-request distribution. +(def (benchmark-lr-casefold (groups 20) (iterations 20) (workload-names #f)) + (unless (and (exact-integer? groups) (>= groups 20) + (exact-integer? iterations) (positive? iterations)) + (error "CPU qualification requires at least twenty groups and positive batches")) + (let* ((spec (cdr (assq 'lr-spec (parser-machine-ir gql-parser)))) + (rows (index-action-rows (cdr (assq 'actions spec)) #t)) + (count (vector-length rows))) + (def (sweep source optimized?) + (let (result (make-vector count #f)) + (let loop ((state 0)) + (when (< state count) + (let (row (vector-ref rows state)) + (vector-set! result state + (if optimized? + (lookup-casefolded-literal-action-entry row source) + (and (lr-action-row-has-literals? row) + (lookup-literal-action-entry row (string-upcase source)))))) + (loop (+ state 1)))) + result)) + (def (paired name reference candidate expected) + (unless (and (equal? (reference) expected) (equal? (candidate) expected)) + (error "CPU pair warmup changed complete entries" name)) + (##gc) + (let loop ((group 0) (observations '())) + (if (= group groups) + (let ((p10 (field (sample-at-percentile observations 'cpu-ratio 10) 'cpu-ratio)) + (p50 (field (sample-at-percentile observations 'cpu-ratio 50) 'cpu-ratio)) + (p90 (field (sample-at-percentile observations 'cpu-ratio 90) 'cpu-ratio))) + (write (list 'LR-CPU-CONTROL-SUMMARY name 'groups groups + 'cpu-ratio-p10 p10 'cpu-ratio-p50 p50 'cpu-ratio-p90 p90)) + (newline) (force-output) + (list p10 p50 p90 + (andmap (lambda (observation) + (andmap (lambda (batch) (>= (field (cdr batch) 'cpu-ms) 1.0)) + (field observation 'batches))) observations))) + (let* ((order (if (even? group) '(reference candidate candidate reference) + '(candidate reference reference candidate))) + (batches (map (lambda (variant) + (cons variant + (measure-parser-batch name group (* iterations count) + (if (eq? variant 'reference) reference candidate) + expected count))) order)) + (sum (lambda (variant) + (apply + (map (lambda (batch) (field (cdr batch) 'cpu-ms)) + (filter (lambda (batch) (eq? (car batch) variant)) batches))))) + (denominator (sum 'reference))) + (unless (positive? denominator) (error "CPU reference interval is not positive" name group)) + (let (observation (list (cons 'sample group) + (cons 'cpu-ratio (/ (sum 'candidate) denominator)) + (cons 'batches batches))) + (write (list 'LR-CPU-CONTROL-PAIR name observation)) (newline) (force-output) + (loop (+ group 1) (cons observation observations))))))) + (let (workloads + (list (list 'early-miss-43 (string-append "identifier_" (make-string 32 #\x))) + (list 'early-miss-267 (string-append "identifier_" (make-string 256 #\x))) + (list 'keyword-hit "match") + (list 'late-miss "matching") + (list 'unicode-fallback "éclair"))) + (when workload-names + (unless (and (list? workload-names) (pair? workload-names) + (andmap (lambda (name) (assq name workloads)) workload-names)) + (error "unknown or empty CPU workload selection" workload-names))) + (for-each + (lambda (workload) + (let* ((name (car workload)) (source (cadr workload)) + (reference (lambda () (sweep source #f))) + (candidate (lambda () (sweep source #t))) + (expected (reference))) + (let ((actual (candidate))) + (let loop ((state 0)) + (when (< state count) + (unless (eq? (vector-ref expected state) (vector-ref actual state)) + (error "CPU qualification lost original entry identity" name state)) + (loop (+ state 1))))) + ;; Identical-operation control brackets each workload. Admission needs + ;; both controls, not merely a faster optimized batch. + (let* ((before (paired (list name 'control-before) reference reference expected)) + (result (paired (list name 'production) reference candidate expected)) + (after (paired (list name 'control-after) reference reference expected)) + (stable? (lambda (ratios) + (and (cadddr ratios) (>= (car ratios) 0.90) (<= (caddr ratios) 1.10)))) + (controls? (and (stable? before) (stable? after)))) + (write (list 'LR-CPU-ADMISSION name 'states count 'groups groups + 'iterations iterations 'controls-stable controls? + 'cpu-ratio-p50 (cadr result) 'cpu-ratio-p90 (caddr result) + 'timing-resolved (cadddr result) + 'component-benefit (and controls? (cadddr result) (< (caddr result) 0.95)))) + (newline) (force-output)))) + (if workload-names + (map (lambda (name) (assq name workloads)) workload-names) + workloads)))) + (displayln "LR-CASEFOLD-CPU-CONTROLS-OK") (force-output)) diff --git a/t/benchmarks/lr-request-stack/constructors.ss b/t/benchmarks/lr-request-stack/constructors.ss new file mode 100644 index 00000000..0ebbdc03 --- /dev/null +++ b/t/benchmarks/lr-request-stack/constructors.ss @@ -0,0 +1,137 @@ +;;; Complete source requests against the construction algorithm at 4f50dd3. +;;; Compile with -O, matching the measured production constructor modules. +(import (only-in :gerbil-parser/src/runtime/lr-parser/state + lr-runtime-action-selector lr-runtime-reduction-plans-set!)) +(import (only-in :std/vector/vector vector-map/index) + (only-in :gerbil-parser/src/compiler/lr production-table production-rhs operand-actions lr-spec-ref) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime parser-machine-ir + parser-machine-with-lr-action-selector) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-event-program-enabled?) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/parse-cost current-parser-cost-observer) + (only-in :gerbil-parser/src/runtime/recognition make-recognition-child make-recognition-node + recognition-child-field recognition-child-value) + (only-in :gerbil-parser/src/runtime/funcs recognition-sequence-arity + recognition-sequence->list recognition-sequence-for-action + recognition-sequence-start recognition-sequence-end) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid-for-source?) + (only-in :gerbil-parser/languages/gql/parser gql-parser) + (only-in :gerbil-parser/languages/fhirpath/parser fhirpath-parser) + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-parser) + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark measure-parser-cpu-pairs)) +(export benchmark-lr-constructor-case) + +;;; Frozen reference: normalize first, then independently read both bounds. +;;; The production constructors, stack driver and publication stay authoritative. +(def (reference-field name children offset constructor) + (case (recognition-sequence-arity children) + ((0) '()) + ((1) + (let (child (car (recognition-sequence->list children))) + (if (recognition-child-field child) + (list (make-recognition-child name + (constructor (recognition-sequence-start children offset) + (recognition-sequence-end children offset) children))) + (list (make-recognition-child name (recognition-child-value child)))))) + (else + (list (make-recognition-child name + (constructor (recognition-sequence-start children offset) + (recognition-sequence-end children offset) children)))))) + +(def (reference-alias kind children offset) + (list (make-recognition-child #f + (make-recognition-node kind (recognition-sequence-start children offset) + (recognition-sequence-end children offset) children)))) + +(def (reference-action action) + (let (name (cadr action)) + (case (car action) + ((field) (lambda (value offset constructor) + (reference-field name (recognition-sequence-for-action value) offset constructor))) + ((alias) (lambda (value offset _constructor) + (reference-alias name (recognition-sequence-for-action value) offset)))))) + +(def (reference-chain actions) + (and (pair? actions) + (let ((first (reference-action (car actions))) (remaining (reference-chain (cdr actions)))) + (if remaining + (lambda (value offset constructor) + (remaining (first value offset constructor) offset constructor)) + first)))) + +(def (reference-plans table) + (vector-map/index + (lambda (_id production) + (list->vector (map (lambda (operand) (reference-chain (operand-actions operand))) + (production-rhs production)))) table)) + +;;; These two private record bindings are resolved once, outside timing. Only +;;; newly cloned qualification runtimes are modified; grammar tables and the +;;; public language machines remain shared and untouched. Record migrations +;;; belong in this qualification owner. +(def runtime-selector lr-runtime-action-selector) +(def install-reference-plans! lr-runtime-reduction-plans-set!) + +(def (source-executor-witness machine source expected) + (let (stages '()) + (parameterize ((current-parser-cost-observer + (lambda (stage row) + (set! stages (cons (cons stage (cdr (assq 'completed row))) stages))))) + (unless (equal? (parse-source machine source) expected) + (error "constructor executor witness changes complete artifact"))) + (reverse stages))) + +(def (compare-constructor-request family units machine source calls) + (let* ((clone (lambda () + (parser-machine-with-lr-action-selector machine + (lambda (_rows _index _casefold _layout) + (runtime-selector (parser-machine-runtime machine)))))) + (original (clone)) (candidate (clone)) + (spec (cdr (assq 'lr-spec (parser-machine-ir machine)))) + (expected (parse-source machine source))) + (install-reference-plans! (parser-machine-runtime original) + (reference-plans (production-table (lr-spec-ref spec 'productions)))) + (let ((public-stages (source-executor-witness machine source expected)) + (original-stages (source-executor-witness original source expected)) + (candidate-stages (source-executor-witness candidate source expected))) + (unless (and (parse-artifact-valid-for-source? expected source) + (equal? original-stages candidate-stages) + (or (eq? family 'arithmetic-addition) (equal? public-stages candidate-stages))) + (error "constructor comparison changes source executor" family units)) + (write (list 'LR-CONSTRUCTOR-EXECUTORS family units + (if (eq? family 'arithmetic-addition) 'representation-control 'public-source-request) + public-stages original-stages candidate-stages)) + (newline) (force-output)) + (write (list 'LR-CONSTRUCTOR-CPU family units + (measure-parser-cpu-pairs 'lr-construction (list family units) 20 calls expected + (lambda () (parse-source original source)) (lambda () (parse-source candidate source))))) + (newline) (force-output))) + +;;; Separate cases fit the unchanged ninety-second native process budget. +;;; Larger small-request batches retain natural GC while resolving its variance. +(def (benchmark-lr-constructor-case family units) + (unless (memq units '(1 64 512)) (error "unsupported constructor input scale" units)) + (when (memq family '(gql-match-filter fhirpath-property-invocation)) + (unless (= units 1) (error "structured request has one fixed source scale" family units))) + (let ((addition (string-join (make-list units "1") " + ")) + (calls (case units ((1) 16384) ((64) 256) (else 64)))) + (parameterize ((current-lr-event-program-enabled? #f)) + (case family + ((gql-return) + (compare-constructor-request family units gql-parser + (string-append "RETURN " + (string-join (map (lambda (i) (string-append "v" (number->string i))) (iota units)) ", ") + "\n") calls)) + ((fhirpath-addition) + (compare-constructor-request family units fhirpath-parser addition + (if (= units 1) 32768 calls))) + ((arithmetic-addition) + (compare-constructor-request family units arithmetic-parser addition calls)) + ((gql-match-filter) + (compare-constructor-request family units gql-parser + "MATCH (n) WHERE n.score > 2 AND n.active = TRUE RETURN n" 512)) + ((fhirpath-property-invocation) + (compare-constructor-request family units fhirpath-parser + "Patient.name.where(use = 'official').given" 2048)) + (else (error "unsupported constructor request family" family))))) + (displayln "LR-CONSTRUCTOR-CASE-OK " family " " units) (force-output)) diff --git a/t/benchmarks/lr-request-stack/derivation.ss b/t/benchmarks/lr-request-stack/derivation.ss new file mode 100644 index 00000000..b5a0e3e5 --- /dev/null +++ b/t/benchmarks/lr-request-stack/derivation.ss @@ -0,0 +1,184 @@ +;;; Untimed fresh-request LR derivation diagnostics, shared across languages. +(import (only-in :gerbil-parser/src/compiler/machine parser-machine-ir parser-machine-runtime) + (only-in :gerbil-parser/src/compiler/lr lr-spec-ref production-table production-rhs operand-actions) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-parse/prepared current-lr-recognition-observer + lr-recognition-fragment? lr-recognition-fragment-production-id + lr-recognition-fragment-children lr-recognition-fragment-value + lr-recognition-fragment-offset current-lr-event-program-enabled?) + (only-in :gerbil-parser/src/runtime/reduction-plan + prepare-lr-reduction-plans reduce-lr-stack-plan) + (only-in :gerbil-parser/src/runtime/recognition + make-recognition-child recognition-child-value recognition-child-field) + (only-in :gerbil-parser/src/runtime/reduce recognition-children-field recognition-children-alias) + (only-in :gerbil-parser/src/runtime/funcs recognition-sequence-root-child + recognition-sequence-for-action recognition-sequence-append recognition-sequence-start) + (only-in :std/list/list-builder with-list-builder)) +(export profile-lr-request-reductions prepare-lr-semantic-replay + replay-lr-semantics replay-lr-semantic-control lr-semantic-replay-count + profile-lr-semantic-actions) + +;;; Captured inputs are diagnostic fixtures, never a second parser. Named +;;; records keep the prepared kernel, borrowed operands and witness distinct. +(defstruct semantic-invocation (plan actions stack offset expected)) +(defstruct lr-semantic-replay (invocations)) +(def (lr-semantic-replay-count replay) + (vector-length (lr-semantic-replay-invocations replay))) + +(def (prepare-lr-semantic-replay runtime spec tokens) + (let* ((table (production-table (lr-spec-ref spec 'productions))) + (plans (prepare-lr-reduction-plans 'recognition table)) + (root #f)) + (parameterize ((current-lr-event-program-enabled? #f) + (current-lr-recognition-observer (lambda (value) (set! root value)))) + (let-values (((_value rest) (lr-parse/prepared runtime tokens))) + (unless (null? rest) (error "semantic replay left trailing tokens")))) + (unless (lr-recognition-fragment? root) + (error "semantic replay requires a deterministic recognition derivation")) + (let (invocations + (with-list-builder (emit!) + (let visit ((pending (list root))) + (when (pair? pending) + (let (piece (car pending)) + (if (lr-recognition-fragment? piece) + (let* ((children (lr-recognition-fragment-children piece)) + (stack (reverse + (map (lambda (child) + (if (lr-recognition-fragment? child) + (lr-recognition-fragment-value child) + (list (make-recognition-child #f child)))) + children))) + (invocation + (make-semantic-invocation + (vector-ref plans (lr-recognition-fragment-production-id piece)) + (map operand-actions + (production-rhs (vector-ref table + (lr-recognition-fragment-production-id piece)))) + stack (lr-recognition-fragment-offset piece) + (lr-recognition-fragment-value piece)))) + ;; Validate every kernel, including nullable and decorated + ;; operands, before measuring with observers disabled. + (unless (equal? (execute-semantic-invocation invocation) + (semantic-invocation-expected invocation)) + (error "semantic replay differs from the captured reduction" + (lr-recognition-fragment-production-id piece))) + (emit! invocation) + (visit (append children (cdr pending)))) + (visit (cdr pending)))))))) + (make-lr-semantic-replay (list->vector invocations))))) + +(def (execute-semantic-invocation invocation) + (reduce-lr-stack-plan (semantic-invocation-plan invocation) + (semantic-invocation-stack invocation) + (semantic-invocation-offset invocation))) + +;;; Both traversals retain only their first result (the complete root). The +;;; control measures traversal/result access, not a parser with AST disabled. +;;; Cached input sharing makes replay a component experiment, not additive +;;; attribution of time inside a fresh complete source request. +(defrule (define-semantic-replay name evaluate) + (def (name replay) + (let* ((invocations (lr-semantic-replay-invocations replay)) + (root (evaluate (vector-ref invocations 0)))) + (let loop ((index 1)) + (when (fx< index (vector-length invocations)) + (evaluate (vector-ref invocations index)) + (loop (fx+ index 1)))) + (let (child (recognition-sequence-root-child root)) + (unless child + (error "semantic replay requires one complete recognition root")) + (recognition-child-value child))))) + +(define-semantic-replay replay-lr-semantics execute-semantic-invocation) +(define-semantic-replay replay-lr-semantic-control semantic-invocation-expected) + +;;; Reexecute the selected derivation's actions outside timing. The reference +;;; normalizes before decoration, so its actual list sizes expose the extra +;;; tail scan. Counts describe work eligibility, never an additive CPU share. +(def (profile-lr-semantic-actions replay) + (let ((fields 0) (aliases 0) (materializations 0) (children 0) + (tail-links 0) (avoidable-tail-links 0) (widest 0) (singleton-bounds 0)) + (def (apply-action action input offset) + (let* ((flat? (or (null? input) (pair? input))) + (ready (recognition-sequence-for-action input)) + (width (length ready)) + (field? (eq? (car action) 'field)) + (bounded? (or (not field?) (> width 1) + (and (= width 1) (recognition-child-field (car ready)))))) + (if field? (set! fields (+ fields 1)) (set! aliases (+ aliases 1))) + (unless flat? + (set! materializations (+ materializations 1)) + (set! children (+ children width))) + (set! widest (max widest width)) + (when bounded? + (when (= width 1) (set! singleton-bounds (+ singleton-bounds 1))) + (let (links (max 0 (- width 1))) + (set! tail-links (+ tail-links links)) + (unless flat? (set! avoidable-tail-links (+ avoidable-tail-links links))))) + ((if field? recognition-children-field recognition-children-alias) + (cadr action) ready offset))) + (vector-for-each + (lambda (invocation) + (let loop ((actions (reverse (semantic-invocation-actions invocation))) + (stack (semantic-invocation-stack invocation)) (result '())) + (if (pair? actions) + (let* ((input (car stack)) + (offset (if (null? input) + (recognition-sequence-start result (semantic-invocation-offset invocation)) + (semantic-invocation-offset invocation))) + (value (foldl (lambda (action current) (apply-action action current offset)) + input (car actions)))) + (loop (cdr actions) (cdr stack) (recognition-sequence-append value result))) + (unless (equal? result (semantic-invocation-expected invocation)) + (error "semantic action diagnostics change the captured reduction"))))) + (lr-semantic-replay-invocations replay)) + (list (cons 'scope 'selected-derivation-action-reexecution) + (cons 'fieldActions fields) (cons 'aliasActions aliases) + (cons 'materializations materializations) (cons 'materializedChildren children) + (cons 'listTailLinks tail-links) (cons 'avoidableListTailLinks avoidable-tail-links) + (cons 'singletonBoundedActions singleton-bounds) + (cons 'widestActionInput widest)))) + +(def (profile-lr-derivation table root) + (let ((reductions 0) (empty 0) (unary 0) (multiple 0) + (identity-unary 0) (decorated 0) (longest 0)) + (let visit ((pending (list (cons root 0)))) + (when (pair? pending) + (let ((piece (caar pending)) (chain (cdar pending))) + (if (lr-recognition-fragment? piece) + (let* ((rhs (production-rhs + (vector-ref table (lr-recognition-fragment-production-id piece)))) + (width (length rhs)) + (next-chain (if (= width 1) (+ chain 1) 0))) + (set! reductions (+ reductions 1)) + (case width + ((0) (set! empty (+ empty 1))) + ((1) (set! unary (+ unary 1)) + (when (null? (operand-actions (car rhs))) + (set! identity-unary (+ identity-unary 1)))) + (else (set! multiple (+ multiple 1)))) + (set! longest (max longest next-chain)) + (for-each (lambda (operand) + (when (pair? (operand-actions operand)) + (set! decorated (+ decorated 1)))) rhs) + (visit (append (map (lambda (child) (cons child next-chain)) + (lr-recognition-fragment-children piece)) + (cdr pending)))) + (visit (cdr pending)))))) + (list (cons 'scope 'selected-fresh-derivation) + (cons 'reductions reductions) (cons 'emptyReductions empty) + (cons 'unaryReductions unary) (cons 'multiOperandReductions multiple) + (cons 'identityUnaryReductions identity-unary) + (cons 'decoratedOperands decorated) (cons 'longestUnaryChain longest)))) + +(def (profile-lr-request-reductions machine tokens (required? #t)) + (let* ((spec (cdr (assq 'lr-spec (parser-machine-ir machine)))) + (table (production-table (lr-spec-ref spec 'productions))) (root #f)) + (parameterize ((current-lr-recognition-observer (lambda (value) (set! root value)))) + (let-values (((_value rest) (lr-parse/prepared (parser-machine-runtime machine) tokens))) + (unless (null? rest) (error "derivation diagnostics left trailing tokens")))) + (cond + ((lr-recognition-fragment? root) (profile-lr-derivation table root)) + (required? (error "derivation diagnostics require a deterministic fresh request")) + (else (list (cons 'scope 'unavailable) + (cons 'reason 'no-deterministic-recognition-derivation)))))) diff --git a/t/benchmarks/lr-request-stack/lookahead.ss b/t/benchmarks/lr-request-stack/lookahead.ss new file mode 100644 index 00000000..b1ec488d --- /dev/null +++ b/t/benchmarks/lr-request-stack/lookahead.ss @@ -0,0 +1,83 @@ +;;; Complete requests compare production selection with the independent +;;; original row selector. No measurement callback enters the timed executor. +(import (only-in :gerbil-parser/src/compiler/machine parser-machine-with-lr-action-selector + parser-machine-runtime) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/token token-lexeme) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid-for-source?) + (only-in :gerbil-parser/src/runtime/parse-cost current-parser-cost-observer) + (only-in :gerbil-parser/src/runtime/lr-parser lr-runtime-direct-step lr-runtime-layout?) + (only-in :gerbil-parser/src/runtime/lr-action-index lr-action-row-has-literals?) + (only-in :gerbil-parser/t/fixtures/lr-action-selection reference-lr-action-selector) + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark + measure-parser-cpu-pairs measure-parser-component)) +(export benchmark-lr-lookahead profile-lr-lookahead) + +;;; Count original selection work outside timing. Counts establish reuse +;;; eligibility, not an additive CPU decomposition or a measured speedup. +(def (profile-lr-lookahead machine source expected) + (let ((observations 0) (literal-observations 0) (eof-observations 0) + (lookaheads (make-table test: eq?)) + (literal-lookaheads (make-table test: eq?)) + (literal-characters 0) (unique-literal-characters 0)) + (let* ((factory + (lambda (rows index casefold? layout?) + (let (select (reference-lr-action-selector rows index casefold? layout?)) + (lambda (state input) + (if (not input) + (set! eof-observations (+ eof-observations 1)) + (begin + (set! observations (+ observations 1)) + (when (lr-action-row-has-literals? (vector-ref index state)) + (set! literal-observations (+ literal-observations 1)) + (let (width (string-length (token-lexeme input))) + (set! literal-characters (+ literal-characters width)) + (unless (table-ref literal-lookaheads input #f) + (table-set! literal-lookaheads input #t) + (set! unique-literal-characters (+ unique-literal-characters width))))) + (table-set! lookaheads input #t))) + (select state input))))) + (observed (parse-source (parser-machine-with-lr-action-selector machine factory) source))) + (unless (equal? observed expected) (error "lookahead observation changes complete product")) + (list (cons 'actionObservations observations) + (cons 'literalRowObservations literal-observations) + (cons 'lookaheads (table-length lookaheads)) + (cons 'literalLookaheads (table-length literal-lookaheads)) + (cons 'literalCharactersObserved literal-characters) + (cons 'literalCharactersOnce unique-literal-characters) + (cons 'eofObservations eof-observations))))) + +(def (benchmark-lr-lookahead workload machine source (groups 20) (calls 100)) + (let* ((stages '()) + (witness + (parameterize + ((current-parser-cost-observer + (lambda (name receipt) + (when (cdr (assq 'completed receipt)) (set! stages (cons name stages)))))) + (parse-source machine source))) + (checkpoint? (memq 'prepared-checkpoint-execution stages)) + (generated? (or (memq 'generated-drive-execution stages) + (and (memq 'generated-source-product stages) (not checkpoint?)))) + (runtime (parser-machine-runtime machine)) + (unchanged? (or generated? (lr-runtime-direct-step runtime) (lr-runtime-layout? runtime))) + ;; An installed generated request is an identical-executor control. + ;; Substituting a selector would remove its generated driver and + ;; compare different executors rather than the proposed classifier. + (reference (if unchanged? machine + (parser-machine-with-lr-action-selector machine reference-lr-action-selector))) + (expected (parse-source reference source)) + (left (lambda () (parse-source reference source))) + (right (lambda () (parse-source machine source)))) + (unless (and (or generated? checkpoint?) (equal? witness expected)) + (error "lookahead request lacks an equivalent executor witness" workload stages)) + (unless (parse-artifact-valid-for-source? expected source) + (error "lookahead benchmark requires a source-bound complete product" workload)) + (write (list 'LR-LOOKAHEAD-WORK workload + (if unchanged? 'identical-executor-control + (profile-lr-lookahead machine source expected)))) + (newline) (force-output) + (measure-parser-component (list workload 'reference-request) 20 calls left expected) + (measure-parser-component (list workload 'production-request) 20 calls right expected) + (let (ratios (measure-parser-cpu-pairs 'lr-lookahead workload groups calls expected left right)) + (write (list 'LR-ACTION-SELECTION-CONTROL-CPU-RATIOS workload ratios)) + (newline) (force-output)))) diff --git a/t/benchmarks/lr-request-stack/record-access.ss b/t/benchmarks/lr-request-stack/record-access.ss new file mode 100644 index 00000000..3db86911 --- /dev/null +++ b/t/benchmarks/lr-request-stack/record-access.ss @@ -0,0 +1,55 @@ +;;; Closed engine records: prepared LR and complete source requests. +(import (only-in :gerbil-parser/languages/gql/parser gql-parser parse-gql +gql-representative-query+) + (only-in :gerbil-parser/languages/fhirpath/parser fhirpath-parser parse-fhirpath) + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-parser parse-arithmetic) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime parser-machine-grammar-digest parser-machine-trivia) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse/prepared lr-parse/prepared/receipt) + (only-in :gerbil-parser/src/runtime/significant parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/artifact make-success-parse-artifact parse-artifact-valid-for-source? + parse-artifact-events token-event? token-event-token-kind token-event-lexeme event-start event-end sha256-text) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in ../parser-stage-cost/benchmark measure-parser-component)) +(import (only-in ./sdk-control call-with-record-sdk-control require-record-sdk-code-identity!)) +(export main) + +(def (measure-record-request family machine parse source) + (let* ((expected (parse source)) + (tokens (map (lambda (event) (make-token (token-event-token-kind event) (token-event-lexeme event) + (event-start event) (event-end event))) + (filter token-event? (parse-artifact-events expected)))) + (runtime (parser-machine-runtime machine)) + (significant (parser-significant-tokens machine tokens)) + (prepared (lambda () (call-with-values (lambda () (lr-parse/prepared runtime significant)) list))) + (recognized (prepared))) + (unless (and (parse-artifact-valid-for-source? expected source) (null? (cadr recognized))) + (error "record request lacks a valid complete artifact" family)) + (let-values (((root rest _receipt) (lr-parse/prepared/receipt runtime significant))) + (unless (and (null? rest) + (equal? (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens root (parser-machine-trivia machine)) expected)) + (error "independent GLR publication differs" family))) + (unless (equal? (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens (car recognized) (parser-machine-trivia machine)) expected) + (error "prepared publication differs" family)) + ;; Canonical public data is portable across independently loaded products. + (write (list 'RECORD-REQUEST-ARTIFACT family (sha256-text (object->string expected)))) + (newline) (force-output) + (measure-parser-component (list family 'prepared-lr) 40 1000 prepared recognized) + (measure-parser-component (list family 'full-source) 40 1000 (lambda () (parse source)) expected))) + +(def (main) + (measure-record-request 'gql-request gql-parser parse-gql +gql-representative-query+) + (measure-record-request 'gql-return gql-parser parse-gql + (string-append "RETURN " (string-join (map (lambda (i) (string-append "v" (number->string i))) (iota 64)) ", ") "\n")) + (measure-record-request 'fhirpath-addition fhirpath-parser parse-fhirpath (string-join (make-list 64 "1") " + ")) + (measure-record-request 'arithmetic-control arithmetic-parser parse-arithmetic "1") + ;; The SDK encoder is an identical implementation control. Arithmetic still + ;; exercises the generated parser, whose shared machine getters may change. + (call-with-record-sdk-control + (lambda (encode expected) + (measure-parser-component '(sdk-control utf8) 40 1000 encode expected sample-output: 'deferred) + (call-with-record-sdk-control + (lambda (peer _expected) + (require-record-sdk-code-identity! encode peer))) + (displayln "RECORD-REQUESTS-OK") (force-output) + encode))) diff --git a/t/benchmarks/lr-request-stack/record-evidence.ss b/t/benchmarks/lr-request-stack/record-evidence.ss new file mode 100644 index 00000000..77e2615a --- /dev/null +++ b/t/benchmarks/lr-request-stack/record-evidence.ss @@ -0,0 +1,127 @@ +;;; Native admission of independently loaded complete-request receipts. +(import (only-in ../parser-stage-cost/benchmark sample-at-percentile)) +(export main) + +(defstruct record-receipt (summaries artifacts) final: #t) + +(def +record-families+ '(gql-request gql-return fhirpath-addition arithmetic-control)) +(def (field row name) (cdr (assq name row))) +(def (require-record! condition . details) + (unless condition (apply error "record request evidence rejected" details))) + +(def (require-cpu-clock! rows summary path key) + (require-record! + (and (andmap (lambda (row) + (let (ticks (field row 'cpu-ns)) + (and (exact-integer? ticks) (>= ticks 0) + (= (field row 'cpu-ms) (exact->inexact (/ ticks 1000000)))))) rows) + (= (field summary 'cpuP50Ns) (field (sample-at-percentile rows 'cpu-ns 50) 'cpu-ns)) + (= (field summary 'cpuP95Ns) (field (sample-at-percentile rows 'cpu-ns 95) 'cpu-ns))) + path key 'cpu-clock)) + +(def (read-record-receipt path candidate? image-marker) + (let ((summaries (make-hash-table)) (samples (make-hash-table)) + (artifacts (make-hash-table)) (complete? #f) (exited? #f) (class? #f) (sdk-entry? #f)) + (call-with-input-file path + (lambda (port) + (let loop ((line (read-line port))) + (unless (eof-object? line) + (cond + ((string-prefix? "(PARSER-COMPONENT-SAMPLE " line) + (let* ((datum (call-with-input-string line read)) + (key (cadr datum)) (row (caddr datum)) (old (hash-ref samples key '())) + (index (field row 'sample))) + (require-record! (and (exact-integer? index) (<= 0 index 39) + (not (find (lambda (row) (= index (field row 'sample))) old))) path key 'sample) + (hash-put! samples key (cons row old)))) + ((string-prefix? "(PARSER-COMPONENT-SUMMARY " line) + (let* ((row (cadr (call-with-input-string line read))) (key (field row 'stage))) + (require-record! (not (hash-get summaries key)) path key 'duplicate) + (hash-put! summaries key row))) + ((string-prefix? "(RECORD-REQUEST-ARTIFACT " line) + (let (row (call-with-input-string line read)) + (require-record! (not (hash-get artifacts (cadr row))) path 'duplicate-artifact) + (hash-put! artifacts (cadr row) (caddr row)))) + ((string-prefix? (string-append image-marker " ") line) + (require-record! (and (not class?) + (equal? line (string-append image-marker (if candidate? " #t" " #f")))) path 'class) + (set! class? #t)) + ((equal? line "SDK-ENCODER-ENTRY-IDENTICAL") + (require-record! (not sdk-entry?) path 'duplicate-sdk-entry) + (set! sdk-entry? #t)) + ((equal? line "RECORD-REQUESTS-OK") + (require-record! (not complete?) path 'duplicate-completion) + (set! complete? #t)) + ((string-prefix? "PROCESS-RESULT " line) + (require-record! (and (not exited?) (string-contains line "exit=0 reason=#f missing=()")) path 'exit) + (set! exited? #t))) + (loop (read-line port)))))) + (require-record! (and complete? exited? class? sdk-entry? (= (hash-length artifacts) 4) + (= (hash-length summaries) 9) (= (hash-length samples) 9)) path 'incomplete) + (for-each + (lambda (family) + (require-record! (hash-get artifacts family) path family 'artifact) + (for-each + (lambda (stage) + (let* ((key (list family stage)) (rows (hash-ref samples key '())) (summary (hash-get summaries key))) + (require-cpu-clock! rows summary path key) + (require-record! (and summary (= (length rows) 40) (= (field summary 'sampleCount) 40) + (= (field summary 'parsesPerSample) 1000) + (andmap (lambda (row) (>= (field row 'cpu-ms) 0)) rows) + (= (field summary 'cpuP50Ms) (field (sample-at-percentile rows 'cpu-ms 50) 'cpu-ms)) + (= (field summary 'cpuP95Ms) (field (sample-at-percentile rows 'cpu-ms 95) 'cpu-ms))) path key 'summary))) + '(prepared-lr full-source))) +record-families+) + (let* ((key '(sdk-control utf8)) (rows (hash-ref samples key '())) (summary (hash-get summaries key))) + (require-cpu-clock! rows summary path key) + (require-record! (and summary (= (length rows) 40) (= (field summary 'sampleCount) 40) + (= (field summary 'parsesPerSample) 1000) + (andmap (lambda (row) (>= (field row 'cpu-ms) 0)) rows) + (= (field summary 'cpuP50Ms) (field (sample-at-percentile rows 'cpu-ms 50) 'cpu-ms)) + (= (field summary 'cpuP95Ms) (field (sample-at-percentile rows 'cpu-ms 95) 'cpu-ms))) + path key 'sdk-control)) + (make-record-receipt summaries artifacts))) + +(def (main reference-a candidate-a candidate-b reference-b + (image-marker "RECORD-CLASS-FINAL") + (targets '(gql-request gql-return fhirpath-addition)) + (controls '(arithmetic-control))) + ;; Four admitted families and four roles imply a unique complete partition. + (let (roles (append targets controls)) + (require-record! (and (pair? targets) + (member image-marker '("RECORD-CLASS-FINAL" "OWNED-STACK" "UNARY-CHAIN" "LOOKAHEAD-VOCABULARY" "AST-BOUNDS" "ROW-CONSTRUCTION" "COMPACT-CHILD" "TOKEN-INLINE" "REQUEST-BOUNDS")) + (= (length roles) (length +record-families+)) + (andmap (lambda (family) (memq family roles)) +record-families+)) + 'scope)) + (let* ((references (map (lambda (path) (read-record-receipt path #f image-marker)) (list reference-a reference-b))) + (candidates (map (lambda (path) (read-record-receipt path #t image-marker)) (list candidate-a candidate-b))) + (all (append references candidates))) + (for-each + (lambda (family) + (let (digest (hash-get (record-receipt-artifacts (car all)) family)) + (require-record! (andmap (lambda (receipt) (equal? digest (hash-get (record-receipt-artifacts receipt) family))) all) family 'artifact-drift))) + +record-families+) + (def (cpu receipts family stage rank) + (let (ticks (case rank ((cpuP50Ms) 'cpuP50Ns) ((cpuP95Ms) 'cpuP95Ns))) + (map (lambda (receipt) + (/ (field (hash-get (record-receipt-summaries receipt) (list family stage)) ticks) 1000000)) + receipts))) + (let* ((control-r (cpu references 'sdk-control 'utf8 'cpuP50Ms)) + (control-c (cpu candidates 'sdk-control 'utf8 'cpuP50Ms)) + (ratio (/ (apply + control-c) (apply + control-r)))) + (require-record! (and (andmap (lambda (value) (>= value 1)) control-r) + (<= 19/20 ratio 21/20) + (<= (apply min control-c) (apply max control-r)) + (<= (apply min control-r) (apply max control-c))) 'control ratio) + (write (list 'RECORD-CPU-CONTROL ratio)) (newline)) + (for-each + (lambda (family) + (for-each + (lambda (stage) + (let* ((r (cpu references family stage 'cpuP50Ms)) (c (cpu candidates family stage 'cpuP50Ms)) + (ratio (/ (apply max c) (apply min r)))) + (write (list 'RECORD-CPU-BOUND family stage ratio)) (newline) (force-output) + (require-record! (if (memq family targets) (< ratio 19/20) (<= ratio 21/20)) family stage 'repeated-median) + (require-record! (<= (apply max (cpu candidates family stage 'cpuP95Ms)) + (* 21/20 (apply max (cpu references family stage 'cpuP95Ms)))) family stage 'tail))) + '(prepared-lr full-source))) (append targets controls)) + (displayln "RECORD-CPU-ADMISSION-OK") (force-output))) diff --git a/t/benchmarks/lr-request-stack/record-repeatability.ss b/t/benchmarks/lr-request-stack/record-repeatability.ss new file mode 100644 index 00000000..cfed8377 --- /dev/null +++ b/t/benchmarks/lr-request-stack/record-repeatability.ss @@ -0,0 +1,17 @@ +;;; Controlled attribution only; these extra phases cannot replace admission. +(import (rename-in ./record-access (main record-request-main)) + (only-in ./sdk-control call-with-record-sdk-control require-record-sdk-code-identity!) + (only-in ../parser-stage-cost/benchmark measure-parser-component)) +(export main) + +(def (main) + (call-with-record-sdk-control + (lambda (encode expected) + (measure-parser-component '(sdk-native-before utf8) 40 1000 encode expected) + (let (ordinary-encode (record-request-main)) + (require-record-sdk-code-identity! encode ordinary-encode)) + (measure-parser-component '(sdk-immediate-a utf8) 40 1000 encode expected) + (measure-parser-component '(sdk-deferred utf8) 40 1000 encode expected sample-output: 'deferred) + (measure-parser-component '(sdk-immediate-b utf8) 40 1000 encode expected))) + (displayln "SDK-REPEATABILITY-DIAGNOSTIC-OK") + (force-output)) diff --git a/t/benchmarks/lr-request-stack/request-cost.ss b/t/benchmarks/lr-request-stack/request-cost.ss new file mode 100644 index 00000000..61e0cd65 --- /dev/null +++ b/t/benchmarks/lr-request-stack/request-cost.ss @@ -0,0 +1,63 @@ +;;; Real-request boundary diagnostics; the four-stage CPU gate is separate. +(import (only-in :gerbil-parser/languages/gql/parser + gql-parser parse-gql +gql-representative-query+) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-runtime parser-machine-grammar-digest parser-machine-trivia) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-parse/prepared lr-parse/prepared/receipt lr-initial-checkpoint lr-checkpoint-drive) + (only-in :gerbil-parser/src/runtime/significant parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact parse-artifact-valid-for-source? parse-artifact-events + token-event? token-event-token-kind token-event-lexeme event-start event-end) + (only-in ../parser-stage-cost/benchmark measure-parser-component)) +(export main) + +;;; Supplies the admitted significant tokens through the production input +;;; callback protocol. No scanner or token observer is timed in this control. +(def (recognize-stream runtime tokens) + (let (remaining tokens) + (def (next-input _mode) + (and (pair? remaining) + (let (input (car remaining)) + (set! remaining (cdr remaining)) input))) + (let-values (((status payload) + (lr-checkpoint-drive (lr-initial-checkpoint runtime '()) next-input #f))) + (unless (and (eq? status 'accepted) (null? remaining)) + (error "boundary diagnostic requires complete deterministic recognition" status)) + payload))) + +(def (main) + (let* ((source +gql-representative-query+) (machine gql-parser) + (artifact (parse-gql source)) + (tokens (map (lambda (event) + (make-token (token-event-token-kind event) (token-event-lexeme event) + (event-start event) (event-end event))) + (filter token-event? (parse-artifact-events artifact)))) + (runtime (parser-machine-runtime machine)) + (significant (parser-significant-tokens machine tokens)) + (prepared (lambda () (call-with-values (lambda () (lr-parse/prepared runtime significant)) list))) + (recognized (prepared)) + (digest (parser-machine-grammar-digest machine)) + (trivia? (parser-machine-trivia machine))) + (def (publish root) (make-success-parse-artifact digest source tokens root trivia?)) + (unless (and (parse-artifact-valid-for-source? artifact source) + (null? (cadr recognized)) (equal? (publish (car recognized)) artifact) + (equal? (recognize-stream runtime significant) recognized)) + (error "request boundary diagnostic changed the complete product")) + (let-values (((root rest _receipt) (lr-parse/prepared/receipt runtime significant))) + (unless (and (null? rest) (equal? (publish root) artifact)) + (error "independent GLR boundary product differs"))) + ;; Separate component intervals are diagnostic. Their medians/GC observations + ;; must not be subtracted to claim an additive decomposition or CPU benefit. + (measure-parser-component '(gql-request boundary-prepared-lr) 40 1000 prepared recognized) + (measure-parser-component '(gql-request boundary-streamed-lr) 40 1000 + (lambda () (recognize-stream runtime significant)) recognized) + (measure-parser-component '(gql-request boundary-publication) 40 1000 + (lambda () (publish (car recognized))) artifact) + (measure-parser-component '(gql-request boundary-lr-publication) 40 1000 + (lambda () (let-values (((root rest) (lr-parse/prepared runtime significant))) + (unless (null? rest) (error "unexpected boundary remainder")) (publish root))) artifact) + (measure-parser-component '(gql-request boundary-full-source) 40 1000 + (lambda () (parse-gql source)) artifact) + (displayln "REQUEST-COST-BOUNDARIES-OK") (force-output))) diff --git a/t/benchmarks/lr-request-stack/request-profile.ss b/t/benchmarks/lr-request-stack/request-profile.ss new file mode 100644 index 00000000..d0e7fe33 --- /dev/null +++ b/t/benchmarks/lr-request-stack/request-profile.ss @@ -0,0 +1,50 @@ +(import (only-in :gerbil-parser/languages/gql/parser gql-parser parse-gql +gql-representative-query+) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime parser-machine-ir) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-events token-event? + token-event-token-kind token-event-lexeme event-start event-end) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in ./lookahead profile-lr-lookahead) + (only-in ./derivation profile-lr-request-reductions prepare-lr-semantic-replay + profile-lr-semantic-actions) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse/prepared) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/significant parser-significant-tokens)) +(export main profile-work) +;;; Statistical attribution is independent of the matched CPU admission. +;;; The workload does not assume a profiler frontend is installed. +(def (main (stage "source")) + (unless (member stage '("source" "prepared")) + (error "invalid request profile stage" stage)) + (let* ((source +gql-representative-query+) + (runtime (parser-machine-runtime gql-parser)) + (tokens (parser-significant-tokens gql-parser (lex-source gql-parser source))) + (request (if (equal? stage "prepared") + (lambda () (call-with-values (lambda () (lr-parse/prepared runtime tokens)) list)) + (lambda () (parse-gql source)))) + (expected (request))) + (let loop ((remaining 30000) (result expected)) + (if (zero? remaining) + (unless (equal? result expected) (error "profile changed complete request" stage)) + (begin + (when (zero? (modulo remaining 3000)) (displayln "PROFILE-REQUESTS " remaining) (force-output)) + (loop (fx- remaining 1) (request))))) + (displayln "REQUEST-PROFILE-OK") (force-output))) + + +;;; Count work from the actual complete product outside every timed interval. +;;; Replay validates canonical operands; these counts are not CPU percentages. +(def (profile-work) + (let* ((source +gql-representative-query+) (expected (parse-gql source)) + (tokens (map (lambda (event) + (make-token (token-event-token-kind event) (token-event-lexeme event) + (event-start event) (event-end event))) + (filter token-event? (parse-artifact-events expected)))) + (significant (parser-significant-tokens gql-parser tokens)) + (runtime (parser-machine-runtime gql-parser)) + (spec (cdr (assq 'lr-spec (parser-machine-ir gql-parser))))) + (write (list 'LR-REQUEST-WORK 'gql-request + (profile-lr-lookahead gql-parser source expected) + (profile-lr-request-reductions gql-parser significant) + (profile-lr-semantic-actions + (prepare-lr-semantic-replay runtime spec significant)))) + (newline) (displayln "REQUEST-WORK-OK") (force-output))) diff --git a/t/benchmarks/lr-request-stack/sdk-control.ss b/t/benchmarks/lr-request-stack/sdk-control.ss new file mode 100644 index 00000000..dff11614 --- /dev/null +++ b/t/benchmarks/lr-request-stack/sdk-control.ss @@ -0,0 +1,15 @@ +;;; One separately compiled encoder factory supplies every measurement caller. +(export call-with-record-sdk-control require-record-sdk-code-identity!) + +(def (call-with-record-sdk-control continuation) + ;; One compiled encoder closure owns its payload and complete expected result. + ;; Diagnostics may reuse it across phases without replacing native imports. + (let* ((source (make-string 4096 #\a)) (expected (string->utf8 source))) + (continuation (lambda () (string->utf8 source)) expected))) + +(def (require-record-sdk-code-identity! retained measured) + (unless (and (##closure? retained) (##closure? measured) + (eq? (##closure-code retained) (##closure-code measured))) + (error "SDK measurement callers do not share one encoder entry")) + (displayln "SDK-ENCODER-ENTRY-IDENTICAL") + (force-output)) diff --git a/t/benchmarks/lr1-partition/benchmark.ss b/t/benchmarks/lr1-partition/benchmark.ss new file mode 100644 index 00000000..922583e3 --- /dev/null +++ b/t/benchmarks/lr1-partition/benchmark.ss @@ -0,0 +1,273 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Isolated Scheme parser-generator construction receipt. + +(import (only-in :gerbil-parser/src/compiler/lr + compute-first compute-nullable lower-rules lr-spec-ref + production-table) + (only-in :gerbil-parser/src/compiler/lr-conflict-candidates + forward-reachable-follow-masks + initial-backward-follow-partitions + lr0-conflict-candidates) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/t/fixtures/lr1-construction + lr1-not-lalr-rules shared-lookahead-rules + precedence-expression-rules mixed-context-rules + lr1-context-family-rules inactive-core-conflict-rules)) + +(def (timing-fields started repetitions) + (let (total-ms (* 1000.0 (- (##current-time-point) started))) + (list (cons 'elapsed-total-ms total-ms) + (cons 'mean-us (/ (* 1000.0 total-ms) repetitions))))) + +(def (chain-rules count) + (def (rule-name index) + (string->symbol (string-append "chain-" (number->string index)))) + (cons + (list 'source-file + (list 'alias 'SourceFile (list 'reference (rule-name 0)))) + (map + (lambda (index) + (list (rule-name index) + (if (= index (- count 1)) + (list 'literal (number->string index)) + (list 'sequence + (list 'literal (number->string index)) + (list 'reference (rule-name (+ index 1))))))) + (iota count)))) + +(def (measure grammar rules construction repetitions) + (let ((started (##current-time-point)) + (last #f)) + (let loop ((remaining repetitions)) + (unless (zero? remaining) + (set! last + (compile-lr-spec rules 'source-file 'reject #f construction)) + (loop (- remaining 1)))) + (let (timing (timing-fields started repetitions)) + (list (cons 'grammar grammar) + (cons 'construction construction) + (cons 'repetitions repetitions) + (cons 'state-count (lr-spec-ref last 'state-count)) + (cons 'construction-items + (or (lr-spec-ref last 'output-item-count) + (lr-spec-ref last 'lookahead-item-visit-count))) + (car timing) (cadr timing))))) + +(def (character-tokens source) + (let loop ((index 0) (tokens '())) + (if (= index (string-length source)) + (reverse tokens) + (loop (+ index 1) + (cons (make-token 'punctuation + (string (string-ref source index)) + index (+ index 1)) + tokens))))) + +;; The table and token stream are prepared outside the timed loop. This +;; measures fresh LR parsing, not grammar construction or lexing. +(def (measure-parse grammar rules construction source repetitions) + (let* ((spec (compile-lr-spec rules 'source-file 'reject #f construction)) + (tokens (character-tokens source)) + (last #f)) + (def (run-batch count) + (let (started (##current-time-point)) + (let loop ((remaining count)) + (unless (zero? remaining) + (let-values (((root rest) (lr-parse spec tokens))) + (unless (null? rest) + (error "parser left a token suffix" grammar source rest)) + (set! last root)) + (loop (- remaining 1)))) + (/ (* 1000000.0 (- (##current-time-point) started)) count))) + (run-batch (min 100 repetitions)) + (let* ((first (run-batch repetitions)) + (second (run-batch repetitions)) + (third (run-batch repetitions)) + (minimum (min first second third)) + (maximum (max first second third))) + (unless last (error "parser produced no root" grammar source)) + (list (cons 'workload 'fresh-lr-parse) + (cons 'grammar grammar) + (cons 'construction construction) + (cons 'source source) + (cons 'repetitions repetitions) + (cons 'batches 3) + (cons 'median-us (- (+ first second third) minimum maximum)) + (cons 'min-us minimum) + (cons 'max-us maximum))))) + +;; Phase-only timing: the lowered grammar and FIRST/nullability are fixed. +;; Compare this with itself across changes, not with full compile-lr-spec time. +(def (measure-forward-follows grammar rules repetitions) + (let* ((productions (lower-rules rules 'source-file)) + (table (production-table productions))) + (let-values (((nullable nullable-index) + (compute-nullable productions))) + (let-values (((first first-index) + (compute-first productions nullable-index))) + (let ((started (##current-time-point)) + (last #f)) + (let loop ((remaining repetitions)) + (unless (zero? remaining) + (let-values (((follows terminals) + (forward-reachable-follow-masks + productions table first-index nullable-index))) + (set! last follows)) + (loop (- remaining 1)))) + (let (timing (timing-fields started repetitions)) + (list (cons 'grammar grammar) + (cons 'construction 'forward-reachable-follows) + (cons 'repetitions repetitions) + (cons 'follow-mask-sum + (foldl + 0 (vector->list last))) + (car timing) (cadr timing)))))))) + +;; Phase-only timing: the lowered grammar and FIRST/nullability are fixed. +;; Compare this with itself across changes, not with full compile-lr-spec time. +(def (measure-direct-seed grammar rules repetitions) + (let* ((productions (lower-rules rules 'source-file)) + (table (production-table productions))) + (let-values (((nullable nullable-index) + (compute-nullable productions))) + (let-values (((first first-index) + (compute-first productions nullable-index))) + (let ((started (##current-time-point)) + (last #f)) + (let loop ((remaining repetitions)) + (unless (zero? remaining) + (let-values (((partitions candidates terminals) + (initial-backward-follow-partitions + productions table first-index nullable-index))) + (set! last partitions)) + (loop (- remaining 1)))) + (let (timing (timing-fields started repetitions)) + (list (cons 'grammar grammar) + (cons 'construction 'lr0-initial-backward-seed) + (cons 'repetitions repetitions) + (cons 'block-count + (foldl (lambda (blocks count) + (+ count (if blocks (length blocks) 0))) + 0 (vector->list last))) + (car timing) (cadr timing)))))))) + +;; Isolate LR(0) construction plus raw candidate detection from the follow +;; partition refinements. The candidate count is a semantic receipt alongside +;; the timing, so an apparent speedup cannot silently drop conflict cells. +(def (measure-candidates grammar rules repetitions) + (let* ((productions (lower-rules rules 'source-file)) + (table (production-table productions))) + (let-values (((nullable nullable-index) + (compute-nullable productions))) + (let-values (((first first-index) + (compute-first productions nullable-index))) + (let ((started (##current-time-point)) + (last #f)) + (let loop ((remaining repetitions)) + (unless (zero? remaining) + (let-values (((states count terminals candidates) + (lr0-conflict-candidates + productions table first-index nullable-index))) + (set! last candidates)) + (loop (- remaining 1)))) + (let (timing (timing-fields started repetitions)) + (list (cons 'grammar grammar) + (cons 'construction 'lr0-conflict-candidates) + (cons 'repetitions repetitions) + (cons 'conflict-state-count + (foldl (lambda (mask count) + (+ count (if (zero? mask) 0 1))) + 0 (vector->list last))) + (car timing) (cadr timing)))))))) + +(def (main . args) + (let (repetitions + (if (null? args) 100 (string->number (car args)))) + (unless (and (integer? repetitions) (> repetitions 0)) + (error "expected a positive repetition count" args)) + (for-each + (lambda (entry) + (write (measure (car entry) (cadr entry) (caddr entry) + repetitions)) + (newline)) + (list + (list 'shared-lookahead shared-lookahead-rules 'lalr) + (list 'shared-lookahead shared-lookahead-rules 'lalr-then-follow) + (list 'shared-lookahead shared-lookahead-rules 'canonical-lr1) + (list 'shared-lookahead shared-lookahead-rules 'partitioned-lr1) + (list 'shared-lookahead shared-lookahead-rules 'follow-partition-lr1) + (list 'lr1-not-lalr lr1-not-lalr-rules 'canonical-lr1) + (list 'lr1-not-lalr lr1-not-lalr-rules 'lalr-then-follow) + (list 'lr1-not-lalr lr1-not-lalr-rules 'partitioned-lr1) + (list 'lr1-not-lalr lr1-not-lalr-rules 'follow-partition-lr1) + (list 'precedence precedence-expression-rules 'canonical-lr1) + (list 'precedence precedence-expression-rules + 'follow-partition-lr1) + (list 'mixed-context mixed-context-rules 'canonical-lr1) + (list 'mixed-context mixed-context-rules 'follow-partition-lr1))) + (for-each + (lambda (entry) + (write (measure-forward-follows + (car entry) (cadr entry) repetitions)) + (newline)) + (list (list 'shared-lookahead shared-lookahead-rules) + (list 'mixed-context mixed-context-rules) + (list 'lr1-context-family-16 (lr1-context-family-rules 16)))) + (for-each + (lambda (entry) + (write (measure-direct-seed (car entry) (cadr entry) repetitions)) + (newline)) + (list (list 'shared-lookahead shared-lookahead-rules) + (list 'lr1-not-lalr lr1-not-lalr-rules) + (list 'inactive-core-conflict inactive-core-conflict-rules))) + (for-each + (lambda (entry) + (write (measure-candidates (car entry) (cadr entry) repetitions)) + (newline)) + (list (list 'lr1-not-lalr lr1-not-lalr-rules) + (list 'mixed-context mixed-context-rules))) + (let ((large (chain-rules 64)) + (large-repetitions (max 1 (quotient repetitions 10)))) + (for-each + (lambda (construction) + (write (measure 'chain-64 large construction + large-repetitions)) + (newline)) + '(lalr canonical-lr1 follow-partition-lr1)) + (write (measure-direct-seed 'chain-64 large large-repetitions)) + (newline) + (write (measure-candidates 'chain-64 large large-repetitions)) + (newline)) + (for-each + (lambda (size) + (let ((rules (lr1-context-family-rules size)) + (family-repetitions (max 1 (quotient repetitions 10)))) + (for-each + (lambda (construction) + (write (measure + (list 'lr1-context-family size) rules construction + family-repetitions)) + (newline)) + '(canonical-lr1 follow-partition-lr1)))) + '(4 16)) + (let (parse-repetitions (* repetitions 10)) + (for-each + (lambda (entry) + (write (measure-parse (car entry) (cadr entry) (caddr entry) + (cadddr entry) parse-repetitions)) + (newline)) + (list + (list 'shared-lookahead shared-lookahead-rules 'lalr "cdd") + (list 'shared-lookahead shared-lookahead-rules + 'follow-partition-lr1 "cdd") + (list 'lr1-not-lalr lr1-not-lalr-rules 'canonical-lr1 "acd") + (list 'lr1-not-lalr lr1-not-lalr-rules + 'follow-partition-lr1 "acd") + (list 'precedence precedence-expression-rules + 'canonical-lr1 "a+a*a") + (list 'precedence precedence-expression-rules + 'follow-partition-lr1 "a+a*a")))))) + +(export main) diff --git a/t/benchmarks/lr1-partition/construction.ss b/t/benchmarks/lr1-partition/construction.ss new file mode 100644 index 00000000..e3d7a7c5 --- /dev/null +++ b/t/benchmarks/lr1-partition/construction.ss @@ -0,0 +1,72 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Complete LR construction for the real TLA+ layout grammar. +;;; CPU timing excludes imports, explicit GC, and complete LRSpec comparison. + +(import (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr + lower-rules production-table lr-spec-ref) + (only-in :gerbil-parser/src/compiler/parser-ir parser-ir-ref) + (only-in :gerbil-parser/languages/tla-plus/parser + tla-plus-layout-language-grammar) + (only-in :gerbil-parser/src/language/descriptor language-grammar-ir)) + +(def (median values) + (let* ((sorted (list-sort < values)) + (count (length sorted)) + (middle (quotient count 2))) + (if (odd? count) + (list-ref sorted middle) + (/ (+ (list-ref sorted (- middle 1)) + (list-ref sorted middle)) 2.0)))) + +(def (main . args) + (let ((samples (if (pair? args) (string->number (car args)) 20)) + (construction (if (>= (length args) 2) (string->symbol (cadr args)) 'lalr))) + (unless (and (<= (length args) 2) + (integer? samples) (positive? samples) + (memq construction '(lalr follow-partition-lr1))) + (error "expected positive samples and optional lalr or follow-partition-lr1" args)) + (write (list 'workload 'tla-layout 'construction construction 'phase 'warmup)) + (newline) (force-output) + (let* ((rules (parser-ir-ref (language-grammar-ir tla-plus-layout-language-grammar) 'rules)) + (productions (vector-length + (production-table (lower-rules rules 'source-file))))) + ;; Default mode continues to exercise the production LALR selection. + (def (compile) + (if (eq? construction 'lalr) + (compile-lr-spec rules 'source-file 'selective-glr) + (compile-lr-spec rules 'source-file 'selective-glr #f construction))) + (let (reference (compile)) + (let loop ((sample 0) (times '()) (gc-counts '()) (gc-times '())) + (if (= sample samples) + (begin + (write (list 'summary 'workload 'tla-layout 'construction construction + 'samples samples 'rules (length rules) + 'productions productions + 'states (lr-spec-ref reference 'state-count) + 'cpu-median-ms (median times) + 'in-call-gcs-median (median gc-counts) + 'in-call-gc-cpu-median-ms (median gc-times))) + (newline) (force-output)) + (begin + (##gc) + (let* ((before (##process-statistics)) + (started (cpu-time)) + (spec (compile)) + (elapsed (* 1000.0 (- (cpu-time) started))) + (after (##process-statistics)) + (gc-count (- (##f64vector-ref after 6) (##f64vector-ref before 6))) + (gc-ms (* 1000.0 + (+ (- (##f64vector-ref after 3) (##f64vector-ref before 3)) + (- (##f64vector-ref after 4) (##f64vector-ref before 4)))))) + (unless (equal? reference spec) + (error "complete LRSpec changed between samples" sample)) + (write (list 'sample sample 'workload 'tla-layout + 'construction construction 'cpu-ms elapsed + 'in-call-gcs gc-count 'in-call-gc-cpu-ms gc-ms)) + (newline) (force-output) + (loop (+ sample 1) (cons elapsed times) + (cons gc-count gc-counts) (cons gc-ms gc-times)))))))))) + +(export main) diff --git a/t/benchmarks/lr1-partition/context-scale.ss b/t/benchmarks/lr1-partition/context-scale.ss new file mode 100644 index 00000000..e5f921b7 --- /dev/null +++ b/t/benchmarks/lr1-partition/context-scale.ss @@ -0,0 +1,120 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Complete Scheme LR construction as independent context count grows. + +(import (only-in :gerbil-parser/src/compiler/funcs + compiler-index-set-for-each) + (only-in :gerbil-parser/src/compiler/lr + compute-first compute-nullable lower-rules lr-spec-ref + production-table) + (only-in :gerbil-parser/src/compiler/lr-conflict-candidates + forward-reachable-follow-masks raw-conflict-cells + initial-backward-follow-partitions) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr-lookahead + build-states-via-lr0) + (only-in :gerbil-parser/t/fixtures/lr1-construction + lr1-context-family-rules)) + +(def (measure size rules construction sample) + (let* ((started (##current-time-point)) + (spec (compile-lr-spec rules 'source-file 'reject #f construction)) + (elapsed-ms (* 1000.0 (- (##current-time-point) started)))) + (list (cons 'workload 'grammar-construction) + (cons 'grammar-contexts size) + (cons 'construction construction) + (cons 'sample sample) + (cons 'elapsed-ms elapsed-ms) + (cons 'state-count (lr-spec-ref spec 'state-count)) + (cons 'follow-block-count + (or (lr-spec-ref spec 'follow-block-count) 0)) + (cons 'construction-items + (or (lr-spec-ref spec 'output-item-count) + (lr-spec-ref spec 'lookahead-item-visit-count)))))) + +(def (measure-seed size rules) + (let* ((productions (lower-rules rules 'source-file)) + (table (production-table productions))) + (let-values (((nullable nullable-index) + (compute-nullable productions))) + (let-values (((first first-index) + (compute-first productions nullable-index))) + (let ((started (##current-time-point)) + (production-follow-pairs 0) + (nfa-vertices 0) + (lr0-states 0) + (conflict-states 0)) + (let-values (((states count lookaheads offsets transitions + terminal-values layout core-symbols + state-visits item-visits) + (build-states-via-lr0 + productions table first-index nullable-index))) + (set! lr0-states count) + (let (candidates + (raw-conflict-cells + states count lookaheads offsets terminal-values + layout core-symbols)) + (for-each + (lambda (mask) + (unless (zero? mask) + (set! conflict-states (+ conflict-states 1)))) + (vector->list candidates)))) + (let-values (((follows terminals) + (forward-reachable-follow-masks + productions table first-index nullable-index))) + (for-each + (lambda (mask) + (compiler-index-set-for-each + mask (lambda (terminal) + (set! production-follow-pairs + (+ production-follow-pairs 1))))) + (vector->list follows))) + (let-values (((partitions candidates terminals) + (initial-backward-follow-partitions + productions table first-index nullable-index))) + (for-each + (lambda (blocks) + (when blocks + (for-each + (lambda (block) + (compiler-index-set-for-each + (cdr block) + (lambda (terminal) + (set! nfa-vertices (+ nfa-vertices 1))))) + blocks))) + (vector->list partitions)) + (list (cons 'workload 'grammar-construction) + (cons 'grammar-contexts size) + (cons 'construction 'initial-backward-seed) + (cons 'elapsed-ms + (* 1000.0 (- (##current-time-point) started))) + (cons 'production-follow-pairs production-follow-pairs) + (cons 'productions (vector-length table)) + (cons 'lr0-states lr0-states) + (cons 'conflict-states conflict-states) + (cons 'nfa-vertices nfa-vertices) + (cons 'initial-blocks + (foldl (lambda (blocks count) + (+ count (if blocks (length blocks) 0))) + 0 (vector->list partitions)))))))))) + +(def (main . args) + (let ((size (if (pair? args) (string->number (car args)) 64)) + (samples (if (and (pair? args) (pair? (cdr args))) + (string->number (cadr args)) 3))) + (unless (and (integer? size) (> size 0) + (integer? samples) (> samples 0)) + (error "expected positive context count and sample count" args)) + (let (rules (lr1-context-family-rules size)) + (write (measure-seed size rules)) + (newline) + (for-each + (lambda (construction) + (let loop ((sample 1)) + (when (<= sample samples) + (write (measure size rules construction sample)) + (newline) + (loop (+ sample 1))))) + '(canonical-lr1 follow-partition-lr1))))) + +(export main) diff --git a/t/benchmarks/lr1-partition/item-space.ss b/t/benchmarks/lr1-partition/item-space.ss new file mode 100644 index 00000000..90fcf4e4 --- /dev/null +++ b/t/benchmarks/lr1-partition/item-space.ss @@ -0,0 +1,102 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Complete compiler products, not a request-latency proxy. Each observation +;;; checks the entire LRSpec outside the measured construction interval. +(import (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr-automaton make-item-layout make-core-symbol-catalog) + (only-in :gerbil-parser/src/compiler/lr lower-rules production-table production-rhs production-terminal-catalog) + (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation) + (only-in :gerbil-parser/t/fixtures/lr1-construction skew-width-rules)) +(export main observe-lr-construction observe-lr-layout) + +;;; Separate the newly added inverse-owner preparation from total compilation. +;;; Retain the last product and consume its symbols outside the timed interval. +;;; The same source entry runs against original and current native producers. +(def (observe-lr-layout workload rules root (samples 20) (calls 2000)) + (unless (and (exact-integer? samples) (positive? samples) + (exact-integer? calls) (positive? calls)) + (error "layout observation requires positive integral samples and calls")) + (let* ((productions (lower-rules rules root)) (table (production-table productions))) + (let-values (((terminals _index) (production-terminal-catalog productions))) + (for-each + (lambda (catalog?) + (def (prepare) + (let (layout (make-item-layout table terminals)) + (if catalog? (make-core-symbol-catalog table layout) layout))) + (def (project product) + (filter values (vector->list + (if catalog? product (make-core-symbol-catalog table product))))) + (let (expected (project (prepare))) + (let sample ((index 0)) + (when (< index samples) + (let* ((before (##process-statistics)) (started (cpu-time)) + (last (let repeat ((remaining calls) (last #f)) + (if (zero? remaining) last + (repeat (- remaining 1) (prepare))))) + (cpu-ms (* 1000.0 (- (cpu-time) started))) + (after (##process-statistics))) + (unless (equal? (project last) expected) + (error "layout preparation changed symbols" workload catalog? index)) + (write (list 'ITEM-LAYOUT-SAMPLE workload catalog? index 'calls calls + 'cpu-ms-per-call (/ cpu-ms calls) + 'allocated-bytes-per-call + (let (bytes (admit-parser-allocation + (- (##f64vector-ref after 7) (##f64vector-ref before 7)) + (- (##f64vector-ref after 6) (##f64vector-ref before 6)))) + (and bytes (/ bytes calls))))) + (newline) (force-output)) + (sample (+ index 1)))))) + '(#f #t))))) + +(def (observe-lr-construction workload rules root route samples (calls 1)) + (unless (and (exact-integer? calls) (positive? calls)) + (error "construction observation requires positive integral calls" calls)) + (let* ((table (production-table (lower-rules rules root))) + (positions (foldl (lambda (production count) + (+ count 1 (length (production-rhs production)))) + 0 (vector->list table))) + (padded (* (vector-length table) + (foldl (lambda (production width) + (max width (+ 1 (length (production-rhs production))))) + 0 (vector->list table))))) + (def (compile) (compile-lr-spec rules root 'selective-glr #f route)) + (let (reference (compile)) + (write (list 'ITEM-SPACE-PRODUCT workload route reference)) + (newline) (force-output) + (let loop ((sample 0)) + (when (< sample samples) + (let* ((before (##process-statistics)) + (started (cpu-time)) + (found (let repeat ((remaining calls) (last #f)) + (if (zero? remaining) last + (repeat (- remaining 1) (compile))))) + (cpu-ms (* 1000.0 (- (cpu-time) started))) + (after (##process-statistics))) + (when (and (> calls 1) (< cpu-ms 1.0)) + (error "construction CPU batch does not resolve one millisecond" workload route calls)) + (unless (equal? reference found) + (error "complete compiler product changed" workload route sample)) + (write (list 'ITEM-SPACE-SAMPLE workload route sample + 'positions positions 'padded-positions padded 'cpu-ms (/ cpu-ms calls) + 'allocated-bytes + (let (bytes (admit-parser-allocation + (- (##f64vector-ref after 7) (##f64vector-ref before 7)) + (- (##f64vector-ref after 6) (##f64vector-ref before 6)))) + (and bytes (/ bytes calls))) + 'in-call-gcs (- (##f64vector-ref after 6) (##f64vector-ref before 6)) + 'calls calls)) + (newline) (force-output)) + (loop (+ sample 1))))))) + +(def (main . args) + (let (samples (if (pair? args) (string->number (car args)) 10)) + (unless (and (<= (length args) 1) (exact-integer? samples) (positive? samples)) + (error "expected one positive sample count" args)) + (for-each + (lambda (size) + (for-each (lambda (route) + (observe-lr-construction size (skew-width-rules (car size) (cadr size)) + 'source-file route samples)) + '(lalr canonical-lr1 follow-partition-lr1))) + '((32 16) (128 64) (512 256))) + (displayln "ITEM-SPACE-BENCHMARK-OK") (force-output))) diff --git a/t/benchmarks/lr1-partition/route-threshold.ss b/t/benchmarks/lr1-partition/route-threshold.ss new file mode 100644 index 00000000..352cec84 --- /dev/null +++ b/t/benchmarks/lr1-partition/route-threshold.ss @@ -0,0 +1,81 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Paired CPU samples for the bounded canonical trial in follow construction. + +(import (only-in :gerbil-parser/src/compiler/lr lower-rules lr-spec-ref + production-table) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/t/fixtures/lr1-construction + lr1-context-family-rules mixed-context-family-rules + acyclic-mixed-context-family-rules + state-local-candidate-family-rules + split-shared-context-family-rules + unreachable-repeated-context-family-rules)) + +(def (main . args) + (let* ((contexts (if (pair? args) (string->number (car args)) 48)) + (samples (if (and (pair? args) (pair? (cdr args))) + (string->number (cadr args)) 20)) + (shape (if (and (pair? args) (pair? (cdr args)) + (pair? (cddr args))) + (string->symbol (caddr args)) 'independent))) + (unless (and (integer? contexts) (positive? contexts) + (integer? samples) (positive? samples) + (memq shape '(independent mixed acyclic-mixed local-candidate split-shared unreachable-repeated))) + (error "expected positive context and sample counts, then independent/mixed/acyclic-mixed/local-candidate/split-shared/unreachable-repeated" + args)) + (let* ((rules ((case shape + ((mixed) mixed-context-family-rules) + ((acyclic-mixed) acyclic-mixed-context-family-rules) + ((local-candidate) + state-local-candidate-family-rules) + ((split-shared) split-shared-context-family-rules) + ((unreachable-repeated) + unreachable-repeated-context-family-rules) + (else lr1-context-family-rules)) + contexts)) + (productions (vector-length + (production-table (lower-rules rules 'source-file))))) + (def (measure construction) + (##gc) + (let* ((started (cpu-time)) + (spec (compile-lr-spec rules 'source-file + (if (eq? shape 'local-candidate) + 'selective-glr 'reject) #f + construction))) + (vector (* 1000.0 (- (cpu-time) started)) + (lr-spec-ref spec 'state-count) + (or (lr-spec-ref spec 'follow-block-count) 0) + (lr-spec-ref spec 'output-item-count)))) + ;; Warm both routes before recording paired, alternating batches. + (measure 'canonical-lr1) + (measure 'follow-partition-lr1) + (let loop ((sample 0)) + (when (< sample samples) + (let* ((first (measure (if (odd? sample) + 'follow-partition-lr1 'canonical-lr1))) + (second (measure (if (odd? sample) + 'canonical-lr1 'follow-partition-lr1))) + (canonical (if (odd? sample) second first)) + (follow (if (odd? sample) first second))) + (unless (if (memq shape '(mixed acyclic-mixed local-candidate)) + (and (< (vector-ref follow 1) + (vector-ref canonical 1)) + (< (vector-ref follow 2) + (vector-ref follow 3))) + (= (vector-ref canonical 1) (vector-ref follow 1))) + (error "construction topology differs" shape contexts sample)) + (write (list (cons 'shape shape) + (cons 'contexts contexts) + (cons 'productions productions) + (cons 'sample sample) + (cons 'canonical-states (vector-ref canonical 1)) + (cons 'follow-states (vector-ref follow 1)) + (cons 'canonical-cpu-ms (vector-ref canonical 0)) + (cons 'follow-cpu-ms (vector-ref follow 0)) + (cons 'follow-block-count (vector-ref follow 2)) + (cons 'follow-items (vector-ref follow 3)))) + (newline)) + (loop (+ sample 1))))))) + +(export main) diff --git a/t/benchmarks/module-loading/benchmark.ss b/t/benchmarks/module-loading/benchmark.ss new file mode 100644 index 00000000..6e90c002 --- /dev/null +++ b/t/benchmarks/module-loading/benchmark.ss @@ -0,0 +1,85 @@ +;;; Complete repeated Rust generation and already-loaded runtime admission. +(displayln "MODULE-BENCHMARK-START") (force-output) +(import :gerbil/expander + (only-in :gerbil-parser/src/language/module-input language-module-descriptor) + (only-in :gerbil-parser/src/compiler/rust-runtime language-rust-runtime-module-source) + (only-in :gerbil-parser/src/ffi/rust-runtime-aot rust-runtime-source)) +(load "t/fixtures/tla-sany-differential/preload.ss") +(load "t/benchmarks/module-loading/reference-preload.ss") +(preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(preload-module "gerbil-parser/t/support/qualification/benchmark-qos") +(import (only-in :gerbil-parser/t/support/qualification/benchmark-qos admit-benchmark-qos!)) +(def report-port (current-output-port)) +(def (report row) + (parameterize ((current-output-port report-port)) + (write row) (newline) (force-output))) +(def grammar-path "t/fixtures/runtime-record-assignments/languages/records/grammar.ss") +(def expected (rust-runtime-source grammar-path)) +(def (reference-generate) + (import-module ':gerbil-parser/rust-runtime-grammar-support #t #t) + (language-rust-runtime-module-source (language-module-descriptor grammar-path reload?: #t))) +(def (production-generate) (rust-runtime-source grammar-path)) +(def (cpu) + (let (stats (##process-statistics)) + (+ (f64vector-ref stats 0) (f64vector-ref stats 1)))) +(def (batch thunk count check) + (let (before (cpu)) + (let loop ((left count)) + (unless (zero? left) (check (thunk)) (loop (- left 1)))) + (let (elapsed (- (cpu) before)) + (report (list 'batch-complete 'calls count 'cpu-seconds elapsed)) + elapsed))) +(def (percentile xs percent) + (list-ref (list-sort < xs) (- (ceiling (/ (* percent (length xs)) 100)) 1))) +(def (phase name left right count check) + (let loop ((group 0) (ratios '())) + (if (= group 4) + (let ((p10 (percentile ratios 10)) (p50 (percentile ratios 50)) (p90 (percentile ratios 90))) + (report (list name 'p10 p10 'p50 p50 'p90 p90 'groups 4 'legs 80 'calls-per-leg count)) + (list p10 p50 p90)) + ;; Opposite ABBA and BAAB quartets balance GC and order within each group. + ;; Every observation contributes; no individual leg or group is dropped. + (let legs ((order (apply append (make-list 10 (if (even? group) '(a b b a b a a b) '(b a a b a b b a))))) + (a 0) (b 0)) + (if (null? order) + (let (ratio (/ a b)) + (report (list name 'group group 'ratio ratio)) + (loop (+ group 1) (cons ratio ratios))) + (let* ((left? (eq? (car order) 'a)) + (cost (batch (if left? left right) count check))) + (legs (cdr order) (if left? (+ a cost) a) (if left? b (+ b cost))))))))) +(def (qualify name production reference count check) + ;; Warm both paths; no collection forcing or discarded observations. + (batch production count check) (batch reference count check) + ;; Allocate the bounded budget to the reference stability controls. Both + ;; sides within each phase still receive exactly the same call count. + (let* ((before (phase (list name 'before) reference reference (+ count 1) check)) + (candidate (phase (list name 'candidate) production reference 1 check)) + (after (phase (list name 'after) reference reference (+ count 1) check))) + (unless (and (>= (car before) .90) (<= (caddr before) 1.10) + (>= (car after) .90) (<= (caddr after) 1.10) + (< (caddr candidate) .95)) + (error "module loading CPU gate failed" name before candidate after)))) + +(def (main mode) + (admit-benchmark-qos!) + (report '(benchmark-host-qos admitted)) + (cond + ((equal? mode "generation") + (qualify 'complete-rust-generation production-generate reference-generate 4 + (lambda (source) (unless (equal? source expected) (error "generated Rust changed"))))) + ((equal? mode "preload") + ;; Model first diagnostic preparation of a closure already loaded by the + ;; host. Each control has fresh local state; the SDK cache remains warm. + ;; This measures redundant graph inspection, not cold module startup. + (preload-runtime-module "gerbil-parser/src/ffi/rust-runtime-aot") + (let ((module "gerbil-parser/src/ffi/rust-runtime-aot") (sink (open-output-file "/dev/null"))) + (parameterize ((current-output-port sink)) + (qualify 'warm-runtime-preparation + (lambda () (preload-runtime-module module) #t) + (lambda () ((make-reference-preloader) module #f) #t) + 4 (lambda (ok) (unless ok (error "preload failed"))))))) + (else (error "unknown benchmark mode" mode))) + (displayln "MODULE-LOADING-CPU-PROOF-OK " mode) (force-output) + (test-child-process-exit! 0)) diff --git a/t/benchmarks/module-loading/cold-host.ss b/t/benchmarks/module-loading/cold-host.ss new file mode 100644 index 00000000..4cfcbffa --- /dev/null +++ b/t/benchmarks/module-loading/cold-host.ss @@ -0,0 +1,14 @@ +;;; Use the existing disposable native-child contract after all ABI assertions. +(displayln "MODULE-COLD-HOST-START") (force-output) +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(preload-module "gerbil-parser/t/fixtures/ffi/abi-probe") +(preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") +(eval '(import + (rename-in (only-in :gerbil-parser/t/fixtures/ffi/abi-probe main) + (main run-abi-probe)) + (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process + test-child-process-exit!))) +(eval '(run-abi-probe)) +(displayln "MODULE-COLD-HOST-OK") (force-output) +(eval '(test-child-process-exit! 0)) diff --git a/t/benchmarks/module-loading/controls.ss b/t/benchmarks/module-loading/controls.ss new file mode 100644 index 00000000..b260c851 --- /dev/null +++ b/t/benchmarks/module-loading/controls.ss @@ -0,0 +1,106 @@ +;;; File content changes invalidate source admission without per-parse reload. +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(preload-module "gerbil-parser/src/language/module-input") +(preload-module "gerbil-parser/src/ffi/rust-runtime-aot") +(preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") +(import :gerbil/expander + (only-in :std/io/tempfile make-temporary-file-name) + (only-in :gerbil-parser/src/language/module-input language-module-descriptor) + (only-in :gerbil-parser/src/language/descriptor language-grammar-language) + (only-in :gerbil-parser/src/ffi/rust-runtime-aot rust-runtime-source) + (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) + +(def (require! value message) + (unless value (error message))) +(def (fails? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def (write-text file text) + (call-with-output-file file (lambda (port) (display text port)))) +(def (grammar-source language) + (string-append + "(import (only-in :gerbil-parser/src/language/descriptor make-language-grammar +language-grammar-schema+))\n" + "(export grammar)\n(def grammar (make-language-grammar +language-grammar-schema+ \"" + language "\" \"v1\" \"loader.test.v1\" #f #f #f #f))\n")) + +(let (file (string-append (make-temporary-file-name "parser-loader-") ".ss")) + (dynamic-wind void + (lambda () + (require! (fails? (lambda () (language-module-descriptor file))) "missing module accepted") + (write-text file (grammar-source "first")) + (let (first (language-module-descriptor file)) + (require! (eq? first (language-module-descriptor file)) "module snapshot not reused") + (write-text file (grammar-source "second")) + (let (second (language-module-descriptor file)) + (require! (equal? (language-grammar-language second) "second") "content invalidation missed source change") + (require! (not (eq? first second)) "reloaded descriptor retained stale identity"))) + (write-text file "(export grammar)\n(def grammar (error \"initializer failed\"))\n") + (require! (fails? (lambda () (language-module-descriptor file))) "initializer failure swallowed") + (require! (fails? (lambda () (language-module-descriptor file))) "failed snapshot admitted") + ;; Restore the last admitted content after a failed replacement. + (write-text file (grammar-source "second")) + (require! (equal? (language-grammar-language (language-module-descriptor file)) "second") + "restored source retained failed replacement context") + (displayln "MODULE-RESTORED-SOURCE-OK") (force-output) + (write-text file (grammar-source "recovered")) + (require! (equal? (language-grammar-language (language-module-descriptor file)) "recovered") + "explicit reload failed to recover") + (require! (fails? (lambda () (language-module-descriptor (string-append file "#missing")))) "missing export accepted") + (displayln "MODULE-SNAPSHOT-CHANGE-FAILURE-OK")) + (lambda () (when (file-exists? file) (delete-file file))))) + +(def (syntax-source spelling) + (string-append + "(import (only-in :gerbil-parser/language-support/grammar deflanguage)\n" + " (only-in :gerbil-parser/src/language/descriptor language-grammar-with-identity))\n" + "(export grammar)\n" + "(deflanguage sourceedit (syntax (lexical (root value) (lex (mark Mark (literals \"" + spelling "\"))))) (rules (value (node Value mark))))\n" + "(def grammar (language-grammar-with-identity sourceedit-syntax \"source-edit\" \"v1\" \"source-edit.v1\"))\n")) +(let (file (string-append (make-temporary-file-name "parser-source-edit-") ".ss")) + (dynamic-wind void + (lambda () + (write-text file (syntax-source "!")) + (let (first (rust-runtime-source file)) + (require! (equal? first (rust-runtime-source file)) "unchanged generation differs") + (write-text file (syntax-source "?")) + (let (second (rust-runtime-source file)) + (require! (not (equal? first second)) "same-size grammar edit kept stale Rust") + (require! (equal? second (rust-runtime-source file reload?: #t)) "automatic and explicit generation differ"))) + (displayln "SOURCE-EDIT-RUST-GENERATION-OK")) + (lambda () (when (file-exists? file) (delete-file file))))) + +;; The official runtime deliberately retains an initializer failure until +;; reload-module! is requested. The preloader must propagate it on every call. +(let* ((directory (make-temporary-file-name "parser-runtime-loader-")) + (name (path-strip-directory directory)) + (file (string-append directory ".scm")) + (prior-path (load-path))) + (dynamic-wind + (lambda () (add-load-path! (path-directory directory))) + (lambda () + (write-text file "(error \"runtime initializer failed\")\n") + (require! (fails? (lambda () (preload-runtime-module name))) "runtime failure swallowed") + (require! (fails? (lambda () (preload-runtime-module name))) "retry skipped failed runtime") + (write-text file "(define parser-loader-recovered (cons 'loaded '()))\n") + (require! (fails? (lambda () (preload-runtime-module name))) "official failure cache bypassed") + (reload-module! name) + (let (loaded (eval 'parser-loader-recovered)) + (preload-runtime-module name) + (preload-module name) + (require! (eq? loaded (eval 'parser-loader-recovered)) "runtime initializer repeated")) + (displayln "RUNTIME-LOAD-FAILURE-RECOVERY-OK")) + (lambda () (set-load-path! prior-path) (when (file-exists? file) (delete-file file))))) + +(let (file (make-temporary-file-name "parser-preload-source-")) + (dynamic-wind void + (lambda () + (write-text file "(import (\n") + (require! (fails? (lambda () (preload-test-imports file))) "malformed source accepted") + (require! (fails? (lambda () (preload-test-imports file))) "source retry skipped failure") + (write-text file "(import)\n") + (preload-test-imports file) + (displayln "SOURCE-PRELOAD-RETRY-OK")) + (lambda () (when (file-exists? file) (delete-file file))))) +(displayln "MODULE-LOADING-CONTROLS-OK") (force-output) +(test-child-process-exit! 0) diff --git a/t/benchmarks/module-loading/reference-preload.ss b/t/benchmarks/module-loading/reference-preload.ss new file mode 100644 index 00000000..6937fdf2 --- /dev/null +++ b/t/benchmarks/module-loading/reference-preload.ss @@ -0,0 +1,93 @@ +;;; Frozen pre-change preloader; benchmark control only. +(def (make-reference-preloader) +;;; -*- Gerbil -*- +;;; Report real compiled-module admission before the test entry imports it. +;;; Dependency boundaries keep cold imports subject to the same silence gate. +(def +preloaded-modules+ (make-table test: equal?)) +(def +compiled-files+ (make-table test: equal?)) + +(def (prefer-compiled-interfaces!) + ;; Called only by the native suite; source benchmark entries retain their + ;; source overlay. FFI C forms require the compiled module interface. + (let (compiled-root (getenv "GERBIL_PATH" #f)) + (when compiled-root + (let (library (path-expand "lib" compiled-root)) + (when (file-exists? library) + (add-load-path! library)))))) + +(def (compiled-file module suffix) + ;; Native products are admitted before the child starts. Cache successful + ;; lookups only, so a later load-path addition can still resolve a miss. + (let (key (cons module suffix)) + (or (table-ref +compiled-files+ key #f) + (let (path (find file-exists? + (map (lambda (root) + (path-expand (string-append module suffix) root)) + (cons (path-expand "lib" (gerbil-home)) (load-path))))) + (when path (table-set! +compiled-files+ key path)) + path)))) + +(def (compiled-dependencies value) + (cond ((and (pair? value) (eq? (car value) 'load-module) + (pair? (cdr value)) (string? (cadr value))) + (list (cadr value))) + ((pair? value) + (append (compiled-dependencies (car value)) + (compiled-dependencies (cdr value)))) + (else '()))) + +;; Runtime wrappers do not list compile-time imports from .ssi interfaces. +;; Use the expander's import parameter to report the real nested admission +;; work. Delegate unchanged and restore the parameter on return or exception. +(def (call-with-compiled-interface-trace thunk) + (let (importer (gx#current-expander-module-import)) + (parameterize + ((gx#current-expander-module-import + (lambda (path reload?) + (displayln "INTERFACE-IMPORT " path) (force-output) + (let (context (importer path reload?)) + (displayln "INTERFACE-IMPORTED " path) (force-output) + context)))) + (thunk)))) + +(def (import-compiled-interface! module) + (call-with-compiled-interface-trace + (lambda () + (gx#import-module (string->symbol (string-append ":" module)) #f #t)))) + +(def (preload-compiled-module! module interfaces?) + (let (key (cons module interfaces?)) + (unless (table-ref +preloaded-modules+ key #f) + (table-set! +preloaded-modules+ key #t) + ;; Generated wrapper modules list dependencies without evaluating their + ;; initializers. Phase bodies (~0/~1) are admitted directly, never decoded. + (unless (string-contains module "~") + (let (source (compiled-file module ".scm")) + (when source + (for-each (lambda (dependency) (preload-compiled-module! dependency interfaces?)) + (call-with-input-file source + (lambda (port) + (let loop ((dependencies '())) + (let (value (read port)) + (if (eof-object? value) dependencies + (loop (append dependencies (compiled-dependencies value)))))))))))) + (displayln "MODULE-LOAD " module) (force-output) + (load-module module) + (displayln "MODULE-LOADED " module) (force-output) + ;; Admit phase bindings incrementally as well. Deferring all .ssi imports + ;; until gxtest loads its first source module hides cold macro work. + (when (and interfaces? (not (string-contains module "~")) + (compiled-file module ".ssi")) + (displayln "MODULE-IMPORT " module) (force-output) + (import-compiled-interface! module) + (displayln "MODULE-IMPORTED " module) (force-output))))) + +(def (preload-module module) (preload-compiled-module! module #t)) + +;;; Compiled benchmark entries execute phase-zero procedures only. Loading +;;; source expander interfaces would retain unrelated compiler state in their +;;; runtime heap; dependency initialization still reports actual progress. +(def (preload-runtime-module module) (preload-compiled-module! module #f)) + + + preload-compiled-module!) diff --git a/t/benchmarks/parser-evidence.ss b/t/benchmarks/parser-evidence.ss new file mode 100644 index 00000000..41f05d66 --- /dev/null +++ b/t/benchmarks/parser-evidence.ss @@ -0,0 +1,9 @@ +;;; Evidence workload composition; statistics and semantic admission stay upstream. +(import (only-in ./gql/runtime/matched-stages main) + (only-in ./lr-request-stack/benchmark benchmark-lr-request-backends)) +(export (rename: evidence-main main)) +(def (evidence-main) + (main "40" "100") + (benchmark-lr-request-backends 20 20) + ;; Completion does not imply that the sampler admitted a performance benefit. + (displayln "PARSER-EVIDENCE-REQUESTS-OK") (force-output)) diff --git a/t/benchmarks/parser-stage-cost/benchmark.ss b/t/benchmarks/parser-stage-cost/benchmark.ss new file mode 100644 index 00000000..9c7c078d --- /dev/null +++ b/t/benchmarks/parser-stage-cost/benchmark.ss @@ -0,0 +1,326 @@ +;;; Shared CPU/GC/allocation sampling and complete-product diagnostic controls. +(import (only-in :asp-gerbil-scheme/src/benchmark/statistics benchmark-percentile-index) + (only-in :std/list/list-builder with-list-builder) + (only-in ../../../src/runtime/parse-cost + current-parser-cost-observer admit-parser-allocation) + (only-in ../../../src/runtime/artifact + parse-artifact-valid-for-source? parse-artifact-events parse-artifact-ref)) +(export measure-parser-stages measure-artifact-event-storage copy-canonical-event-storage + measure-parser-component measure-parser-batch measure-parser-cpu-pairs cpu-pair-request-plan sample-at-percentile + gc-statistics-snapshot sample-gc-snapshot component-allocation-summary parser-cpu-milliseconds) + +(def (require-parser-batch! batch-count calls-per-thunk) + (unless (and (and (integer? batch-count) (exact? batch-count)) (positive? batch-count) + (and (integer? calls-per-thunk) (exact? calls-per-thunk)) (positive? calls-per-thunk) + (zero? (modulo batch-count calls-per-thunk))) + (error "parser batch requires positive integral calls without truncation"))) + +;; Admit complete intervals, not raw counter differences across collections. +;; Thread-switching batches remain process observations without caller ownership. +(def (component-allocation-summary rows batch-count allocation-scope) + (require-parser-batch! batch-count 1) + (let* ((admitted (if (eq? allocation-scope 'single-caller) + (filter (lambda (row) (number? (cdr (assq 'allocated-bytes row)))) rows) + '())) + (per-parse (lambda (rank) + (and (pair? admitted) + (/ (row-percentile admitted 'allocated-bytes rank) batch-count))))) + (list (cons 'allocationScope allocation-scope) + (cons 'allocationSampleCount (length admitted)) + (cons 'allocatedBytesPerParse (per-parse 50)) + (cons 'allocationP95BytesPerParse (per-parse 95))))) + +(def (row-percentile rows key rank) + (percentile (map (lambda (row) (cdr (assq key row))) rows) rank)) + +;; Select the actual wall-ranked observation, preserving its CPU/GC counters. +;; Sample number breaks ties deterministically; independent CPU percentiles +;; cannot explain what happened in the wall P95 observation. +(def (sample-at-percentile rows key rank) + (let (ordered + (list-sort + (lambda (left right) + (let ((a (cdr (assq key left))) (b (cdr (assq key right)))) + (if (= a b) + (< (cdr (assq 'sample left)) (cdr (assq 'sample right))) + (< a b)))) rows)) + (list-ref ordered (benchmark-percentile-index (length ordered) rank)))) + +;; Gambit _kernel.scm process-statistics slots 12..19 describe the latest +;; collection in the entire VM, not allocations attributable to this parser. +;; These are snapshots, never differences or sums across collections. +(def (gc-statistics-snapshot statistics) + (list (cons 'scope 'whole-vm-latest-collection) + (cons 'gcCount (f64vector-ref statistics 6)) + (cons 'cpuMs (* 1000 (+ (f64vector-ref statistics 12) + (f64vector-ref statistics 13)))) + (cons 'wallMs (* 1000 (f64vector-ref statistics 14))) + (cons 'heapBytes (f64vector-ref statistics 15)) + (cons 'allocatedBytes (f64vector-ref statistics 16)) + (cons 'liveBytes (f64vector-ref statistics 17)) + (cons 'movableBytes (f64vector-ref statistics 18)) + (cons 'stillBytes (f64vector-ref statistics 19)))) + +(def (sample-gc-snapshot before after) + ;; A no-GC batch must not inherit the previous batch's collection as its own. + (and (> (f64vector-ref after 6) (f64vector-ref before 6)) + (gc-statistics-snapshot after))) + +;; One warmup and one initial GC per component; timed samples retain naturally +;; occurring GC. Validation and logging are outside the measured batch. +(def (measure-parser-component name samples batch-count thunk expected (calls-per-thunk 1) (allocation-scope 'single-caller) sample-output: (sample-output 'immediate)) + (unless (and (integer? samples) (exact? samples) (positive? samples)) + (error "parser component requires a positive integral sample count" samples)) + (unless (memq sample-output '(immediate deferred)) + (error "invalid parser sample output policy" sample-output)) + (require-parser-batch! batch-count calls-per-thunk) + (unless (equal? (thunk) expected) + (error "parser warmup changed its semantic result" name)) + (##gc) + (let (baseline (gc-statistics-snapshot (##process-statistics))) + (write (list 'PARSER-GC-BASELINE name baseline)) (newline) (force-output) + (let loop ((sample 0) (rows '())) + (if (= sample samples) + (let (summary + (append + (list (cons 'stage name) (cons 'sampleCount samples) + (cons 'parsesPerSample batch-count) + (cons 'wallP50Ms (row-percentile rows 'wall-ms 50)) + (cons 'wallP95Ms (row-percentile rows 'wall-ms 95)) + (cons 'cpuP50Ms (row-percentile rows 'cpu-ms 50)) + (cons 'cpuP50Ns (row-percentile rows 'cpu-ns 50)) + (cons 'cpuP95Ms (row-percentile rows 'cpu-ms 95)) + (cons 'cpuP95Ns (row-percentile rows 'cpu-ns 95)) + (cons 'wallP50Sample (sample-at-percentile rows 'wall-ms 50)) + (cons 'wallP95Sample (sample-at-percentile rows 'wall-ms 95)) + (cons 'maxWallSample (sample-at-percentile rows 'wall-ms 100)) + (cons 'wallP50MsPerParse (/ (row-percentile rows 'wall-ms 50) batch-count)) + (cons 'wallP95MsPerParse (/ (row-percentile rows 'wall-ms 95) batch-count)) + (cons 'gcBaseline baseline) + (cons 'samples (reverse rows))) + (component-allocation-summary rows batch-count allocation-scope))) + (when (eq? sample-output 'deferred) + (for-each (lambda (row) + (write (list 'PARSER-COMPONENT-SAMPLE name row)) (newline)) + (reverse rows))) + (write (list 'PARSER-COMPONENT-SUMMARY + (filter (lambda (row) (not (eq? (car row) 'samples))) summary))) (newline) (force-output) + summary) + (let (row (measure-parser-batch name sample batch-count thunk expected calls-per-thunk)) + (when (eq? sample-output 'immediate) + (write (list 'PARSER-COMPONENT-SAMPLE name row)) (newline) (force-output)) + (loop (+ sample 1) (cons row rows))))))) + +;; Canonicalize cumulative CPU counters before subtraction. Bounded benchmark +;; processes retain nanosecond integer precision; equal OS ticks must not yield +;; different interval endpoints through floating-point cancellation. +(def (parser-cpu-ticks statistics index) + (inexact->exact (round (* 1000000000 (f64vector-ref statistics index))))) + +(def (parser-cpu-nanoseconds before after (user-index 0) (system-index 1)) + (+ (- (parser-cpu-ticks after user-index) (parser-cpu-ticks before user-index)) + (- (parser-cpu-ticks after system-index) (parser-cpu-ticks before system-index)))) + +(def (parser-cpu-milliseconds before after (user-index 0) (system-index 1)) + (exact->inexact (/ (parser-cpu-nanoseconds before after user-index system-index) 1000000))) + +;;; Shared timed batch primitive. It owns no warmup, forced GC or output, so +;;; matched callers can alternate variants without perturbing GC at each switch. +(def (measure-parser-batch name sample batch-count thunk expected (calls-per-thunk 1)) + (require-parser-batch! batch-count calls-per-thunk) + (let* ((before (##process-statistics)) + (wall-start (##current-time-point)) + (result + (let repeat ((remaining (quotient batch-count calls-per-thunk)) (last-result #f)) + (if (zero? remaining) last-result + (repeat (- remaining 1) (thunk))))) + (wall-ms (* 1000 (- (##current-time-point) wall-start))) + (after (##process-statistics)) + (cpu-ns (parser-cpu-nanoseconds before after)) + (cpu-ms (exact->inexact (/ cpu-ns 1000000))) + (delta (lambda (index) + (- (f64vector-ref after index) (f64vector-ref before index)))) + (row (list (cons 'sample sample) + (cons 'wall-ms wall-ms) + (cons 'cpu-ms cpu-ms) + (cons 'cpu-ns cpu-ns) + ;; Signed observation, not a scheduler attribution. It + ;; includes counter/timing noise and may be negative. + (cons 'wall-minus-cpu-ms + (- wall-ms cpu-ms)) + (cons 'gc-count (delta 6)) + (cons 'gc-wall-ms (* 1000 (delta 5))) + (cons 'gc-cpu-ms (parser-cpu-milliseconds before after 3 4)) + (cons 'latest-gc (sample-gc-snapshot before after)) + (cons 'allocation-counter-delta (delta 7)) + (cons 'allocated-bytes + (admit-parser-allocation (delta 7) (delta 6)))))) + (unless (equal? result expected) + (error "parser batch changed its semantic result" name sample)) + row)) + +;;; A representation control, not a parser or a retained-heap measurement. +;;; Reuse payloads exactly as publication does; allocate fresh vectors/list cells. +(def (copy-canonical-event-storage events) + (with-list-builder (emit!) + (for-each (lambda (event) (emit! (vector-copy event))) events))) + +(def (artifact-with-events artifact events) + (map (lambda (entry) + (if (eq? (car entry) 'events) (cons 'events events) entry)) artifact)) + +(def (measure-artifact-event-storage artifact source (samples 11)) + (unless (and (string? source) (integer? samples) (positive? samples) + (parse-artifact-valid-for-source? artifact source)) + (error "event storage control requires a source-admitted artifact and positive sample count")) + (let ((events (parse-artifact-events artifact)) (observations '())) + (let loop ((sample 0)) + (when (< sample samples) + (let* ((before (##process-statistics)) + (copy (copy-canonical-event-storage events)) + (after (##process-statistics)) + (delta (lambda (index) + (- (f64vector-ref after index) (f64vector-ref before index)))) + (row (list (cons 'cpu-ms (* 1000 (+ (delta 0) (delta 1)))) + (cons 'allocated-bytes (admit-parser-allocation (delta 7) (delta 6))) + (cons 'allocation-counter-delta (delta 7)) + (cons 'gc-count (delta 6))))) + ;; Reconstruct and admit the entire product outside the copy interval. + (let (product (artifact-with-events artifact copy)) + (unless (and (equal? product artifact) + (parse-artifact-valid-for-source? product source)) + (error "event storage control changed the complete admitted product" sample))) + (set! observations (cons row observations)) + (write (list 'artifact-event-storage-sample sample row)) + (newline) (force-output)) + (loop (+ sample 1)))) + (let* ((allocations (filter number? (map (lambda (row) (cdr (assq 'allocated-bytes row))) observations))) + (summary + (list (cons 'samples samples) (cons 'events (length events)) + (cons 'vector-slots (foldl (lambda (event n) (+ n (vector-length event))) 0 events)) + (cons 'cpu-p50-ms (median (map (lambda (row) (cdr (assq 'cpu-ms row))) observations))) + (cons 'cpu-p95-ms (percentile (map (lambda (row) (cdr (assq 'cpu-ms row))) observations) 95)) + (cons 'allocation-p50-bytes (median allocations)) + (cons 'allocation-p95-bytes (percentile allocations 95)) + (cons 'allocation-samples (length allocations))))) + (write (list 'artifact-event-storage-summary summary)) (newline) (force-output) + summary))) + +(def (median values) + (let* ((ordered (list-sort < values)) (n (length ordered)) (middle (quotient n 2))) + (and (positive? n) + (if (odd? n) (list-ref ordered middle) + (/ (+ (list-ref ordered (- middle 1)) (list-ref ordered middle)) 2.0))))) + +(def (percentile values rank) + (and (pair? values) + (let (ordered (list-sort < values)) + (list-ref ordered (benchmark-percentile-index (length ordered) rank))))) + +;;; Observer overhead is included; these samples are not latency comparisons. +(def (measure-parser-stages parser source (samples 11)) + (unless (and (procedure? parser) (string? source) + (integer? samples) (positive? samples)) + (error "stage diagnostics require a parser, source and positive sample count")) + (let ((reference (parser source)) (observations '())) + (unless (parse-artifact-valid-for-source? reference source) + (error "stage reference failed source-bound admission")) + (let loop ((sample 0)) + (when (< sample samples) + (let* ((stages '()) + (before (##process-statistics)) + (artifact + (parameterize ((current-parser-cost-observer + (lambda (stage receipt) + (set! stages (cons (cons stage receipt) stages))))) + (parser source))) + (after (##process-statistics)) + (delta (lambda (index) + (- (f64vector-ref after index) (f64vector-ref before index)))) + (whole + (list (cons 'cpu-ms (* 1000 (+ (delta 0) (delta 1)))) + (cons 'allocated-bytes (admit-parser-allocation (delta 7) (delta 6))) + (cons 'allocation-counter-delta (delta 7)) + (cons 'gc-count (delta 6)) (cons 'completed #t))) + (rows (cons (cons 'observed-complete-request whole) (reverse stages)))) + ;; Full equality and source admission are outside the measured request. + (unless (and (equal? reference artifact) + (parse-artifact-valid-for-source? artifact source)) + (error "stage instrumentation changed the complete admitted product" sample)) + (set! observations (append rows observations)) + (write (list 'parser-stage-sample sample rows)) (newline) (force-output)) + (loop (+ sample 1)))) + (let (summary + (map (lambda (stage) + (let* ((receipts (map cdr (filter (lambda (row) (eq? stage (car row))) observations))) + (allocations (filter number? (map (lambda (r) (cdr (assq 'allocated-bytes r))) receipts))) + (completed (foldl (lambda (r total) + (if (cdr (assq 'completed r)) (+ total 1) total)) + 0 receipts))) + (list (cons 'stage stage) (cons 'request-samples samples) + (cons 'samples (length receipts)) + (cons 'completed-samples completed) + (cons 'incomplete-samples (- (length receipts) completed)) + (cons 'cpu-p50-ms (median (map (lambda (r) (cdr (assq 'cpu-ms r))) receipts))) + (cons 'cpu-p95-ms + (percentile (map (lambda (r) (cdr (assq 'cpu-ms r))) receipts) 95)) + (cons 'allocation-p50-bytes (median allocations)) + (cons 'allocation-p95-bytes (percentile allocations 95)) + (cons 'allocation-samples (length allocations))))) + (foldl (lambda (row stages) + (if (memq (car row) stages) stages (cons (car row) stages))) + '() observations))) + (write (list 'parser-stage-summary 'input-characters (string-length source) + 'events (length (parse-artifact-events reference)) + 'source-digest (parse-artifact-ref reference 'sourceDigest) + 'grammar-digest (parse-artifact-ref reference 'grammarDigest) summary)) + (newline) (force-output) + (displayln "PARSER-STAGE-COST-OK samples=" samples) (force-output) + summary))) + +;;; Eight full-quanta legs balance constant, linear and quadratic time position. +;;; Keep four nominal quanta per variant without splitting the timed intervals; +;;; a whole natural collection is less dominant in each retained observation. +(def (cpu-pair-request-plan calls group) + (require-parser-batch! calls 1) + (let (order (if (even? group) '(left right right left right left left right) + '(right left left right left right right left))) + (map (lambda (variant) (cons variant calls)) order))) + +;;; Same-process CPU comparison with complete products and alternating order. +(def (cpu-pair-field row key) (cdr (assq key row))) +(def (measure-parser-cpu-pairs name phase groups calls expected left right) + (unless (and (exact-integer? groups) (>= groups 20)) + (error "paired CPU proof requires at least twenty groups" groups)) + (require-parser-batch! calls 1) + (unless (and (equal? (left) expected) (equal? (right) expected)) + (error "paired CPU warmup differs from complete expected product" name)) + (##gc) + (let loop ((group 0) (rows '())) + (if (= group groups) + (map (lambda (rank) (cpu-pair-field (sample-at-percentile rows 'ratio rank) 'ratio)) '(10 50 90)) + (let* ((plan (cpu-pair-request-plan calls group)) + (batches + (map + (lambda (work) + (let* ((variant (car work)) + (batch (measure-parser-batch (list 'paired-cpu name phase) group (cdr work) + (if (eq? variant 'left) left right) expected))) + ;; Publish completed work outside timing; a whole group can + ;; exceed the watchdog interval on a contended host. + (write (list 'PARSER-CPU-BATCH name phase group variant + (cons (cons 'calls (cdr work)) batch))) + (newline) (force-output) + (cons variant (cons (cons 'calls (cdr work)) batch)))) plan)) + (sum (lambda (variant) + (apply + (map (lambda (batch) (cpu-pair-field (cdr batch) 'cpu-ms)) + (filter (lambda (batch) (eq? (car batch) variant)) batches))))) + (cpu (sum 'left))) + (unless (and (positive? cpu) + (andmap (lambda (batch) (>= (cpu-pair-field (cdr batch) 'cpu-ms) 1.0)) batches)) + (error "parser CPU batch does not resolve one millisecond" name phase group)) + (let (row (list (cons 'sample group) (cons 'ratio (/ (sum 'right) cpu)) + (cons 'legs (length plan)) (cons 'requests-per-variant (* 4 calls)))) + (write (list 'PARSER-CPU-PAIR name phase row 'batches batches)) (newline) (force-output) + (loop (+ group 1) (cons row rows))))))) + diff --git a/t/benchmarks/parser-stage-cost/receipt.ss b/t/benchmarks/parser-stage-cost/receipt.ss new file mode 100644 index 00000000..1e7d1d49 --- /dev/null +++ b/t/benchmarks/parser-stage-cost/receipt.ss @@ -0,0 +1,56 @@ +;;; Shared admission of complete sampler receipts; no language-specific policy. +(import :std/string/misc :std/misc/ports + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark sample-at-percentile)) +(export read-parser-component-receipt) +(def (field row key) (cdr (assq key row))) +(def (require! condition . detail) + (unless condition (apply error "parser component receipt rejected" detail))) +(def (read-parser-component-receipt path completion stage-count batch-count sample-count) + (let ((samples (make-hash-table)) (raw-samples (make-hash-table)) (summaries (make-hash-table)) + (complete? #f) (exit? #f) (admitted 0)) + (call-with-input-file path + (lambda (port) + (let loop () + (let (line (read-line port)) + (unless (eof-object? line) + (cond + ((string-prefix? "(PARSER-COMPONENT-SAMPLE " line) + (let* ((datum (call-with-input-string line read)) + (key (cadr datum)) (row (caddr datum)) + (bytes (field row 'allocated-bytes))) + (require! (and (exact-integer? (field row 'sample)) + (<= 0 (field row 'sample)) (< (field row 'sample) sample-count) + (not (find (lambda (old) (= (field old 'sample) (field row 'sample))) + (hash-ref raw-samples key '())))) key 'sample-identity) + (hash-put! raw-samples key (cons row (hash-ref raw-samples key '()))) + (when (number? bytes) + (require! (and (= (field row 'gc-count) 0) (integer? bytes) (>= bytes 0) + (= bytes (field row 'allocation-counter-delta))) key 'counter) + (hash-put! samples key (cons (inexact->exact bytes) (hash-ref samples key '()))) + (set! admitted (+ admitted 1))))) + ((string-prefix? "(PARSER-COMPONENT-SUMMARY " line) + (let* ((datum (call-with-input-string line read)) (row (cadr datum))) + (require! (not (hash-get summaries (field row 'stage))) path 'duplicate-summary) + (hash-put! summaries (field row 'stage) row))) + ((equal? line completion) (set! complete? #t)) + ((and (string-prefix? "PROCESS-RESULT " line) + (string-contains line "exit=0 reason=#f missing=()")) (set! exit? #t))) + (loop)))))) + (require! (and complete? exit? (= (hash-length summaries) stage-count)) path 'incomplete) + (for-each + (lambda (key) + (let* ((row (hash-get summaries key)) (values (hash-ref samples key '())) + (count (field row 'parsesPerSample))) + (require! (and (= count batch-count) (= (field row 'sampleCount) sample-count) + (eq? (field row 'allocationScope) 'single-caller) + (= (length (hash-ref raw-samples key '())) sample-count) + (= (field (sample-at-percentile (hash-get raw-samples key) 'cpu-ms 50) 'cpu-ms) + (field row 'cpuP50Ms)) + (= (length values) (field row 'allocationSampleCount)) + (pair? values) + (andmap (lambda (value) (= value (car values))) values) + ;; The sampler reports a flonum quotient. Compare that representation; + ;; exact integer batch counters remain the savings authority. + (= (exact->inexact (/ (car values) count)) (field row 'allocatedBytesPerParse))) key 'nonuniform))) + (hash-keys summaries)) + (list summaries samples admitted))) diff --git a/t/benchmarks/parser-throughput/README.org b/t/benchmarks/parser-throughput/README.org new file mode 100644 index 00000000..7826f746 --- /dev/null +++ b/t/benchmarks/parser-throughput/README.org @@ -0,0 +1,145 @@ +#+title: Engine-owned parser throughput qualification + +* Hard performance gate + +These are project acceptance targets, not a universal parser standard. + +| Changing requests per batch | Maximum batch wall | Minimum throughput | +|----------------------------+--------------------+--------------------| +| 1000 | 0.010 s | 100000 requests/s | +| 10000 | 0.100 s | 100000 requests/s | +| 100000 | 1.000 s | 100000 requests/s | + +0.5 seconds per 100000 requests and 300000 requests/s are stretch targets; +neither is reported as achieved without measured evidence. P50/P95/P99/max +are per-request nearest-rank latencies in microseconds. The human selected +reporting tail latency without an additional P95/P99 hard threshold. + +* Test owner and boundaries + +=protocol.ss= owns changing inputs, warmup, full semantic comparisons, CPU/GC +observations, percentiles and failure propagation. Native adapters supply the +compiled parser, an independent oracle and an immutable source identity. +Compilation, input generation and oracle construction precede timing. +Request latency covers only the compiled parser call. Batch wall also includes +timestamp/sample recording and complete semantic comparisons; its throughput +is conservative, not an isolated Scheme Fold or FFI service measurement. + +The precise clock is the installed SDK's =:std/os/time= integer seconds and +nanoseconds, not subtraction of floating-point epoch seconds. It is an OS +realtime clock, NOT a monotonic-clock claim; backwards intervals fail. Clock +reads and their allocation/instrumentation costs are not subtracted. This +protocol currently requires the SDK's POSIX time API; no Windows qualification +is claimed. + +The included =engine-line-fold-not-org= fixture is a small generated engine +grammar with Unicode and CR/LF input. It does NOT qualify complete Org syntax, +Rust/Tokio concurrency, Python response delivery or SMP processor activation. +The real Org adapter must use this same protocol with the actual compiled Org +product and source identity; an engine-fixture pass cannot replace that gate. + +Process CPU and batch wall are separate. Raw allocation-counter deltas and GC +counts are retained. Allocated bytes are admitted only for nonnegative deltas +without GC. This does not measure resident memory, Rust reserved capacity or +exclusive parser allocation. Inputs and oracle results remain retained during +timing; memory claims must include this benchmark storage. There is no claimed +memory hard gate or Scheme parallelism based on these counters. + +* Native Just entry points + +#+begin_src sh +just build-parser-throughput OUTPUT LOAD_PATH +just test-parser-throughput-protocol OUTPUT LOAD_PATH +just benchmark-parser-throughput OUTPUT/parser-throughput preflight +just benchmark-parser-throughput OUTPUT/parser-throughput full +just report-parser-throughput OUTPUT/parser-throughput.full.log +#+end_src + +A one-batch 1000-request preflight rejects grossly inadequate candidates before +an expensive matrix. Full acceptance requires twenty batches at EACH of 1000, +10000 and 100000 requests (sixty batches total), after preflight. Any semantic +or performance failure stops the candidate with a nonzero exit. A truncated +run is never accepted as a successful matrix. Preserve the complete stdout +including failures; compare identical adapter, corpus, compiled inputs, SDK, +host conditions and metric boundaries before promoting an optimization. + +Input indices change between acceptance batches, not merely within a batch; +the source producer must incorporate its index. Clock-zero intervals are +reported as =sub-resolution-request-samples=, not zero-cost parsing. A P50 of +zero is unresolved at the clock's effective precision, not a latency win. +For sub-microsecond attribution, a separately qualified finer clock is needed; +batch wall/throughput gates remain distinct from that instrumentation limit. + +* Local protocol qualification, not Org acceptance + +The changing-input native line fixture completed all sixty acceptance batches +after preflight. Medians of twenty per-batch measurements: + +| Requests | Batch wall (s) | Throughput (requests/s) | Request P95/P99 (us) | +|----------+----------------+-------------------------+----------------------| +| 1000 | 0.001096 | 913058 | 1/1 | +| 10000 | 0.009857 | 1014559 | 1/1 | +| 100000 | 0.136987 | 729997 | 1/1 | + +The tail values are quantized at this SDK clock's observable precision; +1168786 of the 2000000 large-load request intervals were unresolved. They are +NOT zero-cost calls or proof of sub-microsecond P95. The large-load median GC +count was two, so its raw allocation delta is NOT an admitted allocation total. +Resident memory and Rust capacity remain unmeasured by this engine fixture. +Receipt: =.controls/parser-throughput-changing.full.log=. This result only +qualifies the protocol and this tiny grammar; it does not establish an Org +speedup, an optimization against the frozen baseline, SMP or end-to-end +FFI/Python performance. + +* Actual Org preflight: rejected, not accepted + +The downstream thin adapter binds the real compiled Org product digest +=sha256:c39aca6c8ce0b5c55c16911267238905ffd5e224c419640aba43ee7bd43e12a3= +and uses three actual Org fixtures with changing request suffixes. An independent +interpreted Fold constructs expected complete events before timing. + +The 1000-request preflight completed semantic comparisons but failed the hard +performance gate: batch wall =0.216867 s=, throughput =4611.121 requests/s=, +per-request P50/P95/P99/max =121/242/447/6782 us=. Process CPU was =0.215603 s= +and ten collections occurred. Raw allocation-counter delta =392847656= is +NOT an admitted allocated-byte total. No twenty-batch acceptance is claimed; +the native runner exited nonzero immediately, preserving the failure receipt. + +This boundary is =org-scheme-fold-not-ffi-or-python=. It excludes FFI entry, +event transport, graph projection, Tokio queues and Python response delivery. +Batch CPU includes instrumentation and comparisons; it is not exclusive Fold +CPU. Receipt: downstream =target/org-throughput-gated/preflight-real.log=. +The real Org performance gate remains open, and its baseline is unchanged. + +* Upstream actual-Fold attribution and rejected reuse experiment + +=real.ss= consumes the actual generated engine product and data-only grammar +program published by the thin grammar owner. It uses the same changing fixtures +and protocol without importing an Orgize API, FFI, Tokio or Python runtime. +=profile.ss= requires a diagnostic product; its binder refuses performance +admission. Twenty-four complete controls emitted 1767 events and entered shared +procedures 84538 times, including 4518 frame-start whitespace scans. +Instrumentation costs and nested timings are not ordinary latency shares. + +Matched observer-free 1000-request preflights (one batch each, NOT acceptance): + +| Product | Batch wall (s) | Requests/s | Request P95/P99 (us) | Process maximum RSS (bytes) | GC | +|---------+----------------+------------+---------------------+-----------------------------+----| +| Control | 0.197398 | 5065.907 | 227/397 | 191987712 | 7 | +| Reuse | 0.195332 | 5119.489 | 219/280 | 192413696 | 8 | + +The first candidate preflight was 0.201576 s. These singleton differences do +not establish a stable benefit, and both products fail the 0.010 s gate. The +source-fact cache/private frame cell and successor collapse were conservatively +reverted; diagnostic tooling and negative receipts remain. Raw allocation under +GC is not admitted as allocated bytes. RSS includes retained input/oracle +storage, not exclusive Fold memory or Rust reserved capacity. Whole process +times of 49.47/48.67 s include expensive interpreted oracle preparation before +batch timing and must not be reported as request service latency. + +Use =just run-real-fold-throughput= for admission and +=just run-real-fold-resources= for process RSS under the same preflight. The +resource wrapper propagates failure rather than turning rejection into a pass. +Receipts: =.fold-value-control/real-resources.log= and +=.fold-value-reuse/real-resources.log=. Rejected compiled artifacts differ from +the reverted editable source and must not be loaded as current controls. diff --git a/t/benchmarks/parser-throughput/allocation.ss b/t/benchmarks/parser-throughput/allocation.ss new file mode 100644 index 00000000..f3c4f327 --- /dev/null +++ b/t/benchmarks/parser-throughput/allocation.ss @@ -0,0 +1,35 @@ +;;; Diagnostic only: request-local CPU, GC and admissible allocation deltas. +;;; Semantic comparison and aggregation are outside the observed parser call. +(import :gerbil-parser/src/runtime/parse-cost + :gerbil-parser/benchmarks/real-fold/parse-org-compiled-events) +(export main) +(def (main corpus-path) + (let* ((corpus (call-with-input-file corpus-path read)) + (inputs (caddr corpus)) (expected (cadddr corpus)) + (rows (vector (vector 0 0 0 0 0) (vector 0 0 0 0 0) (vector 0 0 0 0 0))) + (index 0)) + (unless (and (eq? (car corpus) 'real-fold-preflight.v1) + (= (vector-length inputs) 1000) (= (vector-length expected) 1000)) + (error "invalid allocation diagnostic corpus")) + (parameterize ((current-parser-cost-observer + (lambda (stage metrics) + (let* ((row (vector-ref rows (modulo index 3))) + (bytes (cdr (assq 'allocated-bytes metrics)))) + (unless (cdr (assq 'completed metrics)) (error "incomplete parser call")) + (vector-set! row 0 (+ 1 (vector-ref row 0))) + (vector-set! row 1 (+ (cdr (assq 'cpu-ms metrics)) (vector-ref row 1))) + (vector-set! row 2 (+ (cdr (assq 'gc-count metrics)) (vector-ref row 2))) + (when bytes + (vector-set! row 3 (+ 1 (vector-ref row 3))) + (vector-set! row 4 (+ bytes (vector-ref row 4)))))))) + (let loop () + (when (< index 1000) + (let (actual (with-parser-cost-stage 'complete-real-fold + (parse-org-compiled-events (vector-ref inputs index)))) + (unless (equal? actual (vector-ref expected index)) + (error "allocation diagnostic complete tape mismatch" index))) + (set! index (+ index 1)) (loop)))) + (displayln "DIAGNOSTIC-ONLY request CPU includes instrumentation; allocation includes observation overhead") + (displayln "FIELDS fixture-residue requests cpu-ms gc-count no-gc-requests no-gc-allocation-sum-bytes") + (for-each (lambda (residue) (write (cons residue (vector->list (vector-ref rows residue)))) (newline)) '(0 1 2)) + (displayln "ALLOCATION-DIAGNOSTIC-OK complete tapes; NOT PERFORMANCE ACCEPTANCE"))) diff --git a/t/benchmarks/parser-throughput/body.ss b/t/benchmarks/parser-throughput/body.ss new file mode 100644 index 00000000..7f330931 --- /dev/null +++ b/t/benchmarks/parser-throughput/body.ss @@ -0,0 +1,18 @@ +;;; Native engine-line-fold adapter, explicitly not the complete Org parser. +(import (only-in :gerbil-parser/t/benchmarks/parser-throughput/protocol + run-parser-throughput) + (only-in :gerbil-parser/src/runtime/source-lines source-line-events)) +(export main) + +(def (input index) + (string-append "heading " (number->string index) "\r\n" + "changing α中🦀 payload\nlast " (number->string index))) + +(def (oracle source) + (source-line-events source 'Document + (lambda (_ from until) + (list '(start Text) (list 'token 'Line from until) '(finish))))) + +(def (main (mode "preflight")) + (run-parser-throughput "engine-line-fold-not-org" fixture-identity + throughput-fold oracle input (string->symbol mode))) diff --git a/t/benchmarks/parser-throughput/call-graph.ss b/t/benchmarks/parser-throughput/call-graph.ss new file mode 100644 index 00000000..38694ab5 --- /dev/null +++ b/t/benchmarks/parser-throughput/call-graph.ss @@ -0,0 +1,60 @@ +;;; Static call sites, not dynamic counts, exclusive CPU or allocation proof. +(export main) + +(def (main product) + (let ((definitions (make-hash-table-eq)) (calls 0) (tail-calls 0) + (argument-slots 0) (owner-assignments 0)) + (for-each + (lambda (file) + (when (and (string-prefix? "parse-org-compiled-events" file) + (string-suffix? ".ss" file)) + (call-with-input-file (path-expand file product) + (lambda (port) + (let read-next () + (let (form (read port)) + (unless (eof-object? form) + (when (and (pair? form) (eq? (car form) 'def) (pair? (cadr form))) + (hash-put! definitions (caadr form) (cddr form))) + (read-next)))))))) + (directory-files product)) + (def (sequence forms tail?) + (unless (null? forms) + (visit (car forms) (and tail? (null? (cdr forms)))) + (sequence (cdr forms) tail?))) + (def (visit code tail?) + (when (pair? code) + (case (car code) + ((quote) (void)) + ((begin) (sequence (cdr code) tail?)) + ((if) (visit (cadr code) #f) (for-each (cut visit <> tail?) (cddr code))) + ((let let*) + (cond + ((symbol? (cadr code)) + (if (and (pair? (caddr code)) (pair? (caaddr code))) + (begin + (for-each (lambda (binding) (for-each (cut visit <> #f) (cdr binding))) (caddr code)) + (sequence (cdddr code) tail?)) + (begin (visit (caddr code) #f) (sequence (cdddr code) tail?)))) + ((and (pair? (cadr code)) (symbol? (caadr code))) + (for-each (cut visit <> #f) (cdadr code)) (sequence (cddr code) tail?)) + (else + (for-each (lambda (binding) (for-each (cut visit <> #f) (cdr binding))) (cadr code)) + (sequence (cddr code) tail?)))) + ((lambda) (sequence (cddr code) #t)) + ((set!) + (when (eq? (cadr code) 'events) (set! owner-assignments (+ owner-assignments 1))) + (visit (caddr code) #f)) + (else + (when (and (symbol? (car code)) (hash-get definitions (car code))) + (set! calls (+ calls 1)) + (set! argument-slots (+ argument-slots (length (cdr code)))) + (when tail? (set! tail-calls (+ tail-calls 1)))) + ;; Other syntax is conservatively non-tail; do not overstate edges. + (for-each (cut visit <> #f) code))))) + (hash-for-each (lambda (_ body) (sequence body #t)) definitions) + (write (list 'static-fold-call-sites + (cons 'definitions (hash-length definitions)) + (cons 'private-calls calls) (cons 'syntactic-tail-calls tail-calls) + (cons 'static-private-argument-slots argument-slots) + (cons 'events-assignments owner-assignments))) + (newline))) diff --git a/t/benchmarks/parser-throughput/generate.ss b/t/benchmarks/parser-throughput/generate.ss new file mode 100644 index 00000000..2f922174 --- /dev/null +++ b/t/benchmarks/parser-throughput/generate.ss @@ -0,0 +1,18 @@ +;;; Build-only engine fixture; never substitute this grammar for Org acceptance. +(import (only-in :std/misc/ports read-all-as-string) + (only-in :gerbil-parser/t/event-strategy-fixture event-lines-language-grammar) + (only-in :gerbil-parser/src/compiler/event-fold-ir event-fold-ir) + (only-in :gerbil-parser/src/compiler/event-fold-scheme event-fold-scheme-source)) +(export main) + +(def (main output) + (let* ((forms '((start-node Text) (token Line start end) (finish-node))) + (identity (hash-get (event-fold-ir 'throughput-fold event-lines-language-grammar + 'Document '() forms '()) "parser_digest"))) + (call-with-output-file output + (lambda (port) + (display (event-fold-scheme-source 'throughput-fold event-lines-language-grammar + 'Document '() forms '()) port) + (write `(def fixture-identity ,identity) port) (newline port) + (display (call-with-input-file "t/benchmarks/parser-throughput/body.ss" + read-all-as-string) port))))) diff --git a/t/benchmarks/parser-throughput/link-locks-test.ss b/t/benchmarks/parser-throughput/link-locks-test.ss new file mode 100644 index 00000000..e3d53f0f --- /dev/null +++ b/t/benchmarks/parser-throughput/link-locks-test.ss @@ -0,0 +1,29 @@ +(import (only-in :std/test test-suite test-case check check-exception) + (only-in :std/test/base TestHarness TestConfig TestModule test-run! test-result-ok?) + (only-in :std/io/tempfile make-temporary-file-name) + (rename-in (only-in "./link-locks.ss" static-link-locks main) (main check-locks))) +(export main link-locks-test) + +(def link-locks-test + (test-suite "AOT lock rejection and evidence preservation" + (test-case "dirty static closures are rejected without altering lock evidence" + (let (directory (make-temporary-file-name "parser-aot-lock-test-")) + (create-directory directory) + (try + (let (lock (path-expand "part.o.lock" directory)) + (check-locks directory) + (call-with-output-file lock (lambda (port) (display "retained evidence" port))) + (check (static-link-locks directory) => (list lock)) + (check-exception (check-locks directory) exception?) + (check (call-with-input-file lock read-line) => "retained evidence") + (delete-file lock) + (check-locks directory)) + (finally (delete-file-or-directory directory #t))))))) + +(def (main) + (exit + (if (test-result-ok? + (test-run! + (TestHarness "AOT lock control" (TestConfig verbosity: 5 capture-output?: #f) + (list (TestModule "AOT lock control" (list link-locks-test) '() void void))))) + 0 42))) diff --git a/t/benchmarks/parser-throughput/link-locks.ss b/t/benchmarks/parser-throughput/link-locks.ss new file mode 100644 index 00000000..dd074d09 --- /dev/null +++ b/t/benchmarks/parser-throughput/link-locks.ss @@ -0,0 +1,14 @@ +;;; Official compile-exe waits indefinitely for object locks. Reject a dirty +;;; static closure before starting; never remove locks owned by another build. +(export main static-link-locks) + +(def (static-link-locks directory) + (if (file-exists? directory) + (map (cut path-expand <> directory) + (filter (cut string-suffix? ".o.lock" <>) (directory-files directory))) + '())) + +(def (main . directories) + (let (locks (foldr append '() (map static-link-locks directories))) + (unless (null? locks) + (error "AOT object locks exist; verify their owners before recovery" locks)))) diff --git a/t/benchmarks/parser-throughput/ordinary-plan.ss b/t/benchmarks/parser-throughput/ordinary-plan.ss new file mode 100644 index 00000000..40190bda --- /dev/null +++ b/t/benchmarks/parser-throughput/ordinary-plan.ss @@ -0,0 +1,18 @@ +;;; Inspect admitted common-byte effects before designing a run transition. +(import (only-in :gerbil-parser/src/compiler/event-fold-zero-state fold-ordinary-state-plan) + (only-in :gerbil-parser/src/compiler/event-fold-analysis fold-statement-analysis)) +(export main) + +(def (main path) + (def (walk value) + (when (pair? value) + (when (eq? (car value) 'for-line-bytes) + (let (plan (fold-ordinary-state-plan (list-ref value 4) (cadr value))) + (when plan + (write (list 'ordinary-run-admission (cadr value) + (vector-ref plan 0) (vector-ref plan 1) + (fold-statement-analysis (vector-ref plan 2)) + (vector-ref plan 2))) + (newline)))) + (for-each walk value))) + (walk (call-with-input-file path read))) diff --git a/t/benchmarks/parser-throughput/prepare-tape-preflight.ss b/t/benchmarks/parser-throughput/prepare-tape-preflight.ss new file mode 100644 index 00000000..6e54f972 --- /dev/null +++ b/t/benchmarks/parser-throughput/prepare-tape-preflight.ss @@ -0,0 +1,28 @@ +;;; Independent changing-input oracle preparation, never timed parser work. +(import :std/misc/ports + :gerbil-parser/src/compiler/event-fold-runtime + :gerbil-parser/t/fold-tape-oracle + :orgize/languages/org/grammar) +(export main) + +(def (main fixtures program-path digest output) + (let* ((program (call-with-input-file program-path read)) + (sources (list->vector + (map (lambda (name) + (call-with-input-file (path-expand name fixtures) read-all-as-string)) + '("doc.org" "plain-links.org" "quote-heavy.org")))) + (inputs (list->vector + (map (lambda (index) + (string-append (vector-ref sources (modulo index 3)) + "\nchanging request " (number->string index) " α中🦀\n")) + (iota 1000 1000)))) + (expected (vector-map + (lambda (source) + (canonical-fold-tape org-mode-language-grammar + (apply run-event-fold source program))) + inputs))) + (call-with-output-file output + (lambda (port) + (write (list 'real-fold-preflight.v1 digest inputs expected) port) + (newline port))) + (displayln "PREPARATION-OK changing-inputs=1000 input-index-base=1000"))) diff --git a/t/benchmarks/parser-throughput/prepared-tape.ss b/t/benchmarks/parser-throughput/prepared-tape.ss new file mode 100644 index 00000000..c88c95b5 --- /dev/null +++ b/t/benchmarks/parser-throughput/prepared-tape.ss @@ -0,0 +1,28 @@ +;;; Keep the untimed interpreter/compiler closure out of the native product. +;;; The existing qualification protocol and complete tape comparison stay intact. +(import :gerbil-parser/t/benchmarks/parser-throughput/protocol + :gerbil-parser/benchmarks/real-fold/parse-org-compiled-events) +(export main) + +(def (main corpus-path digest) + (unless (eq? (bind-parse-org-compiled-events digest) parse-org-compiled-events) + (error "direct real Fold binding mismatch")) + (let (corpus (call-with-input-file corpus-path read)) + (unless (and (list? corpus) (= (length corpus) 4) + (eq? (car corpus) 'real-fold-preflight.v1) + (equal? (cadr corpus) digest) + (vector? (caddr corpus)) (vector? (cadddr corpus)) + (= (vector-length (caddr corpus)) 1000) + (= (vector-length (cadddr corpus)) 1000)) + (error "invalid independently prepared preflight corpus")) + (let ((inputs (caddr corpus)) (expected (cadddr corpus)) (oracle-index 0)) + (run-parser-throughput "upstream-real-org-fold-oev1-direct-prepared" digest + parse-org-compiled-events + (lambda (source) + (unless (and (< oracle-index 1000) + (equal? source (vector-ref inputs oracle-index))) + (error "prepared oracle input ordering mismatch" oracle-index)) + (let (result (vector-ref expected oracle-index)) + (set! oracle-index (+ oracle-index 1)) result)) + (lambda (index) (vector-ref inputs (- index 1000))) + 'preflight)))) diff --git a/t/benchmarks/parser-throughput/profile.ss b/t/benchmarks/parser-throughput/profile.ss new file mode 100644 index 00000000..9b94de83 --- /dev/null +++ b/t/benchmarks/parser-throughput/profile.ss @@ -0,0 +1,56 @@ +;;; Upstream engine attribution over an externally admitted real Fold product. +;;; No Orgize API, FFI, Tokio or Python runtime participates. +(import :gerbil-parser/src/runtime/fold-cost + :gerbil-parser/src/runtime/parse-cost + :gerbil-parser/benchmarks/real-fold/parse-org-compiled-events) +(export main) +(def (main controls (mode "timed") (expected-digest #f)) + (unless (member mode '("timed" "counts" "prepared-counts")) + (error "invalid Fold attribution mode" mode)) + (let* ((input (call-with-input-file controls read)) + (prepared? (equal? mode "prepared-counts")) + (count-only? (member mode '("counts" "prepared-counts"))) + (cases + (if prepared? + (begin + (unless (and (list? input) (= (length input) 4) + (eq? (car input) 'real-fold-preflight.v1) + ;; Diagnostic products intentionally reject the + ;; production throughput binder. Bind the corpus + ;; here; receipts separately seal generated IR and + ;; binary bytes. This never admits performance. + (string? expected-digest) (equal? (cadr input) expected-digest) + (vector? (caddr input)) (vector? (cadddr input)) + (= (vector-length (caddr input)) 1000) + (= (vector-length (cadddr input)) 1000)) + (error "invalid independent attribution corpus")) + (map (lambda (index) + (list (vector-ref (caddr input) index) '() + (vector-ref (cadddr input) index))) (iota 1000))) input)) + (session (make-fold-cost-session)) + (events 0)) + (let (before (##process-statistics)) + (parameterize ((current-fold-cost-session session)) + (for-each + (lambda (control) + (let (actual (call-with-fold-cost 'whole-fold + (lambda () (parse-org-compiled-events (car control) (cadr control))))) + (unless (equal? actual (caddr control)) (error "real Fold diagnostic mismatch")) + (set! events (+ events (if (u8vector? actual) (u8vector-length actual) (length actual)))))) cases)) + (let* ((after (##process-statistics)) + (delta (lambda (index) (- (f64vector-ref after index) (f64vector-ref before index)))) + (raw (delta 7)) (gc (delta 6))) + (write (list 'diagnostic-only (cons 'requests (length cases)) + (cons 'events events) (cons 'cpu-ms (* 1000 (+ (delta 0) (delta 1)))) + (cons 'gc-count gc) (cons 'allocation-counter-delta raw) + (cons 'allocated-bytes (admit-parser-allocation raw gc)))) (newline))) + (displayln (if count-only? + "FIELDS stage count; ordinary-run-scanned-bytes and summary-byte-visits use bytes, other ordinary-run stages use counts; leaf timing unavailable (zero placeholders)" + "FIELDS stage calls inclusive-us exclusive-instrumented-us failures")) + (let (rows (fold-cost-rows session)) + (for-each (lambda (row) (write row) (newline)) + (if count-only? + (list-sort (lambda (left right) (> (cadr left) (cadr right))) rows) rows))) + (when prepared? + (write (list 'diagnostic-corpus-identity expected-digest)) (newline)) + (displayln "DIAGNOSTIC-OK real-fold; NOT PERFORMANCE ACCEPTANCE"))) diff --git a/t/benchmarks/parser-throughput/protocol.ss b/t/benchmarks/parser-throughput/protocol.ss new file mode 100644 index 00000000..061104cc --- /dev/null +++ b/t/benchmarks/parser-throughput/protocol.ss @@ -0,0 +1,114 @@ +;;; Engine-owned qualification protocol; adapters supply a compiled parser. +(import (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation) + (only-in :std/os/time current-system-time-precise + timespec-tv_sec timespec-tv_nsec)) +(export parser-throughput-percentile parser-throughput-gate + parser-throughput-batch run-parser-throughput) + +(def loads '(1000 10000 100000)) +(def minimum-throughput 100000) + +;; Use the installed SDK's precise OS clock, not floating-point epoch seconds +;; whose subtraction quantizes sub-microsecond requests to zero. This is a +;; realtime clock: reject backwards movement; do not call it monotonic. +(def (clock-ns) + (let (time (current-system-time-precise)) + (+ (* (timespec-tv_sec time) 1000000000) (timespec-tv_nsec time)))) + +(def (parser-throughput-percentile sorted percent) + (unless (and (vector? sorted) (positive? (vector-length sorted)) + (exact-integer? percent) (<= 1 percent 100)) + (error "invalid nearest-rank percentile input" percent)) + (vector-ref sorted + (- (quotient (+ (* percent (vector-length sorted)) 99) 100) 1))) + +(def (parser-throughput-gate completed wall-s) + (and (exact-integer? completed) (positive? completed) + (real? wall-s) (> wall-s 0) + (<= wall-s (/ completed minimum-throughput)))) + +(def (parser-throughput-batch parser inputs expected) + (let* ((count (vector-length inputs)) + (latencies (make-vector count 0)) + (unresolved 0) + (stats (##process-statistics)) + (begin (clock-ns))) + (unless (and (> count 0) (= count (vector-length expected))) + (error "benchmark input/oracle extent mismatch")) + (let loop ((index 0)) + (unless (= index count) + (let* ((start (clock-ns)) + (result (parser (vector-ref inputs index))) + (elapsed (- (clock-ns) start))) + (when (< elapsed 0) (error "benchmark clock moved backwards")) + ;; A zero interval is below this clock's observable resolution, + ;; never proof of a zero-cost parser call. + (when (= elapsed 0) (set! unresolved (+ unresolved 1))) + (vector-set! latencies index (/ elapsed 1000.0)) + ;; Full semantic validation is outside request timing, inside the + ;; conservative batch wall. Never label this as bare Fold cost. + (unless (equal? result (vector-ref expected index)) + (error "benchmark semantic mismatch" index)) + (loop (+ index 1))))) + (let* ((wall (/ (- (clock-ns) begin) 1000000000.0)) + (after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) + (f64vector-ref stats slot)))) + (sorted (list->vector (list-sort < (vector->list latencies))))) + (unless (> wall 0) (error "invalid batch wall clock" wall)) + `((completed . ,count) (batch-wall-s . ,wall) + (throughput-requests/s . ,(/ count wall)) + (request-p50-us . ,(parser-throughput-percentile sorted 50)) + (request-p95-us . ,(parser-throughput-percentile sorted 95)) + (request-p99-us . ,(parser-throughput-percentile sorted 99)) + (request-max-us . ,(vector-ref sorted (- count 1))) + (sub-resolution-request-samples . ,unresolved) + (process-cpu-s . ,(+ (delta 0) (delta 1))) + (gc-count . ,(delta 6)) + (allocation-counter-delta . ,(delta 7)) + (allocated-bytes . ,(admit-parser-allocation (delta 7) (delta 6))) + (throughput-wall-gate . ,(parser-throughput-gate count wall)))))) + +(def (run-parser-throughput label identity parser oracle input mode) + (unless (and (string? label) (not (equal? label "")) + (string? identity) (not (equal? identity "")) + (memq mode '(preflight full))) + (error "benchmark requires source identity and preflight/full mode")) + (def (run-load count batches phase) + ;; Generate changing inputs and independent expected values before timing. + ;; Both stay retained: memory evidence includes benchmark oracle storage. + (let loop ((batch 1)) + (when (<= batch batches) + (let* ((inputs (list->vector + (map (lambda (index) (input (+ index (* count batch)))) + (iota count)))) + (expected (list->vector (map oracle (vector->list inputs))))) + (unless (equal? (parser (vector-ref inputs 0)) (vector-ref expected 0)) + (error "parser warmup semantic mismatch")) + (let (receipt (parser-throughput-batch parser inputs expected)) + (write `((schema . parser-throughput.v1) (label . ,label) + (source-identity . ,identity) (phase . ,phase) + (requests . ,count) (batch . ,batch) + (input-index-base . ,(* count batch)) + (minimum-throughput-requests/s . ,minimum-throughput) + (maximum-batch-wall-s . ,(/ count minimum-throughput)) + (request-boundary . compiled-parser-call) + (batch-boundary . parser-calls-plus-complete-semantic-comparison) + (percentile-method . nearest-rank-per-request) + (clock . sdk-clock-realtime-integer-nanoseconds) + (tail-latency-gate . not-configured) + (memory-boundary . process-allocation-counter-and-gc-not-rss) + (input-and-oracle-retained . #t) + (samples . ,receipt))) + (newline) (force-output) + (unless (cdr (assq 'throughput-wall-gate receipt)) + (error "PERFORMANCE-GATE-FAILED; stop rejected candidate" count batch))) + (loop (+ batch 1)))))) + ;; A failing preflight is a rejection, never a truncated successful matrix. + (run-load 1000 1 'preflight) + (if (eq? mode 'full) + (begin + (for-each (lambda (count) (run-load count 20 'acceptance)) loads) + (displayln "PARSER-THROUGHPUT-FULL-OK loads=3 batches-per-load=20")) + (displayln "PARSER-THROUGHPUT-PREFLIGHT-OK; full acceptance not run")) + (force-output)) diff --git a/t/benchmarks/parser-throughput/real-tape.ss b/t/benchmarks/parser-throughput/real-tape.ss new file mode 100644 index 00000000..51f4890f --- /dev/null +++ b/t/benchmarks/parser-throughput/real-tape.ss @@ -0,0 +1,28 @@ +;;; Actual generated Org Fold directly consumed as the production OEV1 tape. +;;; Independent interpreter/list encoding is outside measured requests. +(import :std/misc/ports + :gerbil-parser/src/compiler/event-fold-runtime + :gerbil-parser/t/fold-tape-oracle + :gerbil-parser/t/benchmarks/parser-throughput/protocol + :orgize/languages/org/grammar + :gerbil-parser/benchmarks/real-fold/parse-org-compiled-events) +(export main) +(def (main fixtures program-path digest (mode "preflight") (consumer "direct")) + (unless (member consumer '("direct" "list-control")) (error "invalid Fold consumer lane")) + (unless (eq? (bind-parse-org-compiled-events digest) parse-org-compiled-events) + (error "direct real Fold binding mismatch")) + (let ((program (call-with-input-file program-path read)) + (legacy-consumer (make-legacy-fold-tape-consumer org-mode-language-grammar)) + (sources (list->vector + (map (lambda (name) + (call-with-input-file (path-expand name fixtures) read-all-as-string)) + '("doc.org" "plain-links.org" "quote-heavy.org"))))) + (run-parser-throughput (string-append "upstream-real-org-fold-oev1-" consumer) digest + (if (equal? consumer "direct") parse-org-compiled-events + (lambda (source) (legacy-consumer (parse-org-compiled-events source)))) + (lambda (source) (canonical-fold-tape org-mode-language-grammar + (apply run-event-fold source program))) + (lambda (index) + (string-append (vector-ref sources (modulo index 3)) + "\nchanging request " (number->string index) " α中🦀\n")) + (string->symbol mode)))) diff --git a/t/benchmarks/parser-throughput/real.ss b/t/benchmarks/parser-throughput/real.ss new file mode 100644 index 00000000..3413f41b --- /dev/null +++ b/t/benchmarks/parser-throughput/real.ss @@ -0,0 +1,20 @@ +;;; Actual generated Fold; all runtime, measurement and oracle ownership is upstream. +(import :std/misc/ports + :gerbil-parser/src/compiler/event-fold-runtime + :gerbil-parser/t/benchmarks/parser-throughput/protocol + :gerbil-parser/benchmarks/real-fold/parse-org-compiled-events) +(export main) +(def (main fixtures program-path digest (mode "preflight")) + (unless (eq? (bind-parse-org-compiled-events digest) parse-org-compiled-events) + (error "real Fold binding mismatch")) + (let ((program (call-with-input-file program-path read)) + (sources (list->vector + (map (lambda (name) + (call-with-input-file (path-expand name fixtures) read-all-as-string)) + '("doc.org" "plain-links.org" "quote-heavy.org"))))) + (run-parser-throughput "upstream-real-org-fold" digest parse-org-compiled-events + (lambda (source) (apply run-event-fold source program)) + (lambda (index) + (string-append (vector-ref sources (modulo index 3)) + "\nchanging request " (number->string index) " α中🦀\n")) + (string->symbol mode)))) diff --git a/t/benchmarks/parser-throughput/report.ss b/t/benchmarks/parser-throughput/report.ss new file mode 100644 index 00000000..d639c81e --- /dev/null +++ b/t/benchmarks/parser-throughput/report.ss @@ -0,0 +1,57 @@ +;;; Native receipt admission; no external statistics or JSON translation. +(export main) + +(def (field row key) + (let (entry (assq key row)) + (unless entry (error "missing benchmark field" key)) + (cdr entry))) + +(def (median values) + (let* ((ordered (list-sort < values)) (count (length ordered)) + (middle (quotient count 2))) + (if (odd? count) (list-ref ordered middle) + (/ (+ (list-ref ordered (- middle 1)) (list-ref ordered middle)) 2)))) + +(def (main path) + (let ((rows '()) (finished #f)) + (call-with-input-file path + (lambda (port) + (let loop () + (let (line (read-line port)) + (unless (eof-object? line) + (let (value (call-with-input-string line read)) + (cond + ((and (pair? value) (assq 'schema value)) + (when (eq? (field value 'phase) 'acceptance) + (set! rows (cons value rows)))) + ((eq? value 'PARSER-THROUGHPUT-FULL-OK) (set! finished #t)) + (else (error "unexpected or failed benchmark output" line)))) + (loop)))))) + (unless (and finished (= (length rows) 60)) + (error "complete sixty-batch receipt required")) + (for-each + (lambda (count) + (let (group (filter (lambda (row) (= (field row 'requests) count)) rows)) + (unless (and (= (length group) 20) + (equal? (list-sort < (map (cut field <> 'batch) group)) (iota 20 1)) + (every (lambda (row) + (and (= (field row 'input-index-base) + (* count (field row 'batch))) + (equal? (field row 'source-identity) + (field (car rows) 'source-identity)) + (field (field row 'samples) 'throughput-wall-gate) + (= (field (field row 'samples) 'completed) count))) group)) + (error "missing, mismatched or failed acceptance batch" count)) + (let (metrics (map (cut field <> 'samples) group)) + (write `((requests . ,count) (batches . 20) + (boundary . ,(field (car group) 'label)) + (aggregation . medians-of-per-batch-request-percentiles-not-pooled) + ,@(map (lambda (key) + (cons key (median (map (cut field <> key) metrics)))) + '(batch-wall-s throughput-requests/s request-p95-us request-p99-us + gc-count allocation-counter-delta)) + (sub-resolution-request-samples . + ,(foldl + 0 (map (cut field <> 'sub-resolution-request-samples) metrics))))) + (newline)))) + '(1000 10000 100000)) + (displayln "PARSER-THROUGHPUT-REPORT-OK") (force-output))) diff --git a/t/benchmarks/parser-throughput/run-plan.ss b/t/benchmarks/parser-throughput/run-plan.ss new file mode 100644 index 00000000..b7e2c450 --- /dev/null +++ b/t/benchmarks/parser-throughput/run-plan.ss @@ -0,0 +1,54 @@ +;;; Static admission coverage, not executed counts or a throughput result. +(import (only-in :gerbil-parser/src/compiler/event-fold-ordinary-run fold-ordinary-run-plan fold-consumer-run-plan) + (only-in :gerbil-parser/src/compiler/event-fold-zero-state fold-zero-state-plan fold-consumer-state-body) + (only-in :gerbil-parser/src/compiler/event-fold-consumer-state fold-consumer-selector-body) + (only-in :gerbil-parser/src/compiler/event-fold-byte-dispatch fold-byte-regions) + (only-in :gerbil-parser/src/compiler/event-fold-analysis fold-eliminate-inert-branches fold-statement-analysis)) +(export main) + +(def (main path) + (let ((seen (make-table test: equal?)) (loops 0) (eligible 0) + (refined 0) (before-selectors 0) (after-selectors 0) (selective 0)) + (def (selectors value) + (if (not (pair? value)) 0 + (+ (if (memq (car value) '(uint-positive? uint-equal? uint-not-equal?)) 1 0) + (foldl + 0 (map selectors value))))) + (def (walk value) + (when (pair? value) + (when (eq? (car value) 'for-line-bytes) + (set! loops (+ loops 1)) + (let (plan (fold-consumer-run-plan (list-ref value 4) (cadr value))) + (when plan (set! selective (+ selective 1)))) + (let* ((original (list-ref value 4)) + (local (fold-consumer-selector-body (fold-consumer-state-body original)))) + (unless (equal? original local) (set! refined (+ refined 1))) + (set! before-selectors (+ before-selectors (selectors original))) + (set! after-selectors (+ after-selectors (selectors local)))) + (let (plan (fold-ordinary-run-plan (list-ref value 4) (cadr value))) + (when (and (not plan) (eq? (cadr value) 'inline-byte-index)) + (let* ((idle (fold-zero-state-plan (list-ref value 4) #t)) + (regions (fold-byte-regions (if idle (vector-ref idle 1) (list-ref value 4)) + (cadr value))) + (common (and regions (fold-eliminate-inert-branches (vector-ref regions 1) (cadr value))))) + (unless (table-ref seen common #f) + (table-set! seen common #t) + (write (list 'inline-run-rejection common)) (newline)))) + (when plan + (set! eligible (+ eligible 1)) + (let (row (list (cadr value) (length (vector-ref plan 0)) + (length (vector-ref plan 1)) (vector-ref plan 2) + (map cadr (vector-ref plan 3)))) + (unless (table-ref seen row #f) + (table-set! seen row #t) (write (cons 'run-admission row)) (newline) + (write (list 'run-guard-cells (cadr value) (vector-ref plan 0))) + (newline) + (let (writes (vector-ref (fold-statement-analysis (list-ref value 4)) 1)) + (write (list 'run-loop-invariant-guards (cadr value) + (filter (lambda (name) (not (memq name writes))) (vector-ref plan 0)))) + (newline))))))) + (for-each walk value))) + (walk (call-with-input-file path read)) + (write (list 'static-run-coverage loops eligible)) (newline) + (write (list 'static-selective-consumer-coverage selective)) (newline) + (write (list 'static-consumer-selector-coverage refined before-selectors after-selectors)) + (newline))) diff --git a/t/benchmarks/parser-throughput/stalls.ss b/t/benchmarks/parser-throughput/stalls.ss new file mode 100644 index 00000000..db01af9e --- /dev/null +++ b/t/benchmarks/parser-throughput/stalls.ss @@ -0,0 +1,85 @@ +;;; Diagnostic only. Read the installed Gambit header's VM GC counters; +;;; never alter the collector or scheduler. Per-request probes add overhead. +(import :std/ffi + (only-in :std/os/time current-system-time-precise timespec-tv_sec timespec-tv_nsec) + :gerbil-parser/benchmarks/real-fold/parse-org-compiled-events) +(export main) +(C-ffi-macrology) +(C-include "") +(C-include "") +;; Independent monotonic clock checks realtime sampling, not the collector. +(def-C-lambda monotonic-us () double + "struct timespec t; if (clock_gettime(CLOCK_MONOTONIC, &t)) ___return(-1.0); + ___return(t.tv_sec * 1e6 + t.tv_nsec * 1e-3);") +(def-C-lambda gc-real () double "___return(___VMSTATE->mem.gc_real_time_);") +(def-C-lambda gc-cpu () double + "___return(___VMSTATE->mem.gc_user_time_ + ___VMSTATE->mem.gc_sys_time_);") +(def-C-lambda gc-count () double "___return(___VMSTATE->mem.nb_gcs_);") +(def-C-lambda cpu () double + "struct rusage r; if (getrusage(RUSAGE_SELF, &r)) ___return(-1.0); + ___return(r.ru_utime.tv_sec + r.ru_stime.tv_sec + + (r.ru_utime.tv_usec + r.ru_stime.tv_usec) * 1e-6);") +(def-C-lambda switches () long + "struct rusage r; if (getrusage(RUSAGE_SELF, &r)) ___return(-1); + ___return(r.ru_nivcsw);") +(def-C-lambda voluntary-switches () long + "struct rusage r; if (getrusage(RUSAGE_SELF, &r)) ___return(-1); + ___return(r.ru_nvcsw);") +(def (clock-us) + (let (t (current-system-time-precise)) + (+ (* (timespec-tv_sec t) 1000000) (quotient (timespec-tv_nsec t) 1000)))) +(def (main corpus-path (mode "stalls")) + (let* ((corpus (call-with-input-file corpus-path read)) + (inputs (caddr corpus)) (expected (cadddr corpus)) (rows '()) + (comparison-wall-us 0) (batch-wall 0) (batch-cpu 0)) + (unless (and (eq? (car corpus) 'real-fold-preflight.v1) + (= (vector-length inputs) 1000) (= (vector-length expected) 1000)) + (error "invalid diagnostic corpus")) + (unless (member mode '("stalls" "profile")) (error "unknown diagnostic mode" mode)) + (when (equal? mode "profile") + (displayln "FOLD-PROFILE-READY") (force-output) + ;; Keep one AOT process alive for the OS sampler, not a timed matrix. + ;; Sampled stack owners distinguish complete tape comparison from Fold. + (for-each + (lambda (_batch) + (for-each + (lambda (index) + (unless (equal? (parse-org-compiled-events (vector-ref inputs index)) + (vector-ref expected index)) + (error "profile tape mismatch" index))) + (iota 1000))) + (iota 100)) + (displayln "FOLD-PROFILE-OK complete tapes; NOT PERFORMANCE ACCEPTANCE") + (exit 0)) + (set! batch-cpu (cpu)) + (set! batch-wall (monotonic-us)) + (for-each + (lambda (index) + (let* ((c0 (cpu)) (g0 (gc-real)) (gc0 (gc-cpu)) + (n0 (gc-count)) (s0 (switches)) (v0 (voluntary-switches)) + (m0 (monotonic-us)) (t0 (clock-us)) + (result (parse-org-compiled-events (vector-ref inputs index))) + (wall (- (clock-us) t0)) (c1 (cpu)) + (m1 (monotonic-us)) + (g1 (gc-real)) (gc1 (gc-cpu)) (n1 (gc-count)) (s1 (switches)) + (v1 (voluntary-switches))) + (let (comparison-start (monotonic-us)) + (unless (equal? result (vector-ref expected index)) + (error "diagnostic tape mismatch" index)) + (set! comparison-wall-us + (+ comparison-wall-us (- (monotonic-us) comparison-start)))) + (set! rows (cons (list wall index (* 1000000 (- c1 c0)) + (* 1000000 (- g1 g0)) (* 1000000 (- gc1 gc0)) + (- n1 n0) (- s1 s0) (- v1 v0) (- m1 m0)) rows)))) + (iota 1000)) + (set! batch-wall (- (monotonic-us) batch-wall)) + (set! batch-cpu (* 1000000 (- (cpu) batch-cpu))) + (displayln "BATCH-DIAGNOSTIC wall-us=" batch-wall " cpu-us=" batch-cpu + " parse-realtime-us=" (foldl + 0 (map car rows)) + " parse-monotonic-us=" (foldl + 0 (map (cut list-ref <> 8) rows)) + " comparison-wall-us=" comparison-wall-us) + (displayln "DIAGNOSTIC-ONLY probes add overhead; NOT PERFORMANCE ACCEPTANCE") + (displayln "FIELDS realtime-wall-us input-index cpu-us gc-wall-us gc-cpu-us gc-count involuntary-switches voluntary-switches monotonic-wall-us") + (for-each (lambda (row) (write row) (newline)) + (take (list-sort (lambda (a b) (> (car a) (car b))) rows) 20)) + (displayln "FOLD-STALLS-OK complete tapes; historical spikes not retrospectively classified"))) diff --git a/t/benchmarks/prefix-families/README.org b/t/benchmarks/prefix-families/README.org new file mode 100644 index 00000000..4a95a14b --- /dev/null +++ b/t/benchmarks/prefix-families/README.org @@ -0,0 +1,64 @@ +#+title: Native prefix-family staging + +* Engine boundary + +The existing homogeneous =or= of four to 32 nonempty =line-starts-with= or +=line-starts-with-ascii-ci= predicates is staged into shared byte decisions. +The aggregate encoded prefix length is bounded at 512 bytes. The compiler +uses pure grouping, mapping and folding over immutable build-time values; +requests allocate no trie. Mixed predicates retain ordinary short-circuit +lowering. No grammar, admitted IR, ABI, runtime cache or public DSL changes. + +This follows the engine scanner's prepared-literal approach and Gerbil's +named functional combinators: factor repeated traversal into one mechanism, +not more nested handwritten loops in downstream grammar declarations. + +* Matched controls + +The rich family uses Org's six TITLE/SUBTITLE/AUTHOR/DATE/CAPTION/DESCRIPTION +prefixes. A separate 32-prefix family is a stress control. The linear route +uses equivalent nested single-prefix conditions, generated by the same +current compiler, not a historical executable. All routes include encoding, +line traversal and complete event publication, with identical hit/miss actions. + +The generator embeds independent =run-event-fold= interpreter expectations +for each route: every prefix, lowercase variants, UTF-8 payloads, CRLF and +blank lines, truncated prefixes, empty source and EOF. Before timing, every +document's complete output is compared across both native routes. Eight +samples rotate route positions and verify all 20 events per document. + +#+begin_src sh +just generate-prefix-family-benchmark OUTPUT LOAD_PATH +just benchmark-prefix-families OUTPUT/prefix-family-benchmark 8 +#+end_src + +Link the generated source with the project's native executable recipe against +the same compiled engine library. Generation and compilation are outside +request timing; requests do not launch processes or exchange JSON. + +* Local observation + +CPU medians in milliseconds, eight samples per route at each scale: + +| Documents | Rich linear | Rich staged | Stress linear | Stress staged | +|-----------+-------------+-------------+---------------+---------------| +| 1000 | 2.298 | 1.107 | 8.769 | 0.998 | +| 2000 | 4.816 | 2.076 | 17.503 | 2.034 | +| 10000 | 20.277 | 8.596 | 84.867 | 7.652 | + +All 96 timed rows and complete parity controls passed. Original stdout is +retained as =prefix-family-benchmark.log= beside the local native executable. +At 10,000 documents rich wall medians were 20.395/9.686ms; stress medians were +85.357/7.670ms. Individual stress wall samples reached 319.606ms (linear) and +83.401ms (staged), so host scheduling noise is retained rather than hidden. +Every sample collected garbage: raw allocation-counter deltas are retained +but not admitted as allocation totals. + +These are engine mechanism measurements, not full Org grammar, Rust/Tokio or +Scheme actor scheduling, FFI packets, query, or filesystem measurements. +They do not close the complete consumer's separate, still-unpassed 100ms gate. + +The subsequent interpreter-qualified repetition also passed all 96 rows. +Rich CPU medians were 2.243/1.121ms, 4.441/2.083ms and 20.302/8.929ms +at 1,000/2,000/10,000 documents. That receipt is retained separately as +=prefix-family-benchmark-oracle.log=; it does not replace the first run. diff --git a/t/benchmarks/prefix-families/body.ss b/t/benchmarks/prefix-families/body.ss new file mode 100644 index 00000000..4f08aade --- /dev/null +++ b/t/benchmarks/prefix-families/body.ss @@ -0,0 +1,50 @@ +;;; Native callers, complete event parity and request-cost boundaries. +(import (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation)) +(export main) + +(def routes (list (cons 'rich-linear rich-linear) (cons 'rich-fused rich-fused) + (cons 'stress-linear stress-linear) (cons 'stress-fused stress-fused))) + +(def (document index) + (string-append "#+TITLE: " (number->string index) "\r\n#+description: λ中🦀\n" + "#+KEY1: value\r#+key31: é\r\n\n#+CAPTION:")) + +(def (positive-count text) + (let (count (string->number text)) + (unless (and (exact-integer? count) (positive? count)) (error "invalid benchmark count" text)) + count)) + +(def (measure documents route) + (let* ((before (##process-statistics)) (start (##current-time-point)) + (count (foldl (lambda (text sum) (+ sum (length ((cdr route) text)))) 0 documents)) + (wall (* 1000 (- (##current-time-point) start))) (after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) (f64vector-ref before slot))))) + (unless (= count (* 20 (length documents))) (error "native prefix fold lost events" route count)) + (list 'wall-ms wall 'cpu-ms (* 1000 (+ (delta 0) (delta 1))) 'gc-count (delta 6) + 'allocation-counter-delta (delta 7) 'allocated-bytes (admit-parser-allocation (delta 7) (delta 6))))) + +(def (main (samples-text "8") . sizes-text) + (for-each (lambda (check) (check)) + (list rich-linear-oracle rich-fused-oracle stress-linear-oracle stress-fused-oracle)) + (displayln "PREFIX-INTERPRETER-OK") (force-output) + (let ((samples (positive-count samples-text)) + (sizes (map positive-count (if (null? sizes-text) '("1000" "2000" "10000") sizes-text)))) + (for-each + (lambda (size) + (let (documents (map document (iota size))) + (for-each + (lambda (text) + (unless (and (equal? (rich-linear text) (rich-fused text)) + (equal? (stress-linear text) (stress-fused text))) + (error "native prefix family changed complete events" text))) + (append '("" "#+T" "#+TITLE:" "#+TITLE:x" "#+DESCRIPTIO:" "é中🦀" "#+KEY31:\n") documents)) + (displayln "PREFIX-PARITY-OK documents=" size) (force-output) + (for-each + (lambda (sample) + (let (offset (modulo sample 4)) + (for-each + (lambda (route) + (write (append (list 'prefix-family 'documents size 'sample sample 'mode (car route)) + (measure documents route))) (newline) (force-output)) + (append (drop routes offset) (take routes offset))))) (iota samples)))) sizes)) + (displayln "PREFIX-BENCHMARK-OK") (force-output)) diff --git a/t/benchmarks/prefix-families/generate.ss b/t/benchmarks/prefix-families/generate.ss new file mode 100644 index 00000000..3b9e2165 --- /dev/null +++ b/t/benchmarks/prefix-families/generate.ss @@ -0,0 +1,50 @@ +;;; Build-time generation; benchmark requests call real native Fold products. +(import (only-in :std/misc/ports read-all-as-string) + (only-in :gerbil-parser/t/event-strategy-fixture event-lines-language-grammar) + (only-in :gerbil-parser/src/compiler/event-fold-runtime run-event-fold) + (only-in :gerbil-parser/src/compiler/event-fold-scheme event-fold-scheme-source)) +(export main) + +(def (family-condition names) + (cons 'or (map (lambda (name) `(line-starts-with-ascii-ci ,name)) names))) + +(def (linear-forms names) + (if (null? names) + '((start-node Text) (token Line start end) (finish-node)) + `((if (line-starts-with-ascii-ci ,(car names)) + ((start-node Heading) (token Line start end) (finish-node)) ,(linear-forms (cdr names)))))) + +(def (main output) + (call-with-output-file output + (lambda (port) + (for-each + (lambda (family) + (let ((label (car family)) (names (cdr family))) + (for-each + (lambda (fused?) + (let (name (string->symbol + (string-append label (if fused? "-fused" "-linear")))) + (let (forms (if fused? + `((if ,(family-condition names) + ((start-node Heading) (token Line start end) (finish-node)) + ((start-node Text) (token Line start end) (finish-node)))) + (linear-forms names))) + (display (event-fold-scheme-source name event-lines-language-grammar 'Document '() + forms '()) port) + (write `(def (,(string->symbol (string-append (symbol->string name) "-oracle"))) + ,@(map (lambda (text) + `(unless (equal? (,name ,text) + ',(run-event-fold text 'Document '() forms '() '())) + (error "native prefix interpreter mismatch" ',name ,text))) + (append '("" "#+T" "#+TITLE:x" "#+DESCRIPTIO:" "é中🦀") + (foldr append '() (map + (lambda (prefix) + (list prefix (string-downcase prefix) + (string-append prefix " λ中🦀\r\n\n") + (substring prefix 0 (- (string-length prefix) 1)))) + names))))) port) + (newline port)))) + '(#f #t)))) + (list (cons "rich" '("#+TITLE:" "#+SUBTITLE:" "#+AUTHOR:" "#+DATE:" "#+CAPTION:" "#+DESCRIPTION:")) + (cons "stress" (map (lambda (index) (string-append "#+KEY" (number->string index) ":")) (iota 32))))) + (display (call-with-input-file "t/benchmarks/prefix-families/body.ss" read-all-as-string) port)))) diff --git a/t/benchmarks/ranked-literal/benchmark.ss b/t/benchmarks/ranked-literal/benchmark.ss new file mode 100644 index 00000000..504dd70e --- /dev/null +++ b/t/benchmarks/ranked-literal/benchmark.ss @@ -0,0 +1,290 @@ +;;; Shared lexer publication: prefix chains, filtered modes and scalar controls. +(import (only-in :gerbil-parser/src/runtime/scan make-ranked-literal-scanner make-literal-trie + make-bounded-literal-end-scanner literal-trie-child) + (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation) + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/benchmark measure-parser-component measure-parser-batch sample-at-percentile measure-parser-cpu-pairs)) +(export benchmark-ranked-literal benchmark-ranked-literal-requests benchmark-literal-edge-cpu qualify-literal-edge-cpu + benchmark-ranked-edge-cpu qualify-ranked-edge-cpu + benchmark-ranked-publication-cpu qualify-ranked-publication-cpu + ranked-cpu-admission) +(def (benchmark-ranked-literal (samples 11) (iterations 1000)) + (unless (and (exact-integer? samples) (positive? samples) + (exact-integer? iterations) (positive? iterations)) + (error "ranked literal benchmark requires positive exact counts")) + (let* ((entries (map (lambda (n) (list (make-string (+ n 1) #\a) n n n)) (iota 32))) + (scan (make-ranked-literal-scanner entries)) (text (make-string 32 #\a)) + (single (make-ranked-literal-scanner '(("a" single 0 0)))) + (short (make-vector 32 #f)) (none (make-vector 32 #f))) + (vector-set! short 0 #t) + (for-each (lambda (row) + (let ((scanner (car row)) (source (cadr row)) (admitted (caddr row)) + (width (cadddr row)) (name (car (cddddr row)))) + (let sample ((index 0)) + (when (< index samples) + (let* ((before (##process-statistics)) + (sum (let repeat ((count 0) (total 0)) + (if (= count iterations) total + (let (found (scanner source 0 admitted)) + (repeat (+ count 1) (+ total (if found (cadr found) 0))))))) + (after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) (f64vector-ref before slot))))) + (unless (= sum (* iterations width)) (error "ranked literal endpoint changed" name)) + (write (list 'ranked-literal name index 'iterations iterations + 'cpu-ms-per-call (/ (* 1000 (+ (delta 0) (delta 1))) iterations) + 'allocated-bytes-per-call + (let (bytes (admit-parser-allocation (delta 7) (delta 6))) + (and bytes (/ bytes iterations))))) + (newline) (force-output)) + (sample (+ index 1)))))) + (list (list scan text #f 32 'prefix-chain) (list scan text short 1 'short-admitted) + (list single "a" #f 1 'single-terminal) (list scan text none 0 'all-filtered)))) + (displayln "RANKED-LITERAL-OK") (force-output)) + +;;; Compiled complete-result controls across accepting-prefix depths. +(def (benchmark-ranked-literal-requests) + (for-each + (lambda (width) + (let* ((entries (map (lambda (index) + (list (make-string (+ index 1) #\a) 'literal index index)) + (iota width))) + (scan (make-ranked-literal-scanner entries)) + (source (string-append (make-string width #\a) "!")) + (mask (make-vector width #f))) + (vector-set! mask 0 #t) + (measure-parser-component (list 'accepting-prefixes width) 20 5000 + (lambda () (scan source 0)) (list 'literal width (- width 1) (- width 1))) + (measure-parser-component (list 'filtered-prefixes width) 20 5000 + (lambda () (scan source 0 mask)) '(literal 1 0 0)) + (measure-parser-component (list 'missing-prefix width) 20 5000 + (lambda () (scan "!" 0)) #f))) + '(1 4 16 64)) + (displayln "RANKED-LITERAL-REQUESTS-OK") (force-output)) + +;;; The reference changes edge storage only, retaining the complete bounded +;;; scanner protocol. It is a primitive comparator, not a second parser engine. +(def (table-edge-copy node) + (let ((edges (vector-ref node 1)) (children (make-table test: eqv?))) + (for-each (lambda (edge) + (table-set! children (car edge) (table-edge-copy (cdr edge)))) + (cond ((pair? edges) (list edges)) + ((not edges) '()) + (else (table->list edges)))) + (vector (vector-ref node 0) children))) +(def (table-edge-scanner literals) + (let (root (table-edge-copy (make-literal-trie (map (lambda (text) (cons text #t)) literals)))) + (lambda (source start limit) + (and (exact-integer? start) (exact-integer? limit) + (<= 0 start limit (string-length source)) + (let loop ((node root) (offset start) (selected #f)) + (if (= offset limit) selected + (let (child (table-ref (vector-ref node 1) (string-ref source offset) #f)) + (if child + (let (next (+ offset 1)) + (loop child next (if (vector-ref child 0) next selected))) + selected)))))))) +(def (edge-field row key) (cdr (assq key row))) +(def (benchmark-literal-edge-cpu (groups 20) (characters-per-batch 1000000) (widths '(1 8 32 128))) + (unless (and (exact-integer? groups) (>= groups 20) + (exact-integer? characters-per-batch) (>= characters-per-batch 128) + (list? widths) (pair? widths) + (andmap (lambda (width) (memv width '(1 8 32 128))) widths)) + (error "edge CPU benchmark requires twenty groups and positive resolved batches")) + (let (summaries '()) + (for-each + (lambda (width) + (let* ((literals (list (make-string width #\a) (string-append "β" (make-string width #\a)))) + (source (string-append (car literals) "!")) + (limit (string-length source)) + (calls (quotient characters-per-batch width)) + (reference-scan (table-edge-scanner literals)) + (candidate-scan (make-bounded-literal-end-scanner literals)) + (reference (lambda () (reference-scan source 0 limit))) + (candidate (lambda () (candidate-scan source 0 limit)))) + ;; Validate the production primitive against direct string matching at + ;; all boundaries, including Unicode and failure, outside timed batches. + (for-each (lambda (text) + (let positions ((start 0)) + (when (<= start (string-length text)) + (let limits ((end start)) + (when (<= end (string-length text)) + (let (expected + (let linear ((rest literals) (best #f)) + (if (null? rest) best + (let (next (+ start (string-length (car rest)))) + (linear (cdr rest) + (if (and (<= next end) + (string=? (car rest) (substring text start next))) + (if best (max best next) next) best)))))) + (unless (and (equal? (reference-scan text start end) expected) + (equal? (candidate-scan text start end) expected)) + (error "bounded edge result differs from linear oracle" width start end))) + (limits (+ end 1)))) + (positions (+ start 1))))) + (list source "!" (cadr literals) "β!")) + (let* ((before (measure-parser-cpu-pairs width 'control-before groups calls width reference reference)) + (result (measure-parser-cpu-pairs width 'production groups calls width reference candidate)) + (after (measure-parser-cpu-pairs width 'control-after groups calls width reference reference)) + (stable? (lambda (ratios) (and (>= (car ratios) 0.9) (<= (caddr ratios) 1.1)))) + (admitted? (and (stable? before) (stable? after) (< (caddr result) 0.95)))) + (write (list 'LITERAL-EDGE-CPU-SUMMARY width 'groups groups 'calls-per-batch calls + 'before before 'production result 'after after 'cpu-benefit admitted?)) + (set! summaries + (cons (cons width (list (cons 'before before) (cons 'production result) + (cons 'after after) (cons 'cpu-benefit admitted?))) summaries)) + (newline) (force-output)))) widths) + (displayln "LITERAL-EDGE-CPU-OK") (force-output) + (reverse summaries))) + +;;; Explicit performance test: correctness/completion alone cannot pass it. +(def (qualify-literal-edge-cpu (groups 20) (characters-per-batch 2000000)) + (let (summaries (benchmark-literal-edge-cpu groups characters-per-batch '(128))) + (unless (andmap (lambda (entry) (edge-field (cdr entry) 'cpu-benefit)) summaries) + (error "literal edge CPU qualification failed its unchanged controls and benefit gate" summaries)) + (displayln "LITERAL-EDGE-CPU-PROOF-OK") (force-output) + summaries)) + +;;; Preserve the old ranked traversal and terminal publication exactly; change +;;; only prepared edge storage to tables for the matched CPU reference. +(def (table-ranked-scanner entries) + (def (best-admitted candidates admitted) + (let loop ((remaining candidates) (best #f)) + (if (null? remaining) best + (let (candidate (car remaining)) + (loop (cdr remaining) + (if (and (or (not admitted) (vector-ref admitted (caddr candidate))) + (or (not best) (> (cadr candidate) (cadr best)) + (and (= (cadr candidate) (cadr best)) + (< (caddr candidate) (caddr best))))) + candidate best)))))) + (let (root (table-edge-copy + (make-literal-trie + (map (lambda (entry) (cons (car entry) (cdr entry))) entries) + '() (lambda (previous payload) (cons payload previous))))) + (lambda (source start (admitted #f)) + (let (limit (string-length source)) + (let scan ((node root) (offset start) (selected #f)) + (if (= offset limit) selected + (let (child (table-ref (vector-ref node 1) (string-ref source offset) #f)) + (if child + (let* ((next (+ offset 1)) (terminal (best-admitted (vector-ref child 0) admitted))) + (scan child next + (if terminal (list (car terminal) next (cadr terminal) (caddr terminal)) selected))) + selected)))))))) + +;;; Independent catalog oracle, including all four public result fields. +(def (ranked-edge-oracle entries source start admitted) + (let loop ((rest entries) (best #f)) + (if (null? rest) + (and best (list (cadr best) (+ start (string-length (car best))) (caddr best) (cadddr best))) + (let* ((entry (car rest)) (width (string-length (car entry)))) + (loop (cdr rest) + (if (and (or (not admitted) (vector-ref admitted (cadddr entry))) + (<= (+ start width) (string-length source)) + (string=? (car entry) (substring source start (+ start width))) + (or (not best) (> width (string-length (car best))) + (and (= width (string-length (car best))) + (or (> (caddr entry) (caddr best)) + (and (= (caddr entry) (caddr best)) + (< (cadddr entry) (cadddr best))))))) entry best)))))) + +(def (benchmark-ranked-edge-cpu (groups 20) (characters-per-batch 2000000) + (names '(ranked-hit ranked-filtered ranked-rejected branch-hit))) + (unless (and (exact-integer? groups) (>= groups 20) + (exact-integer? characters-per-batch) (>= characters-per-batch 128) + (list? names) (pair? names) + (andmap (lambda (name) (memq name '(ranked-hit ranked-filtered ranked-rejected branch-hit))) names)) + (error "ranked CPU benchmark requires declared workloads and resolved batches")) + (map (lambda (name) + (let* ((literal (make-string 128 #\a)) + (entries (if (eq? name 'branch-hit) + (map (lambda (code) (list (string (integer->char code)) 'branch 0 (- code 32))) (iota 96 32)) + (list (list literal 'first 2 0) (list literal 'later 2 1) + (list literal 'higher 3 2) '("β😀" unicode 1 3)))) + (source (if (eq? name 'branch-hit) "x!" (string-append literal "!"))) + (mask (case name + ((ranked-filtered) '#(#t #f #f #t)) + ((ranked-rejected) '#(#f #f #f #f)) + (else #f))) + (calls (quotient characters-per-batch (if (eq? name 'branch-hit) 1 128))) + (reference-scan (table-ranked-scanner entries)) + (candidate-scan (make-ranked-literal-scanner entries)) + (reference (lambda () (reference-scan source 0 mask))) + (candidate (lambda () (candidate-scan source 0 mask))) + (expected (ranked-edge-oracle entries source 0 mask))) + (for-each (lambda (text) + (let offsets ((start 0)) + (when (<= start (string-length text)) + (for-each (lambda (admitted) + (let (oracle (ranked-edge-oracle entries text start admitted)) + (unless (and (equal? (reference-scan text start admitted) oracle) + (equal? (candidate-scan text start admitted) oracle)) + (error "ranked edge complete result differs from catalog" name start)))) + (list #f mask)) + (offsets (+ start 1))))) (list source "!β😀!" "a!" "")) + (let* ((before (measure-parser-cpu-pairs name 'control-before groups calls expected reference reference)) + (result (measure-parser-cpu-pairs name 'production groups calls expected reference candidate)) + (after (measure-parser-cpu-pairs name 'control-after groups calls expected reference reference)) + (stable? (lambda (ratios) (and (>= (car ratios) 0.90) (<= (caddr ratios) 1.10)))) + (admitted? (and (stable? before) (stable? after) (< (caddr result) 0.95))) + (summary (list (cons 'before before) (cons 'production result) + (cons 'after after) (cons 'cpu-benefit admitted?)))) + (write (list 'RANKED-EDGE-CPU-SUMMARY name 'groups groups 'calls-per-batch calls summary)) + (newline) (force-output) + (cons name summary)))) names)) + +(def (qualify-ranked-edge-cpu (groups 20) (characters-per-batch 2000000)) + (let (summaries (benchmark-ranked-edge-cpu groups characters-per-batch '(ranked-hit))) + (unless (andmap (lambda (entry) (edge-field (cdr entry) 'cpu-benefit)) summaries) + (error "ranked edge CPU controls or benefit gate failed" summaries)) + (displayln "RANKED-EDGE-CPU-PROOF-OK") (force-output) + summaries)) + +;;; Matched primitive reference: retain compact edges and the former complete +;;; terminal selection/publication algorithm. No parser is duplicated. +(import (only-in ../../fixtures/ranked-publication reference-ranked-publication-scanner)) + +(def (ranked-cpu-admission name groups calls expected reference candidate) + (let* ((before (measure-parser-cpu-pairs name 'control-before groups calls expected reference reference)) + (result (measure-parser-cpu-pairs name 'production groups calls expected reference candidate)) + (after (measure-parser-cpu-pairs name 'control-after groups calls expected reference reference)) + (stable? (lambda (ratios) (and (>= (car ratios) 0.90) (<= (caddr ratios) 1.10)))) + (summary (list (cons 'before before) (cons 'production result) (cons 'after after) + (cons 'cpu-benefit (and (stable? before) (stable? after) (< (caddr result) 0.95)))))) + (write (list 'RANKED-PUBLICATION-CPU-SUMMARY name 'groups groups 'calls-per-batch calls summary)) + (newline) (force-output) + (cons name summary))) + +(def (benchmark-ranked-publication-cpu (groups 20) (calls 20000) + (names '(prefix-hit prefix-filtered short-hit rank-fanout branch-hit))) + (unless (and (list? names) (pair? names) + (andmap (lambda (name) (memq name '(prefix-hit prefix-filtered short-hit rank-fanout branch-hit))) names)) + (error "ranked publication requires declared workload names" names)) + (map (lambda (name) + (let* ((entries (case name + ((branch-hit) (append (map (lambda (code) (list (string (integer->char code)) 'branch 0 (- code 32))) (iota 96 32)) '(("λ😀" unicode 0 96)))) + ((rank-fanout) (map (lambda (rank) (list "word" 'word rank rank)) (iota 16))) + ((short-hit) '(("word" word 0 0) ("while" while 0 1))) + (else '(("+" plus 9 0) ("++" increment 2 1) ("+++" triple 1 2) + ("++++" quad 0 3) ("++" later 2 4))))) + (source (case name ((short-hit rank-fanout) "word!") ((branch-hit) "x!") (else "++++!"))) + (mask (and (eq? name 'prefix-filtered) '#(#t #f #f #f #f))) + (old (reference-ranked-publication-scanner entries)) + (new (make-ranked-literal-scanner entries)) + (expected (ranked-edge-oracle entries source 0 mask))) + (for-each (lambda (text) + (let offsets ((offset 0)) + (when (<= offset (string-length text)) + (let (oracle (ranked-edge-oracle entries text offset mask)) + (unless (and (equal? (old text offset mask) oracle) (equal? (new text offset mask) oracle)) + (error "ranked publication differs from complete catalog oracle" name offset))) + (offsets (+ offset 1))))) (list source "!β😀" "" "+" "++" "word")) + (ranked-cpu-admission name groups calls expected + (lambda () (old source 0 mask)) (lambda () (new source 0 mask))))) names)) + +(def (qualify-ranked-publication-cpu (groups 20) (calls 20000) + (names '(prefix-hit rank-fanout branch-hit))) + (let (summaries (benchmark-ranked-publication-cpu groups calls names)) + (unless (andmap (lambda (entry) (edge-field (cdr entry) 'cpu-benefit)) summaries) + (error "ranked publication CPU controls or benefit gate failed" summaries)) + (displayln "RANKED-PUBLICATION-CPU-PROOF-OK") (force-output) + summaries)) diff --git a/t/benchmarks/ranked-literal/cpu-proof.ss b/t/benchmarks/ranked-literal/cpu-proof.ss new file mode 100644 index 00000000..63b53fa5 --- /dev/null +++ b/t/benchmarks/ranked-literal/cpu-proof.ss @@ -0,0 +1,56 @@ +#!/usr/bin/env gxi +;;; CPU assertions use compiled benchmark owners and one isolated Scheme VM. +(port-settings-set! (current-output-port) '(buffering: #f)) +(displayln "CPU-PROOF-BOOTSTRAP") (force-output) +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(call-with-compiled-interface-trace + (lambda () + (preload-runtime-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") + (preload-runtime-module "gerbil-parser/t/benchmarks/ranked-literal/benchmark"))) +(def leave (eval 'gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process#test-child-process-exit!)) +(def (positive-exact-count text) + (let (count (string->number text)) + (unless (and (exact-integer? count) (positive? count)) + (error "CPU proof requires positive exact counts" text)) + count)) +(def (main (workload "bounded") (groups "20") (batch-text #f) (scenario #f) (clauses-text #f)) + (with-catch + (lambda (condition) + (write (list 'CPU-PROOF-REJECTED (error-message condition) (error-irritants condition))) + (newline) (force-output) (leave 1)) + (lambda () + (when (and (equal? workload "lexical") (not scenario)) + (set! scenario "keyword")) + (when (member workload '("parser" "lexical")) + (call-with-compiled-interface-trace + (lambda () (preload-runtime-module + (string-append "gerbil-parser/t/benchmarks/ranked-literal/" + (if (equal? workload "parser") "parser-cpu" "lexical-cpu")))))) + (let (entry + (cond ((equal? workload "bounded") + 'gerbil-parser/t/benchmarks/ranked-literal/benchmark#qualify-literal-edge-cpu) + ((equal? workload "ranked") + 'gerbil-parser/t/benchmarks/ranked-literal/benchmark#qualify-ranked-edge-cpu) + ((equal? workload "publication") + 'gerbil-parser/t/benchmarks/ranked-literal/benchmark#qualify-ranked-publication-cpu) + ((equal? workload "lexical") + 'gerbil-parser/t/benchmarks/ranked-literal/lexical-cpu#qualify-ranked-lexical-cpu) + ((equal? workload "parser") + 'gerbil-parser/t/benchmarks/ranked-literal/parser-cpu#qualify-ranked-parser-cpu) + (else (error "unknown CPU proof workload" workload)))) + (let ((count (positive-exact-count groups)) + (batch (positive-exact-count + (or batch-text (cond ((equal? workload "parser") "256") + ((equal? workload "publication") "100000") + ((equal? workload "lexical") "250000") + (else "2000000")))))) + (cond (clauses-text + (unless (equal? workload "parser") + (error "clause count belongs to complete parser requests" workload)) + ((eval entry) count batch (if scenario (string->symbol scenario) 'layout) + (positive-exact-count clauses-text))) + (scenario ((eval entry) count batch (string->symbol scenario))) + (else ((eval entry) count batch))))) + (displayln "SCANNER-CPU-PROOF-OK " workload) (force-output) + (leave 0)))) diff --git a/t/benchmarks/ranked-literal/evidence.ss b/t/benchmarks/ranked-literal/evidence.ss new file mode 100644 index 00000000..aa104496 --- /dev/null +++ b/t/benchmarks/ranked-literal/evidence.ss @@ -0,0 +1,45 @@ +;;; Exact allocation evidence and conservative CPU admission from native receipts. +(import (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/receipt read-parser-component-receipt)) +(export compare-ranked-literal-receipts) +(def (field row key) (cdr (assq key row))) +(def (require! condition . detail) + (unless condition (apply error "ranked scanner evidence rejected" detail))) +(def (read-request-receipt path) + (read-parser-component-receipt path "RANKED-LITERAL-REQUESTS-OK" 12 5000 20)) + +(def (compare-ranked-literal-receipts reference-a candidate-a candidate-b reference-b) + (let* ((ra (read-request-receipt reference-a)) (ca (read-request-receipt candidate-a)) + (cb (read-request-receipt candidate-b)) (rb (read-request-receipt reference-b))) + (def (cpu receipt key) (field (hash-get (car receipt) key) 'cpuP50Ms)) + (def (control? key) + (or (= (cadr key) 1) (eq? (car key) 'missing-prefix) + (eq? (car key) 'filtered-prefixes))) + (def (stable? key) + (let* ((c [(cpu ca key) (cpu cb key)]) (r [(cpu ra key) (cpu rb key)]) + (ratio (/ (apply + c) (apply + r)))) + (and (<= 0.95 ratio 1.05) + (<= (/ (apply min c) (apply max r)) 1 (/ (apply max c) (apply min r)))))) + (let* ((keys (apply append + (map (lambda (width) + (map (lambda (name) (list name width)) + '(accepting-prefixes filtered-prefixes missing-prefix))) '(1 4 16 64)))) + (controls-stable? (andmap stable? (filter control? keys)))) + (for-each + (lambda (key) + (let* ((baseline (car (hash-get (cadr ra) key))) + (candidate (car (hash-get (cadr ca) key))) + (saved (/ (- baseline candidate) 5000)) + (expected (if (eq? (car key) 'accepting-prefixes) (* 192 (- (cadr key) 1)) 0)) + (benefit? (< (max (cpu ca key) (cpu cb key)) + (* 0.95 (min (cpu ra key) (cpu rb key)))))) + (require! (= baseline (car (hash-get (cadr rb) key))) key 'reference-drift) + (require! (= candidate (car (hash-get (cadr cb) key))) key 'candidate-drift) + (require! (= saved expected) key 'allocation-model) + (write (list 'RANKED-LITERAL-EVIDENCE key 'saved-bytes saved + 'cpu-ratio (/ (+ (cpu ca key) (cpu cb key)) (+ (cpu ra key) (cpu rb key))) + 'repeated-benefit? benefit? + 'cpu-benefit-admitted? (and controls-stable? benefit?))) + (newline) (force-output))) keys) + (write (list 'RANKED-LITERAL-ALLOCATION-OK 'admitted-intervals + (+ (caddr ra) (caddr ca) (caddr cb) (caddr rb)) + 'controls-stable? controls-stable?)) (newline) (force-output)))) diff --git a/t/benchmarks/ranked-literal/lexical-cpu.ss b/t/benchmarks/ranked-literal/lexical-cpu.ss new file mode 100644 index 00000000..f5d3a858 --- /dev/null +++ b/t/benchmarks/ranked-literal/lexical-cpu.ss @@ -0,0 +1,28 @@ +;;; Complete ranked lexical competition, including rule identity and endpoints. +(import (only-in ./benchmark ranked-cpu-admission) + (only-in ../../fixtures/ranked-publication reference-ranked-lexical-scanner-factory) + (only-in :gerbil-parser/src/runtime/scan make-ranked-lexical-scanner-factory)) +(export benchmark-ranked-lexical-cpu qualify-ranked-lexical-cpu) +(def (benchmark-ranked-lexical-cpu (groups 20) (calls 500000) (scenario #f)) + (let* ((entries '(("if" keyword 2 0) ("ifelse" longer 1 1) ("+" plus 9 2) + ("+++" triple 0 3) ("α😀" wide 1 4) ("1e" prefix 2 7))) + (regular '((identifier name 0 5) (number number 0 6))) + (admitted (make-vector 8 #t)) + (old ((reference-ranked-lexical-scanner-factory entries) regular admitted)) + (new ((make-ranked-lexical-scanner-factory entries) regular admitted))) + (unless (or (not scenario) (memq scenario '(keyword identifier literal-prefix))) + (error "unknown complete lexical CPU scenario" scenario)) + (map (lambda (row) + (let ((name (car row)) (source (cadr row)) (expected (caddr row))) + (ranked-cpu-admission (list 'ranked-lexical name) groups calls expected + (lambda () (old source 0)) (lambda () (new source 0))))) + (filter (lambda (row) (or (not scenario) (eq? scenario (car row)))) + '((keyword "ifelse!" (longer 6 1 1)) + (identifier "ifelsewhere!" (name 11 0 5)) + (literal-prefix "+++!" (triple 3 0 3))))))) +(def (qualify-ranked-lexical-cpu (groups 20) (calls 500000) (scenario #f)) + (let (summaries (benchmark-ranked-lexical-cpu groups calls scenario)) + (unless (andmap (lambda (entry) (cdr (assq 'cpu-benefit (cdr entry)))) summaries) + (error "complete ranked lexical CPU controls or benefit gate failed" summaries)) + (displayln "RANKED-LEXICAL-CPU-PROOF-OK") (force-output) + summaries)) diff --git a/t/benchmarks/ranked-literal/parser-cpu.ss b/t/benchmarks/ranked-literal/parser-cpu.ss new file mode 100644 index 00000000..2aea08bc --- /dev/null +++ b/t/benchmarks/ranked-literal/parser-cpu.ss @@ -0,0 +1,52 @@ +;;; Complete public parser CPU assertions load language owners only here. +(import (only-in ./benchmark ranked-cpu-admission) + (only-in ../../fixtures/ranked-publication reference-ranked-lexical-scanner-factory) + (only-in :gerbil-parser/src/runtime/scan make-ranked-lexical-scanner-factory)) +(import (only-in ../../fixtures/lr-action-selection reference-lr-action-selector)) +(export benchmark-ranked-parser-cpu qualify-ranked-parser-cpu) + +(import (only-in :gerbil-parser/src/compiler/machine parser-machine-with-ranked-scanner parser-machine-with-lr-action-selector) + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid-for-source? parse-artifact-success?) + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-layout-language-grammar tla-plus-core-language-grammar)) + +;;; Both variants execute the public fresh parse entry: source traversal, +;;; lexical modes, GLR/layout and complete artifact publication stay shared. +;;; Preparation substitutes ranked scanning and, for layout, the former +;;; action projection primitive. Timed requests keep the public pipeline. +(def (benchmark-ranked-parser-cpu (groups 20) (batch-count 256) (scenario 'layout) (clauses 12)) + (unless (and (memq scenario '(layout core assumptions)) (exact-integer? clauses) (positive? clauses)) + (error "ranked parser CPU workload requires positive exact clauses")) + (let* ((machine (language-grammar-machine + (if (eq? scenario 'layout) tla-plus-layout-language-grammar + tla-plus-core-language-grammar))) + (old-lexical (parser-machine-with-ranked-scanner machine reference-ranked-lexical-scanner-factory)) + (old (if (eq? scenario 'layout) + (parser-machine-with-lr-action-selector old-lexical reference-lr-action-selector) + old-lexical)) + (new (parser-machine-with-ranked-scanner machine make-ranked-lexical-scanner-factory)) + (source + (string-append "---- MODULE Ranked ----\n" + (case scenario + ((layout) (string-append "Init ==\n" + (apply string-append (map (lambda (_) " /\\ TRUE\n") (iota clauses))))) + ((core) (string-append "Init == TRUE" + (apply string-append (map (lambda (_) " /\\ TRUE") (iota clauses))) "\n")) + ((assumptions) (apply string-append (map (lambda (_) "ASSUME TRUE\n") (iota clauses))))) + "====\n")) + (expected (parse-source old source))) + (unless (and (parse-artifact-success? expected) + (parse-artifact-valid-for-source? expected source) + (equal? expected (parse-source new source)) + (equal? expected (parse-source machine source))) + (error "ranked parser variants differ from complete production artifact")) + (list (ranked-cpu-admission (list 'public-tla-plus-parser scenario clauses) groups batch-count expected + (lambda () (parse-source old source)) (lambda () (parse-source new source)))))) + +(def (qualify-ranked-parser-cpu (groups 20) (batch-count 256) (scenario 'layout) (clauses 12)) + (let (summaries (benchmark-ranked-parser-cpu groups batch-count scenario clauses)) + (unless (andmap (lambda (entry) (cdr (assq 'cpu-benefit (cdr entry)))) summaries) + (error "complete public parser CPU controls or benefit gate failed" summaries)) + (displayln "RANKED-PARSER-CPU-PROOF-OK") (force-output) + summaries)) diff --git a/t/benchmarks/region-depth/benchmark.ss b/t/benchmarks/region-depth/benchmark.ss new file mode 100644 index 00000000..ffad2387 --- /dev/null +++ b/t/benchmarks/region-depth/benchmark.ss @@ -0,0 +1,37 @@ +;;; Request-owned region counters versus flat and independently nested frames. +(import (only-in :gerbil-parser/src/runtime/region-scanner + prepare-region-plan region-plan-pair-end) + (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation)) +(export benchmark-region-depth) + +(def (benchmark-region-depth (samples 11) (iterations 10)) + (unless (and (exact-integer? samples) (positive? samples) + (exact-integer? iterations) (positive? iterations)) + (error "region depth benchmark requires positive exact counts")) + (let* ((plan (prepare-region-plan '((";") ((#\" #t ())) (("@{" #\{ #\} 1)) #f))) + (deep (string-append "@{" (make-string 4096 #\{) "λ" (make-string 4096 #\}) "}")) + (nested (string-append (string-concatenate (make-list 512 "@{")) "λ" (make-string 512 #\})))) + (for-each + (lambda (row) + (let ((text (car row)) (name (cadr row))) + (unless (= (region-plan-pair-end plan text 0) (string-length text)) + (error "invalid region reference endpoint" name)) + (let sample ((index 0)) + (when (< index samples) + (let* ((before (##process-statistics)) + (sum (let repeat ((count 0) (total 0)) + (if (= count iterations) total + (repeat (+ count 1) (+ total (region-plan-pair-end plan text 0)))))) + (after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) (f64vector-ref before slot))))) + (unless (= sum (* iterations (string-length text))) + (error "region endpoint changed" name)) + (write (list 'region-depth name index 'iterations iterations + 'cpu-ms-per-call (/ (* 1000 (+ (delta 0) (delta 1))) iterations) + 'allocated-bytes-per-call + (let (bytes (admit-parser-allocation (delta 7) (delta 6))) + (and bytes (/ bytes iterations))))) + (newline) (force-output)) + (sample (+ index 1)))))) + (list (list deep 'counter-depth) (list "@{λ}" 'flat) (list nested 'distinct-frames)))) + (displayln "REGION-DEPTH-OK") (force-output)) diff --git a/t/benchmarks/regular-mode/benchmark.ss b/t/benchmarks/regular-mode/benchmark.ss new file mode 100644 index 00000000..0ce880f7 --- /dev/null +++ b/t/benchmarks/regular-mode/benchmark.ss @@ -0,0 +1,90 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Pure Scheme comparison with the previous ASCII-prefiltered mode scan. + +(import (only-in :std/vector/vector vector-map/index) + (only-in :gerbil-parser/src/runtime/scan + make-ranked-regular-scanner + scan-whitespace scan-horizontal-whitespace scan-newline + scan-decimal-digits scan-identifier scan-number-literal)) + +(def entries + (list + (list scan-whitespace char-whitespace? 'white 0 0) + (list scan-horizontal-whitespace + (lambda (ch) (or (char=? ch #\space) (char=? ch #\tab))) + 'horizontal 3 1) + (list scan-newline + (lambda (ch) (or (char=? ch #\newline) (char=? ch #\return))) + 'newline 2 2) + (list scan-decimal-digits char-numeric? 'digits 1 3) + (list scan-identifier + (lambda (ch) (or (char-alphabetic? ch) (char=? ch #\_))) + 'name 0 4) + (list scan-number-literal char-numeric? 'number 0 5))) + +(def ascii-candidates + (vector-map/index + (lambda (index _) + (filter (lambda (entry) ((cadr entry) (integer->char index))) + entries)) + (make-vector 128))) + +(def combined + (make-ranked-regular-scanner + '((whitespace+ white 0 0) + (horizontal-whitespace+ horizontal 3 1) + (newline+ newline 2 2) + (decimal-digit+ digits 1 3) + (identifier name 0 4) + (number number 0 5)))) + +(def (old-mode-scan source offset) + (let* ((ch (string-ref source offset)) + (candidates + (if (< (char->integer ch) 128) + (vector-ref ascii-candidates (char->integer ch)) + entries))) + (fold + (lambda (entry best) + (let (end ((car entry) source offset)) + (if (and end + (or (not best) + (> end (cadr best)) + (and (= end (cadr best)) + (or (> (cadddr entry) (caddr best)) + (and (= (cadddr entry) (caddr best)) + (< (car (cddddr entry)) + (cadddr best))))))) + (list (caddr entry) end (cadddr entry) + (car (cddddr entry))) + best))) + #f candidates))) + +(def (measure label scanner source repetitions) + (let ((started (##current-time-point)) (last #f)) + (let loop ((remaining repetitions)) + (unless (zero? remaining) + (set! last (scanner source 0)) + (loop (- remaining 1)))) + (write (list (cons 'scanner label) + (cons 'source source) + (cons 'repetitions repetitions) + (cons 'last last) + (cons 'elapsed-total-ms + (* 1000.0 (- (##current-time-point) started))))) + (newline))) + +(def (main . args) + (let (repetitions + (if (null? args) 50000 (string->number (car args)))) + (for-each + (lambda (source) + (unless (equal? (old-mode-scan source 0) (combined source 0)) + (error "regular scanners disagree" source)) + (measure 'old-mode old-mode-scan source repetitions) + (measure 'combined combined source repetitions)) + '("alpha-123+" "1234567890+" "12.34e+5;" + "12e+;" " \t \t\n" "αβ-٣+")))) + +(export main) diff --git a/t/benchmarks/regular-mode/request-cost.ss b/t/benchmarks/regular-mode/request-cost.ss new file mode 100644 index 00000000..a883a9ae --- /dev/null +++ b/t/benchmarks/regular-mode/request-cost.ss @@ -0,0 +1,66 @@ +;;; Native caller qualification for shared regular scans and lexer preparation. +(import (only-in :gerbil-parser/src/runtime/scan make-ranked-regular-scanner) + (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation) + (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/compiler/hcl-source direct-lex-hcl) + (only-in :gerbil-parser/src/compiler/machine parser-machine-prepare-lexer)) +(export benchmark-regular-mode benchmark-lexical-preparation) +(def (measure-regular-mode name run valid? samples iterations) + (let sample ((index 0)) + (when (< index samples) + (let* ((before (##process-statistics)) + (last (let repeat ((count 0) (last #f)) + (if (= count iterations) last (repeat (+ count 1) (run))))) + (after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) (f64vector-ref before slot))))) + (unless (valid? last) (error "regular mode product mismatch" name)) + (write (list 'regular-mode name index 'iterations iterations + 'cpu-ms-per-call (/ (* 1000 (+ (delta 0) (delta 1))) iterations) + 'allocated-bytes-per-call + (let (bytes (admit-parser-allocation (delta 7) (delta 6))) + (and bytes (/ bytes iterations))))) + (newline) (force-output)) + (sample (+ index 1))))) +(def (benchmark-regular-mode (samples 11) (iterations 100)) + (unless (and (exact-integer? samples) (positive? samples) + (exact-integer? iterations) (positive? iterations)) + (error "regular mode benchmark requires positive exact counts")) + (let* ((scan (make-ranked-regular-scanner + '((whitespace+ white 0 0) (horizontal-whitespace+ horizontal 3 1) + (newline+ newline 2 2) (decimal-digit+ digits 1 3) + (identifier name 0 4) (number number 0 5)))) + (ascii (make-string 4096 #\a)) (wide (make-string 4096 #\α)) + (space (make-string 4096 #\space)) (digits (make-string 4096 #\1))) + (for-each (lambda (row) + (let ((text (car row)) (expected (cadr row)) (name (caddr row))) + (measure-regular-mode name (lambda () (scan text 0)) + (lambda (found) (equal? found expected)) samples iterations))) + (list (list ascii '(name 4096 0 4) 'ascii-identifier) + (list wide '(name 4096 0 4) 'unicode-identifier) + (list space '(horizontal 4096 3 1) 'whitespace) + (list digits '(digits 4096 1 3) 'numeric) + (list "a" '(name 1 0 4) 'singleton) + (list "?" #f 'unmatched)))) + (let* ((source (call-with-output-string (lambda (port) + (for-each (lambda (n) (display "key" port) (display n port) + (display " = " port) (display n port) (newline port)) (iota 64))))) + (expected (direct-lex-hcl source))) + (unless (equal? (lex-source hcl-parser source) expected) + (error "independent HCL lexer reference mismatch")) + (measure-regular-mode 'complete-hcl-lexer (lambda () (lex-source hcl-parser source)) + (lambda (found) (equal? found expected)) samples 20)) + (displayln "REGULAR-MODE-OK") (force-output)) + +;;; Preparation remains visible when scan dispatch moves out of the request. +;;; Validate the returned native lexer after timing, against the independent lexer. +(def (benchmark-lexical-preparation (samples 11)) + (unless (and (exact-integer? samples) (positive? samples)) + (error "lexical preparation benchmark requires a positive exact count")) + (let* ((source "alpha = 12\n") (expected (direct-lex-hcl source))) + (measure-regular-mode 'hcl-lexer-preparation + (lambda () + (let-values (((lexer plans certificates) (parser-machine-prepare-lexer hcl-parser))) + lexer)) + (lambda (lexer) (equal? (lexer source) expected)) samples 1)) + (displayln "LEXICAL-PREPARATION-OK") (force-output)) diff --git a/t/benchmarks/runtime-closure-diagnostic/benchmark.ss b/t/benchmarks/runtime-closure-diagnostic/benchmark.ss new file mode 100644 index 00000000..38aa6791 --- /dev/null +++ b/t/benchmarks/runtime-closure-diagnostic/benchmark.ss @@ -0,0 +1,47 @@ +(import :gerbil-parser/src/modules/parser/interface + :gerbil-parser/src/runtime/artifact + :gerbil-parser/src/runtime/lexer + :gerbil-parser/src/runtime/significant + :gerbil-parser/src/compiler/machine + :gerbil-parser/src/language/descriptor + :gerbil-parser/languages/gql/parser) +(export main) +(def (measure label count thunk) + (let loop ((sample 0)) + (when (< sample 5) + (##gc) + (let ((started (##current-time-point)) (cpu (cpu-time))) + (let run ((n count)) + (unless (zero? n) (thunk) (run (- n 1)))) + (write (list (cons 'phase label) (cons 'sample sample) (cons 'count count) + (cons 'cpu-ms (* 1000 (- (cpu-time) cpu))) + (cons 'wall-ms (* 1000 (- (##current-time-point) started))))) + (newline) (force-output)) + (loop (+ sample 1))))) +(def (main . args) + (let* ((machine (language-grammar-machine gql-language-grammar)) + (source +gql-representative-query+) + (tokens (lex-source machine source)) + (significant (parser-significant-tokens machine tokens))) + (let-values (((root rest) ((parser-machine-parse machine) significant #f))) + (unless (null? rest) (error "incomplete parse")) + (let (artifact (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens root (parser-machine-trivia machine))) + (unless (and (parse-artifact-valid? artifact) + (equal? (parse-artifact-roundtrip artifact) source)) + (error "invalid complete artifact"))) + (measure 'gql-complete 100 (lambda () (parse-gql source))) + (measure 'gql-lex 100 (lambda () (lex-source machine source))) + (measure 'gql-significant 100 (lambda () (parser-significant-tokens machine tokens))) + (measure 'gql-lr 100 (lambda () ((parser-machine-parse machine) significant #f))) + (measure 'gql-publication 100 + (lambda () (make-success-parse-artifact (parser-machine-grammar-digest machine) + source tokens root (parser-machine-trivia machine)))))) + (let* ((make-role (lambda () (make-grammar-role 'observability-role '() '() '() '() '() '() '() '() '()))) + (role (make-role)) + (grammar (make-grammar 'observability-grammar '() (list role)))) + (measure 'poo-role 50 make-role) + (measure 'poo-grammar 50 (lambda () (make-grammar 'observability-grammar '() (list role)))) + (measure 'poo-role-predicate 50 (lambda () (grammar-role? role))) + (measure 'poo-grammar-predicate 50 (lambda () (grammar? grammar)))) + (displayln "DIAGNOSTIC-OK") (force-output)) diff --git a/t/benchmarks/semantic-publication/benchmark.ss b/t/benchmarks/semantic-publication/benchmark.ss new file mode 100644 index 00000000..ca994183 --- /dev/null +++ b/t/benchmarks/semantic-publication/benchmark.ss @@ -0,0 +1,81 @@ +#!/usr/bin/env gxi +;;; Hold source/token binding constant and isolate complete artifact publication. +(import (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-for-current-semantic-backend + parser-machine-grammar-digest parser-machine-trivia) + (only-in :gerbil-parser/src/runtime/parser parse-source/checkpoints) + (only-in :gerbil-parser/src/runtime/lr-parser + current-lr-event-program-enabled? current-lr-recognition-observer + lr-recognition-fragment-value lr-recognition-project) + (only-in :gerbil-parser/src/runtime/recognition recognition-child-value) + (only-in :gerbil-parser/src/runtime/funcs recognition-sequence->list) + (only-in :gerbil-parser/src/runtime/event-program event-program-value?) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact parse-artifact-events parse-artifact-valid?)) +(def (root-value sequence) + (let (children (recognition-sequence->list sequence)) + (unless (and (pair? children) (null? (cdr children))) + (error "publication requires one root")) + (recognition-child-value (car children)))) +(def (median values) + (let* ((sorted (list-sort < values)) (mid (quotient (length sorted) 2))) + (/ (+ (list-ref sorted (- mid 1)) (list-ref sorted mid)) 2.0))) +(def (measure publish family units mode) + (let (times + (map (lambda (sample) + (##gc) + (let* ((start (cpu-time)) (artifact (publish)) + (elapsed (* 1000.0 (- (cpu-time) start)))) + (unless artifact (error "publication returned no artifact")) + (write (list 'publication-sample family units mode sample 'cpu-ms elapsed)) + (newline) (force-output) + elapsed)) (iota 20))) + (list (cons 'mode mode) (cons 'samples 20) + (cons 'cpu-samples-ms times) (cons 'cpu-median-ms (median times))))) +(def (measure-source family units source) + (let ((captured #f) + (machine (parser-machine-for-current-semantic-backend hcl-parser))) + (parameterize ((current-lr-recognition-observer (lambda (root) (set! captured root)))) + (let-values (((artifact tokens modes checkpoints) + (parse-source/checkpoints machine source 64))) + (unless (and captured (parse-artifact-valid? artifact)) + (error "publication baseline lacks captured grammar")) + (let* ((program (root-value (lr-recognition-fragment-value captured))) + (canonical (root-value (lr-recognition-project captured tokens))) + (publish (lambda (root) + (make-success-parse-artifact + (parser-machine-grammar-digest machine) source tokens root + (parser-machine-trivia machine)))) + (control (lambda () (publish canonical))) + (candidate (lambda () (publish program)))) + (unless (and (event-program-value? program) + (equal? (control) artifact) (equal? (candidate) artifact)) + (error "publication roots differ from complete captured artifact")) + (for-each + (lambda (order) + (for-each + (lambda (mode) + (write (append + (list (cons 'workload 'complete-semantic-publication) + (cons 'family family) (cons 'input-units units) + (cons 'order order) (cons 'complete-artifact-equal? #t) + (cons 'events (length (parse-artifact-events artifact)))) + (measure (if (eq? mode 'canonical) control candidate) family units mode))) + (newline) (force-output)) + (if (= order 1) '(canonical unified-event) '(unified-event canonical)))) + '(1 2))))))) +(def (main . args) + (parameterize ((current-lr-event-program-enabled? #t)) + (for-each + (lambda (units) + (measure-source 'hcl-siblings units (string-join (make-list units "a = 1\n") ""))) + '(400 800)) + (for-each + (lambda (units) + (measure-source 'hcl-nested-siblings units + (string-join (make-list units + (string-append "resource \"r\" {\n" (string-join (make-list 80 "a = 1\n") "") "}\n")) ""))) + '(8 16))) + (displayln "SEMANTIC-PUBLICATION-ALL-OK")) +(export main) diff --git a/t/benchmarks/semantic-sequence/benchmark.ss b/t/benchmarks/semantic-sequence/benchmark.ss new file mode 100644 index 00000000..03ff9355 --- /dev/null +++ b/t/benchmarks/semantic-sequence/benchmark.ss @@ -0,0 +1,80 @@ +#!/usr/bin/env gxi +;;; Same-head join policies: tokens are prebuilt; every construction root stays live. +(import :gerbil-parser/src/runtime/event-program + (only-in :gerbil-parser/src/runtime/token make-token)) +(def (median values) + (let* ((sorted (list-sort < values)) (mid (quotient (length sorted) 2))) + (/ (+ (list-ref sorted (- mid 1)) (list-ref sorted mid)) 2.0))) +(def (build append-code tokens count) + (let loop ((i 0) (code #f) (roots '())) + (if (= i count) roots + (let (next (append-code code (vector-ref tokens i))) + (loop (+ i 1) next (cons next roots)))))) +(def (check-sequence sequence tokens count) + (let (i 0) + (event-program-sequence-for-each + (lambda (field value delta moved?) + (unless (and (not field) (eq? value (vector-ref tokens i)) (= delta 0) (not moved?)) + (error "producer token identity or movement changed")) + (set! i (+ i 1))) sequence) + (unless (and (= i count) (= (event-program-sequence-start sequence 0) 0) + (= (event-program-sequence-end sequence 0) count)) + (error "producer count/bounds changed")))) +(def (measure family units mode tokens) + (let* ((append-code (if (eq? mode 'avl) event-program-append/avl event-program-append)) + (parent (and (eq? family 'retained-root-forks) (car (build append-code tokens units)))) + (operations (if parent 128 (* 8 units))) + (times '()) (allocations '()) (lives '())) + (for-each + (lambda (sample) + (##gc) + (let* ((before (##process-statistics)) (start (cpu-time)) + (roots (if parent + (map (lambda (_) (append-code parent (vector-ref tokens units))) (iota 128)) + (map (lambda (_) (build append-code tokens units)) (iota 8)))) + (elapsed (* 1000.0 (- (cpu-time) start)))) + ;; Collection is outside CPU timing and settles process allocation/live counters. + (##gc) + (let* ((after (##process-statistics)) + (allocated (- (f64vector-ref after 16) (f64vector-ref before 16))) + (live (- (f64vector-ref after 17) (f64vector-ref before 17)))) + (if parent + (begin (check-sequence parent tokens units) + (check-sequence (car roots) tokens (+ units 1)) + (check-sequence (last roots) tokens (+ units 1))) + (for-each + (lambda (history) + (check-sequence (car history) tokens units) + (check-sequence (list-ref history (quotient units 2)) tokens (- units (quotient units 2))) + (check-sequence (last history) tokens 1)) roots)) + (set! times (cons elapsed times)) + (set! allocations (cons allocated allocations)) + (set! lives (cons live lives)) + (write (list 'sequence-sample family units mode sample elapsed allocated live)) + (newline) (force-output)))) (iota 20)) + (list (cons 'mode mode) (cons 'samples 20) (cons 'operations operations) + (cons 'cpu-samples-ms (reverse times)) (cons 'cpu-median-ms (median times)) + (cons 'allocation-samples-bytes (reverse allocations)) + (cons 'allocation-median-bytes (median allocations)) + (cons 'retained-live-samples-bytes (reverse lives)) + (cons 'retained-live-median-bytes (median lives))))) +(def (main . args) + (for-each + (lambda (family) + (for-each + (lambda (units) + (let (tokens (list->vector (map (lambda (i) (make-token 'number "1" i (+ i 1))) (iota (+ units 1))))) + (for-each + (lambda (order) + (for-each + (lambda (mode) + (write (append (list (cons 'workload 'persistent-sequence-construction) + (cons 'family family) (cons 'input-units units) + (cons 'order order) (cons 'retained-roots-equal? #t)) + (measure family units mode tokens))) + (newline) (force-output)) + (if (= order 1) '(avl append-buffer) '(append-buffer avl)))) '(1 2)))) + (if (eq? family 'linear-retained-roots) '(400 800 1600) '(256 512 1024)))) + '(linear-retained-roots retained-root-forks)) + (displayln "SEMANTIC-SEQUENCE-ALL-OK")) +(export main) diff --git a/t/benchmarks/semantic-stack/benchmark.ss b/t/benchmarks/semantic-stack/benchmark.ss new file mode 100644 index 00000000..d50e2cc5 --- /dev/null +++ b/t/benchmarks/semantic-stack/benchmark.ss @@ -0,0 +1,90 @@ +;;; Prepared shared-engine reductions; scanner and grammar construction excluded. +(import (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-parse/prepared lr-parse/prepared/receipt + current-lr-event-program-enabled? lr-runtime-for-current-semantic-backend) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact parse-artifact-valid-for-source?) + (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation)) +(export benchmark-semantic-stack) + +;;; Preparation is measured separately; cached runtime requests must not hide +;;; the allocation or CPU cost of admitting and staging their action plans. +(def (benchmark-preparation spec width decorated? samples iterations) + (let sample ((index 0)) + (when (< index samples) + (let (before (##process-statistics)) + (let repeat ((count 0)) + (when (< count iterations) (lr-prepare spec) (repeat (+ count 1)))) + (let* ((after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) + (f64vector-ref before slot))))) + (write (list 'semantic-stack-prepare width decorated? index + 'cpu-ms-per-call (/ (* 1000 (+ (delta 0) (delta 1))) iterations) + 'allocated-bytes-per-call + (let (bytes (admit-parser-allocation (delta 7) (delta 6))) + (and bytes (/ bytes iterations))))) + (newline) (force-output))) + (sample (+ index 1))))) + +(def (benchmark-semantic-stack (samples 20) (iterations 100)) + (unless (and (integer? samples) (positive? samples) + (integer? iterations) (positive? iterations)) + (error "semantic stack benchmark requires positive sample and iteration counts")) + (for-each + (lambda (width) + (for-each + (lambda (decorated?) + (let* ((operand (if decorated? + '(field outer (alias Renamed (field inner (token word)))) + '(token word))) + (rules (list (list 'source-file (list 'alias 'SourceFile + (cons 'sequence (make-list width operand)))))) + (spec (compile-lr-spec rules 'source-file)) + (runtime (lr-prepare spec)) + (tokens (map (lambda (index) (make-token 'word "a" index (+ index 1))) + (iota width))) + (source (make-string width #\a)) + (digest (string-append "sha256:" (make-string 64 #\0)))) + (benchmark-preparation spec width decorated? samples iterations) + (let-values (((reference rest receipt) (lr-parse/prepared/receipt runtime tokens))) + (let (expected (make-success-parse-artifact digest source tokens reference false)) + (unless (and (null? rest) (parse-artifact-valid-for-source? expected source)) + (error "invalid independent stack reference" width decorated?)) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (let (selected (lr-runtime-for-current-semantic-backend runtime)) + (def (run publish?) + (let-values (((root rest) (lr-parse/prepared selected tokens))) + (unless (null? rest) (error "stack benchmark did not complete")) + (if publish? + (make-success-parse-artifact digest source tokens root false) + root))) + (unless (equal? (run #t) expected) + (error "stack benchmark changed complete product" width decorated? events?)) + (for-each + (lambda (publish?) + (let warm ((count 0)) + (when (< count iterations) (run publish?) (warm (+ count 1)))) + (let sample ((index 0)) + (when (< index samples) + (let (before (##process-statistics)) + (let repeat ((count 0)) + (when (< count iterations) (run publish?) (repeat (+ count 1)))) + (let* ((after (##process-statistics)) + (delta (lambda (slot) (- (f64vector-ref after slot) + (f64vector-ref before slot))))) + (write (list 'semantic-stack width decorated? events? publish? index + 'cpu-ms-per-call (/ (* 1000 (+ (delta 0) (delta 1))) iterations) + 'allocated-bytes-per-call + (let (bytes (admit-parser-allocation (delta 7) (delta 6))) + (and bytes (/ bytes iterations))))) + (newline) (force-output))) + (sample (+ index 1))))) + '(#f #t))))) + '(#f #t)))))) + '(#f #t))) + '(1 8 32)) + (displayln "SEMANTIC-STACK-OK") (force-output)) diff --git a/t/benchmarks/source-edits/README.org b/t/benchmarks/source-edits/README.org new file mode 100644 index 00000000..81d06e31 --- /dev/null +++ b/t/benchmarks/source-edits/README.org @@ -0,0 +1,33 @@ +#+title: Native Source edit cost benchmark + +After the normal native package and =build-ffi-tests.ss= builds: + +#+begin_src shell +just benchmark-source-edits 20 128 +#+end_src + +Five fixed edit kinds use two independent Bash Source handles. Fresh history +is cleared by an empty parse, while the session handle is primed with the base +text. Preparation stays outside timed calls. Every call uses the actual C ABI, +reads back/releases its C result and compares all GPA1 bytes with fresh Scheme +execution. Alternating lane order and natural GC preserve negative results. + +=SOURCE-EDIT-SAMPLE= gives complete-call wall/CPU, Scheme allocations and GC. +=SOURCE-EDIT-SUMMARY= reports P50/P95 and the actual wall-P95 observation. +=SOURCE-EDIT-STAGES= comes from a separate observed call, includes nested costs, +scanned/reused token counts, history source/token dimensions and payload size. +It is not an additive decomposition or exact retained heap measurement. +CI runs 8 samples with 32 tail lines on both native platforms. Timing is +informational; payload equality, failure behavior and handle release are gates. + +See [[file:../../../docs/research/source-edit-cost-contract.org][the measurement contract]] for boundaries and current receipts. + +The C caller copies input into C-owned storage before entering Scheme, then +reads the result into a freshly borrowed Scheme buffer after callback return. +Input/state/result cleanup is included in each complete call. A separate +=verify-callback-gc= child exercises 80 large parses with natural collection; +this semantic control changes neither GC policy nor timed sample preparation. + +=allocation-counter-delta= retains raw signed SDK observations. Allocation +summaries admit only nonnegative zero-GC intervals and report their count. +Wall/CPU/GC quantiles and the wall-P95 sample retain all observations. diff --git a/t/benchmarks/source-edits/abi-call.ss b/t/benchmarks/source-edits/abi-call.ss new file mode 100644 index 00000000..6c4b9ca1 --- /dev/null +++ b/t/benchmarks/source-edits/abi-call.ss @@ -0,0 +1,67 @@ +;;; Real C transport; C-owned input/result storage survives Scheme callbacks. +(import :gerbil-parser/src/ffi/language-abi) +(export parse-source/abi!) +(extern allocate-call invoke-call readback-call release-call) +(def (parse-source/abi! handle source output) + (unless (and (u8vector? source) (u8vector? output)) + (error "expected Source ABI byte buffers")) + (let (call (allocate-call source)) + (unless call (error "Source ABI caller allocation failed")) + (dynamic-wind void + (lambda () + (let (status (invoke-call handle call)) + (if (zero? status) (readback-call call output) status))) + (lambda () (release-call call))))) +(begin-foreign + (namespace ("gerbil-parser/t/benchmarks/source-edits/abi-call#" + allocate-call invoke-call readback-call release-call)) + (c-declare #< +#include +#include +#include +#define GP_BYTES(obj) ___CAST(uint8_t*, ___BODY_AS((obj), ___tSUBTYPED)) +#define GP_SIZE(obj) ___HD_BYTES(___HEADER(obj)) +typedef struct source_edit_call { + uint8_t *source; + size_t length; + gerbil_parser_result result; +} source_edit_call; +static source_edit_call *source_edit_allocate(___SCMOBJ source) { + size_t length = GP_SIZE(source); + source_edit_call *call = malloc(sizeof(*call)); + if (!call) return NULL; + call->source = length ? malloc(length) : NULL; + if (length && !call->source) { free(call); return NULL; } + if (length) memcpy(call->source, GP_BYTES(source), length); + call->length = length; + gerbil_parser_result_init(&call->result); + return call; +} +static int source_edit_invoke(uint64_t handle, source_edit_call *call) { + int status = gerbil_parser_language_parse(handle, call->source, call->length, &call->result); + if (status) fprintf(stderr, "Source transport error: %.*s\n", (int)call->result.length, (const char*)call->result.payload); + return status; +} +/* A fresh C call receives the relocated Scheme buffer after callback return. + No Scheme callback or allocation occurs while this buffer is borrowed. */ +static int source_edit_readback(source_edit_call *call, ___SCMOBJ output) { + if (call->result.length != GP_SIZE(output)) { + fprintf(stderr, "Source payload length: got=%zu expected=%zu\n", call->result.length, (size_t)GP_SIZE(output)); + return -2; + } + if (call->result.length) memcpy(GP_BYTES(output), call->result.payload, call->result.length); + return 0; +} +static void source_edit_release(source_edit_call *call) { + gerbil_parser_result_release(&call->result); + free(call->source); + free(call); +} +END-C + ) + (c-define-type source-edit-call-ptr (pointer "source_edit_call")) + (define allocate-call (c-lambda (scheme-object) source-edit-call-ptr "source_edit_allocate")) + (define invoke-call (c-lambda (unsigned-int64 source-edit-call-ptr) int "source_edit_invoke")) + (define readback-call (c-lambda (source-edit-call-ptr scheme-object) int "source_edit_readback")) + (define release-call (c-lambda (source-edit-call-ptr) void "source_edit_release"))) diff --git a/t/benchmarks/source-edits/benchmark.ss b/t/benchmarks/source-edits/benchmark.ss new file mode 100644 index 00000000..57629ff8 --- /dev/null +++ b/t/benchmarks/source-edits/benchmark.ss @@ -0,0 +1,163 @@ +#!/usr/bin/env gxi +;;; Matched edit calls over the actual C ABI; independent fresh payload control. +(import (only-in :gerbil-parser/languages/bash/parser bash-source-language) + (only-in :gerbil-parser/src/ffi/language-artifact-codec bind-language-abi abi-parse-binary-payload) + (only-in :gerbil-parser/src/ffi/language-handles register-language! release-language! + language-handle-count language-handle-scan-counts) + (only-in :gerbil-parser/src/runtime/parse-cost current-parser-cost-observer with-parser-cost-stage admit-parser-allocation) + (only-in :std/vector/u8vector little u8vector-u32-ref) + (only-in :asp-gerbil-scheme/src/benchmark/statistics benchmark-percentile-index) + (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + ./abi-call) +(export main verify-callback-gc) +(def (fixtures lines) + (let (tail (string-join (make-list lines "echo tail\n") "")) + (list + (list 'utf8-prefix (string-append "cat <utf8 edited)) (base-bytes (string->utf8 base)) + (expected (abi-parse-binary-payload language edited)) + (base-result (abi-parse-binary-payload language base)) + (prime-output (make-u8vector (u8vector-length base-result))) + (empty-output (make-u8vector (u8vector-length (abi-parse-binary-payload language "")))) + (fresh-output (make-u8vector (u8vector-length expected))) + (session-output (make-u8vector (u8vector-length expected)))) + (unless (= (u8vector-u32-ref expected 8 little) (cadddr row)) + (error "benchmark fixture acceptance changed" name)) + (let loop ((sample -2) (fresh-rows '()) (session-rows '())) + (if (= sample samples) + (begin + (write (list 'SOURCE-EDIT-SUMMARY name 'fresh (summary (reverse fresh-rows)))) (newline) + (write (list 'SOURCE-EDIT-SUMMARY name 'session (summary (reverse session-rows)))) (newline)) + (begin + ;; Preparation is excluded equally: fresh has no prior tokens, session has base. + (call! fresh #u8() empty-output) + (call! session base-bytes prime-output) + (let-values (((fresh-row session-row) + (if (even? sample) + (values (measure-call fresh bytes fresh-output) + (measure-call session bytes session-output)) + (let (s (measure-call session bytes session-output)) + (values (measure-call fresh bytes fresh-output) s))))) + ;; Compare every byte after the complete uninstrumented call. + (unless (and (equal? fresh-output expected) (equal? session-output expected)) + (error "fresh/session GPA1 mismatch" name sample)) + (when (>= sample 0) + (write (list 'SOURCE-EDIT-SAMPLE name sample 'fresh fresh-row 'session session-row)) + (newline) (force-output)) + (loop (+ sample 1) + (if (negative? sample) fresh-rows (cons fresh-row fresh-rows)) + (if (negative? sample) session-rows (cons session-row session-rows))))))) + ;; A separate instrumented call explains boundaries, never the A/B latency. + (for-each + (lambda (lane) + (let ((handle (if (eq? lane 'fresh) fresh session)) (stages '())) + (if (eq? lane 'fresh) (call! handle #u8() empty-output) + (call! handle base-bytes prime-output)) + (parameterize ((current-parser-cost-observer + (lambda (stage receipt) (set! stages (cons (cons stage receipt) stages))))) + (with-parser-cost-stage 'full-call (call! handle bytes session-output))) + (unless (equal? session-output expected) (error "observed call GPA1 mismatch" name lane)) + (for-each (lambda (stage) + (unless (assq stage stages) (error "missing native cost stage" name lane stage))) + '(full-call ffi-input-copy utf8-decode source-copy source-prepare source-scan + artifact-publication artifact-validation payload-encoding ffi-result-copy)) + (when (and (not (eq? name 'lexical-rejection)) (not (assq 'command-recognition stages))) + (error "missing command recognition stage" name lane)) + (when (and (eq? name 'lexical-rejection) + (cdr (assq 'completed (cdr (assq 'source-scan stages))))) + (error "lexical failure reported a completed scan" lane)) + (let-values (((scanned reused) (language-handle-scan-counts handle))) + (when (and (eq? lane 'fresh) (positive? reused)) (error "fresh lane reused history" name)) + (write (list 'SOURCE-EDIT-STAGES name lane + (cons 'inclusive #t) (cons 'stages (reverse stages)) + (cons 'scanned scanned) (cons 'reused reused) + (cons 'history-token-records (if scanned (+ scanned reused) 0)) + (cons 'history-source-bytes (if scanned (u8vector-length bytes) 0)) + (cons 'payload-bytes (u8vector-length expected)))) + (newline) (force-output)))) '(fresh session)))) +;;; Separate semantic stress child; all collections arise from real parses. +;;; This control never changes GC policy or the timed benchmark samples. +(def (verify-callback-gc) + (let* ((baseline (language-handle-count)) + (handle (register-language! bash-source-language)) + (language (bind-language-abi bash-source-language)) + (collections 0)) + (dynamic-wind void + (lambda () + (for-each + (lambda (row) + (let* ((source (caddr row)) (bytes (string->utf8 source)) + (expected (abi-parse-binary-payload language source)) + (output (make-u8vector (u8vector-length expected)))) + (for-each + (lambda (_sample) + (let (before (f64vector-ref (##process-statistics) 6)) + (call! handle bytes output) + (set! collections (+ collections + (- (f64vector-ref (##process-statistics) 6) before)))) + (unless (equal? output expected) + (error "C callback collection changed GPA1 bytes" (car row)))) + (iota 16)) + (displayln "SOURCE-CALLBACK-GC-CASE-OK " (car row)) (force-output))) + (fixtures 1024))) + (lambda () (release-language! handle))) + (unless (and (positive? collections) (= baseline (language-handle-count))) + (error "callback collection control lacks a GC receipt" collections)) + (displayln "SOURCE-CALLBACK-GC-OK collections=" collections) (force-output)) + (test-child-process-exit! 0)) +(def (main . args) + (let ((samples (if (pair? args) (string->number (car args)) 20)) + (lines (if (> (length args) 1) (string->number (cadr args)) 128))) + (unless (and (integer? samples) (<= 2 samples 200) + (integer? lines) (<= 1 lines 1024)) + (error "expected samples in 2..200 and tail lines in 1..1024")) + (let* ((baseline (language-handle-count)) + (fresh (register-language! bash-source-language)) + (session (register-language! bash-source-language)) + (language (bind-language-abi bash-source-language))) + (dynamic-wind void + (lambda () + (for-each (lambda (row) (run-case row samples fresh session language)) (fixtures lines))) + (lambda () (release-language! fresh) (release-language! session))) + (unless (= baseline (language-handle-count)) (error "benchmark leaked handles"))) + (displayln "SOURCE-EDIT-BENCHMARK-OK") (force-output) + (test-child-process-exit! 0))) diff --git a/t/benchmarks/source-edits/checkpoint-publication.ss b/t/benchmarks/source-edits/checkpoint-publication.ss new file mode 100644 index 00000000..f6d2229f --- /dev/null +++ b/t/benchmarks/source-edits/checkpoint-publication.ss @@ -0,0 +1,82 @@ +#!/usr/bin/env gxi +;;; Matched publication component; real session parity is checked by source-scanner-test. +(import (only-in :std/vector/vector subvector-for-each subvector->list) + (only-in :std/list/list-builder with-list-builder)) +(export main) +(def (rebind old) (vector 'checkpoint (vector-ref old 1))) +(def (copied states prefix index) + (let (tail (map rebind (list-tail (vector->list states) (+ index 1)))) + (list->vector (append (reverse prefix) tail)))) +(def (iterated states prefix index) + (let (tail (with-list-builder (emit) + (subvector-for-each (lambda (old) (emit (rebind old))) + states start: (+ index 1)))) + (list->vector (append-reverse prefix tail)))) +(def (bounded states prefix index) + (let (tail (map rebind (vector->list states (+ index 1)))) + (list->vector (append-reverse prefix tail)))) +(def (observe name publish states prefix index) + (##gc) + (let* ((before (##process-statistics)) (cpu (cpu-time)) + (result (let loop ((n 100) (result #f)) + (if (zero? n) result + (loop (- n 1) (publish states prefix index))))) + (cpu-ms (* 1000 (- (cpu-time) cpu))) + (after (##process-statistics))) + (unless (equal? result states) (error "checkpoint publication changed order" name index)) + (when (< (+ index 1) (vector-length states)) + (when (eq? (vector-ref result (+ index 1)) (vector-ref states (+ index 1))) + (error "publication retained an unrebound checkpoint" name index))) + (write (list name 'prefix (+ index 1) 'suffix (- (vector-length states) index 1) + 'cpu-ms cpu-ms + 'gc-count (- (f64vector-ref after 6) (f64vector-ref before 6)) + 'allocation-counter-delta (- (f64vector-ref after 7) (f64vector-ref before 7)))) + (newline) (force-output))) +(def (copied-join head tail) (append (reverse head) tail)) +(def (folded-join head tail) (foldl cons tail head)) +(def (native-join head tail) (append-reverse head tail)) +(def (observe-join name join head tail expected) + (##gc) + (let* ((before (##process-statistics)) (cpu (cpu-time)) + (result (let loop ((n 1000) (result #f)) + (if (zero? n) result + (loop (- n 1) (join head tail))))) + (cpu-ms (* 1000 (- (cpu-time) cpu))) + (after (##process-statistics))) + (unless (and (equal? result expected) + (eq? (list-tail result (length head)) tail)) + (error "parser join changed order or retained tail" name)) + (write (list 'parser-join name 'prefix (length head) 'suffix (length tail) + 'cpu-ms cpu-ms + 'gc-count (- (f64vector-ref after 6) (f64vector-ref before 6)) + 'allocation-counter-delta (- (f64vector-ref after 7) (f64vector-ref before 7)))) + (newline) (force-output))) +(def (measure-parser-joins) + (for-each + (lambda (sizes) + (let* ((head (iota (car sizes))) (tail (iota (cadr sizes))) + (retained-head (map values head)) + (expected (copied-join head tail))) + (for-each + (lambda (control) + (for-each + (lambda (method) (observe-join (car method) (cdr method) head tail expected)) + (list control (cons 'native native-join) (cons 'native native-join) control))) + (list (cons 'copied-fragment copied-join) (cons 'folded-fork folded-join))) + (unless (equal? head retained-head) (error "parser join mutated its prefix")))) + '((0 4) (4 0) (1 4) (256 256) (1024 16))) + (displayln "PARSER-JOINS-OK") (force-output)) + +(def (main . _args) + (let (states (list->vector (map (lambda (n) (vector 'checkpoint n)) (iota 4097)))) + (for-each + (lambda (index) + (let (prefix (reverse (subvector->list states end: (+ index 1)))) + (for-each + (lambda (method) (observe (car method) (cdr method) states prefix index)) + (list (cons 'copied copied) (cons 'bounded bounded) + (cons 'bounded bounded) (cons 'copied copied) + (cons 'iterated iterated) (cons 'iterated iterated))))) + '(0 2048 4095 4096))) + (measure-parser-joins) + (displayln "CHECKPOINT-PUBLICATION-OK") (force-output)) diff --git a/t/benchmarks/source-edits/session-parser.ss b/t/benchmarks/source-edits/session-parser.ss new file mode 100644 index 00000000..618ef61c --- /dev/null +++ b/t/benchmarks/source-edits/session-parser.ss @@ -0,0 +1,40 @@ +#!/usr/bin/env gxi +;;; Complete Scheme session workload; fresh parsing is an independent semantic control. +(import (only-in :gerbil-parser/languages/bash/parser bash-source-language) + (only-in :gerbil-parser/src/language/source + parse-source-language parse-source-language/session source-language-session-artifact + source-language-session-scanned-token-count source-language-session-reused-token-count) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid? parse-artifact-roundtrip)) +(export main) +(def (main . _args) + (let* ((prefix (string-join (make-list 64 "echo prefix\n") "")) + (tail (string-join (make-list 64 "echo tail\n") "")) + (before (string-append prefix "cat <exact (ceiling (* percentile (length sorted)))) 1))) +(def (main . args) + (unless (<= (length args) 2) (error "expected sample count and optional entry" args)) + (let* ((samples (if (null? args) 20 (string->number (car args)))) + (selected (if (and (pair? args) (pair? (cdr args))) + (let (row (assoc (string->symbol (cadr args)) entries)) + (unless row (error "unknown all-language benchmark entry" (cadr args))) + [row]) + entries))) + (unless (and (integer? samples) (exact? samples) (positive? samples)) (error "expected positive sample count" args)) + (for-each + (lambda (row) + (let* ((entry (cadr row)) (parse (.ref entry '.parse)) + (sources (map syntax-fixture-source (language-loader-fixtures (caddr row) 'accepted)))) + (parse-batch parse sources) + (let loop ((remaining samples) (times '())) + (if (positive? remaining) + (begin + (##gc) + (let (started (##current-time-point)) + (parse-batch parse sources) + (let (elapsed (* 1000.0 (- (##current-time-point) started))) + (write (list 'sample (car row) (- samples remaining) elapsed)) + (newline) (force-output) + (loop (- remaining 1) (cons elapsed times))))) + (let (sorted (list-sort < times)) + (write (list (cons 'schema "gerbil-parser.all-language-batch.v1") + (cons 'entry (car row)) (cons 'metadata (language-metadata-ref (.ref entry 'metadata) 'digest)) + (cons 'phase 'warm-parse-and-roundtrip) + (cons 'fixture-count (length sources)) + (cons 'source-characters (apply + (map string-length sources))) + (cons 'sample-count samples) (cons 'quantile 'nearest-rank) + (cons 'p50-ms (nearest-rank sorted 0.50)) + (cons 'p95-ms (nearest-rank sorted 0.95)))) + (newline) (force-output)))))) selected) + (exit 0))) diff --git a/t/benchmarks/versioned-languages/benchmark.ss b/t/benchmarks/versioned-languages/benchmark.ss index 1cbe9b96..e47a789c 100644 --- a/t/benchmarks/versioned-languages/benchmark.ss +++ b/t/benchmarks/versioned-languages/benchmark.ss @@ -8,7 +8,7 @@ (rule . GERBIL-PARSER-VERSIONED-LANGUAGES) (feature . versioned-language-contracts) (optimizationFocus . "one generic generated machine across configuration and graph-query syntax families") - (inputShape . "the admitted corpus: 12 HCL v2.24.0 sources, 14 openGQL 1.9.0 examples, four openCypher 2024.1 queries, and five accepted TLA+ native-core v1 sources per sample") + (inputShape . "the admitted corpus: 12 HCL v2.24.0 sources, 14 openGQL 1.9.0 examples, four openCypher 2024.1 queries, and five accepted TLA+ core grammar sources per sample") (expectedOutcome . "all pinned language contracts roundtrip with no runtime language dispatch") (measurementPhases collect-before collect-after policy-before policy-after assert-time-gate observe-runtime-memory) diff --git a/t/benchmarks/versioned-languages/matched-batch.ss b/t/benchmarks/versioned-languages/matched-batch.ss new file mode 100644 index 00000000..8db4a326 --- /dev/null +++ b/t/benchmarks/versioned-languages/matched-batch.ss @@ -0,0 +1,82 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Complete ParseArtifact workload for matched parser-revision comparisons. +;;; Run in separate processes with each revision's own GERBIL_PATH. Dynamic +;;; import and GC are outside the timed region; each sample parses all sources. + +(import (only-in :gerbil-parser/languages/hcl/parser-test hcl-test-language) + (only-in :gerbil-parser/languages/cypher/parser-test opencypher-test-language) + (only-in :gerbil-parser/languages/tla-plus/parser-test tla-plus-core-test-language) + (only-in :gerbil-parser/language-support/development + language-loader-fixtures language-loader-fixture-count) + (only-in :gerbil-parser/language-support/fixture syntax-fixture-source) + (only-in :gerbil-parser/languages/gql/parser-test gql-test-language) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-roundtrip) + (only-in :gerbil-parser/languages/hcl/parser + parse-hcl) + (only-in :gerbil-parser/languages/gql/parser + gql-language parse-gql) + (only-in :gerbil-parser/languages/cypher/parser + parse-opencypher) + (only-in :gerbil-parser/languages/tla-plus/parser + parse-tla-plus-core)) + +(def corpus + (list + (cons parse-hcl + (map syntax-fixture-source (language-loader-fixtures hcl-test-language 'accepted))) + (cons parse-gql + (map syntax-fixture-source (language-loader-fixtures gql-test-language 'accepted))) + (cons parse-opencypher + (map syntax-fixture-source (language-loader-fixtures opencypher-test-language 'accepted))) + (cons parse-tla-plus-core + (map syntax-fixture-source (language-loader-fixtures tla-plus-core-test-language 'accepted))))) + +(def (run-batch) + (for-each + (lambda (group) + (for-each + (lambda (source) + (let (artifact ((car group) source)) + (unless (and (parse-artifact-success? artifact) + (equal? (parse-artifact-roundtrip artifact) source)) + (error "matched ParseArtifact batch failed" source)))) + (cdr group))) + corpus)) + +(def (main . args) + (let ((label (if (null? args) "unnamed" (car args))) + (samples (if (or (null? args) (null? (cdr args))) + 20 (string->number (cadr args))))) + (unless (and (integer? samples) (> samples 0)) + (error "expected positive sample count" args)) + (run-batch) + (run-batch) + (run-batch) + (write + (list (cons 'label label) + (cons 'fixture-count + (fold (lambda (group total) (+ total (length (cdr group)))) + 0 corpus)) + (cons 'source-characters + (fold (lambda (group total) + (+ total + (fold (lambda (source n) + (+ n (string-length source))) + 0 (cdr group)))) + 0 corpus)))) + (newline) + (let loop ((sample 0)) + (when (< sample samples) + (##gc) + (let (started (##current-time-point)) + (run-batch) + (write + (list (cons 'sample sample) + (cons 'elapsed-ms + (* 1000.0 (- (##current-time-point) started))))) + (newline)) + (loop (+ sample 1)))))) + +(export main) diff --git a/t/bootstrap-test.ss b/t/bootstrap-test.ss new file mode 100644 index 00000000..729f9d0a --- /dev/null +++ b/t/bootstrap-test.ss @@ -0,0 +1,18 @@ +(import :std/test :std/encoding/json :std/io/tempfile :std/misc/ports + (only-in "../tools/ci/bootstrap.ss" admit-release! append-lines!)) +(export bootstrap-test) +(def bootstrap-test + (test-suite "SDK release bootstrap" + (test-case "immutable release platform admission" + (let (metadata (string->json "{\"schema\":\"gerbil-bazel.toolchain-release.v1\",\"upstreamRevision\":\"2591dcd9b7c6d2c4e9dd8611a17c5b1a5d82bbdb\",\"platform\":{\"os\":\"darwin\",\"arch\":\"aarch64\"}}" + (JSONReadOptions object-as-hash: #t))) + (admit-release! metadata "darwin" "aarch64") + (check-exception (admit-release! metadata "linux" "x86_64") true))) + (test-case "environment publication preserves append bytes" + (let (file (make-temporary-file-name "parser-bootstrap-env-")) + (try + (append-lines! file '("FIRST=one" "SECOND=two")) + (append-lines! file '("THIRD=three")) + (check (call-with-input-file file read-all-as-string) + => "FIRST=one\nSECOND=two\nTHIRD=three\n") + (finally (when (file-exists? file) (delete-file file)))))))) diff --git a/t/bound-ir-index-test.ss b/t/bound-ir-index-test.ss index 5c1b9703..faee7fb7 100644 --- a/t/bound-ir-index-test.ss +++ b/t/bound-ir-index-test.ss @@ -3,9 +3,11 @@ ;;; Section-local declaration and reference indexes preserve Bound IR v1. (import (only-in :std/test check check-exception test-case test-suite) + (only-in :std/list/list-builder with-list-builder) (only-in :gerbil-parser/src/compiler/bound-ir bind-grammar-ir bound-grammar-ir-ref - bound-grammar-ir-section)) + bound-grammar-ir-section bound-grammar-ir-binding) + (only-in :gerbil-parser/src/runtime/identity sha256-text)) (export bound-ir-index-test) (def (grammar-ir syntax-kinds terminals) @@ -16,8 +18,62 @@ (cons 'lexical-rules '()) (cons 'rules '()))) +;;; Independent text writer keeps the previous canonical identity contract. +(def (text-digest datum) + (sha256-text (call-with-output-string (lambda (port) (write datum port))))) + (def bound-ir-index-test (test-suite "Bound IR indexed admission" + (test-case "empty reference digest reuse keeps independent caller-owned strings" + (let* ((grammar (grammar-ir '((Left node ()) (Right node ())) '())) + (result (bind-grammar-ir grammar "empty-reference-copy" '())) + (left (bound-grammar-ir-ref (bound-grammar-ir-binding result 'syntax-kind 'Left) 'referenceDigest)) + (right (bound-grammar-ir-ref (bound-grammar-ir-binding result 'syntax-kind 'Right) 'referenceDigest)) + (expected (text-digest '()))) + (check left => expected) (check right => expected) + (check (eq? left right) => #f) + (string-set! left 0 (if (char=? (string-ref left 0) #\0) #\1 #\0)) + (check right => expected) + (let (again (bind-grammar-ir grammar "empty-reference-copy" '())) + (check (bound-grammar-ir-ref (bound-grammar-ir-binding again 'syntax-kind 'Left) 'referenceDigest) + => expected)))) + (test-case "byte publication preserves Unicode and every canonical digest" + (let* ((grammar (grammar-ir '((|根| node (|值|)) (Word token ())) + '((word Word)))) + (origin "source/λ/中文") + (lineage '(author "\"escaped\"\nline" #u8(0 255))) + (result (bind-grammar-ir grammar origin lineage)) + (sections (bound-grammar-ir-ref result 'sections)) + (references + (with-list-builder (emit) + (for-each + (lambda (section) + (for-each + (lambda (binding) + (for-each emit (bound-grammar-ir-ref binding 'references))) + (cdr section))) + sections)))) + (check (bound-grammar-ir-ref result 'grammarDigest) => (text-digest grammar)) + (check (bound-grammar-ir-ref result 'identityDigest) => (text-digest sections)) + (check (bound-grammar-ir-ref result 'referenceDigest) => (text-digest references)) + (for-each + (lambda (section) + (for-each + (lambda (binding) + (let* ((namespace (car section)) + (name (bound-grammar-ir-ref binding 'name)) + (row (if (eq? namespace 'field) + (list name (bound-grammar-ir-ref binding 'scope)) + (assq name (cdr (assq (case namespace + ((syntax-kind) 'syntax-kinds) + ((terminal) 'terminals)) grammar))))) + (refs (bound-grammar-ir-ref binding 'references))) + (check (bound-grammar-ir-ref binding 'declarationId) + => (text-digest (list 'bound-index-witness namespace row + (bound-grammar-ir-ref binding 'source) lineage refs))) + (check (bound-grammar-ir-ref binding 'referenceDigest) => (text-digest refs)))) + (cdr section))) + sections))) (test-case "closed references preserve declaration order and canonical IR" (let* ((source (grammar-ir '((SourceFile node ()) (Word token ())) '((word Word)))) @@ -36,9 +92,49 @@ (grammar-ir '((SourceFile node ()) (SourceFile node ())) '()) "bound-index-test" '()) true)) + (test-case "source indexing retains first entries qualified fields and false values" + (let* ((source (grammar-ir '((Left node (value)) (Right node (value))) '())) + (locations '((syntax-kind (Left . #f) (Left . ignored) + (Right . ((path . "right")))) + (field ((Left value) . ((path . "left-field"))) + ((Right value) . ((path . "right-field")))))) + (result (bind-grammar-ir source "fallback" '() locations))) + (check (bound-grammar-ir-ref + (bound-grammar-ir-binding result 'syntax-kind 'Left) 'source) => #f) + (check (bound-grammar-ir-ref + (bound-grammar-ir-binding result 'field 'value 'Left) 'source) + => '((path . "left-field"))) + (check (bound-grammar-ir-ref + (bound-grammar-ir-binding result 'field 'value 'Right) 'source) + => '((path . "right-field"))) + (check (bound-grammar-ir-ref result 'referenceCount) => 4) + (check locations => '((syntax-kind (Left . #f) (Left . ignored) + (Right . ((path . "right")))) + (field ((Left value) . ((path . "left-field"))) + ((Right value) . ((path . "right-field")))))))) + (test-case "empty first source section wins and missing entries keep defaults" + (let* ((source (grammar-ir '((Root node ())) '())) + (result (bind-grammar-ir source "fallback" '(expansion) + '((syntax-kind) (syntax-kind (Root . ignored)))))) + (check (bound-grammar-ir-ref + (bound-grammar-ir-binding result 'syntax-kind 'Root) 'source) + => '((path . "fallback") (location . "fallback") (generated? . #f))))) + (test-case "large binding catalogs avoid variadic reference flattening" + (let* ((names (map (lambda (index) (string->symbol (number->string index))) (iota 20000))) + (source (grammar-ir (map (lambda (name) (list name 'node '())) names) '())) + (result (bind-grammar-ir source "large-bound-source" '()))) + (check (bound-grammar-ir-ref result 'bindingCount) => 20000) + (check (bound-grammar-ir-ref result 'referenceCount) => 0) + (check (map (lambda (binding) (bound-grammar-ir-ref binding 'name)) + (bound-grammar-ir-section result 'syntax-kind)) => names))) (test-case "unresolved references fail closed" (check-exception (bind-grammar-ir (grammar-ir '((SourceFile node ())) '((word Missing))) "bound-index-test" '()) true)))) + +;; The 20000-binding scalability Case owns its CPU/GC interval. Other owners +;; must not consume that interval while its real work is silent. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/build-product-contract-test.ss b/t/build-product-contract-test.ss index 2736da0b..a1dab768 100644 --- a/t/build-product-contract-test.ss +++ b/t/build-product-contract-test.ss @@ -3,12 +3,13 @@ (import (only-in :std/misc/ports read-all-as-string) (only-in :std/test check test-case test-suite) + (only-in :std/text/pregexp pregexp-match) (only-in :asp-gerbil-scheme/building-api asp-gerbil-scheme-package-native-capabilities) - (only-in :gerbil-parser/src/build-support/rust-rowan-aot - rust-rowan-aot-runtime-modules - rust-rowan-aot-package - rust-rowan-aot-build-spec)) + (only-in :gerbil-parser/src/build-support/rust-runtime-aot + rust-runtime-aot-runtime-modules + rust-runtime-aot-package + rust-runtime-aot-build-spec)) (def (native-item-kind? kind item) (and (pair? item) (eq? (car item) kind))) @@ -16,7 +17,9 @@ (def build-product-contract-tests (test-suite "build product ownership" (test-case "default package build excludes optional commands" - (let ((library-source + (let* ((cargo-source (call-with-input-file "Cargo.toml" read-all-as-string)) + (bridge (pregexp-match "gerbil-scheme-rust-ir = [^\n]*rev = \"([0-9a-f]{40})\"" cargo-source)) + (library-source (call-with-input-file "build.ss" read-all-as-string)) (package-source (call-with-input-file "gerbil.pkg" read-all-as-string)) @@ -25,23 +28,29 @@ (check (and (string-contains library-source "\"build-gparse.ss\"") (string-contains library-source - "\"build-rust-rowan-aot.ss\"") + "\"build-rust-runtime-aot.ss\"") (string-contains library-source ":asp-gerbil-scheme/building-api") (string-contains library-source "\"src/cli.ss\"") (string-contains library-source - "\"src/ffi/rust-rowan-aot-main.ss\"") + "\"src/ffi/rust-runtime-aot-main.ss\"") #t) => #t) - (check (and (string-contains package-source - "poo-flow@a321c63") - (not (string-contains package-source - "asp-gerbil-scheme@"))) - => #t) + (check (and bridge #t) => #t) + (check (cadr (member 'depend: (call-with-input-string package-source read))) + => (list "github.com/tao3k/poo-flow-core@e85fd45303b208b23e5957fd316e7004994109a4" + "github.com/tao3k/asp-gerbil-scheme@f5b7c009d2cab61144a8af18008a0bfba3056584")) (check (string-contains library-source "(exe: \"src/main\"") => #f) + ;; A foreign context must be excluded from generic discovery and + ;; included in the explicit FFI specs that carry Darwin bundle flags. + (check (and (string-contains library-source "\"src/ffi/language-abi-context.ss\"") + (string-contains library-source "\"src/ffi/language-abi-context\"") + (string-contains library-source "-Wl,-undefined,dynamic_lookup") + #t) + => #t) (check (and (string-contains command-source "defbuild-script") (string-contains command-source @@ -53,24 +62,28 @@ (check (string-contains command-source "asp-gerbil-scheme-package-spec!") => #f))) - (test-case "ASP Building API projects the Rowan executable closure" - (let* ((spec (rust-rowan-aot-build-spec)) + (test-case "ASP Building API projects the Rust generator executable closure" + (let* ((spec (rust-runtime-aot-build-spec)) (gxc-items (filter (lambda (item) (native-item-kind? 'gxc: item)) spec)) (exe-items (filter (lambda (item) (native-item-kind? 'exe: item)) spec))) - (check (map cadr gxc-items) => rust-rowan-aot-runtime-modules) + (check (map cadr gxc-items) => rust-runtime-aot-runtime-modules) (check (length exe-items) => 1) (let (executable (car exe-items)) - (check (cadr executable) => "src/ffi/rust-rowan-aot-main") + (check (cadr executable) => "src/ffi/rust-runtime-aot-main") (check (cadr (member 'bin: executable)) - => "gerbil-parser-rowan-aot") + => "gerbil-parser-runtime-aot") (check (and (member "-cc-options" executable) (member "-ld-options" executable) #t) => #t)) (check (asp-gerbil-scheme-package-native-capabilities - rust-rowan-aot-package) + rust-runtime-aot-package) => '(tls)))))) (export build-product-contract-tests) + +;; gxtest discovers only exported names ending in -test. +(def build-product-contract-test build-product-contract-tests) +(export build-product-contract-test) diff --git a/t/build-runtime-organization-tests.ss b/t/build-runtime-organization-tests.ss new file mode 100644 index 00000000..317b2784 --- /dev/null +++ b/t/build-runtime-organization-tests.ss @@ -0,0 +1,37 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Prepare the declared native gxtest closure through the existing PackageSpec. +(import (only-in :clan/poo/object .o) + (only-in :std/make make) + (only-in :std/misc/process run-process) + (only-in :std/string/misc string-trim) + (only-in :asp-gerbil-scheme/building-api + asp-gerbil-scheme-library-package-prototype + asp-gerbil-scheme-package-policy-native-spec)) +(export main) +(def tests + (.o (:: @ asp-gerbil-scheme-library-package-prototype) + public-entry-modules: + '("t/runtime-module-organization-test.ss" + "t/test-style-contract-test.ss" + "t/benchmarks/parser-throughput/link-locks-test.ss" + "t/check-runtime-organization.ss" + "t/lr-semantic-preparation-test.ss" + "t/lr-stack-reduction-test.ss" + "t/lr-action-preparation-test.ss" + "t/lr-fragment-transfer-test.ss" + "t/recovery-incremental-test.ss"))) +(def (main) + (let (spec (asp-gerbil-scheme-package-policy-native-spec tests)) + (make spec srcdir: (current-directory) + libdir: (path-expand "lib" (getenv "GERBIL_PATH")) + optimize: #t parallelize: 4) + (make (list (list 'exe: "t/runtime-organization-closure.ss" + bin: (path-expand "bin/runtime-organization-tests" (getenv "GERBIL_PATH")) + "-ld-options" + (string-append + (cond-expand (darwin "-Wl,-ld_classic ") (else "")) + (string-trim (run-process '("pkg-config" "--libs" "openssl")))))) + srcdir: (current-directory) parallelize: 4 verbose: 1) + (displayln "RUNTIME-ORGANIZATION-TESTS-PREPARED") + (force-output))) diff --git a/t/build-strategy-test.ss b/t/build-strategy-test.ss new file mode 100644 index 00000000..3e48d584 --- /dev/null +++ b/t/build-strategy-test.ss @@ -0,0 +1,135 @@ +;;; Two real output targets share one engine admission and Loader dispatch. +(import :std/test + (only-in :clan/poo/object .o .cc .ref .slot?) + (only-in :clan/poo/mop validate) + (only-in :gerbil-parser/language-build-support + BuildStrategy. BuildStrategyContract RustRuntimeStrategy. FusedReductionStrategy. + make-bound-build-strategy make-fused-reduction-strategy make-rust-runtime-strategy + make-recursive-source-strategy + declare-language-build-strategy emit-build-strategy emit-language-build-strategy) + (only-in :gerbil-parser/language-support/development LanguageDevelopmentLoaderContract deflanguage-development-loader) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid?) + (only-in :gerbil-parser/src/compiler/build-strategy declare-build-strategy-provider) + (only-in :gerbil-parser/src/compiler/rust-runtime language-rust-runtime-module-source) + (only-in :gerbil-parser/src/language/descriptor language-grammar-with-parser-policy) + (only-in :gerbil-parser/languages/hcl/parser hcl-language-grammar) + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-language-grammar)) +(export build-strategy-test) + +;;; This owner exercises engine admission; language corpus suites own fixtures. +(deflanguage-development-loader hcl-test-language + (grammar hcl-language-grammar) + (parse parse-hcl-strategy-test) + (slots build-strategies: + (list (declare-language-build-strategy 'fused-reductions + (make-fused-reduction-strategy hcl-language-grammar)) + (declare-language-build-strategy 'rust-runtime + (make-rust-runtime-strategy hcl-language-grammar))))) +(deflanguage-development-loader arithmetic-test-language + (grammar arithmetic-language-grammar) + (parse parse-arithmetic-strategy-test) + (slots build-strategies: + (list (declare-language-build-strategy 'fused-reductions + (make-fused-reduction-strategy arithmetic-language-grammar)) + (declare-language-build-strategy 'rust-runtime + (make-rust-runtime-strategy arithmetic-language-grammar))))) + +(def (loader-output loader name) + (call-with-output-string (lambda (port) (emit-language-build-strategy loader name port)))) +(def (strategy-output strategy) + (call-with-output-string (lambda (port) (emit-build-strategy strategy port)))) + +(def build-strategy-test + (test-suite "common POO build capability protocol" + (test-case "both languages declare two different admitted providers" + (for-each + (lambda (loader) + (let* ((rows (.ref loader 'build-strategies)) + (scheme (cdr (assq 'fused-reductions rows))) + (rust (cdr (assq 'rust-runtime rows)))) + (check (length rows) => 2) + (validate BuildStrategyContract scheme) + (validate BuildStrategyContract rust) + (check (.ref scheme 'kind) => 'fused-reductions) + (check (.ref scheme 'format) => 'scheme) + (check (.ref rust 'kind) => 'rust-runtime) + (check (.ref rust 'format) => 'rust))) + (list hcl-test-language arithmetic-test-language))) + (test-case "the same Loader dispatcher preserves canonical Rust output in both languages" + (for-each + (lambda (loader) + (check (loader-output loader 'rust-runtime) + => (language-rust-runtime-module-source (.ref loader 'descriptor)))) + (list hcl-test-language arithmetic-test-language))) + (test-case "generic declaration supports downstream names and inherited POO metadata" + (let* ((prototype (.o (:: self RustRuntimeStrategy.) metadata: (.o consumer: 'downstream))) + (strategy (make-rust-runtime-strategy arithmetic-language-grammar prototype)) + (loader (.cc arithmetic-test-language 'build-strategies + (list (declare-language-build-strategy 'my-rust-output strategy))))) + (validate LanguageDevelopmentLoaderContract loader) + (check (.ref (.ref strategy 'metadata) 'consumer) => 'downstream) + (check (loader-output loader 'my-rust-output) => (strategy-output strategy)))) + (test-case "a recipe method override cannot replace the registered engine writer" + (let* ((strategy (.cc (make-rust-runtime-strategy arithmetic-language-grammar) + '.emit (lambda (_) (error "unchecked writer must not execute")))) + (loader (.cc arithmetic-test-language 'build-strategies + (list (declare-language-build-strategy 'rust-output strategy))))) + (check (loader-output loader 'rust-output) + => (language-rust-runtime-module-source arithmetic-language-grammar)))) + (test-case "unregistered providers and forged target identity reject before writing" + (let ((strategy (make-rust-runtime-strategy arithmetic-language-grammar)) + (port (open-output-string))) + (for-each + (lambda (invalid) + (check-exception (validate BuildStrategyContract invalid) true) + (check-exception (emit-build-strategy invalid port) true)) + (list BuildStrategy. + (.cc strategy 'provider (.o kind: 'rust-runtime .emit: (lambda (_) (error "unchecked")))) + (.cc strategy 'kind 'fused-reductions) + (.cc strategy 'format 'scheme) + (.cc strategy 'digest "sha256:stale"))) + (check (get-output-string port) => "") + (check-exception (make-rust-runtime-strategy arithmetic-language-grammar FusedReductionStrategy.) true))) + (test-case "an engine extension adds a constrained recipe and format without dispatcher edits" + (let* ((admissions 0) (emissions 0) + (engine-provider + (declare-build-strategy-provider + 'test-receipt 'test-format + (lambda (candidate) + (set! admissions (+ admissions 1)) + (and (.slot? candidate 'receipt) (eq? (.ref candidate 'receipt) 'qualified))) + (lambda (candidate port) + (set! emissions (+ emissions 1)) + (let (form (list 'engine-receipt (.ref candidate 'receipt))) + (write form port))))) + (prototype (.o (:: self BuildStrategy.) provider: engine-provider receipt: 'qualified)) + (strategy (make-bound-build-strategy arithmetic-language-grammar prototype)) + (loader (.cc arithmetic-test-language 'build-strategies + (list (declare-language-build-strategy 'custom-receipt strategy))))) + (check (.ref strategy 'format) => 'test-format) + (check (loader-output loader 'custom-receipt) => "(engine-receipt qualified)") + (let ((before-admissions admissions) (before-emissions emissions)) + (for-each (lambda (_) (check (parse-artifact-valid? ((.ref loader '.parse) "1 + 2")) => #t)) (iota 8)) + (check admissions => before-admissions) + (check emissions => before-emissions)) + (check-exception (make-bound-build-strategy arithmetic-language-grammar + (.cc prototype 'receipt 'unknown)) true))) + (test-case "the common registry rejects foreign descriptors across provider kinds" + (check-exception + (validate LanguageDevelopmentLoaderContract + (.cc hcl-test-language 'build-strategies + (list (declare-language-build-strategy 'rust-output + (make-rust-runtime-strategy arithmetic-language-grammar))))) true)) + (test-case "recursive emission admits only its reviewed grammar before writing" + (let* ((strategy (make-recursive-source-strategy hcl-language-grammar)) + (port (open-output-string))) + (validate BuildStrategyContract strategy) + (check-exception (make-recursive-source-strategy arithmetic-language-grammar) true) + (check-exception + (emit-build-strategy (.cc strategy 'descriptor arithmetic-language-grammar) port) true) + (check (get-output-string port) => ""))) + (test-case "Rust policy admission remains distinct from Scheme reduction admission" + (let (descriptor (language-grammar-with-parser-policy + arithmetic-language-grammar "test-policy" 16 (lambda (_) #f))) + (check-exception (make-rust-runtime-strategy descriptor) true) + (validate BuildStrategyContract (make-fused-reduction-strategy descriptor)))))) diff --git a/t/canonical-lr1-reference-test.ss b/t/canonical-lr1-reference-test.ss new file mode 100644 index 00000000..3a714628 --- /dev/null +++ b/t/canonical-lr1-reference-test.ss @@ -0,0 +1,956 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; LR(1)-but-not-LALR(1) grammar checks the state-splitting reference. + +(import :std/test + (only-in :core/observability/testing-case poo-flow-test-case) + (only-in :gerbil-parser/src/compiler/funcs + compiler-index-set-difference compiler-index-set-union + compiler-index-set-for-each compiler-index-set-reachable + compiler-index-set-reachable/memo + compiler-index-partition-refine) + (only-in :gerbil-parser/src/compiler/lr + compute-first compute-nullable lower-rules lr-spec-ref + production-rhs production-table production-terminal-catalog) + (only-in :gerbil-parser/src/compiler/lr-automaton + build-lr0-automaton make-item-layout make-core-symbol-catalog + core-item-production-id core-item-dot) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr-conflict-candidates + forward-reachable-follow-masks + initial-backward-follow-partitions + lr0-conflict-candidates raw-conflict-cells) + (only-in :gerbil-parser/src/compiler/lr-lookahead + build-states-via-canonical-lr1 build-states-via-lr0) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-parse lr-prepare lr-parse/prepared lr-rejection-condition?) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/t/fixtures/lr1-construction + lr1-not-lalr-rules shared-lookahead-rules + genuine-reduce-conflict-rules precedence-expression-rules + mixed-context-rules lr1-context-family-rules + state-local-candidate-family-rules + mixed-context-family-rules acyclic-mixed-context-family-rules + split-shared-context-family-rules + unreachable-repeated-context-family-rules + inactive-core-conflict-rules)) + +(def (character-tokens text (base 0)) + (let loop ((index 0) (found '())) + (if (= index (string-length text)) + (reverse found) + (loop (+ index 1) + (cons (make-token 'punctuation + (string (string-ref text index)) + (+ base index) (+ base index 1)) + found))))) + +(def (parse-token-result spec tokens) + (with-catch + (lambda (condition) + (if (lr-rejection-condition? condition) + 'rejected + (raise condition))) + (lambda () + (let-values (((root rest) + (lr-parse spec tokens))) + (unless (null? rest) + (error "LR parser left a token suffix" rest)) + root)))) + +(def (parse-result spec text) + (parse-token-result spec (character-tokens text))) + +(def (prepared-result-parser spec) + ;; One immutable grammar/capability basis per sweep; requests still own fresh + ;; token vectors and LR state, including recovery after a rejected word. + (let (runtime (lr-prepare spec)) + (lambda (tokens) + (with-catch + (lambda (condition) + (if (lr-rejection-condition? condition) 'rejected (raise condition))) + (lambda () + (let-values (((root rest) (lr-parse/prepared runtime tokens))) + (unless (null? rest) (error "LR parser left a token suffix" rest)) + root)))))) + +(def (check-parse-equivalence canonical other alphabet length) + (let ((left (prepared-result-parser canonical)) (right (prepared-result-parser other))) + (let walk ((prefix "") (remaining length)) + (if (zero? remaining) + (check (equal? (left (character-tokens prefix)) + (right (character-tokens prefix))) + => #t) + (for-each + (lambda (part) + (walk (string-append prefix part) (- remaining 1))) + alphabet))))) + +(def (check-lr0-candidates-cover-canonical rules) + (let* ((productions (lower-rules rules 'source-file)) + (table (production-table productions))) + (let-values (((nullable nullable-index) + (compute-nullable productions))) + (let-values (((first first-index) + (compute-first productions nullable-index))) + (let-values (((lr0-states lr0-count terminals candidates) + (lr0-conflict-candidates + productions table first-index nullable-index))) + (let-values (((canonical-states canonical-count lookaheads offsets + transitions canonical-terminals layout core-symbols + state-visits item-visits) + (build-states-via-canonical-lr1 + productions table first-index nullable-index))) + (let ((index (make-table test: equal?)) + (canonical-conflicts + (raw-conflict-cells + canonical-states canonical-count lookaheads offsets + canonical-terminals layout core-symbols))) + (check (equal? terminals canonical-terminals) => #t) + (let index-loop ((state 0)) + (when (< state lr0-count) + (table-set! index (vector-ref lr0-states state) state) + (index-loop (+ state 1)))) + (let check-loop ((state 0)) + (when (< state canonical-count) + (let (lr0-state + (table-ref index (vector-ref canonical-states state) + #f)) + (check (number? lr0-state) => #t) + (check + (compiler-index-set-difference + (vector-ref canonical-conflicts state) + (vector-ref candidates lr0-state)) + => 0)) + (check-loop (+ state 1)))) + (values candidates canonical-conflicts)))))))) + +(def (check-forward-follows-match-lr0 rules) + (let* ((productions (lower-rules rules 'source-file)) + (table (production-table productions))) + (let-values (((nullable nullable-index) + (compute-nullable productions))) + (let-values (((first first-index) + (compute-first productions nullable-index))) + (let-values (((follows follow-terminals) + (forward-reachable-follow-masks + productions table first-index nullable-index))) + (let-values (((states count lookaheads offsets transitions + lr0-terminals layout core-symbols + state-visits item-visits) + (build-states-via-lr0 + productions table first-index nullable-index))) + (let (completed (make-vector (vector-length table) 0)) + (check (equal? follow-terminals lr0-terminals) => #t) + (let state-loop ((state 0)) + (when (< state count) + (let ((node (vector-ref offsets state))) + (for-each + (lambda (core) + (unless (vector-ref core-symbols core) + (let (production-id (core-item-production-id core layout)) + (vector-set! + completed production-id + (compiler-index-set-union + (vector-ref completed production-id) + (vector-ref lookaheads node))))) + (set! node (+ node 1))) + (vector-ref states state))) + (state-loop (+ state 1)))) + (let production-loop ((id 0)) + (when (< id (vector-length table)) + (check (vector-ref follows id) + => (vector-ref completed id)) + (production-loop (+ id 1))))))))))) + +(def (check-initial-backward-partitions rules) + (let* ((productions (lower-rules rules 'source-file)) + (table (production-table productions))) + (let-values (((nullable nullable-index) + (compute-nullable productions))) + (let-values (((first first-index) + (compute-first productions nullable-index))) + (let-values (((follows terminals) + (forward-reachable-follow-masks + productions table first-index nullable-index))) + (let-values (((partitions candidates partition-terminals) + (initial-backward-follow-partitions + productions table first-index nullable-index))) + (check (equal? terminals partition-terminals) => #t) + ;; Walk actual RHS intervals independently of the layout decoder. + ;; A constant stride inferred from vector size hides padding. + (let production-loop ((id 0) (start 0)) + (when (< id (vector-length table)) + (let dot-loop ((dot 0)) + (when (<= dot (length + (production-rhs (vector-ref table id)))) + (let (blocks (vector-ref partitions (+ start dot))) + (check + (foldl (lambda (block mask) + (compiler-index-set-union mask (cdr block))) + 0 blocks) + => (vector-ref follows id))) + (dot-loop (+ dot 1)))) + (production-loop + (+ id 1) + (+ start 1 (length (production-rhs (vector-ref table id))))))) + partitions)))))) + +;; Kernel identity is independent of closure construction and key encoding. +;; Check its graph invariants on recursive, nullable, and shared contexts. +(def (check-lr0-kernel-identity rules) + (let* ((productions (lower-rules rules 'source-file)) + (table (production-table productions))) + (let-values (((terminals terminal-index) + (production-terminal-catalog productions))) + (let* ((layout (make-item-layout table terminals)) + (symbols (make-core-symbol-catalog table layout))) + (let-values (((states transitions count by-lhs visits) + (build-lr0-automaton productions table layout symbols))) + (let ((kernels (make-vector count '())) + (seen (make-table test: equal?))) + (check visits => count) + (let index ((state 0)) + (when (< state count) + (let (kernel + (filter (lambda (item) + (not (zero? (core-item-dot item layout)))) + (vector-ref states state))) + (vector-set! kernels state kernel) + (if (zero? state) + (check kernel => '()) + (begin + (check (pair? kernel) => #t) + (check (table-ref seen kernel #f) => #f) + (table-set! seen kernel state)))) + (index (+ state 1)))) + (let edges ((state 0)) + (when (< state count) + ;; Every after-dot symbol must be published exactly once, in + ;; sorted-item discovery order, including shared predictions. + (let ((expected '()) (symbols-seen (make-table test: equal?))) + (for-each + (lambda (item) + (let (symbol (vector-ref symbols item)) + (when (and symbol (not (table-ref symbols-seen symbol #f))) + (table-set! symbols-seen symbol #t) + (set! expected (cons symbol expected))))) + (vector-ref states state)) + (check (map car (vector-ref transitions state)) => (reverse expected))) + (for-each + (lambda (edge) + (let* ((symbol (car edge)) (target (cdr edge)) + (expected + (map (lambda (item) (+ item 1)) + (filter (lambda (item) + (equal? (vector-ref symbols item) symbol)) + (vector-ref states state))))) + (check (> target 0) => #t) + (check (equal? expected (vector-ref kernels target)) => #t))) + (vector-ref transitions state)) + (edges (+ state 1)))))))))) + +(def canonical-lr1-reference-test + (test-suite "canonical LR(1) reference" + (poo-flow-test-case "prepared corpus readers preserve fresh rejection and recovery" + (let* ((spec (compile-lr-spec lr1-not-lalr-rules 'source-file 'reject #f 'canonical-lr1)) + (parse (prepared-result-parser spec))) + (check (eq? (parse-result spec "acd") 'rejected) => #f) + (for-each + (lambda (text) + (check (parse (character-tokens text)) => (parse-result spec text))) + '("" "acd" "bce" "bad" "ace" "bcd" "acd" "")))) + (poo-flow-test-case "affected partition refinement matches exhaustive synchronous oracle" + (def (signature predecessors node ids) + (filter-map + (lambda (label) + (let (groups + (filter (lambda (group) + (ormap (lambda (edge) + (and (= label (car edge)) + (= group (vector-ref ids (cdr edge))))) + (vector-ref predecessors node))) + (iota (vector-length ids)))) + (and (pair? groups) (cons label groups)))) + '(0 2))) + (def (oracle initial initial-count predecessors record) + (let refine ((ids initial) (groups initial-count)) + (let ((next (make-vector (vector-length ids) 0)) + (keys '()) (next-count 0)) + (for-each + (lambda (node) + ;; Match the existing engine's singleton elision when counting + ;; work, while the oracle still recomputes every node's key. + (when (> (length (filter (lambda (group) + (= group (vector-ref ids node))) + (vector->list ids))) 1) + (record)) + (let* ((key (cons (vector-ref ids node) + (signature predecessors node ids))) + (known (assoc key keys))) + (unless known + (set! known (cons key next-count)) + (set! keys (cons known keys)) + (set! next-count (+ next-count 1))) + (vector-set! next node (cdr known)))) + (iota (vector-length ids))) + (if (= groups next-count) (values next next-count) + (refine next next-count))))) + (def (compare predecessors initial initial-count) + (let ((saved (vector->list initial)) (expected-calls 0) (actual-calls 0)) + (let-values (((expected expected-count) + (oracle initial initial-count predecessors + (lambda () (set! expected-calls (+ expected-calls 1))))) + ((actual actual-count) + (compiler-index-partition-refine + initial initial-count + (lambda (node ids) (signature predecessors node ids)) + (lambda (source visit) + (for-each + (lambda (target) + (for-each (lambda (edge) + (when (= source (cdr edge)) (visit target))) + (vector-ref predecessors target))) + (iota (vector-length predecessors)))) + (lambda (groups nodes examined) + (set! actual-calls (+ actual-calls examined)))))) + ;; One graph comparison carries all four facts. Keep full vector, + ;; count, input immutability and examined-work checks, without four + ;; separately synchronized progress transactions per tiny graph. + (check (list actual actual-count (vector->list initial) + (<= actual-calls expected-calls)) + => (list expected expected-count saved #t)) + (values actual-calls expected-calls)))) + (let graphs ((graph 0)) + (when (< graph 512) + (let (predecessors (make-vector 3 '())) + (for-each (lambda (bit) + (when (not (zero? (bitwise-and graph (arithmetic-shift 1 bit)))) + (let ((target (modulo bit 3)) (source (quotient bit 3))) + (vector-set! predecessors target + (cons (cons 0 source) (vector-ref predecessors target)))))) + (iota 9)) + (for-each (lambda (entry) (compare predecessors (car entry) (cdr entry))) + (list (cons '#(0 0 0) 1) (cons '#(0 0 1) 2) + (cons '#(0 1 0) 2) (cons '#(0 1 1) 2) (cons '#(0 1 2) 3)))) + (graphs (+ graph 1)))) + (let graphs ((graph 0)) + (when (< graph 64) + (let (predecessors (make-vector 3 '())) + (for-each (lambda (bit) + (when (not (zero? (bitwise-and graph (arithmetic-shift 1 bit)))) + (let* ((source (modulo bit 3)) (target (modulo (+ source 1) 3)) + (edge (cons (if (< bit 3) 0 2) source))) + (vector-set! predecessors target + (cons edge (cons edge (vector-ref predecessors target))))))) + (iota 6)) + (compare predecessors '#(0 0 0) 1) + (compare predecessors '#(0 0 1) 2)) + (graphs (+ graph 1)))) + ;; A split propagates along the chain while the independent self-loop + ;; group remains stable, despite changes to its numeric group identity. + (let (predecessors '#(() ((0 . 0)) ((0 . 1)) ((0 . 2)) ((0 . 3)) ((0 . 4)) ((2 . 6)) ((2 . 7)))) + (let-values (((actual expected) (compare predecessors '#(0 1 1 1 1 1 2 2) 3))) + (check actual => 16) + (check expected => 24) + (check (< actual expected) => #t))) + ;; Sizes belong to group identities, not node order. Exercise dense IDs + ;; that are initially permuted before first-seen canonical publication. + (compare '#(() ((0 . 0)) ((2 . 1))) '#(1 0 0) 2) + (compare '#(() ((0 . 0)) ((2 . 1))) '#(2 0 1) 3) + (compare '#() '#() 0) + ;; Larger interacting groups exercise retained/new identities and both + ;; member/worklist orders. Reverse the entire graph, not just its IDs; + ;; every result is compared with the independent synchronous oracle. + (let graphs ((graph 0)) + (when (< graph 64) + (let ((predecessors (make-vector 6 '())) + (reversed (make-vector 6 '()))) + (for-each + (lambda (source) + (let* ((target (modulo (+ source 1) 6)) + (label (if (zero? (bitwise-and graph (arithmetic-shift 1 source))) 0 2)) + (edge (cons label source))) + (vector-set! predecessors target + (cons edge (cons edge (vector-ref predecessors target)))) + (when (zero? (modulo source 2)) + (let (target (modulo (+ source 3) 6)) + (vector-set! predecessors target + (cons (cons 2 source) (vector-ref predecessors target))))) + (when (not (zero? (bitwise-and graph (arithmetic-shift 1 source)))) + (vector-set! predecessors source + (cons (cons 0 source) (vector-ref predecessors source)))))) + (iota 6)) + (for-each + (lambda (node) + (vector-set! reversed (- 5 node) + (map (lambda (edge) (cons (car edge) (- 5 (cdr edge)))) + (vector-ref predecessors node)))) + (iota 6)) + (for-each + (lambda (entry) + (compare predecessors (car entry) (cdr entry)) + (compare reversed (list->vector (reverse (vector->list (car entry)))) (cdr entry))) + (list (cons '#(0 0 0 0 0 0) 1) (cons '#(1 0 0 1 0 0) 2) + (cons '#(2 0 1 2 0 1) 3) (cons '#(0 1 1 1 1 1) 2)))) + (graphs (+ graph 1)))) + ;; Workspaces must remain local to an invocation. A result produced + ;; after multiple rounds survives a separate refinement call. + (let ((initial '#(0 0 0)) (rounds 0)) + (let-values (((first groups) + (compiler-index-partition-refine + initial 1 (lambda (node ids) (if (= node 0) 0 1)) + (lambda (source visit) (void)) + (lambda (groups nodes examined) (set! rounds (+ rounds 1)))))) + (check first => '#(0 1 1)) + (check groups => 2) + (check rounds => 2) + (let-values (((second groups) + (compiler-index-partition-refine + initial 1 (lambda (node ids) node) + (lambda (source visit) (void)) + (lambda (groups nodes examined) (void))))) + (check second => '#(0 1 2)) + (check groups => 3) + (check (eq? first second) => #f)) + (check first => '#(0 1 1)) + (check initial => '#(0 0 0))))) + (poo-flow-test-case "indexed reachability agrees with exhaustive list graph oracle" + (def (oracle initial rows) + (let visit ((pending (filter (lambda (node) (not (zero? (bitwise-and initial (arithmetic-shift 1 node))))) (iota (vector-length rows)))) + (seen '())) + (if (null? pending) + (foldl (lambda (node mask) (bitwise-ior mask (arithmetic-shift 1 node))) 0 seen) + (let (node (car pending)) + (if (memv node seen) + (visit (cdr pending) seen) + (visit (append (vector-ref rows node) (cdr pending)) (cons node seen))))))) + (let graphs ((graph 0)) + (when (< graph 512) + (let ((edges (make-vector 3 0)) (rows (make-vector 3 '()))) + (for-each (lambda (node) + (let (mask (bitwise-and 7 (arithmetic-shift graph (* -3 node)))) + (vector-set! edges node mask) + (vector-set! rows node (filter (lambda (target) (not (zero? (bitwise-and mask (arithmetic-shift 1 target))))) '(0 1 2))))) '(0 1 2)) + (let (cache (make-vector 3 #f)) + (for-each + (lambda (initial) + (let (expected (oracle initial rows)) + (check (compiler-index-set-reachable initial edges) => expected) + (check (compiler-index-set-reachable/memo initial edges (make-vector 3 #f)) => expected) + (check (compiler-index-set-reachable/memo initial edges cache) => expected))) + (iota 8)) + (for-each + (lambda (node) + (let (closure (vector-ref cache node)) + (when closure (check closure => (oracle (arithmetic-shift 1 node) rows))))) + (iota 3)))) + (graphs (+ graph 1)))) + (check (compiler-index-set-reachable 0 '#()) => 0) + (check (compiler-index-set-reachable/memo 0 '#() '#()) => 0) + (let ((chain (make-vector 1024 0)) (cycle (make-vector 1024 0))) + (let nodes ((node 0)) + (when (< node 1024) + (when (< node 1023) (vector-set! chain node (arithmetic-shift 1 (+ node 1)))) + (vector-set! cycle node (arithmetic-shift 1 (modulo (+ node 1) 1024))) + (nodes (+ node 1)))) + (check (compiler-index-set-reachable 1 chain) => (- (arithmetic-shift 1 1024) 1)) + (check (compiler-index-set-reachable (arithmetic-shift 1 512) cycle) => (- (arithmetic-shift 1 1024) 1)) + ;; High-bit, overlapping roots preserve exact cold/warm results and + ;; graph-specific cache ownership across independent calls. + (let ((cache (make-vector 1024 #f)) + (roots (bitwise-ior 1 (arithmetic-shift 1 512) (arithmetic-shift 1 1023))) + (all (- (arithmetic-shift 1 1024) 1))) + (check (compiler-index-set-reachable/memo roots chain cache) => all) + (check (vector-ref cache 0) => all) + (check (compiler-index-set-reachable/memo roots chain cache) => all) + (check (compiler-index-set-reachable/memo roots cycle (make-vector 1024 #f)) => all)))) + (poo-flow-test-case "indexed traversal agrees with bit membership across fields" + (def (check-mask mask) + (let ((actual '()) (expected '())) + (compiler-index-set-for-each mask (lambda (index) (set! actual (cons index actual)))) + (let bits ((index 0)) + (when (< index (integer-length mask)) + (when (not (zero? (bitwise-and mask (arithmetic-shift 1 index)))) (set! expected (cons index expected))) + (bits (+ index 1)))) + (check actual => expected))) + (let masks ((mask 0)) + (when (< mask 4096) (check-mask mask) (masks (+ mask 1)))) + (let (width (integer-length (##greatest-fixnum))) + (for-each + (lambda (index) + (let (bit (arithmetic-shift 1 index)) + (check-mask bit) + (check-mask (- bit 1)) + (check-mask (bitwise-ior bit 1 (arithmetic-shift 1 (quotient index 2)))))) + [(- width 1) width (+ width 1) (* width 2) 1024 4096])) + ;; Reentrant traversal must not disturb the outer field or order. + (let ((outer '()) (inner '())) + (compiler-index-set-for-each (bitwise-ior 1 (arithmetic-shift 1 1024)) + (lambda (index) + (set! outer (cons index outer)) + (compiler-index-set-for-each 5 (lambda (bit) (set! inner (cons bit inner)))))) + (check outer => '(1024 0)) + (check inner => '(2 0 2 0))) + (let ((outer '()) (nested-count 0)) + (compiler-index-set-for-each (- (arithmetic-shift 1 128) 1) + (lambda (index) + (set! outer (cons index outer)) + (compiler-index-set-for-each 5 (lambda (_) (set! nested-count (+ nested-count 1)))))) + (check outer => (reverse (iota 128))) + (check nested-count => 256)) + (let ((calls 0) (sentinel (list 'callback-failure))) + (check (with-catch identity + (lambda () + (compiler-index-set-for-each (- (arithmetic-shift 1 128) 1) + (lambda (_) (set! calls (+ calls 1)) (raise sentinel))))) + => sentinel) + (check calls => 1))) + (poo-flow-test-case "LR(0) kernels uniquely identify states and goto targets" + (check-lr0-kernel-identity precedence-expression-rules) + (check-lr0-kernel-identity inactive-core-conflict-rules) + (check-lr0-kernel-identity (mixed-context-family-rules 16)) + (check-lr0-kernel-identity (state-local-candidate-family-rules 16)) + (check-lr0-kernel-identity (lr1-context-family-rules 16))) + (poo-flow-test-case "forward reachable follows agree with LR(0) propagation" + (check-forward-follows-match-lr0 lr1-not-lalr-rules) + (check-forward-follows-match-lr0 shared-lookahead-rules) + (check-forward-follows-match-lr0 genuine-reduce-conflict-rules) + (check-forward-follows-match-lr0 mixed-context-rules) + (check-forward-follows-match-lr0 (lr1-context-family-rules 16))) + (poo-flow-test-case "initial backward partition preserves all follow strings" + (let (partitions + (check-initial-backward-partitions lr1-not-lalr-rules)) + (check (vector-any (lambda (blocks) + (and blocks (> (length blocks) 1))) + partitions) + => #t)) + (check-initial-backward-partitions shared-lookahead-rules) + (check-initial-backward-partitions genuine-reduce-conflict-rules)) + (poo-flow-test-case "initial partition skips cores without a conflict action" + (let (blocks (vector->list + (check-initial-backward-partitions + inactive-core-conflict-rules))) + (check (foldl (lambda (entry total) + (+ total (if entry (length entry) 0))) + 0 blocks) + => 41) + (check (foldl (lambda (entry total) + (+ total (if (and entry (> (length entry) 1)) 1 0))) + 0 blocks) + => 3))) + (poo-flow-test-case "LR(0) raw candidates conservatively cover canonical conflicts" + (let-values (((candidates canonical-conflicts) + (check-lr0-candidates-cover-canonical + lr1-not-lalr-rules))) + (check (vector-any (lambda (mask) (not (zero? mask))) candidates) + => #t) + (check (vector-any (lambda (mask) (not (zero? mask))) + canonical-conflicts) + => #f)) + (check-lr0-candidates-cover-canonical shared-lookahead-rules) + (check-lr0-candidates-cover-canonical inactive-core-conflict-rules) + (let-values (((candidates canonical-conflicts) + (check-lr0-candidates-cover-canonical + genuine-reduce-conflict-rules))) + (check (vector-any (lambda (mask) (not (zero? mask))) candidates) + => #t) + (check (vector-any (lambda (mask) (not (zero? mask))) + canonical-conflicts) + => #t)) + (check-exception + (compile-lr-spec genuine-reduce-conflict-rules 'source-file + 'reject #f 'follow-partition-lr1) + true)) + (poo-flow-test-case "split contexts remove an LALR reduce/reduce conflict" + (check-exception + (compile-lr-spec lr1-not-lalr-rules 'source-file) + true) + (let (spec + (compile-lr-spec lr1-not-lalr-rules 'source-file + 'reject #f 'canonical-lr1)) + (check (lr-spec-ref spec 'algorithm) + => 'canonical-lr1-reference-v1) + (check (> (lr-spec-ref spec 'state-count) 0) => #t)) + (let ((canonical + (compile-lr-spec lr1-not-lalr-rules 'source-file + 'reject #f 'canonical-lr1)) + (partitioned + (compile-lr-spec lr1-not-lalr-rules 'source-file + 'reject #f 'partitioned-lr1)) + (direct + (compile-lr-spec lr1-not-lalr-rules 'source-file + 'reject #f 'follow-partition-lr1))) + (check (lr-spec-ref partitioned 'algorithm) + => 'conflict-partitioned-lr1-v1) + (check (lr-spec-ref direct 'algorithm) + => 'follow-partition-lr1-v1) + (check (<= (lr-spec-ref partitioned 'state-count) + (lr-spec-ref canonical 'state-count)) + => #t) + (check (> (lr-spec-ref direct 'follow-block-count) 0) => #t))) + (poo-flow-test-case "independent LR(1) contexts preserve parse products" + (for-each + (lambda (context-count) + (let* ((rules (lr1-context-family-rules context-count)) + (canonical + (compile-lr-spec rules 'source-file 'reject #f + 'canonical-lr1)) + (direct + (compile-lr-spec rules 'source-file 'reject #f + 'follow-partition-lr1))) + (check-exception (compile-lr-spec rules 'source-file) true) + (check (lr-spec-ref canonical 'state-count) + => (lr-spec-ref direct 'state-count)) + (when (memv context-count '(32 48 64)) + (check (lr-spec-ref direct 'follow-block-count) => 0)) + (for-each + (lambda (index) + (let* ((prefix + (string-append "region-" (number->string index) ":")) + (prefix-length (string-length prefix)) + (prefix-token + (make-token 'punctuation prefix 0 prefix-length))) + (for-each + (lambda (body) + (let* ((tokens + (cons prefix-token + (character-tokens body prefix-length))) + (expected (parse-token-result canonical tokens))) + (check (eq? expected 'rejected) => #f) + (check (equal? expected + (parse-token-result direct tokens)) + => #t))) + '("acd" "ace" "bcd" "bce")) + (let (tokens + (cons prefix-token + (character-tokens "acc" prefix-length))) + (check (parse-token-result canonical tokens) => 'rejected) + (check (parse-token-result direct tokens) => 'rejected)))) + (iota context-count)))) + '(4 16 32 48 64))) + (poo-flow-test-case "state-local conflict candidates preserve parse products" + (let* ((rules (state-local-candidate-family-rules 32)) + (canonical + (compile-lr-spec rules 'source-file 'selective-glr #f + 'canonical-lr1)) + (direct + (compile-lr-spec rules 'source-file 'selective-glr #f + 'follow-partition-lr1))) + (check (< (lr-spec-ref direct 'state-count) + (lr-spec-ref canonical 'state-count)) => #t) + (check (> (lr-spec-ref direct 'follow-block-count) 0) => #t) + (for-each + (lambda (index) + (let* ((prefix + (string-append "region-" (number->string index) ":")) + (prefix-length (string-length prefix)) + (prefix-token + (make-token 'punctuation prefix 0 prefix-length))) + (def (tokens body) + (cons prefix-token (character-tokens body prefix-length))) + (for-each + (lambda (body) + (let (input (tokens body)) + (check (equal? (parse-token-result canonical input) + (parse-token-result direct input)) => #t))) + '("axd" "axe" "bxxe" "bxe" "axf" "bxxd" "")))) + '(0 15 31)))) + (poo-flow-test-case "mixed context family compresses forward follow blocks" + (for-each + (lambda (context-count) + (let* ((rules (mixed-context-family-rules context-count)) + (canonical + (compile-lr-spec rules 'source-file 'reject #f + 'canonical-lr1)) + (direct + (compile-lr-spec rules 'source-file 'reject #f + 'follow-partition-lr1))) + (check (< (lr-spec-ref direct 'state-count) + (lr-spec-ref canonical 'state-count)) => #t) + (check (< (lr-spec-ref direct 'follow-block-count) + (lr-spec-ref direct 'output-item-count)) => #t) + (for-each + (lambda (index) + (let* ((prefix + (string-append "region-" (number->string index) ":")) + (prefix-length (string-length prefix)) + (prefix-token + (make-token 'punctuation prefix 0 prefix-length))) + (for-each + (lambda (body) + (let* ((tokens + (cons prefix-token + (character-tokens body prefix-length))) + (expected (parse-token-result canonical tokens))) + (check (eq? expected 'rejected) + => (if (member body '("acc" "cd")) #t #f)) + (check (equal? expected + (parse-token-result direct tokens)) + => #t))) + '("acd" "ace" "bcd" "bce" "dd" "cdd" "acc" "cd")))) + (iota context-count)))) + '(4 16 32 64 128))) + (poo-flow-test-case "FOLLOW stable edges preserve products after production reordering" + ;; Reordering productions changes core/block numbering. Compare with + ;; canonical construction to catch reuse of a previous round's edge IDs, + ;; including singleton and inactive blocks beside the final split. + (def (nullable-tail-family count) + (let (rules (mixed-context-family-rules count)) + (cons (list 'source-file + (list 'sequence (cadar rules) '(optional (literal "z")))) + (cdr rules)))) + (for-each + (lambda (family) + (for-each + (lambda (count) + (let (rules (family count)) + (for-each + (lambda (ordered) + (let ((canonical (compile-lr-spec ordered 'source-file 'reject #f 'canonical-lr1)) + (direct (compile-lr-spec ordered 'source-file 'reject #f 'follow-partition-lr1))) + (check (> (lr-spec-ref direct 'follow-block-count) 0) => #t) + (for-each + (lambda (region) + (let* ((prefix (string-append "region-" (number->string region) ":")) + (size (string-length prefix))) + (for-each + (lambda (body) + (let (tokens (cons (make-token 'punctuation prefix 0 size) + (character-tokens body size))) + (check (parse-token-result direct tokens) + => (parse-token-result canonical tokens)))) + '("acd" "ace" "bcd" "bce" "cc" "cd" "dc" "dd" "acc" "acdz" "cddz" "accz")))) + (iota count)))) + (list rules (reverse rules))))) + '(4 8))) + (list mixed-context-family-rules acyclic-mixed-context-family-rules nullable-tail-family))) + (poo-flow-test-case "acyclic shared contexts still need follow compression" + (for-each + (lambda (context-count) + (let* ((rules (acyclic-mixed-context-family-rules context-count)) + (canonical + (compile-lr-spec rules 'source-file 'reject #f + 'canonical-lr1)) + (direct + (compile-lr-spec rules 'source-file 'reject #f + 'follow-partition-lr1))) + (check (< (lr-spec-ref direct 'state-count) + (lr-spec-ref canonical 'state-count)) => #t) + (check (> (lr-spec-ref direct 'follow-block-count) 0) => #t) + (for-each + (lambda (index) + (let* ((prefix + (string-append "region-" (number->string index) ":")) + (prefix-length (string-length prefix)) + (prefix-token + (make-token 'punctuation prefix 0 prefix-length))) + (for-each + (lambda (body) + (let* ((tokens + (cons prefix-token + (character-tokens body prefix-length))) + (expected (parse-token-result canonical tokens))) + (check (equal? expected + (parse-token-result direct tokens)) => #t))) + '("acd" "ace" "bcd" "bce" "cc" "cd" "dc" "dd" "acc")))) + (list 0 15 (- context-count 1))))) + '(32 64))) + (poo-flow-test-case "split shared productions do not force compression" + (for-each + (lambda (context-count) + (let* ((rules (split-shared-context-family-rules context-count)) + (canonical + (compile-lr-spec rules 'source-file 'reject #f + 'canonical-lr1)) + (direct + (compile-lr-spec rules 'source-file 'reject #f + 'follow-partition-lr1))) + (check (lr-spec-ref direct 'state-count) + => (lr-spec-ref canonical 'state-count)) + (check (zero? (lr-spec-ref direct 'follow-block-count)) + => (>= context-count 64)) + (for-each + (lambda (index) + (let* ((prefix + (string-append "region-" (number->string index) ":")) + (prefix-length (string-length prefix)) + (prefix-token + (make-token 'punctuation prefix 0 prefix-length))) + (for-each + (lambda (body) + (let* ((tokens + (cons prefix-token + (character-tokens body prefix-length))) + (expected (parse-token-result canonical tokens))) + (check (eq? expected 'rejected) + => (equal? body "acc")) + (check (equal? expected + (parse-token-result direct tokens)) => #t))) + '("dx" "dy" "cdx" "cdy" "acc")))) + (list 0 15 (- context-count 1))))) + '(16 64))) + (poo-flow-test-case "unreachable repeated helper keeps canonical route" + (let* ((rules (unreachable-repeated-context-family-rules 64)) + (canonical + (compile-lr-spec rules 'source-file 'reject #f 'canonical-lr1)) + (direct + (compile-lr-spec rules 'source-file 'reject #f + 'follow-partition-lr1))) + (check (lr-spec-ref direct 'state-count) + => (lr-spec-ref canonical 'state-count)) + (check (lr-spec-ref direct 'follow-block-count) => 0) + (for-each + (lambda (index) + (let* ((prefix + (string-append "region-" (number->string index) ":")) + (prefix-length (string-length prefix)) + (prefix-token + (make-token 'punctuation prefix 0 prefix-length))) + (for-each + (lambda (body) + (let (tokens + (cons prefix-token + (character-tokens body prefix-length))) + (check (equal? (parse-token-result canonical tokens) + (parse-token-result direct tokens)) => #t))) + '("acd" "bce" "acc")))) + '(0 31 63)))) + (poo-flow-test-case "bounded canonical trial stops above its state budget" + (let* ((productions (lower-rules lr1-not-lalr-rules 'source-file)) + (table (production-table productions))) + (let-values (((nullable nullable-index) + (compute-nullable productions))) + (let-values (((first first-index) + (compute-first productions nullable-index))) + (check + (call-with-values + (lambda () + (build-states-via-canonical-lr1 + productions table first-index nullable-index 1)) + list) + => '(#f)))))) + (poo-flow-test-case "an LALR grammar remains admitted on both paths" + (let ((lalr (compile-lr-spec shared-lookahead-rules 'source-file)) + (canonical + (compile-lr-spec shared-lookahead-rules 'source-file + 'reject #f 'canonical-lr1)) + (direct + (compile-lr-spec shared-lookahead-rules 'source-file + 'reject #f 'follow-partition-lr1))) + (check (>= (lr-spec-ref canonical 'state-count) + (lr-spec-ref lalr 'state-count)) + => #t) + (check (> (lr-spec-ref canonical 'output-item-count) 0) + => #t) + (check (> (lr-spec-ref direct 'state-count) 0) => #t) + (check (lr-spec-ref direct 'follow-block-count) => 0) + (for-each + (lambda (first-token) + (for-each + (lambda (second-token) + (for-each + (lambda (third-token) + (let* ((text + (string-append first-token second-token third-token)) + (expected (parse-result canonical text))) + (check (equal? (parse-result lalr text) expected) => #t) + (check (equal? (parse-result direct text) expected) => #t))) + '("c" "d"))) + '("c" "d"))) + '("c" "d")) + (let (partitioned + (compile-lr-spec shared-lookahead-rules 'source-file + 'reject #f 'partitioned-lr1)) + (check (<= (lr-spec-ref partitioned 'state-count) + (lr-spec-ref canonical 'state-count)) + => #t) + (check (lr-spec-ref partitioned 'state-count) + => (lr-spec-ref lalr 'state-count))))) + (poo-flow-test-case "canonical and partitioned parsers accept the same fixtures" + (let ((canonical + (compile-lr-spec lr1-not-lalr-rules 'source-file + 'reject #f 'canonical-lr1)) + (partitioned + (compile-lr-spec lr1-not-lalr-rules 'source-file + 'reject #f 'partitioned-lr1)) + (direct + (compile-lr-spec lr1-not-lalr-rules 'source-file + 'reject #f 'follow-partition-lr1))) + (for-each + (lambda (text) + (let-values (((canonical-root canonical-rest) + (lr-parse canonical (character-tokens text))) + ((partitioned-root partitioned-rest) + (lr-parse partitioned (character-tokens text))) + ((direct-root direct-rest) + (lr-parse direct (character-tokens text)))) + (check canonical-rest => '()) + (check partitioned-rest => '()) + (check direct-rest => '()) + (check (equal? canonical-root partitioned-root) => #t) + (check (equal? canonical-root direct-root) => #t))) + '("acd" "ace" "bcd" "bce")) + (check-exception + (lr-parse canonical (character-tokens "acc")) true) + (check-exception + (lr-parse partitioned (character-tokens "acc")) true) + (check-exception + (lr-parse direct (character-tokens "acc")) true) + (let ((alphabet '("a" "b" "c" "d" "e")) + (accepted 0)) + (for-each + (lambda (first) + (for-each + (lambda (second) + (for-each + (lambda (third) + (let* ((text (string-append first second third)) + (left (parse-result canonical text)) + (right (parse-result partitioned text)) + (via-follow (parse-result direct text))) + (unless (eq? left 'rejected) + (set! accepted (+ accepted 1))) + (check (equal? left right) => #t) + (check (equal? left via-follow) => #t))) + alphabet)) + alphabet)) + alphabet) + (check accepted => 4)))) + (poo-flow-test-case "precedence grammar keeps canonical parse products" + (let ((canonical + (compile-lr-spec precedence-expression-rules 'source-file + 'reject #f 'canonical-lr1)) + (direct + (compile-lr-spec precedence-expression-rules 'source-file + 'reject #f 'follow-partition-lr1))) + (check (> (lr-spec-ref direct 'follow-block-count) 0) => #t) + (for-each + (lambda (text) + (check (equal? (parse-result canonical text) + (parse-result direct text)) + => #t)) + '("a" "a+a" "a*a" "a+a*a" "a*a+a" "a+a+a" "a*a*a")))) + (poo-flow-test-case "mixed contexts retain LR(1) recognition" + (let ((canonical + (compile-lr-spec mixed-context-rules 'source-file + 'reject #f 'canonical-lr1)) + (direct + (compile-lr-spec mixed-context-rules 'source-file + 'reject #f 'follow-partition-lr1))) + (check (> (lr-spec-ref direct 'follow-block-count) 0) => #t) + (check (<= (lr-spec-ref direct 'state-count) + (lr-spec-ref canonical 'state-count)) => #t) + (for-each + (lambda (text) + (check (equal? (parse-result canonical text) + (parse-result direct text)) + => #t)) + '("acd" "ace" "bcd" "bce" "dd" "cdd" "cdcd" "acc" "ccc")) + (for-each + (lambda (length) + (check-parse-equivalence + canonical direct '("a" "b" "c" "d" "e") length)) + '(2 3 4)))))) + +(export canonical-lr1-reference-test) diff --git a/t/check-runtime-organization.ss b/t/check-runtime-organization.ss new file mode 100644 index 00000000..fbf36562 --- /dev/null +++ b/t/check-runtime-organization.ss @@ -0,0 +1,23 @@ +;;; -*- Gerbil -*- +;;; Existing native process contract observes the existing native test harness. +(import (only-in :gerbil-parser/t/support/qualification/process + run-observed-process process-result-status process-result-reason process-result-output)) +(export main) +(def (qualify command log expected) + (let* ((result (run-observed-process command log timeout: 120 idle-timeout: 5 error-output?: #f)) + (output (utf8->string (process-result-output result)))) + (unless (and (= (process-result-status result) expected) (not (process-result-reason result))) + (error "runtime organization process rejected" (process-result-status result) (process-result-reason result))) + (if (zero? expected) + (unless (and (string-contains output "\nOK\n") + (not (or (string-contains output "ERROR") (string-contains output "FAILED")))) + (error "runtime organization harness did not pass")) + (unless (string-contains output "\nFAILED\n") (error "failure control missing rejection"))))) +(def (main) + (let* ((environment (getenv "GERBIL_PATH")) + (binary (path-expand "bin/runtime-organization-tests" environment))) + (qualify (list binary) (path-expand "runtime-organization.log" environment) 0) + (qualify (list binary "--failure-control") + (path-expand "runtime-organization-failure.log" environment) 42) + (displayln "RUNTIME-ORGANIZATION-OK") + (force-output))) diff --git a/t/command-profile-test.ss b/t/command-profile-test.ss new file mode 100644 index 00000000..f342fa8f --- /dev/null +++ b/t/command-profile-test.ss @@ -0,0 +1,116 @@ +;;; Admitted command roles are inherited values, never runtime callbacks. +(import :std/test (only-in :clan/poo/object .o .ref) + :gerbil-parser/src/language/command-profile + (only-in :gerbil-parser/src/language/result-profile compile-result-profile) + (only-in :gerbil-parser/src/language/part-profile compile-part-profile) + (only-in :gerbil-parser/src/runtime/shell-parser make-shell-parser) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/recognition recognition-node-kind) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-events parse-artifact-success?) + (only-in :gerbil-parser/src/runtime/source-scanner source-scanner-tokens) + (only-in :gerbil-parser/src/language/source source-language-scanner-factory source-language-digest declare-source-language) + (only-in :gerbil-parser/src/runtime/source-engines ShellSourceStrategy.) + (only-in :gerbil-parser/t/fixtures/bash-products bash-commands bash-results bash-parts bash-command-scanner bash-word-regions bash-source-language)) +(export command-profile-test) +(def (rejects? thunk) (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def (scan source) (source-scanner-tokens ((source-language-scanner-factory bash-source-language) source) 'command)) +(def (plan profile) (admit-command-plan (compile-command-profile profile) (compile-result-profile bash-results) bash-command-scanner)) +(def command-profile-test + (test-suite "closed command profile admission and extension" + (test-case "private rule renaming preserves entry positions and complete source events" + (let* ((source-forms (.ref bash-commands 'forms)) + (names (map (lambda (row) (cons (car row) + (string->symbol (string-append "private/" (symbol->string (car row)))))) source-forms))) + (def (rename id) (cdr (assq id names))) + (def (program data) + (cond ((not (pair? data)) data) + ((eq? (car data) 'call) (list 'call (cadr data) (rename (caddr data)))) + (else (cons (program (car data)) (program (cdr data)))))) + (let* ((profile (.o (:: self bash-commands) + forms: (map (lambda (row) (cons (rename (car row)) (program (cdr row)))) source-forms) + ports: (map (lambda (row) (list (car row) (rename (cadr row)))) (.ref bash-commands 'ports)))) + (control (plan bash-commands)) (candidate (plan profile))) + (let-values (((parse-control _control) (make-shell-parser bash-word-regions (compile-result-profile bash-results) (compile-part-profile bash-parts) control)) + ((parse-candidate _candidate) (make-shell-parser bash-word-regions (compile-result-profile bash-results) (compile-part-profile bash-parts) candidate))) + (for-each (lambda (source) + (let ((expected (parse-control source scan "control")) (actual (parse-candidate source scan "candidate"))) + (check (parse-artifact-success? actual) => (parse-artifact-success? expected)) + (check (parse-artifact-events actual) => (parse-artifact-events expected)))) + '("arr=(α 中) echo ok" "time -p ! echo α | cat" "f ( ) { echo 中; }" + "if echo; then echo; fi" "((1+(2*3)))" "(echo α)" + "cat < #t)) + '(() ((assignment-tail array-tail) (assignment-tail array-tail)) + ((assignment-tail absent) (pipeline-head pipeline-prefix)) + ((assignment-tail pipeline-prefix) (pipeline-head array-tail))))) + (test-case "token roles and complete descriptor profiles retain exact boundaries" + (let (prepared (plan bash-commands)) + (check (and (command-plan-token? prepared 'pipeline (make-token 'operator "|&" 0 2)) #t) => #t) + (check (command-plan-token? prepared 'pipeline (make-token 'word "|&" 0 2)) => #f) + (for-each (lambda (text) (check (command-plan-text? prepared 'descriptor text) => #t)) '("123" "{α_123}")) + (for-each (lambda (text) (check (command-plan-text? prepared 'descriptor text) => #f)) '("" "123x" "{x}tail" "{1x}")))) + (test-case "inherited role overrides use the same engine and owned recipes" + (let* ((declarations (map (lambda (row) (if (eq? (car row) 'pipeline) (list 'pipeline (list 'operator (string-copy "||"))) row)) (.ref bash-commands 'roles))) + (profile (.o (:: self bash-commands) roles: declarations)) (prepared (plan profile)) + (recipe (command-plan-recipe prepared))) + (string-set! (cadr (cadr (assq 'pipeline declarations))) 0 #\!) + (string-set! (cadr (cadr (assq 'pipeline (car recipe)))) 0 #\!) + (check (and (command-plan-token? prepared 'pipeline (make-token 'operator "||" 0 2)) #t) => #t) + (check (command-plan-token? prepared 'pipeline (make-token 'operator "!|" 0 2)) => #f) + (let-values (((parse receipt) (make-shell-parser bash-word-regions (compile-result-profile bash-results) (compile-part-profile bash-parts) prepared))) + (check (parse-artifact-success? (parse "echo x || echo y\n" scan (source-language-digest bash-source-language))) => #t)))) + (test-case "projection values inherit and reject missing fields before parsing" + (let* ((profile (.o (:: self bash-commands) projections: (map (lambda (row) (if (eq? (car row) 'BashFile) '(BashFile CommandList) row)) (.ref bash-commands 'projections)))) + (prepared (plan profile))) + (let-values (((parse receipt) (make-shell-parser bash-word-regions (compile-result-profile bash-results) (compile-part-profile bash-parts) prepared))) + (check (vector-ref (car (parse-artifact-events (parse "echo α\n" scan (source-language-digest bash-source-language)))) 2) => 'CommandList))) + (check (rejects? (lambda () (plan (.o (:: self bash-commands) projections: (map (lambda (row) (if (eq? (car row) 'BashFile) '(BashFile Word) row)) (.ref bash-commands 'projections)))))) => #t)) + (test-case "malformed callback roles and foreign scanner terminals reject at admission" + (for-each (lambda (profile) (check (rejects? (lambda () (plan profile))) => #t)) + (list (.o (:: self bash-commands) roles: '()) + (.o (:: self bash-commands) roles: (append (.ref bash-commands 'roles) '((pipeline (operator "|"))))) + (.o (:: self bash-commands) texts: '((descriptor (run (numeric) 0 #f)))) + (.o (:: self bash-commands) roles: (map (lambda (row) (if (eq? (car row) 'pipeline) '(pipeline (foreign "|")) row)) (.ref bash-commands 'roles))) + (.o (:: self bash-commands) roles: (lambda () '()))))) + (test-case "inherited control programs parse an independent Unicode syntax" + (let* ((declarations (map (lambda (row) + (if (eq? (car row) 'while-loop) + (append '(while-loop (word "重复") 10 WhileCommand (take keyword (word "重复"))) (list-tail row 5)) row)) (.ref bash-commands 'forms))) + (profile (.o (:: self bash-commands) forms: declarations)) (prepared (plan profile))) + (let-values (((parse receipt) (make-shell-parser bash-word-regions (compile-result-profile bash-results) (compile-part-profile bash-parts) prepared))) + (let (artifact (parse "重复 echo α; do echo 中; done\n" scan (source-language-digest bash-source-language))) + (check (parse-artifact-success? artifact) => #t) + (check (length (filter (lambda (event) (and (eq? (vector-ref event 0) 'start-node) (eq? (vector-ref event 2) 'WhileCommand))) (parse-artifact-events artifact))) => 1))))) + (test-case "empty inherited header disables prefixes without changing the executor" + (let (prepared (plan (.o (:: self bash-commands) forms: (map (lambda (row) + (if (eq? (car row) 'pipeline-prefix) '(pipeline-prefix (manual) 0 Pipeline) row)) (.ref bash-commands 'forms))))) + (let-values (((parse receipt) (make-shell-parser bash-word-regions (compile-result-profile bash-results) (compile-part-profile bash-parts) prepared))) + (let (artifact (parse "time echo α\n" scan (source-language-digest bash-source-language))) + (check (parse-artifact-success? artifact) => #t) + (check (length (filter (lambda (event) (and (eq? (vector-ref event 0) 'start-node) (eq? (vector-ref event 2) 'Pipeline))) (parse-artifact-events artifact))) => 0))))) + (test-case "shared call DAG admission visits each form once" + (let* ((name (lambda (index) (string->symbol (string-append "dag-" (number->string index))))) + (rows (map (lambda (index) + (if (= index 31) + (list (name index) '(manual) 0 'Pipeline '(take keyword (word "end"))) + (list (name index) '(manual) 0 'Pipeline (list 'call 'command (name (+ index 1))) + (list 'call 'command (name (+ index 1)))))) (iota 32))) + (prepared (plan (.o (:: self bash-commands) forms: (append (.ref bash-commands 'forms) rows))))) + (check (command-plan-match-form prepared (list (make-token 'word "if" 0 2))) => 'conditional-branch))) + (test-case "invalid fields, cycles and non-consuming repetitions reject before execution" + (for-each (lambda (form-declarations) (check (rejects? (lambda () (plan (.o (:: self bash-commands) forms: form-declarations)))) => #t)) + (list '() + (append (.ref bash-commands 'forms) '((cycle (word "cycle") 10 WhileCommand (call body cycle)))) + (append (.ref bash-commands 'forms) '((spin (word "spin") 10 WhileCommand (take keyword (word "spin")) (many (word) (as WhileCommand))))) + (append (.ref bash-commands 'forms) '((foreign (word "foreign") 10 WhileCommand (raw missing)))) + (append (.ref bash-commands 'forms) '((unknown (word "unknown") 10 WhileCommand (callback body anything)))) + (append (.ref bash-commands 'forms) '((missing (word "missing") 10 WhileCommand (call body absent)))) + (append (.ref bash-commands 'forms) '((empty-call (word "empty-call") 10 Pipeline (call command pipeline-prefix)))) + (append (.ref bash-commands 'forms) '((duplicate (lookahead (word) (operator "(") (operator ")")) 5 FunctionDefinition (word name))))))) + (test-case "command values participate in Source identity without a schema bump" + (let (changed (declare-source-language "bash" "5.3" "bash-5.3-structured-source.v1" + (.o (:: self ShellSourceStrategy.) regions: bash-word-regions scanner: bash-command-scanner results: bash-results parts: bash-parts + commands: (.o (:: self bash-commands) projections: (map (lambda (row) (if (eq? (car row) 'BashFile) '(BashFile CommandList) row)) (.ref bash-commands 'projections)))))) + (check (equal? (source-language-digest changed) (source-language-digest bash-source-language)) => #f))))) diff --git a/t/component-receipt-test.ss b/t/component-receipt-test.ss new file mode 100644 index 00000000..f7790605 --- /dev/null +++ b/t/component-receipt-test.ss @@ -0,0 +1,43 @@ +;;; Forged or incomplete observations must not qualify an optimization. +(import :std/test :std/io/tempfile + (only-in :gerbil-parser/t/benchmarks/parser-stage-cost/receipt read-parser-component-receipt)) +(export component-receipt-test) +(def (sample index (bytes 64)) + (list 'PARSER-COMPONENT-SAMPLE 'control + (list (cons 'sample index) '(cpu-ms . 2.) '(gc-count . 0.) + (cons 'allocated-bytes bytes) (cons 'allocation-counter-delta bytes)))) +(def (summary (cpu 2.)) + (list 'PARSER-COMPONENT-SUMMARY + (list '(stage . control) '(parsesPerSample . 4) '(sampleCount . 2) + '(allocationScope . single-caller) '(allocationSampleCount . 2) + '(allocatedBytesPerParse . 16.) (cons 'cpuP50Ms cpu)))) +(def (admit rows (complete? #t)) + (let (path (make-temporary-file-name "parser-component-receipt-")) + (try + (begin + (call-with-output-file path + (lambda (port) + (for-each (lambda (row) (write row port) (newline port)) rows) + (when complete? (display "CONTROL-OK\n" port)) + (display "PROCESS-RESULT elapsed=1 exit=0 reason=#f missing=()\n" port))) + (read-parser-component-receipt path "CONTROL-OK" 1 4 2)) + (finally (when (file-exists? path) (delete-file path)))))) +(def (rejected? rows (complete? #t)) + (with-catch + (lambda (e) (equal? (error-message e) "parser component receipt rejected")) + (lambda () (admit rows complete?) #f))) +(def component-receipt-test + (test-suite "component evidence admission" + (test-case "complete independent samples retain exact allocation" + (let (result (admit (list (sample 0) (sample 1) (summary)))) + (check (caddr result) => 2) + (check (hash-get (cadr result) 'control) => '(64 64)))) + (test-case "repeated sample identities and summaries are rejected" + (check (rejected? (list (sample 0) (sample 0) (summary))) => #t) + (check (rejected? (list (sample 0) (sample 1) (summary) (summary))) => #t)) + (test-case "changed counters and stale CPU summaries are rejected" + (check (rejected? (list (sample 0) (sample 1 72) (summary))) => #t) + (check (rejected? (list (sample 0) (sample 1) (summary 3.))) => #t)) + (test-case "missing completion and missing observations are rejected" + (check (rejected? (list (sample 0) (sample 1) (summary)) #f) => #t) + (check (rejected? (list (sample 0) (summary))) => #t)))) diff --git a/t/conformance-main.ss b/t/conformance-main.ss new file mode 100644 index 00000000..ee551ec5 --- /dev/null +++ b/t/conformance-main.ss @@ -0,0 +1,122 @@ +#!/usr/bin/env gxi +;;; Native suite entry: run compiled tests without reloading expander interfaces. +(import (only-in :std/test/base TestConfig current-test-config VERBOSITY-CASE + test-suite! test-result-ok?) + (prefix-in (only-in :gerbil-parser/t/benchmarks/versioned-languages/all-languages main) benchmark-) + (prefix-in (only-in :gerbil-parser/t/fixtures/language-pack-research/package-expression-runtime main) concise-runtime-) + (prefix-in (only-in :gerbil-parser/t/fixtures/language-pack-research/composition-history main) history-) + (prefix-in (only-in :gerbil-parser/t/fixtures/language-pack-research/list-runtime main) native-poo-runtime-) + (prefix-in (only-in :gerbil-parser/t/fixtures/language-pack-research/provenance-language main) provenance-) + (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/parser-test parser-test) + (only-in :gerbil-parser/t/test-style-contract-test test-style-contract-test) + (only-in :gerbil-parser/t/language-diagnostics-test language-diagnostics-test) + (only-in :gerbil-parser/t/language-surface-test language-surface-test language-backend-test) + (only-in :gerbil-parser/t/grammar-composition-contract-test grammar-composition-contract-test) + (only-in :gerbil-parser/t/grammar-composition-lowering-test grammar-composition-lowering-test) + (only-in :gerbil-parser/t/grammar-composition-execution-test grammar-composition-execution-test) + (only-in :gerbil-parser/t/resolved-grammar-test resolved-grammar-test) + (only-in :gerbil-parser/t/lr-conflict-origins-test lr-conflict-origins-test) + (only-in :gerbil-parser/t/lr-action-preparation-test lr-action-preparation-test lr-ordinary-action-preparation-test) + (only-in :gerbil-parser/t/ranked-parser-request-test ranked-parser-request-test) + (only-in :gerbil-parser/t/layout-column-storage-test layout-column-storage-test) + (only-in :gerbil-parser/t/language-artifact-test language-artifact-test) + (only-in :gerbil-parser/t/artifact-publication-cost-test artifact-publication-cost-test) + (only-in :gerbil-parser/t/artifact-window-publication-test artifact-window-publication-test) + (only-in :gerbil-parser/t/language-entry-boundary-test language-entry-boundary-test) + (only-in :gerbil-parser/t/language-topology-test language-topology-test) + (only-in :gerbil-parser/t/language-loader-test language-loader-test) + (only-in :gerbil-parser/t/language-loader-value-test language-loader-value-test) + (only-in :gerbil-parser/t/antlr4-source-test antlr4-source-test) + (only-in :gerbil-parser/languages/gql/parser-test gql-parser-test) + (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-parser-test) + (only-in :gerbil-parser/languages/bash/parser-test bash-parser-test) + (only-in :gerbil-parser/languages/cypher/parser-test opencypher-parser-test) + (only-in :gerbil-parser/languages/fhirpath/parser-test fhirpath-parser-test) + (only-in :gerbil-parser/languages/hcl/parser-test hcl-parser-test) + (only-in :gerbil-parser/languages/hl7/parser-test hl7-parser-test) + (only-in :gerbil-parser/languages/tla-plus/parser-test tla-plus-core-parser-test tla-plus-layout-parser-test tla-plus-sany-candidate-parser-test) + (only-in :gerbil-parser/t/gql/benchmark-profile-test gql-benchmark-profile-test) + (only-in :gerbil-parser/t/language-pack-metadata-test language-pack-metadata-test) + (only-in :gerbil-parser/t/source-strategy-test source-strategy-test) + (only-in :gerbil-parser/t/source-grammar-test source-grammar-test) + (only-in :gerbil-parser/t/command-profile-test command-profile-test) + (only-in :gerbil-parser/t/build-strategy-test build-strategy-test) + (only-in :gerbil-parser/t/fixtures/language-pack-research/package-expression-parser-test package-expression-parser-test) + (only-in :gerbil-parser/t/fixtures/language-pack-research/list-parser-test list-parser-test)) +(export main) +;; Source-module admission requires a live expander; retain that Suite there. +(def source-suites + '(("languages/tla-plus/parser-test" sany-closure-test))) +(def suites + (list + (cons "ranked-requests" (list lr-action-preparation-test lr-ordinary-action-preparation-test ranked-parser-request-test)) + (cons "downstream-records" (list parser-test)) + (cons "test-style" (list test-style-contract-test)) + (cons "diagnostics" (list language-diagnostics-test)) + (cons "surface" (list language-surface-test language-backend-test)) + (cons "composition" (list grammar-composition-contract-test + grammar-composition-lowering-test + grammar-composition-execution-test resolved-grammar-test + lr-conflict-origins-test layout-column-storage-test)) + (cons "publication" (list language-artifact-test artifact-publication-cost-test + artifact-window-publication-test)) + (cons "entry-boundaries" (list language-entry-boundary-test)) + (cons "topology" (list language-topology-test)) + (cons "loaders" (list language-loader-test language-loader-value-test)) + (cons "gql" (list antlr4-source-test gql-parser-test)) + (cons "arithmetic" (list arithmetic-parser-test)) + (cons "bash" (list source-grammar-test command-profile-test bash-parser-test)) + (cons "cypher" (list opencypher-parser-test)) + (cons "fhirpath" (list fhirpath-parser-test)) + (cons "hcl" (list hcl-parser-test)) + (cons "hl7" (list hl7-parser-test)) + (cons "tla-plus" (list tla-plus-core-parser-test tla-plus-layout-parser-test tla-plus-sany-candidate-parser-test)) + (cons "gql-profile" (list gql-benchmark-profile-test)) + (cons "pack-metadata" (list language-pack-metadata-test)) + (cons "source-services" (list source-strategy-test)) + (cons "build-services" (list build-strategy-test)) + (cons "concise-package" (list package-expression-parser-test)) + (cons "composed-package" (list list-parser-test)))) +(def (validate-conformance-suites! rows) + (let (seen (make-table test: eq?)) + (for-each (lambda (row) + (for-each (lambda (suite) + (when (table-ref seen suite #f) + (error "duplicate conformance Suite" (car row))) + (table-set! seen suite #t)) (cdr row))) rows))) +(def (run-suite-row! row (budget 10)) + ;; Execute on the caller thread and retain the ten-second group budget. + (let (timer (spawn (lambda () (thread-sleep! budget) + (displayln "CONFORMANCE-GROUP-TIMEOUT " (car row)) + (force-output) (exit 70)))) + (try + (for-each + (lambda (suite) + (let (result + (parameterize ((current-test-config + (TestConfig verbosity: VERBOSITY-CASE capture-output?: #f))) + (test-suite! suite))) + (unless (test-result-ok? result) (exit 1)))) + (cdr row)) + (displayln "DSL-SUITE-OK: " (car row)) (force-output) + (finally (thread-terminate! timer))))) +(def (main name . args) + (when (equal? name "semantic") + (unless (null? args) (error "unexpected semantic arguments" args)) + (validate-conformance-suites! suites) + (for-each run-suite-row! suites) + (displayln "CONFORMANCE-SEMANTIC-OK groups=" (length suites)) + (force-output) (exit 0)) + (let (program (assoc name (list (cons "concise-runtime" concise-runtime-main) + (cons "history" history-main) + (cons "native-poo-runtime" native-poo-runtime-main) + (cons "provenance" provenance-main)))) + (when program + (unless (null? args) (error "unexpected research control arguments" args)) + ((cdr program)))) + (if (equal? name "all-language-benchmark") + (apply benchmark-main (if (null? args) ["1"] args)) + (let (row (assoc name suites)) + (unless (null? args) (error "unexpected native suite arguments" args)) + (unless row (error "unknown native conformance suite" name)) + (run-suite-row! row) (exit 0)))) diff --git a/t/contextual-deferred-scanner-test.ss b/t/contextual-deferred-scanner-test.ss new file mode 100644 index 00000000..3acf051e --- /dev/null +++ b/t/contextual-deferred-scanner-test.ss @@ -0,0 +1,431 @@ +;;; -*- Gerbil -*- +;;; Language declarations exercise one engine-owned deferred delimiter scanner. + +(import (only-in :std/test check check-exception test-case test-suite) + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-method make-contextual-role + make-contextual-scan-rule) + (only-in :gerbil-parser/src/compiler/contextual-dispatch + compile-contextual-dispatch) + (only-in :gerbil-parser/src/compiler/contextual-scanner-ir + compile-contextual-scanner) + (only-in :gerbil-parser/src/runtime/contextual-scanner + prepare-contextual-scanner prepare-contextual-scanner-plan contextual-scanner-initial-state + contextual-scanner-step contextual-scan-state-byte-offset + contextual-scan-state-mode contextual-scan-state-canonical + restore-contextual-scan-state contextual-scan-state-converged? + contextual-scanner-replay-prefix contextual-scan-state-with-mode + prepare-contextual-scan-suffix contextual-scan-suffix-state) + (only-in :gerbil-parser/src/runtime/token + token-kind token-lexeme token-start token-end)) +(export contextual-deferred-scanner-test) + +(def (method name mode position form result) + (make-contextual-method name mode position form result)) + +(def (rule name mode form matcher rank action) + (make-contextual-scan-rule name mode form matcher rank action)) + +(def bash-operators + '(";;&" "&>>" "<<-" "<<<" "&&" "||" "|&" ";;" ";&" + "<<" ">>" "<>" "<&" ">&" ">|" "&>" ";" "&" "|" "(" ")" + "<" ">")) + +(def (deferred-scanner-ir owner marker-policy operators (separator #f)) + (let* ((role + (make-contextual-role + owner + (list (method 'operator 'any 'any 'operator 'operator) + (method 'here-open 'any 'any 'here-open 'here-open) + (method 'word 'any 'any 'word 'word) + (method 'space 'any 'any 'space 'horizontal-whitespace) + (method 'newline 'any 'any 'newline 'newline) + (method 'body 'any 'any 'body 'here-body) + (method 'end 'any 'any 'end 'here-end)))) + (dispatch + (compile-contextual-dispatch + (list role) '(command here-body) '(command) + '(operator here-open word space newline body end))) + (rules + (list (rule 'here-strip 'command 'here-open + '(literal "<<-") 30 '(expect-marker #t)) + (rule 'here-plain 'command 'here-open + '(literal "<<") 30 '(expect-marker #f)) + (rule 'operator 'command 'operator + (list 'literals operators) 20 'keep) + (rule 'word 'command 'word + (list 'balanced-word operators '("'" "\"") '()) + 0 (list 'enqueue-if-expecting marker-policy)) + (rule 'space 'command 'space + '(horizontal-whitespace+) 0 'keep) + (rule 'newline 'command 'newline '(newline-one) 0 + '(activate-next here-body)) + (rule 'end 'here-body 'end + (if separator (list 'marker-line-at (string separator)) + '(marker-line)) 10 + '(finish-marker command here-body)) + (rule 'body 'here-body 'body + (if separator (list 'body-line-at (string separator)) + '(body-line)) 0 'keep))) + (ir (compile-contextual-scanner rules dispatch 'command))) + ir)) + +(def (deferred-scanner source owner marker-policy operators) + (prepare-contextual-scanner + (prepare-contextual-scanner-plan (deferred-scanner-ir owner marker-policy operators)) source)) + +(def (scan-all scanner) + (let loop ((state (contextual-scanner-initial-state scanner)) + (tokens '())) + (let-values (((token next) + (contextual-scanner-step scanner state 'command))) + (if token + (loop next (cons token tokens)) + (values (reverse tokens) next))))) + +(def contextual-deferred-scanner-test + (test-suite "generic deferred delimiter scanner" + (test-case "prefix replay reaches source-owned HCL and Bash delimiter states" + (for-each + (lambda (policy) + (let* ((plan (prepare-contextual-scanner-plan + (deferred-scanner-ir 'prefix-replay policy '("<<" "<<-" "<<<")))) + (old (prepare-contextual-scanner plan "猫 < (contextual-scan-state-canonical (advance new count))) + (check (contextual-scan-state-canonical checkpoint) => receipt) + (check-exception (contextual-scanner-step new checkpoint 'command) true) + (let-values (((token next) (contextual-scanner-step new replayed 'command))) + (check (and token #t) => #t)))) + '(0 4 5)) + (let (checkpoint (advance old 5)) + (check-exception (contextual-scanner-replay-prefix new checkpoint '(command)) true) + (check-exception (contextual-scanner-replay-prefix new checkpoint (make-list 6 'command)) true) + (check-exception (contextual-scanner-replay-prefix new checkpoint '(unknown)) true) + (check-exception + (contextual-scanner-replay-prefix new (contextual-scan-state-with-mode checkpoint 'command) + (make-list 5 'command)) true)) + (for-each + (lambda (sources) + (let* ((a (prepare-contextual-scanner plan (car sources))) + (b (prepare-contextual-scanner plan (cadr sources))) + (checkpoint (advance a 1))) + ;; The prefix bytes match, but actual longest-match/EOF/quote decisions change. + (check-exception (contextual-scanner-replay-prefix b checkpoint '(command)) true))) + '(("cat" "catx") ("< #t) + (check (trace a old (contextual-scan-state-byte-offset old)) + => (trace b new (contextual-scan-state-byte-offset new))) + (let (proof (prepare-contextual-scan-suffix old new)) + (let-values (((body old-next) (contextual-scanner-step a old 'command))) + (let-values (((body new-next) (contextual-scanner-step b new 'command))) + (let (rebound (contextual-scan-suffix-state proof old-next)) + (check (contextual-scan-state-canonical rebound) => (contextual-scan-state-canonical new-next)) + (check (trace b rebound (contextual-scan-state-byte-offset new)) + => (trace b new-next (contextual-scan-state-byte-offset new)))))) + (for-each + (lambda (outside) + (check (with-catch (lambda (condition) (error-message condition)) + (lambda () (contextual-scan-suffix-state proof outside) #f)) + => "checkpoint is outside the admitted scanner suffix")) + (list (contextual-scanner-initial-state a) new))) + ;; Each adversarial receipt changes lexical context with the same suffix. + (for-each + (lambda (replacement) + (let ((changed (restore-contextual-scan-state + b (map (lambda (row) + (if (eq? (car row) (car replacement)) replacement row)) receipt)))) + (check (contextual-scan-state-converged? old changed) => #f))) + '((active . ("B" #f #f)) (active . ("A" #t #f)) + (active . ("A" #f #t)) (expecting . plain) (mode . command) + (pending . (("A" #f #f))))) + (check (contextual-scan-state-converged? + old (advance (prepare-contextual-scanner plan "cat < #f) + (check (contextual-scan-state-converged? + old (advance (prepare-contextual-scanner + (prepare-contextual-scanner-plan + (deferred-scanner-ir 'convergence policy '("<<" "<<-"))) + "cat < #f)) + (let* ((pending (advance a 4)) + (receipt (contextual-scan-state-canonical pending))) + (check (contextual-scan-state-converged? pending (advance b 4)) => #t) + (for-each + (lambda (values) + (let ((changed (restore-contextual-scan-state + a (map (lambda (row) + (if (eq? (car row) 'pending) (cons 'pending values) row)) receipt)))) + (check (contextual-scan-state-converged? pending changed) => #f))) + '((("B" #f #f)) (("A" #t #f)) (("A" #f #t))))) + (let* ((q (prepare-contextual-scanner plan "< #f)))) + '(raw shell-quote-removal))) + (test-case "shared scanner plans keep delimiter queues source-local" + (let* ((ir (deferred-scanner-ir 'plan-queues 'raw '("<<"))) + (plan (prepare-contextual-scanner-plan ir)) + (a (prepare-contextual-scanner plan "< '(("A" #f #f))) + (check (cdr (assq 'pending (contextual-scan-state-canonical (contextual-scanner-initial-state b)))) => '()) + (check (cdr (assq 'pending (contextual-scan-state-canonical (queued b)))) => '(("B" #f #f))) + (let-values (((tokens final) (scan-all a))) + (check (map token-lexeme tokens) => '("<<" "A" "\n" "α\n" "A\n"))) + (let-values (((tokens final) (scan-all b))) + (check (map token-lexeme tokens) => '("<<" "B" "\n" "β\n" "B\n"))))) + (test-case "queued checkpoints preserve FIFO order across independent futures" + (let* ((source "< '(("A" #f #f) ("B" #f #f))) + (check (suffix after-a) => (suffix restored)) + (check (suffix after-b) + => (suffix (restore-contextual-scan-state scanner second))) + (check (map cadr (filter (lambda (row) (eq? (car row) 'here-end)) + (suffix restored))) + => '("A\n" "B\n" "C\n")) + (check (contextual-scan-state-canonical after-a) => receipt)))) + (test-case "Bash here bodies preserve marker order and CRLF bytes" + (let* ((source "cat <<'A' <<-B\r\n$x α\r\nA\r\n\tβ $x\r\n\tB\r\n") + (scanner + (deferred-scanner source 'bash-deferred + 'shell-quote-removal bash-operators))) + (let-values (((tokens final) (scan-all scanner))) + (check (map token-kind + (filter (lambda (token) + (memq (token-kind token) + '(here-body here-end))) + tokens)) + => '(here-body here-end here-body here-end)) + (check (map token-lexeme + (filter (lambda (token) + (eq? (token-kind token) 'here-end)) + tokens)) + => '("A\r\n" "\tB\r\n")) + (check (contextual-scan-state-mode final) => 'command) + (check (contextual-scan-state-byte-offset final) + => (u8vector-length (string->utf8 source))) + (check (cdr (assq 'pending (contextual-scan-state-canonical final))) + => '()) + (check (cdr (assq 'active (contextual-scan-state-canonical final))) + => #f)))) + (test-case "marker slices reject equal-prefix and equal-suffix body lines" + (for-each + (lambda (separator) + (let* ((line-end (string (or separator #\newline))) + (source (string-append "<<-猫" line-end + "\t猫x" line-end "\tx猫" line-end + "\t猫猫" line-end "\t猫")) + (plan (prepare-contextual-scanner-plan + (deferred-scanner-ir 'slice-marker 'raw + '("<<" "<<-") separator))) + (scanner (prepare-contextual-scanner plan source))) + (let-values (((tokens final) (scan-all scanner))) + (check (map token-kind tokens) + => '(here-open word newline here-body here-body here-body here-end)) + (check (map token-lexeme (list-tail tokens 3)) + => (list (string-append "\t猫x" line-end) + (string-append "\tx猫" line-end) + (string-append "\t猫猫" line-end) "\t猫")) + (check (contextual-scan-state-mode final) => 'command) + (check (contextual-scan-state-byte-offset final) + => (u8vector-length (string->utf8 source)))))) + '(#f #\newline #\return))) + (test-case "HCL raw marker uses the same deferred scanner" + (let* ((source "< '(here-open word newline here-body here-end)) + (check (token-lexeme (list-ref tokens 3)) => "α\n") + (check (token-lexeme (list-ref tokens 4)) => "EOF") + (check (contextual-scan-state-byte-offset final) + => (u8vector-length (string->utf8 source)))))) + (test-case "an unfinished marker fails at EOF" + (let (scanner + (deferred-scanner "cat < "unfinished contextual scanner obligation"))) + (test-case "shell marker quote removal preserves ordinary backslashes" + (let* ((source "cat <<\"\\q\"\npayload\n\\q\n") + (scanner (deferred-scanner source 'bash-deferred + 'shell-quote-removal bash-operators))) + (let-values (((tokens final) (scan-all scanner))) + (check (apply string-append (map token-lexeme tokens)) => source) + (check (map token-lexeme + (filter (lambda (token) (eq? (token-kind token) 'here-end)) + tokens)) + => '("\\q\n"))))) + (test-case "empty quoted shell marker closes on the blank line" + (let* ((source "cat <<''\npayload\n\n") + (scanner (deferred-scanner source 'bash-deferred + 'shell-quote-removal bash-operators))) + (let-values (((tokens final) (scan-all scanner))) + (check (apply string-append (map token-lexeme tokens)) => source) + (check (map token-lexeme + (filter (lambda (token) (eq? (token-kind token) 'here-end)) + tokens)) + => '("\n"))))) + (test-case "checkpoint restore counts a Unicode source prefix and resumes its tail" + (let* ((source "< 13) + (check (contextual-scan-state-canonical restored) => receipt) + (let-values (((tail next) (contextual-scanner-step scanner restored 'command))) + (check (token-lexeme tail) => "EOF\n") + (check (token-start tail) => 13) + (check (token-end tail) => 17) + (check (contextual-scan-state-byte-offset next) => 17))))))) + (test-case "scanner requests own source snapshots across declaration policies" + (for-each + (lambda (entry) + (let* ((source (string-copy "< receipt) + (let-values (((tokens final) (scan-all scanner))) + (check (map token-lexeme tokens) => '("<<" "EOF" "\n" "α\n" "EOF\n"))))) + '((hcl-owned raw) (bash-owned shell-quote-removal)))) + (test-case "published and restored checkpoints own pending and active markers" + (for-each + (lambda (entry) + (let* ((scanner (deferred-scanner "< expected) + (check (contextual-scan-state-canonical restored) => expected) + (let-values (((_newline active) (contextual-scanner-step scanner restored 'command))) + (let* ((receipt (contextual-scan-state-canonical active)) + (expected (contextual-scan-state-canonical active)) + (restored (restore-contextual-scan-state scanner receipt))) + (string-set! (car (cdr (assq 'active receipt))) 0 #\X) + (check (contextual-scan-state-canonical active) => expected) + (check (contextual-scan-state-canonical restored) => expected) + (let-values (((body next) (contextual-scanner-step scanner restored 'command))) + (check (token-lexeme body) => "α\n") + (let-values (((end final) (contextual-scanner-step scanner next 'command))) + (check (token-lexeme end) => "EOF\n")))))))))) + '((hcl-checkpoint-owned raw) (bash-checkpoint-owned shell-quote-removal)))) + (test-case "checkpoint restores only for the same source and machine" + (let* ((source "< receipt) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (restore-contextual-scan-state other-source receipt) + #f)) + => "contextual scanner checkpoint identity mismatch") + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (restore-contextual-scan-state + scanner + (map (lambda (row) + (if (eq? (car row) 'byteOffset) + (cons 'byteOffset 1) row)) + receipt)) + #f)) + => "invalid contextual scanner checkpoint"))))))))) diff --git a/t/contextual-dispatch-test.ss b/t/contextual-dispatch-test.ss new file mode 100644 index 00000000..79929b23 --- /dev/null +++ b/t/contextual-dispatch-test.ss @@ -0,0 +1,108 @@ +;;; -*- Gerbil -*- +;;; Compiler decisions for independently varying lexical and syntax axes. + +(import (only-in :std/test check test-case test-suite) + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-method make-contextual-role) + (only-in :gerbil-parser/src/compiler/contextual-dispatch + compile-contextual-dispatch contextual-dispatch-ref)) +(export contextual-dispatch-test) + +(def (method name mode position form result) + (make-contextual-method name mode position form result)) + +(def (cell ir mode position form) + (find (lambda (row) + (equal? (take row 3) (list mode position form))) + (contextual-dispatch-ref ir 'cells))) + +(def contextual-dispatch-test + (test-suite "contextual compiler dispatch" + (test-case "Bash mode and syntax position resolve together" + (let* ((base + (make-contextual-role + 'word-base + (list (method 'bare 'any 'any 'bare 'ordinary-word) + (method 'quoted 'any 'any 'quoted 'quoted-word)))) + (bash + (make-contextual-role + 'bash-command + (list (method 'reserved 'command 'command-start + 'bare 'reserved-word)))) + (ir (compile-contextual-dispatch + (list base bash) + '(command word) '(command-start argument) '(bare quoted)))) + (check (list-ref (cell ir 'command 'command-start 'bare) 3) + => '(reserved-word ((bash-command reserved)))) + (check (list-ref (cell ir 'command 'argument 'bare) 3) + => '(ordinary-word ((word-base bare)))) + (check (list-ref (cell ir 'word 'argument 'quoted) 3) + => '(quoted-word ((word-base quoted)))))) + (test-case "the same compiler resolves an HCL template context" + (let* ((role + (make-contextual-role + 'hcl-template + (list (method 'literal 'any 'any 'text 'template-text) + (method 'interpolation 'template 'expression + 'dollar-open 'interpolation-open)))) + (ir (compile-contextual-dispatch + (list role) + '(template expression) '(text expression) + '(text dollar-open)))) + (check (list-ref (cell ir 'template 'expression 'dollar-open) 3) + => '(interpolation-open ((hcl-template interpolation)))) + (check (list-ref (cell ir 'template 'text 'text) 3) + => '(template-text ((hcl-template literal)))))) + (test-case "incomparable methods with different results fail closed" + (let ((by-mode + (make-contextual-role + 'by-mode (list (method 'm 'command 'any 'bare 'word)))) + (by-position + (make-contextual-role + 'by-position + (list (method 'p 'any 'command-start 'bare 'keyword))))) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (compile-contextual-dispatch + (list by-mode by-position) + '(command) '(command-start) '(bare)) + #f)) + => "ambiguous contextual dispatch"))) + (test-case "a same-result tie retains both method origins" + (let* ((by-mode + (make-contextual-role + 'by-mode (list (method 'm 'command 'any 'bare 'word)))) + (by-position + (make-contextual-role + 'by-position + (list (method 'p 'any 'command-start 'bare 'word)))) + (ir (compile-contextual-dispatch + (list by-mode by-position) + '(command) '(command-start) '(bare)))) + (check (list-ref (cell ir 'command 'command-start 'bare) 3) + => '(word ((by-mode m) (by-position p)))))) + (test-case "method result and role identity bind the digest" + (let* ((a (make-contextual-role + 'one (list (method 'm 'any 'any 'bare 'word)))) + (b (make-contextual-role + 'one (list (method 'm 'any 'any 'bare 'keyword)))) + (c (make-contextual-role + 'two (list (method 'm 'any 'any 'bare 'word)))) + (compile + (lambda (role) + (compile-contextual-dispatch + (list role) '(command) '(command-start) '(bare))))) + (check (string=? + (contextual-dispatch-ref (compile a) 'digest) + (contextual-dispatch-ref (compile a) 'digest)) + => #t) + (check (string=? + (contextual-dispatch-ref (compile a) 'digest) + (contextual-dispatch-ref (compile b) 'digest)) + => #f) + (check (string=? + (contextual-dispatch-ref (compile a) 'digest) + (contextual-dispatch-ref (compile c) 'digest)) + => #f))))) diff --git a/t/contextual-parser-test.ss b/t/contextual-parser-test.ss new file mode 100644 index 00000000..02ccc736 --- /dev/null +++ b/t/contextual-parser-test.ss @@ -0,0 +1,237 @@ +;;; -*- Gerbil -*- +;;; The generic contextual scanner supplies an existing LR ParserMachine. + +(import (only-in :std/test check test-case test-suite) + (only-in :gerbil-parser/languages/hcl/grammar + hcl-parser) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-grammar-digest parser-machine-contextual-ir parser-machine-ir) + (only-in :gerbil-parser/t/benchmarks/contextual-scanner/parser-fixture + contextual-product) + (only-in :gerbil-parser/src/runtime/parser + parse-source/contextual prepare-contextual-parser + parse-source/contextual/prepared) + (only-in :gerbil-parser/src/runtime/identity sha256-text) + (only-in :gerbil-parser/src/runtime/contextual-ir contextual-parser-ir-valid? contextual-ir-ref) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-events + parse-artifact-status parse-artifact-ref + parse-artifact-valid? parse-artifact-roundtrip + token-event? token-event-token-kind token-event-lexeme)) +(export contextual-parser-test) + +(def contextual-parser-test + (test-suite "contextual scanner LR integration" + (test-case "ordinary LR machine owns the common admitted instruction product" + (let (ir (parser-machine-contextual-ir hcl-parser)) + (check (contextual-parser-ir-valid? ir) => #t) + (check (contextual-ir-ref (contextual-ir-ref ir 'recognition) 'program) + => (parser-machine-ir hcl-parser)) + (set-cdr! (assq 'root-kind ir) 'Foreign) + (check (contextual-parser-ir-valid? (parser-machine-contextual-ir hcl-parser)) => #t))) + (test-case "raw and prepared parsers own product and published artifact identities" + (let* ((machine hcl-parser) + (literal (string-copy "1")) + (product (contextual-product machine (parser-machine-grammar-digest machine) + '(line) '((line () ())) literal)) + (plan (prepare-contextual-parser machine product)) + (sources '("x = 1\n" "名字 = 1\n" "x = " "x = @你好\n")) + (expected + (map (lambda (source) + (parse-source/contextual machine product source)) sources))) + (string-set! literal 0 #\2) + (set-cdr! (car (cdr (assq 'state-positions (cdr (assq 'context product))))) 'foreign) + (string-set! (cdr (assq 'digest product)) 0 #\x) + (let (artifact (parse-source/contextual/prepared plan (car sources))) + (string-set! (parse-artifact-ref artifact 'grammarDigest) 0 #\x)) + (for-each + (lambda (source reference) + (let (artifact (parse-source/contextual/prepared plan source)) + (check artifact => reference) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + sources expected) + (check (with-catch (lambda (condition) (error-message condition)) + (lambda () (parse-source/contextual machine product "x = 1\n") #f)) + => "contextual parser product does not match parser machine"))) + (test-case "prepared parser rejects mismatched products and raw plan inputs" + (check (with-catch (lambda (condition) (error-message condition)) + (lambda () (prepare-contextual-parser hcl-parser + (contextual-product hcl-parser "other")) #f)) + => "contextual parser product does not match parser machine") + (check (with-catch (lambda (condition) (error-message condition)) + (lambda () (parse-source/contextual/prepared '() "") #f)) + => "contextual parser requires prepared plan and source")) + (test-case "self-consistent non-symbol positions fail both runtime admissions" + (def (resign datum) + (let (body (filter (lambda (row) (not (eq? (car row) 'digest))) datum)) + (append body (list (cons 'digest (sha256-text + (call-with-output-string (lambda (port) (write body port))))))))) + (let* ((machine hcl-parser) + (product (contextual-product machine (parser-machine-grammar-digest machine))) + (position (string-copy "line")) + (scanner (resign + (map (lambda (row) (if (eq? (car row) 'positions) + (cons 'positions (list position)) row)) + (cdr (assq 'scanner product))))) + (altered (resign + (map (lambda (row) + (case (car row) + ((scanner) (cons 'scanner scanner)) + ((context) + (cons 'context + (list (cons 'state-positions + (map (lambda (entry) (cons (car entry) position)) + (cdr (assq 'state-positions (cdr row)))))))) + (else row))) product)))) + (for-each + (lambda (admit) + (check (with-catch (lambda (condition) (error-message condition)) + (lambda () (admit) #f)) + => "invalid contextual LR position table")) + (list (lambda () (prepare-contextual-parser machine altered)) + (lambda () (parse-source/contextual machine altered "x = 1\n")))))) + (test-case "HCL grammar accepts tokens from the generic scanner" + (let* ((source "x = 1\n") + (machine hcl-parser) + (product + (contextual-product machine + (parser-machine-grammar-digest machine))) + (artifact + (parse-source/contextual machine product source)) + (tokens (filter token-event? (parse-artifact-events artifact)))) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-ref artifact 'grammarDigest) + => (cdr (assq 'digest product))) + (check (map token-event-token-kind tokens) + => '(identifier horizontal-whitespace punctuation + horizontal-whitespace number newline)) + (check (apply string-append + (map token-event-lexeme tokens)) + => source))) + (test-case "LR rejection remains a rejected ParseArtifact" + (let* ((machine hcl-parser) + (artifact + (parse-source/contextual + machine + (contextual-product + machine (parser-machine-grammar-digest machine)) + "x = "))) + (check (parse-artifact-status artifact) => 'rejected) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => "x = "))) + (test-case "rejected lookahead and unscanned suffix retain UTF8 coverage" + (let* ((machine hcl-parser) + (product (contextual-product + machine (parser-machine-grammar-digest machine)))) + (for-each + (lambda (source) + (let (artifact (parse-source/contextual machine product source)) + (check (parse-artifact-status artifact) => 'rejected) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + '("x x = 1\n" "x = @你好\n")))) + (test-case "different grammar identity is rejected before scanning" + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (parse-source/contextual + hcl-parser + (contextual-product hcl-parser "other") "x = 1\n") + #f)) + => "contextual parser product does not match parser machine")) + (test-case "scanner change alters the published grammar identity" + (let* ((machine hcl-parser) + (base (parser-machine-grammar-digest machine)) + (first (contextual-product machine base)) + (second (contextual-product + machine base '(line) '((line () ())) "2"))) + (check (equal? (cdr (assq 'digest first)) + (cdr (assq 'digest second))) => #f))) + (test-case "token requirements select a specific LR position" + (let* ((machine hcl-parser) + (product + (contextual-product + machine (parser-machine-grammar-digest machine) + '(line name) '((line () ()) (name ((token identifier)) ())))) + (positions (cdr (assq 'state-positions (cdr (assq 'context product)))))) + (check (cdr (assq 0 positions)) => 'name) + (check (any (lambda (row) (eq? (cdr row) 'line)) positions) => #t) + (check (parse-artifact-success? + (parse-source/contextual machine product "x = 1\n")) + => #t))) + (test-case "literal requirements select grammar-owned LR positions" + (let* ((machine hcl-parser) + (product + (contextual-product + machine (parser-machine-grammar-digest machine) + '(line assignment) + '((line () ()) (assignment ((literal "=")) ())))) + (positions (cdr (assq 'state-positions (cdr (assq 'context product)))))) + (check (any (lambda (row) (eq? (cdr row) 'assignment)) positions) => #t) + (check (parse-artifact-success? + (parse-source/contextual machine product "x = 1\n")) + => #t))) + (test-case "EOF requirements select an empty-input parser position" + (let* ((machine hcl-parser) + (product + (contextual-product + machine (parser-machine-grammar-digest machine) + '(line finish) '((line () ()) (finish ((eof)) ())))) + (positions (cdr (assq 'state-positions (cdr (assq 'context product)))))) + (check (any (lambda (row) (eq? (cdr row) 'finish)) positions) => #t) + (check (parse-artifact-success? + (parse-source/contextual machine product "")) => #t))) + (test-case "duplicate LR state positions fail runtime preparation" + (let* ((machine hcl-parser) + (product (contextual-product + machine (parser-machine-grammar-digest machine))) + (rows (cdr (assq 'state-positions (cdr (assq 'context product))))) + (body + (map (lambda (row) + (if (eq? (car row) 'context) + (cons 'context (list (cons 'state-positions + (cons (car rows) (cons (car rows) (cddr rows)))))) + row)) + (filter (lambda (row) (not (eq? (car row) 'digest))) product))) + (altered + (append body + (list (cons 'digest + (sha256-text + (call-with-output-string + (lambda (port) (write body port))))))))) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () (parse-source/contextual machine altered "x = 1\n") #f)) + => "invalid contextual LR position table") + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () (prepare-contextual-parser machine altered) #f)) + => "invalid contextual LR position table"))) + (test-case "unknown lookaheads fail declaration admission" + (let ((machine hcl-parser)) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (contextual-product + machine (parser-machine-grammar-digest machine) + '(line unknown) + '((line () ()) (unknown ((literal "@@")) ()))) + #f)) + => "invalid contextual LR position declaration"))) + (test-case "incomparable LR position declarations are rejected" + (let* ((machine hcl-parser) + (base (parser-machine-grammar-digest machine))) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (contextual-product + machine base '(line other) + '((line () ()) (other () ()))) + #f)) + => "unresolved contextual LR position"))))) diff --git a/t/contextual-scanner-test.ss b/t/contextual-scanner-test.ss new file mode 100644 index 00000000..0a6c49d8 --- /dev/null +++ b/t/contextual-scanner-test.ss @@ -0,0 +1,507 @@ +;;; -*- Gerbil -*- +;;; Two language declaration shapes execute through one engine scanner. + +(import (only-in :std/test check test-case test-suite) + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-method make-contextual-role + make-contextual-scan-rule) + (only-in :gerbil-parser/src/compiler/contextual-dispatch + compile-contextual-dispatch) + (only-in :gerbil-parser/src/compiler/contextual-scanner-ir + compile-contextual-scanner contextual-scanner-ir-ref) + (only-in :gerbil-parser/src/runtime/contextual-scanner + prepare-contextual-scanner prepare-contextual-scanner-plan contextual-scanner-initial-state + contextual-scanner-step contextual-scan-state-byte-offset + contextual-scan-state-mode contextual-scan-state-with-mode + contextual-scan-state-canonical) + (only-in :gerbil-parser/src/runtime/token + token-end token-kind token-lexeme token-start) + (only-in :gerbil-parser/src/runtime/identity sha256-text)) +(export contextual-scanner-test contextual-literal-width-test) + +(def (method name mode position form result) + (make-contextual-method name mode position form result)) + +(def (rule name mode form matcher rank (action 'keep)) + (make-contextual-scan-rule name mode form matcher rank action)) + +(def bash-operators + '(";;&" "&>>" "<<-" "<<<" "&&" "||" "|&" ";;" ";&" + "<<" ">>" "<>" "<&" ">&" ">|" "&>" ";" "&" "|" "(" ")" + "<" ">")) + +(def bash-pairs + (list (list "${" #\{ #\}) + (list "$(" #\( #\)) + (list "<(" #\( #\)) + (list ">(" #\( #\)))) + +(def (bash-scanner source) + (let* ((role + (make-contextual-role + 'bash-lexical + (list (method 'operator 'any 'any 'operator 'operator) + (method 'word 'any 'any 'word 'word) + (method 'reserved 'command 'command-start + 'keyword 'reserved-word) + (method 'keyword-argument 'any 'any 'keyword 'word) + (method 'space 'any 'any 'space 'horizontal-whitespace) + (method 'newline 'any 'any 'newline 'newline)))) + (dispatch + (compile-contextual-dispatch + (list role) '(command) '(command-start argument) + '(operator word keyword space newline))) + (rules + (list (rule 'operator 'command 'operator + (list 'literals bash-operators) 20) + (rule 'reserved-if 'command 'keyword '(literal "if") 10) + (rule 'word 'command 'word + (list 'balanced-word bash-operators + '("'" "\"" "`") bash-pairs) 0) + (rule 'space 'command 'space '(horizontal-whitespace+) 0) + (rule 'newline 'command 'newline '(newline) 0))) + (ir (compile-contextual-scanner rules dispatch 'command))) + (values (prepare-contextual-scanner ir source) ir))) + +(def (hcl-template-scanner source) + (let* ((role + (make-contextual-role + 'hcl-template + (list (method 'open 'any 'any 'open 'interpolation-open) + (method 'text 'any 'any 'text 'template-text) + (method 'space 'any 'any 'space 'template-text) + (method 'identifier 'any 'any 'identifier 'identifier) + (method 'close 'any 'any 'close 'interpolation-close)))) + (dispatch + (compile-contextual-dispatch + (list role) '(template expression) '(text expression) + '(open text space identifier close))) + (rules + (list (rule 'open 'template 'open '(literal "${") 10) + (rule 'text 'template 'text + '(balanced-word ("${" "}") ("\"") ()) 0) + (rule 'space 'template 'space '(horizontal-whitespace+) 0) + (rule 'identifier 'expression 'identifier '(identifier) 0) + (rule 'close 'expression 'close '(literal "}") 0))) + (ir (compile-contextual-scanner rules dispatch 'template))) + (values (prepare-contextual-scanner ir source) ir))) + +(def (with-cells ir cells) + (let (body (map (lambda (row) (if (eq? (car row) 'cells) (cons 'cells cells) row)) + (filter (lambda (row) (not (eq? (car row) 'digest))) ir))) + (append body (list (cons 'digest (sha256-text + (call-with-output-string (lambda (port) (write body port))))))))) + +(def (plan-fixture) + (let* ((literal (string-copy "if")) + (role (make-contextual-role 'plan-fixture + (list (method 'literal 'any 'any 'literal 'keyword) + (method 'word 'any 'any 'word 'word)))) + (dispatch (compile-contextual-dispatch (list role) (list 'normal) + (list 'token) (list 'literal 'word))) + (ir (compile-contextual-scanner + (list (rule 'literal 'normal 'literal (list 'literal literal) 10) + (rule 'word 'normal 'word '(identifier) 0)) dispatch 'normal))) + (values ir literal))) + +(def contextual-scanner-test + (test-suite "generic contextual scanner" + (test-case "shared guarded matchers retain rule rank, action and independent request state" + (let* ((role (make-contextual-role 'shared + (list (method 'normal 'normal 'any 'redirect 'redirect) + (method 'plain 'plain 'any 'redirect 'plain)))) + (dispatch (compile-contextual-dispatch + (list role) '(normal plain) '(token) '(redirect))) + (catalog (map (lambda (n) (string-append "padding" (number->string n))) (iota 128))) + (ir (compile-contextual-scanner + (list (rule 'expect 'normal 'redirect '(unless-prefix ("<<<") () (literal "<<")) 20 '(expect-marker #f)) + (rule 'lower 'normal 'redirect '(unless-prefix ("<<<") () (literal "<<")) 0) + (rule 'plain 'plain 'redirect '(unless-prefix ("<<<") () (literal "<<")) 0) + (rule 'catalog 'normal 'redirect (list 'literals catalog) -1)) + dispatch 'normal)) + (plan (prepare-contextual-scanner-plan ir))) + (for-each + (lambda (source) + (let ((a (prepare-contextual-scanner plan source)) + (b (prepare-contextual-scanner plan source))) + (let-values (((token next) (contextual-scanner-step + a (contextual-scanner-initial-state a) 'token))) + (check (token-kind token) => 'redirect) + (check (cdr (assq 'expecting (contextual-scan-state-canonical next))) => 'plain)) + (let-values (((token next) (contextual-scanner-step b + (contextual-scan-state-with-mode + (contextual-scanner-initial-state b) 'plain) 'token))) + (check (token-kind token) => 'plain) + (check (token-lexeme token) => "<<") + (check (cdr (assq 'expecting (contextual-scan-state-canonical next))) => #f)))) + (list "<<" (string-append "<<" (make-string 64 #\space)))))) + (test-case "prepared prefixes preserve full-row outcomes and checkpoint receipts" + (for-each + (lambda (source) + (let-values (((linear ir) (bash-scanner source))) + (let ((indexed (prepare-contextual-scanner (prepare-contextual-scanner-plan ir) source))) + (let loop ((a (contextual-scanner-initial-state linear)) + (b (contextual-scanner-initial-state indexed))) + (let-values (((left next-left) (contextual-scanner-step linear a 'argument)) + ((right next-right) (contextual-scanner-step indexed b 'argument))) + (check (and left (list (token-kind left) (token-lexeme left) + (token-start left) (token-end left))) + => (and right (list (token-kind right) (token-lexeme right) + (token-start right) (token-end right)))) + (check (contextual-scan-state-canonical next-left) + => (contextual-scan-state-canonical next-right)) + (when left (loop next-left next-right))))))) + '("" "if αβ && name\n" "echo \"α\" |& next\r\n" "α $(echo x) >> file\n"))) + (test-case "scanner plans own IR and bind independent checkpoint identities" + (let-values (((ir literal) (plan-fixture))) + (let* ((digest (string-copy (cdr (assq 'digest ir)))) + (plan (prepare-contextual-scanner-plan ir)) + (a (prepare-contextual-scanner plan "if")) + (receipt (contextual-scan-state-canonical (contextual-scanner-initial-state a)))) + (string-set! literal 0 #\x) + (set-car! (cdr (assq 'modes ir)) 'foreign) + (string-set! (cdr (assq 'digest ir)) 0 #\x) + (string-set! (cdr (assq 'scannerDigest receipt)) 0 #\x) + (check + (with-catch (lambda (condition) (error-message condition)) + (lambda () (prepare-contextual-scanner ir "if") #f)) + => "contextual scanner requires compiled IR and source") + (let* ((b (prepare-contextual-scanner plan "if")) + (initial (contextual-scanner-initial-state b)) + (canonical (contextual-scan-state-canonical initial))) + (check (cdr (assq 'scannerDigest canonical)) => digest) + (let-values (((token next) (contextual-scanner-step b initial 'token))) + (check (token-kind token) => 'keyword) + (check (token-lexeme token) => "if"))) + (let (b (prepare-contextual-scanner plan "αβ")) + (let-values (((token next) (contextual-scanner-step b (contextual-scanner-initial-state b) 'token))) + (check (token-kind token) => 'word) + (check (token-end token) => 4)))))) + (test-case "scanner plans reject cycles and callback data" + (let (cycle (list 'cycle)) + (set-cdr! cycle cycle) + (check (with-catch (lambda (condition) (error-message condition)) + (lambda () (prepare-contextual-scanner-plan cycle) #f)) + => "cyclic contextual scanner plan IR")) + (check (with-catch (lambda (condition) (error-message condition)) + (lambda () (prepare-contextual-scanner-plan (list (lambda () #f))) #f)) + => "contextual scanner plan requires closed IR data")) + (test-case "dispatch indexing preserves negative sparse and duplicate cells" + (let-values (((_scanner ir) (bash-scanner ";"))) + (let* ((cells (cdr (assq 'cells ir))) + (target (find (lambda (cell) (equal? (take cell 3) '(command argument operator))) cells)) + (negative (map (lambda (cell) (if (eq? cell target) + (append (take cell 3) '(#f)) cell)) cells)) + (sparse (filter (lambda (cell) (not (eq? cell target))) cells))) + (for-each + (lambda (rows) + (let (scanner (prepare-contextual-scanner (with-cells ir rows) ";")) + (check + (with-catch (lambda (condition) (error-message condition)) + (lambda () (contextual-scanner-step scanner + (contextual-scanner-initial-state scanner) 'argument) #f)) + => "missing contextual scanner dispatch"))) + (list negative sparse)) + (check + (with-catch (lambda (condition) (error-message condition)) + (lambda () (prepare-contextual-scanner (with-cells ir (cons target negative)) ";") #f)) + => "duplicate contextual scanner dispatch cell") + (let* ((wide (append cells + (map (lambda (n) (list 'command 'argument + (string->symbol (string-append "extra" (number->string n))) + '(unused))) (iota 9)))) + (single (filter (lambda (cell) (eq? (cadr cell) 'argument)) cells)) + (structural (cons (list '(foreign mode) 'argument 'foreign '(unused)) cells))) + (for-each + (lambda (rows) + (let (scanner (prepare-contextual-scanner (with-cells ir rows) ";")) + (let-values (((token next) (contextual-scanner-step scanner + (contextual-scanner-initial-state scanner) 'argument))) + (check (token-kind token) => 'operator) + (check (token-lexeme token) => ";")))) + (list wide single structural)))))) + (test-case "position admission rejects undeclared identities even at EOF" + (let-values (((scanner _ir) (bash-scanner "if"))) + (let (initial (contextual-scanner-initial-state scanner)) + (check + (with-catch (lambda (condition) (error-message condition)) + (lambda () (contextual-scanner-step scanner initial (gensym 'argument)) #f)) + => "invalid contextual scanner checkpoint or position") + (let-values (((token next) (contextual-scanner-step scanner initial 'argument))) + (check (token-kind token) => 'word) + (check + (with-catch (lambda (condition) (error-message condition)) + (lambda () (contextual-scanner-step scanner next 'undeclared) #f)) + => "invalid contextual scanner checkpoint or position") + (let-values (((token final) (contextual-scanner-step scanner next 'command-start))) + (check token => #f) + (check (contextual-scan-state-byte-offset final) => 2)))))) + (test-case "Bash nested word scans without a language callback" + (let-values (((scanner ir) + (bash-scanner "if printf %s \"${x:-$(printf '%s' '}')}\"\n"))) + (let loop ((state (contextual-scanner-initial-state scanner)) + (position 'command-start) + (tokens '())) + (let-values (((token next) + (contextual-scanner-step scanner state position))) + (if token + (loop next 'argument (cons token tokens)) + (let (found (reverse tokens)) + (check (map token-kind found) + => '(reserved-word horizontal-whitespace word + horizontal-whitespace word + horizontal-whitespace word newline)) + (check (token-lexeme (list-ref found 6)) + => "\"${x:-$(printf '%s' '}')}\"") + (check (contextual-scan-state-byte-offset next) + => (u8vector-length + (string->utf8 + "if printf %s \"${x:-$(printf '%s' '}')}\"\n"))) + (check (string? (contextual-scanner-ir-ref ir 'digest)) + => #t))))))) + (test-case "HCL template and expression modes share the executor" + (let-values (((scanner _ir) (hcl-template-scanner "hello ${name} α"))) + (let* ((initial (contextual-scanner-initial-state scanner)) + (tokens '())) + (def (take state position) + (let-values (((token next) + (contextual-scanner-step scanner state position))) + (set! tokens (append tokens (list token))) + next)) + (let* ((a (take initial 'text)) + (b (take a 'text)) + (c (take b 'text)) + (d (take (contextual-scan-state-with-mode c 'expression) + 'expression)) + (e (take d 'expression)) + (f (take (contextual-scan-state-with-mode e 'template) 'text)) + (g (take f 'text))) + (check (map token-kind tokens) + => '(template-text template-text interpolation-open + identifier interpolation-close template-text + template-text)) + (check (contextual-scan-state-mode g) => 'template) + (check (token-start (list-ref tokens 6)) => (token-end (list-ref tokens 5))) + (check (contextual-scan-state-byte-offset g) + => (u8vector-length (string->utf8 "hello ${name} α"))))))) + (test-case "literal catalogs preserve longest complete Unicode prefixes across modes" + (let* ((role (make-contextual-role 'prefixes + (list (method 'literal 'any 'any 'literal 'literal)))) + (dispatch (compile-contextual-dispatch + (list role) '(first second) '(token) '(literal))) + (catalog (append '("α" "αβ" "αβγ") + (map (lambda (n) (string-append "padding" (number->string n))) + (iota 128)))) + (ir (compile-contextual-scanner + (list (rule 'first 'first 'literal (list 'literals catalog) 0) + (rule 'second 'second 'literal (list 'literals catalog) 0)) + dispatch 'first)) + (scanner (prepare-contextual-scanner (prepare-contextual-scanner-plan ir) + (string-append "αβαβx" (make-string 64 #\x))))) + (let-values (((a next) (contextual-scanner-step + scanner (contextual-scanner-initial-state scanner) 'token))) + (check (token-lexeme a) => "αβ") + (check (list (token-start a) (token-end a)) => '(0 4)) + (let-values (((b last) (contextual-scanner-step + scanner (contextual-scan-state-with-mode next 'second) 'token))) + (check (token-lexeme b) => "αβ") + (check (list (token-start b) (token-end b)) => '(4 8)) + (check (contextual-scan-state-byte-offset last) => 8) + (check + (with-catch (lambda (condition) (error-message condition)) + (lambda () (contextual-scanner-step scanner last 'token) #f)) + => "contextual scanner has no match"))))) + (test-case "closed guards retain Unicode exceptions and prefix rejection" + (let* ((role (make-contextual-role 'guarded + (list (method 'word 'any 'any 'word 'word) + (method 'literal 'any 'any 'literal 'literal)))) + (dispatch (compile-contextual-dispatch + (list role) '(normal) '(token) '(word literal))) + (ir (compile-contextual-scanner + (list (rule 'guarded-word 'normal 'word + '(unless-prefix ("ab" "λ") ("abc" "λx") (identifier)) 10) + (rule 'fallback 'normal 'literal '(literals ("ab" "λ")) 0)) + dispatch 'normal))) + (for-each + (lambda (input) + (for-each + (lambda (recipe) + (let (scanner (prepare-contextual-scanner recipe (car input))) + (let-values (((token next) (contextual-scanner-step + scanner (contextual-scanner-initial-state scanner) 'token))) + (check (list (token-kind token) (token-lexeme token) (token-end token)) + => (cdr input))))) + (list ir (prepare-contextual-scanner-plan ir)))) + '(("abc" word "abc" 3) ("abx" literal "ab" 2) + ("λx" word "λx" 3) ("λz" literal "λ" 2) + ("z" word "z" 1))))) + (test-case "candidate selection preserves length before rank in either declaration order" + (let* ((role (make-contextual-role 'selection + (list (method 'short 'any 'any 'short 'short) + (method 'long 'any 'any 'long 'long)))) + (dispatch (compile-contextual-dispatch (list role) '(normal) '(token) '(short long)))) + (for-each + (lambda (rows) + (let (ir (compile-contextual-scanner (map (lambda (row) (apply rule row)) rows) dispatch 'normal)) + (for-each + (lambda (recipe) + (let (scanner (prepare-contextual-scanner recipe "λx")) + (let-values (((token next) (contextual-scanner-step scanner + (contextual-scanner-initial-state scanner) 'token))) + (check (list (token-kind token) (token-lexeme token) (token-end token) + (contextual-scan-state-byte-offset next)) => '(long "λx" 3 3))))) + (list ir (prepare-contextual-scanner-plan ir))))) + '(((short normal short (literal "λ") 100) (long normal long (literal "λx") 0)) + ((long normal long (literal "λx") 0) (short normal short (literal "λ") 100)) + ((short normal short (literal "λx") 0) (long normal long (literal "λx") 1)) + ((long normal long (literal "λx") 1) (short normal short (literal "λx") 0)))))) + (test-case "candidate conflicts remain eager even before a longer later match" + (let* ((role (make-contextual-role 'selection + (list (method 'short 'any 'any 'short 'short) + (method 'long 'any 'any 'long 'long)))) + (dispatch (compile-contextual-dispatch (list role) '(normal) '(token) '(short long))) + (ir (compile-contextual-scanner + (list (rule 'first 'normal 'short '(literal "λ") 0) + (rule 'conflict 'normal 'long '(literals ("λ" "Ω")) 0) + (rule 'later 'normal 'long '(literal "λx") 0)) dispatch 'normal))) + (for-each + (lambda (recipe) + (let (scanner (prepare-contextual-scanner recipe "λx")) + (check (with-catch error-message + (lambda () (contextual-scanner-step scanner + (contextual-scanner-initial-state scanner) 'token) #f)) + => "ambiguous contextual scanner match"))) + (list ir (prepare-contextual-scanner-plan ir))))) + (test-case "equivalent ties preserve the first rule in conflict diagnostics" + (let* ((role (make-contextual-role 'selection + (list (method 'short 'any 'any 'short 'short) + (method 'long 'any 'any 'long 'long)))) + (dispatch (compile-contextual-dispatch (list role) '(normal) '(token) '(short long))) + (ir (compile-contextual-scanner + (list (rule 'first 'normal 'short '(literal "λ") 0) + (rule 'equivalent 'normal 'short '(literals ("λ" "Ω")) 0) + (rule 'conflict 'normal 'long '(unless-prefix ("Ω") () (literal "λ")) 0)) + dispatch 'normal))) + (for-each + (lambda (recipe) + (let (scanner (prepare-contextual-scanner recipe "λ")) + (check (with-catch error-irritants + (lambda () (contextual-scanner-step scanner + (contextual-scanner-initial-state scanner) 'token) #f)) + => '(first conflict)))) + (list ir (prepare-contextual-scanner-plan ir))))) + (test-case "equal token forms do not hide conflicting scanner actions" + (let* ((role (make-contextual-role + 'redirect + (list (method 'redirect 'any 'any 'redirect 'redirect)))) + (dispatch (compile-contextual-dispatch + (list role) '(command) '(command) '(redirect))) + (ir (compile-contextual-scanner + (list (rule 'open 'command 'redirect '(literal "<<") 0 + '(expect-marker #f)) + (rule 'plain 'command 'redirect + '(literals ("<<")) 0 'keep)) + dispatch 'command)) + (scanner (prepare-contextual-scanner (prepare-contextual-scanner-plan ir) "<<"))) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (contextual-scanner-step + scanner (contextual-scanner-initial-state scanner) 'command) + #f)) + => "ambiguous contextual scanner match"))) + (test-case "input IR cannot admit the private prepared matchers under guards" + (for-each (lambda (matcher) + (let-values (((_scanner ir) (bash-scanner "if"))) + (let* ((body + (map (lambda (row) + (if (eq? (car row) 'rules) + (cons 'rules + (map (lambda (rule) + (list (car rule) (cadr rule) (caddr rule) + matcher + (list-ref rule 4) (list-ref rule 5))) + (cdr row))) row)) + (filter (lambda (row) (not (eq? (car row) 'digest))) ir))) + (altered (append body + (list (cons 'digest + (sha256-text (call-with-output-string + (lambda (port) (write body port))))))))) + (check + (with-catch (lambda (condition) (error-message condition)) + (lambda () (prepare-contextual-scanner altered "if") #f)) + => "private contextual scanner matcher in input IR")))) + '((literal-trie 42) (prepared-region 42) + (unless-prefix ("#") () (literal-trie 42)) + (unless-prefix ("#") () (prepared-region 42))))) + (test-case "a self-consistent digest cannot admit a foreign opcode contract" + (let-values (((_scanner ir) (bash-scanner "if"))) + (let* ((body + (map (lambda (row) + (if (eq? (car row) 'opcode-contract) + (cons 'opcode-contract "foreign-opcodes.v1") row)) + (filter (lambda (row) (not (eq? (car row) 'digest))) ir))) + (altered + (append body + (list (cons 'digest + (sha256-text + (call-with-output-string + (lambda (port) (write body port))))))))) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () (prepare-contextual-scanner altered "if") #f)) + => "contextual scanner requires compiled IR and source")))) + (test-case "compiler rejects altered dispatch and duplicate scan candidates" + (let* ((role (make-contextual-role + 'validation + (list (method 'first 'any 'any 'first 'first) + (method 'second 'any 'any 'second 'second)))) + (dispatch (compile-contextual-dispatch + (list role) '(normal) '(start) '(first second))) + (first (rule 'first 'normal 'first '(literal "x") 0)) + (second (rule 'second 'normal 'second '(literal "x") 0)) + (altered + (map (lambda (row) + (if (eq? (car row) 'cells) + (cons 'cells '()) row)) + dispatch))) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (compile-contextual-scanner (list first second) + dispatch 'normal) + #f)) + => "duplicate contextual scan candidate") + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (compile-contextual-scanner (list first) altered 'normal) + #f)) + => "invalid contextual scanner declaration"))))) + +(def contextual-literal-width-test + (test-suite "prepared literal scalar byte widths" + (test-case "ASCII and every Unicode width retain byte endpoints" + (let* ((role (make-contextual-role 'widths + (list (method 'literal 'any 'any 'literal 'literal)))) + (dispatch (compile-contextual-dispatch (list role) '(normal) '(token) '(literal))) + (ir (compile-contextual-scanner + (map (lambda (text) (rule (string->symbol text) 'normal 'literal + (list 'literal text) 0)) + '("a" "α" "猫" "😀")) dispatch 'normal))) + (for-each (lambda (input) + (let ((scanner (prepare-contextual-scanner input "aα猫😀")) (start 0)) + (let loop ((state (contextual-scanner-initial-state scanner)) + (expected '(1 3 6 10))) + (unless (null? expected) + (let-values (((token next) (contextual-scanner-step scanner state 'token))) + (check (token-start token) => start) + (check (token-end token) => (car expected)) + (check (contextual-scan-state-byte-offset next) => (car expected)) + (set! start (car expected)) + (loop next (cdr expected))))))) + (list ir (prepare-contextual-scanner-plan ir))))))) diff --git a/t/core-observability-test.ss b/t/core-observability-test.ss index 8fe12851..6736671a 100644 --- a/t/core-observability-test.ss +++ b/t/core-observability-test.ss @@ -9,7 +9,7 @@ :asp-gerbil-scheme/src/benchmark/framework :core/observability/debug :gerbil-parser/src/modules/parser/interface - "./scenarios/observability/opencypher-grammar-phases/scenario") + (only-in :gerbil-parser/t/fixtures/progress report-test-progress!)) (def benchmark-path "t/benchmarks/poo-grammar-objects/benchmark.ss") @@ -24,7 +24,7 @@ (def (make-observed-grammar-batch) (let loop ((remaining 50) (grammars '())) (if (zero? remaining) - grammars + (begin (report-test-progress! "OBJECT-BATCH-OK grammars=50") grammars) (loop (- remaining 1) (cons (make-observed-grammar) grammars))))) @@ -80,12 +80,14 @@ make-observed-grammar-batch)) (write receipt) (newline) - (check (benchmark-contract-receipt-pass? receipt) => #t))) - (test-case "LanguageGrammar atomically configures POO Flow phase observation" - (let (receipt (opencypher-grammar-observability-scenario)) - (write receipt) - (newline) - (check (opencypher-grammar-observability-scenario-pass? receipt) - => #t))))) + (check (benchmark-contract-receipt-pass? receipt) => #t))))) (export core-observability-tests) + +;; gxtest discovers only exported names ending in -test. +(def core-observability-test core-observability-tests) +(export core-observability-test) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/engine-controls.ss b/t/engine-controls.ss new file mode 100644 index 00000000..73bc6c89 --- /dev/null +++ b/t/engine-controls.ss @@ -0,0 +1,21 @@ +;;; -*- Gerbil -*- +;;; Compile source-owned suites into one std/test entry; no interpreter +;;; module expansion or child-process scheduler belongs to these controls. +(import (only-in :std/test/base TestHarness TestModule TestConfig test-run! test-result-ok?) + (only-in ./event-fold-scheme-test event-fold-scheme-test) + (only-in ./ranked-regular-scanner-test ranked-regular-scanner-test) + (only-in ./parser-runtime-test parser-runtime-test)) +(export main) + +(def (main . args) + (let* ((modules + (map (lambda (entry) + (TestModule (car entry) (list (cdr entry)) '() void void)) + (list (cons "fold-source" event-fold-scheme-test) + (cons "ranked-regular-scanner" ranked-regular-scanner-test) + (cons "parser-runtime" parser-runtime-test)))) + (result (test-run! (TestHarness "Engine controls" + (TestConfig verbosity: 5 capture-output?: #f) modules)))) + (if (test-result-ok? result) + (begin (displayln "OK") (force-output)) + (begin (displayln "FAILED engine controls") (force-output) (exit 42))))) diff --git a/t/event-fold-analysis-test.ss b/t/event-fold-analysis-test.ss new file mode 100644 index 00000000..3379c782 --- /dev/null +++ b/t/event-fold-analysis-test.ss @@ -0,0 +1,36 @@ +(import :std/test ../src/compiler/event-fold-analysis) +(export event-fold-analysis-test) +(def event-fold-analysis-test + (test-suite "Fold dependence and effect plan" + (test-case "pure scan dominance is lexical and short-circuit safe" + (let ((scan '(line-skip-horizontal start))) + (check (fold-shared-scans `(and (offset-less? ,scan end) + (offset-less? ,scan (line-trim-end)))) => (list scan)) + (check (fold-shared-scans `(and (state ready) (offset-less? ,scan ,scan))) => '()) + (check (fold-shared-scans `(or (bool #t) (offset-less? ,scan ,scan))) => '()) + (check (fold-shared-scans `(and (offset-less? ,scan end) + (future-heading-title? "*" " " 1 "END") + (offset-less? ,scan end))) => '()) + (check (fold-rewrite-scans `(offset-less? ,scan end) (list (cons scan 'memo))) + => '(offset-less? (line-index memo) end)))) + (test-case "prefix guards cannot skip effects errors or unknown instructions" + (check (fold-prefix-guard '(and (state ready) (line-starts-with-ascii-ci "Ab"))) + => (vector 'start '(65 97))) + (check (fold-prefix-guard '(line-byte-equal? (line-index cursor) 97)) + => (vector '(line-index cursor) '(97))) + (check (fold-prefix-guard '(or (state ready) (line-starts-with "ab"))) => #f) + (check (fold-prefix-guard '(line-starts-with "")) => #f) + (check (fold-prefix-guard '(and (future-heading-title? "*" " " 1 "END") + (line-starts-with "ab"))) => #f) + (check (fold-prefix-guard '(and (line-byte-equal? (state-offset at) 97) + (line-starts-with "ab"))) => #f) + (check (fold-prefix-guard '(and (unknown-query) (line-starts-with "ab"))) => #f)) + (test-case "state effects frame changes and emissions remain distinct" + (let (facts (fold-statement-analysis + '((set-uint at (offset (state-offset previous))) + (with-source-bounds start end ((token Line start end))) + (call-source-helper nested start end)))) + (check (vector-ref facts 0) => '(previous)) + (check (vector-ref facts 1) => '(at)) + (for-each (lambda (effect) (check (not (not (memq effect (vector-ref facts 2)))) => #t)) + '(bounds range-error events helper)))))) diff --git a/t/event-fold-counters.ss b/t/event-fold-counters.ss new file mode 100644 index 00000000..4bc644f1 --- /dev/null +++ b/t/event-fold-counters.ss @@ -0,0 +1,17 @@ +;;; Explicit native catalog: execution parity plus build-time source admission. +(import (only-in :std/test/base + TestHarness TestConfig TestModule test-run! test-result-ok?) + (only-in :gerbil-parser/t/event-fold-test event-fold-test) + (only-in :gerbil-parser/t/event-fold-scheme-test event-fold-scheme-test)) +(export main) + +(def (main . args) + (let (result + (test-run! + (TestHarness "native event fold" + (TestConfig verbosity: 5 capture-output?: #f) + (list (TestModule "event fold" (list event-fold-test event-fold-scheme-test) + '() void void))))) + (unless (test-result-ok? result) (exit 1)) + (displayln "OK") + (force-output))) diff --git a/t/event-fold-fixture.ss b/t/event-fold-fixture.ss index 6eca6b87..f946ba68 100644 --- a/t/event-fold-fixture.ss +++ b/t/event-fold-fixture.ss @@ -2,7 +2,7 @@ ;;; One source-owned stateful parser algorithm for Scheme and Rust verification. (import (only-in "event-strategy-fixture.ss" event-lines-language-grammar) - (only-in :gerbil-parser/rust-rowan-event-support + (only-in :gerbil-parser/rust-runtime-event-support define-event-fold-parser)) (export parse-fold-lines parse_fold_lines parse-outline-lines parse_outline_lines) diff --git a/t/event-fold-scheme-test.ss b/t/event-fold-scheme-test.ss new file mode 100644 index 00000000..81382265 --- /dev/null +++ b/t/event-fold-scheme-test.ss @@ -0,0 +1,201 @@ +;;; -*- Gerbil -*- +;;; The explicit native catalog admits the compiler without per-case processes. +(import (only-in :std/test check check-exception test-case test-suite) + (only-in ./event-strategy-fixture event-lines-language-grammar) + (only-in ../src/compiler/event-fold-scheme event-fold-scheme-source) + (only-in ../src/compiler/event-fold-scheme-context current-fold-scheme-diagnostics) + (only-in ../src/compiler/event-fold-future fold-future-marker-before-boundary? + fold-future-named-index) + (only-in ../src/compiler/event-fold-prefix-family fold-scheme-prefix-family) + (only-in ../src/compiler/event-fold-scheme-modules event-fold-scheme-module-sources) + (only-in ../src/runtime/event-fold-lines fold-line-name-set-contains?) + (only-in ../src/runtime/byte-spans byte-span-skip byte-span-all?) + (only-in ../src/runtime/source-lines fold-source-lines fold-source-byte-lines + fold-source-byte-spans fold-source-frame-boundaries! fold-source-input-bytes)) +(export event-fold-scheme-test) + +(def (source initial line finish (helpers '())) + (event-fold-scheme-source 'native-product event-lines-language-grammar 'Document + initial line finish helpers)) + +(def event-fold-scheme-test + (test-suite "Native EventFold source admission" + (test-case "native byte frames preserve Unicode CR LF CRLF and terminal bounds" + (check-exception (fold-source-input-bytes 42) true) + (for-each + (lambda (invalid) (check-exception (fold-source-input-bytes invalid) true)) + (list (u8vector 255) (u8vector 192 128) (u8vector 237 160 128) + (u8vector 244 144 128 128) (u8vector 226 130) (u8vector 128))) + (check-exception (fold-source-byte-lines "not bytes" '() void) true) + (let (bytes (u8vector 99 32 9 97 98)) + (check (byte-span-skip bytes 1 5 (lambda (byte) (memv byte '(9 32)))) => 3) + (check (byte-span-skip bytes 3 3 void) => 3) + (check (byte-span-all? bytes 1 3 (lambda (byte) (memv byte '(9 32)))) => #t) + (check (byte-span-all? bytes 1 5 (lambda (byte) (memv byte '(9 32)))) => #f)) + (for-each + (lambda (text) + (let* ((bytes (string->utf8 text)) + (before (u8vector->list bytes)) + (oracle (fold-source-lines text '() + (lambda (line start end rows) + (cons (list line start end) rows)))) + (actual (fold-source-byte-lines bytes '() + (lambda (line view start end rows) + (check (string->utf8 line) => view) + (check (u8vector-length view) => (- end start)) + (cons (list line start end) rows))))) + (check (eq? (fold-source-input-bytes bytes) bytes) => #t) + (check (fold-source-input-bytes text) => bytes) + (check actual => oracle) + (check (fold-source-byte-spans bytes '() + (lambda (start end rows) + (fold-source-frame-boundaries! bytes start end) + (cons (list (utf8->string (subu8vector bytes start end)) start end) rows))) + => oracle) + (check (u8vector->list bytes) => before))) + '("" "abc" "\n" "\r" "\r\n" "é中🦀\r\nx\ry\nz" "a\n\n")) + (let (bytes (string->utf8 "é中🦀")) + (for-each (lambda (bounds) + (check-exception + (fold-source-frame-boundaries! bytes (car bounds) (cadr bounds)) true)) + '((-1 0) (0 10) (2 0) (1 2) (0 1) (2 3) (5 6))) + (for-each (lambda (bounds) + (fold-source-frame-boundaries! bytes (car bounds) (cadr bounds))) + '((0 9) (0 2) (2 5) (5 9) (1 1) (9 9))))) + (test-case "runtime marker encoding preserves boundaries and cache fast paths" + (def (query text target stop indent? heading?) + (fold-future-marker-before-boundary? + (string->utf8 text) 0 target stop 42 32 indent? heading? #f + (make-hash-table-eqv))) + (check (query "x\n EnD \r\n" "END" #f #t #f) => #t) + (check (query "x\n EnD \r\n" "END" #f #f #f) => #f) + (check (query "stop\nEND\n" "END" "STOP" #f #f) => #f) + (check (query "END\n" "END" "END" #f #f) => #f) + (check (query "* Heading\nEND\n" "END" #f #f #t) => #f) + (check (query "x\n中\n" "中" #f #f #f) => #t) + (check (fold-future-marker-before-boundary? + (string->utf8 "x\n中\n") 0 (string->utf8 "中") #f + 42 32 #f #f #f (make-hash-table-eqv)) => #t) + (check (fold-future-marker-before-boundary? + (string->utf8 "STOP\nEND\n") 0 (string->utf8 "END") (string->utf8 "STOP") + 42 32 #f #f #f (make-hash-table-eqv)) => #f) + (check (query "ENDING\n" "END" #f #f #f) => #f) + (let* ((bytes (string->utf8 " Stop\r\nEND n\nSTOP\n")) + (index (fold-future-named-index bytes "END " "" "STOP" + 42 32 #t #f #t #f #t))) + (check (vector->list (vector-ref index 2)) => '(14 0))) + (check (fold-future-marker-before-boundary? + (string->utf8 ":KEY: value\nEND\n") 0 "END" #f + 42 32 #f #f 58 (make-hash-table-eqv)) => #t) + (check (fold-future-marker-before-boundary? + (string->utf8 "ordinary body\nEND\n") 0 "END" #f + 42 32 #f #f 58 (make-hash-table-eqv)) => #f) + (let ((bytes (string->utf8 "END\n")) (cache (make-hash-table-eqv))) + (hash-put! cache 0 1) + ;; Invalid marker values deliberately prove that these paths never + ;; invoke the encoder, matching the original cached search semantics. + (check (fold-future-marker-before-boundary? + bytes 0 #f #f 42 32 #f #f #f cache) => #t) + (hash-put! cache 0 2) + (check (fold-future-marker-before-boundary? + bytes 0 #f #f 42 32 #f #f #f cache) => #f) + (check (fold-future-marker-before-boundary? + bytes (u8vector-length bytes) #f #f 42 32 #f #f #f cache) => #f))) + (test-case "generated source is deterministic and has no interpreted request path" + (for-each + (lambda (operator) + (let (generated + (source '() + `((if (,operator start (line-step start) (97 98)) + ((token Line start end)) ())) '())) + (check (not (string-contains generated "(let loop ((cursor from))")) => #t) + (check (not (not (string-contains generated "native fold range outside source line"))) => #t) + (check (not (not (string-contains generated "(u8vector-ref bytes from)"))) => #t))) + '(line-bytes-all-in? line-bytes-any-in?)) + (let* ((family '((line-starts-with-ascii-ci "#+TITLE:") + (line-starts-with-ascii-ci "#+AUTHOR:") + (line-starts-with-ascii-ci "#+DATE:") + (line-starts-with-ascii-ci "#+CAPTION:"))) + (code (fold-scheme-prefix-family family))) + (check (not (not code)) => #t) + (check code => (fold-scheme-prefix-family (reverse family))) + (check (fold-scheme-prefix-family (cons '(state ready) family)) => #f) + (check (fold-scheme-prefix-family (take family 3)) => #f) + (check (fold-scheme-prefix-family (make-list 33 (car family))) => #f) + (check (fold-scheme-prefix-family (make-list 4 (list 'line-starts-with (make-string 129 #\a)))) => #f)) + (let* ((forms '((if (line-bytes-all-in? start end (10 97)) + ((token Line start end)) ()))) + (first (source '() forms '()))) + (check (equal? first (source '() forms '())) => #t) + (check (not (string-contains first "call-with-fold-cost")) => #t) + (let (observed (parameterize ((current-fold-scheme-diagnostics #t)) + (source '() forms '()))) + (check (not (not (string-contains observed "call-with-fold-cost"))) => #t) + (check (not (not (string-contains observed "diagnostic Fold cannot bind"))) => #t)) + (let (counted (parameterize ((current-fold-scheme-diagnostics 'counts)) + (source '() forms '()))) + (check (not (not (string-contains counted "(fold-cost-count!"))) => #t) + (check (not (string-contains counted "(call-with-fold-cost '")) => #t) + (check (not (not (string-contains counted "diagnostic Fold cannot bind"))) => #t)) + (check (source '() forms '()) => first) + (for-each (lambda (forbidden) + (check (not (string-contains first forbidden)) => #t)) + '("event-fold-runtime" "fold-statements" "fold-predicate" "run-event-fold" "eval " + "current-fold-line-view" "fold-line-bytes" "(string->utf8 line)")) + (check (not (not (string-contains first "(def (bind-native-product expected-digest)"))) => #t) + (check (not (not (string-contains first "(reverse! (cons '(finish) events))"))) => #t) + (let (units (event-fold-scheme-module-sources + 'native-product event-lines-language-grammar 'Document + '() forms '() '() '() 1)) + (check units => (event-fold-scheme-module-sources + 'native-product event-lines-language-grammar 'Document + '() forms '() '() '() 1)) + (for-each + (lambda (unit) + ;; An empty import compiles, but runtime expander evaluation rejects it. + (check (not (string-contains (cdr unit) "(import)")) => #t) + (let (definitions + (call-with-input-string (cdr unit) + (lambda (port) + (let loop ((count 0)) + (let (form (read port)) + (if (eof-object? form) count + (loop (+ count + (if (and (pair? form) (eq? (car form) 'def) + (pair? (cadr form)) + (string-prefix? "%event-fold-" (symbol->string (caadr form)))) + 1 0))))))))) + (check (<= definitions 1) => #t))) + units)) + (let (names (list->vector + (map (lambda (index) (string (integer->char (+ 32 index)))) (iota 95)))) + (for-each + (lambda (name) + (let (bytes (string->utf8 (string-append "x" name "y"))) + (check (fold-line-name-set-contains? bytes 1 2 names) => #t))) + (vector->list names)) + (check (fold-line-name-set-contains? (string->utf8 "") 0 0 names) => #f) + (check (fold-line-name-set-contains? (string->utf8 "é") 0 2 names) => #f) + (check (fold-line-name-set-contains? (string->utf8 "aa") 0 2 names) => #f)))) + (test-case "helpers are module-level source-parameterized native procedures" + (let (generated + (source '() '((call-source-helper slice start end)) '() + '((slice () ((token Line start end)))))) + (check (not (not (string-contains generated + "(def (%event-fold-native-product-helper-0 source-bytes"))) => #t) + (check (not (string-contains generated "(utf8->string (subu8vector")) => #t) + (check (not (string-contains generated "subu8vector")) => #t) + (check (not (string-contains generated "utf8->string")) => #t) + (check (not (not (string-contains generated "fold-source-frame-boundaries!"))) => #t))) + (test-case "untyped slots unknown instructions and callbacks fail before generation" + (check-exception (source '((flag #f)) '((set-uint flag (uint 1))) '()) true) + (check-exception (source '() '((foreign-operation)) '()) true) + (check-exception (source '() (list void) '()) true) + (check-exception (source '() '((call-source-helper missing start end)) '()) true)) + (test-case "cyclic input and dormant helper recursion are rejected" + (let (cycle (list '(token Line start end))) + (set-cdr! cycle cycle) + (check-exception (source '() cycle '()) true)) + (check-exception + (source '() '() '() '((cycle () ((call-source-helper cycle start end))))) + true)))) diff --git a/t/event-fold-test.ss b/t/event-fold-test.ss index b9ac15bb..41ee0c5e 100644 --- a/t/event-fold-test.ss +++ b/t/event-fold-test.ss @@ -6,12 +6,326 @@ (only-in "event-strategy-fixture.ss" event-lines-language-grammar) (only-in "event-fold-fixture.ss" parse-fold-lines parse_fold_lines parse-outline-lines parse_outline_lines) - (only-in :gerbil-parser/rust-rowan-event-support - event-fold-ir-json run-event-fold)) + (only-in :gerbil-parser/rust-runtime-event-support + event-fold-ir-json run-event-fold) + (only-in :gerbil-parser/src/compiler/event-fold-runtime + prepare-event-fold-program run-event-fold-program) + (only-in :gerbil-parser/src/compiler/event-strategy-aot + fold-source-lines source-line-events) + (only-in :gerbil-parser/src/compiler/event-fold-state-frame + prepare-fold-state-layout instantiate-fold-state-frame + fold-frame-ref fold-frame-update! fold-frame-bound? + fold-frame-bind! fold-frame-unbind!) + (only-in :gerbil-parser/src/modules/parser/interface + source-offset-after source-pattern-end source-pattern-at? + source-ascii-ci-pattern-at?)) (export event-fold-test) (def event-fold-test (test-suite "Scheme stateful event fold AOT" + (test-case "native source fold retains line endings byte bounds and initial state" + (for-each + (lambda (fixture) + (check + (reverse + (fold-source-lines (car fixture) '(seed) + (lambda (line start end reversed) + (cons (list line start end) reversed)))) + => (cadr fixture))) + '(("" (seed)) + ("tail" (seed ("tail" 0 4))) + ("\n" (seed ("\n" 0 1))) + ("é\r\nλ\rtail" (seed ("é\r\n" 0 4) ("λ\r" 4 7) ("tail" 7 11))) + ("a\n\n" (seed ("a\n" 0 2) ("\n" 2 3))))) + (check (source-line-events "é\r\nλ" 'Document + (lambda (_ start end) (list (list 'token 'Line start end)))) + => '((start Document) (token Line 0 4) (token Line 4 6) (finish)))) + (test-case "native fold chains multiple line emissions before final transitions" + (let ((forms '((start-node Text) (token Line start end) (finish-node))) + (finish '((start-node Ending) (finish-node)))) + (check (run-event-fold "é\r\nλ" 'Document '() forms finish) + => '((start Document) + (start Text) (token Line 0 4) (finish) + (start Text) (token Line 4 6) (finish) + (start Ending) (finish) (finish))) + (check (run-event-fold "" 'Document '() forms finish) + => '((start Document) (start Ending) (finish) (finish))))) + (test-case "prepared state layout isolates values and temporary bindings" + (let* ((layout (prepare-fold-state-layout '((flag . #f) (count . 0) (stack)))) + (first (instantiate-fold-state-frame layout)) + (second (instantiate-fold-state-frame layout))) + (fold-frame-update! first 'flag #t) + (fold-frame-update! first 'count 9) + (fold-frame-update! first 'stack '(4 2)) + (fold-frame-bind! first 'temporary #f) + (check (fold-frame-bound? first 'temporary) => #t) + (check (fold-frame-bound? second 'temporary) => #f) + (check (fold-frame-ref second 'flag) => #f) + (check (fold-frame-ref second 'count) => 0) + (check (fold-frame-ref second 'stack) => '()) + (fold-frame-unbind! first 'temporary) + (check (fold-frame-bound? first 'temporary) => #f) + (check-exception (fold-frame-ref first 'temporary) true))) + (test-case "prepared frames retain first binding and missing update policy" + (let* ((layout (prepare-fold-state-layout '((count . 0) (count . 4)))) + (frame (instantiate-fold-state-frame layout))) + (check (fold-frame-ref frame 'count) => 0) + (fold-frame-update! frame 'missing 10) + (check (fold-frame-bound? frame 'missing) => #f) + (check-exception (fold-frame-bind! frame 'count 2) true) + (fold-frame-unbind! frame 'count) + (check (fold-frame-bound? frame 'count) => #f) + (fold-frame-update! frame 'count 7) + (check (fold-frame-bound? frame 'count) => #f) + (fold-frame-bind! frame 'count 3) + (check (fold-frame-ref frame 'count) => 3) + (check (fold-frame-ref (instantiate-fold-state-frame layout) 'count) => 0))) + (test-case "reusable programs snapshot declarations and preserve UTF-8 events" + (let* ((initial (list (list 'ready #f))) + (forms (list '(set-bool ready (bool #t)) '(token Line start end))) + (program (prepare-event-fold-program 'Document initial forms '()))) + (set-car! (cdr (car initial)) #t) + (set-car! forms '(finish-node)) + (for-each + (lambda (source) + (check (run-event-fold-program program source) + => (run-event-fold source 'Document '((ready #f)) + '((set-bool ready (bool #t)) (token Line start end)) '()))) + '("éx\r\n" "λ\n" "" "tail")))) + (test-case "reused helpers and parameter overrides never retain request state" + (let (program + (prepare-event-fold-program 'Document '((level 1)) + '((call-source-helper piece start end ((state level)))) '() + '((piece ((level 0) (seen #f)) + ((if (state seen) ((finish-node)) + ((set-bool seen (bool #t)) + (if (uint-equal? (state level) (uint 2)) + ((token Other start end)) ((token Line start end)))))) + (level))))) + (check (run-event-fold-program program "a\nb\n" '((level . 2))) + => '((start Document) (token Other 0 2) (token Other 2 4) (finish))) + (check (run-event-fold-program program "λ\n") + => '((start Document) (token Line 0 3) (finish))))) + (test-case "prepared programs reject cycles callbacks and mutable initial values" + (let (cycle (list 'token)) + (set-cdr! cycle cycle) + (check-exception (prepare-event-fold-program 'Document '() cycle '()) true)) + (check-exception (prepare-event-fold-program 'Document '() (list void) '()) true) + (check-exception (prepare-fold-state-layout (list (cons 'state (vector 1)))) true) + (check-exception (prepare-event-fold-program 'Document '((count -1)) '() '()) true)) + (test-case "engine source patterns advance by UTF-8 bytes" + (let (end (source-pattern-end 'start "éx")) + (check end => '(line-step (line-step (line-step start)))) + (check (run-event-fold "éx\n" 'Document '() + `((if ,(source-pattern-at? 'start "éx") + ((token Prefix start ,end) (token Ending ,end end)) + ((token Other start end)))) '()) + => '((start Document) (token Prefix 0 3) + (token Ending 3 4) (finish))))) + (test-case "engine ASCII case-insensitive single markers stay admissible" + (let (predicate (source-ascii-ci-pattern-at? 'start "a")) + (check predicate => '(line-bytes-any-in? start (line-step start) (65 97))) + (check (run-event-fold "A\n" 'Document '() + `((if ,predicate ((token Line start end)) ())) '()) + => '((start Document) (token Line 0 2) (finish))) + (check (string? (event-fold-ir-json + 'single_ci event-lines-language-grammar 'Document '() + `((if ,predicate ((token Line start end)) ())) '())) + => #t))) + (test-case "engine ASCII patterns retain punctuation and matching boundaries" + (let (forms `((if ,(source-ascii-ci-pattern-at? 'start "Ab-_") + ((token Match start end)) ((token Miss start end))))) + (check (run-event-fold "aB-_\n" 'Document '() forms '()) + => '((start Document) (token Match 0 5) (finish))) + (check (run-event-fold "ab-\n" 'Document '() forms '()) + => '((start Document) (token Miss 0 4) (finish))) + (check (run-event-fold "ab+x\n" 'Document '() forms '()) + => '((start Document) (token Miss 0 5) (finish))))) + (test-case "engine offset construction rejects invalid counts before iteration" + (check (source-offset-after 'start 0) => 'start) + (check (source-pattern-end 'start "") => 'start) + (for-each (lambda (count) + (check-exception (source-offset-after 'start count) true)) + '(-1 1/2 1.0 #f)) + (check-exception (source-pattern-end 'start #f) true)) + (test-case "engine pattern admission distinguishes UTF-8 from ASCII folding" + (for-each (lambda (pattern) + (check-exception (source-ascii-ci-pattern-at? 'start pattern) true)) + '("é" "Σ" "" #f)) + (check-exception (source-pattern-at? 'start "") true) + (check-exception (source-pattern-at? 'start #f) true)) + (test-case "one event chain preserves nested helper byte-loop and join order" + (check (run-event-fold + "ab\n" 'Document '() + '((start-node Text) + (token Prefix start (line-prefix-end "a")) + (for-line-bytes index start (line-content-end) + ((with-source-bounds (line-index index) end + ((call-source-helper piece start end))))) + (join-once handled + ((if (bool #f) ((set-bool handled (bool #t))) ()) + (token Ending start end)) + ((token Space start end))) + (finish-node)) '() + '((piece () ((start-node Heading) (token Line start end) + (finish-node))))) + => '((start Document) (start Text) (token Prefix 0 1) + (start Heading) (token Line 0 3) (finish) + (start Heading) (token Line 1 3) (finish) + (token Ending 0 3) (token Space 0 3) (finish) (finish)))) + (test-case "empty emissions and handled joins preserve an existing event chain" + (check (run-event-fold + "x\n" 'Document '() + '((start-node Text) (token Before start end) + (for-line-bytes index start start ((token Dropped start end))) + (token Empty start start) + (join-once handled + ((token Branch start end) (set-bool handled (bool #t))) + ((token Skipped start end))) + (finish-node)) + '((start-node Heading) (finish-node))) + => '((start Document) (start Text) (token Before 0 2) + (token Branch 0 2) (finish) (start Heading) (finish) + (finish)))) + (test-case "mutation-only byte loops retain state without emitting wrappers" + (check (run-event-fold + "aé\n" 'Document '((count 0)) + '((for-line-bytes index start (line-content-end) + ((set-uint count (uint-add (state count) (uint 1))) + (if (bool #f) ((token Skipped start end)) ()))) + (token Line start (state-offset count)) + (token Ending (state-offset count) end)) '()) + => '((start Document) (token Line 0 3) (token Ending 3 4) + (finish)))) + (test-case "join temporary bindings are removed before the next join" + (check (run-event-fold + "a\nb\n" 'Document '((seen #f)) + '((join-once handled + ((set-bool seen (bool #t)) (set-bool handled (bool #t))) + ((token Skipped start end))) + (join-once handled + ((if (bool #f) ((set-bool handled (bool #t))) ())) + ((if (state seen) ((token Line start end)) ())))) '()) + => '((start Document) (token Line 0 2) (token Line 2 4) + (finish)))) + (test-case "repeated helper calls own independent execution slots" + (check (run-event-fold + "a\n" 'Document '((position 2)) + '((call-source-helper head start end) + (call-source-helper head start end) + (token Line start (state-offset position))) '() + '((head ((position 0)) + ((set-uint position (uint-add (state position) (uint 1))) + (token Prefix start (state-offset position)))))) + => '((start Document) (token Prefix 0 1) (token Prefix 0 1) + (token Line 0 2) (finish)))) + (test-case "native UTF-8 length keeps multibyte prefix source offsets" + (check (run-event-fold + "éx\n" 'Document '() + '((token Prefix start (line-prefix-end "é")) + (token Line (line-prefix-end "é") end)) '()) + => '((start Document) (token Prefix 0 2) (token Line 2 4) + (finish)))) + (test-case "prepared byte sets preserve empty ranges and UTF-8 membership" + (check (run-event-fold + "é\n" 'Document '() + '((if (and (line-bytes-all-in? start (line-content-end) (195 169 195 255)) + (line-bytes-any-in? start (line-content-end) (169)) + (line-bytes-all-in? start start ()) + (not (line-bytes-any-in? start start ())) + (not (line-bytes-any-in? start end ()))) + ((token Line start end)) ())) '()) + => '((start Document) (token Line 0 3) (finish)))) + (test-case "prepared byte sets reject malformed values only when evaluated" + (for-each + (lambda (values) + (check-exception + (run-event-fold "a\n" 'Document '() + `((if (line-bytes-any-in? start end ,values) () ())) '()) + true)) + '((256) (-1) (1/2) (a) (1 . 2) "a")) + (check (run-event-fold + "a\n" 'Document '() + '((if (or (bool #t) (line-bytes-any-in? start end (256))) + ((token Line start end)) ())) '()) + => '((start Document) (token Line 0 2) (finish)))) + (test-case "prepared byte sets do not leak across parse requests" + (let* ((values (list 97)) + (forms `((if (line-bytes-any-in? start end ,values) + ((token Line start end)) ())))) + (check (run-event-fold "a\n" 'Document '() forms '()) + => '((start Document) (token Line 0 2) (finish))) + (set-car! values 98) + (check (run-event-fold "a\n" 'Document '() forms '()) + => '((start Document) (finish))))) + (test-case "helper parameter updates preserve duplicate-name semantics" + (check (run-event-fold + "a\n" 'Document '((caller 0)) + '((call-source-helper head start end ((uint 1)))) + '() + '((head ((position 0) (position 2)) + ((token Line start (state-offset position)) + (token Line (state-offset position) end)) + (position)))) + => '((start Document) (token Line 0 1) (token Line 1 2) + (finish)))) + (test-case "boolean predicate loops preserve short-circuit rejection boundaries" + (check (run-event-fold + "a\n" 'Document '() + '((if (and (bool #f) (state undeclared)) () ()) + (if (or (bool #t) (state undeclared)) + ((token Line start end)) ())) '()) + => '((start Document) (token Line 0 2) (finish)))) + (test-case "helper execution restores the independent caller frame" + (check (run-event-fold + "a\n" 'Document '((position 1)) + '((call-source-helper head start (state-offset position)) + (token Line (state-offset position) end)) + '() + '((head ((position 0) (position 2)) + ((set-uint position (uint 1)) + (token Line start (state-offset position)))))) + => '((start Document) (token Line 0 1) (token Line 1 2) + (finish)))) + (test-case "execution slots preserve untouched and repeated bindings" + (let ((initial '((left 0) (middle 0) (right 2) (enabled #f))) + (forms + '((set-uint middle (uint 1)) + (set-uint middle (uint 1)) + (set-bool enabled (bool #t)) + (if (and (uint-equal? (state left) (uint 0)) + (uint-equal? (state right) (uint 2)) + (state enabled)) + ((token Line start (state-offset middle)) + (token Line (state-offset middle) end)) ())))) + (check (run-event-fold "a\n" 'Document initial forms '()) + => '((start Document) (token Line 0 1) (token Line 1 2) + (finish))) + (check initial => '((left 0) (middle 0) (right 2) (enabled #f))) + (check (run-event-fold "b\n" 'Document initial forms '()) + => '((start Document) (token Line 0 1) (token Line 1 2) + (finish))))) + (test-case "duplicate runtime bindings retain the original all-update semantics" + (check (run-event-fold + "a\n" 'Document '((offset 0) (offset 2)) + '((set-uint offset (uint 1)) + (token Line start (state-offset offset)) + (token Line (state-offset offset) end)) '()) + => '((start Document) (token Line 0 1) (token Line 1 2) + (finish)))) + (test-case "nested UTF-8 helper frames restore their caller byte view" + (check (run-event-fold + "αb\nz\n" 'Document '() + '((if (line-byte-equal? start 206) + ((call-source-helper head start (line-step (line-step start))) + (if (line-byte-equal? (line-step (line-step start)) 98) + ((token Line (line-step (line-step start)) end)) ())) + ((token Line start end)))) + '() + '((head () ((if (line-byte-equal? start 206) + ((token Line start end)) ()))))) + => '((start Document) (token Line 0 2) (token Line 2 4) + (token Line 4 6) (finish)))) (test-case "the Scheme algorithm owns multi-line node lifetimes" (check (parse-fold-lines "a\nb\n* α\r\nc") => '((start Document) @@ -85,6 +399,178 @@ (check (hash-ref (hash-ref (vector-ref (hash-ref ir "line") 0) "condition") "kind") => "line_blank"))) + (test-case "checked source slices compare names across physical lines" + (let ((initial '((name-start 0) (name-end 0))) + (forms + '((if (line-starts-with "name:") + ((set-uint name-start (offset (line-prefix-end "name:"))) + (set-uint name-end (offset (line-content-end))) + (token Line start end)) + ((if (source-slices-equal-ascii-ci? + (state-offset name-start) (state-offset name-end) + start (line-content-end)) + ((start-node Heading) (token Line start end) + (finish-node)) + ((start-node Text) (token Line start end) + (finish-node)))))))) + (check (run-event-fold "name:ALPHA\nalpha\nother\n" + 'Document initial forms '()) + => '((start Document) (token Line 0 11) + (start Heading) (token Line 11 17) (finish) + (start Text) (token Line 17 23) (finish) + (finish))) + (let* ((wire (event-fold-ir-json + 'source_slices event-lines-language-grammar + 'Document initial forms '())) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (hash-ref ir "name") => "source_slices")))) + (test-case "named future marker respects names headings and parent closes" + (let* ((name-start '(line-prefix-end "#+BEGIN_")) + (name-end `(line-scan-key ,name-start)) + (future `(future-named-line-marker-before-boundary? + ,name-start ,name-end "#+END_" "" "#+END_CENTER" + "*" " " #t #t #t)) + (forms + `((if (line-starts-with-ascii-ci "#+BEGIN_") + ((if ,future + ((start-node Heading) (token Line start end) + (finish-node)) + ((start-node Text) (token Line start end) + (finish-node)))) + ((token Line start end)))))) + (check (map cadr + (filter (lambda (event) (eq? (car event) 'start)) + (run-event-fold + "#+BEGIN_foo\n#+end_FOO\n#+BEGIN_bar\n* Next\n#+END_bar\n#+BEGIN_baz\n#+END_CENTER\n#+END_baz\n#+BEGIN_qux\n#+END_other\n#+END_QUX\n" + 'Document '() forms '()))) + => '(Document Heading Text Text Heading)) + (check (map cadr + (filter (lambda (event) (eq? (car event) 'start)) + (run-event-fold + "#+BEGIN_foo\n#+BEGIN_foo\n#+END_foo\n" + 'Document '() forms '()))) + => '(Document Heading Heading)) + (check (map cadr + (filter (lambda (event) (eq? (car event) 'start)) + (run-event-fold + "#+BEGIN_foo\r\n #+end_FOO \r\n" + 'Document '() forms '()))) + => '(Document Heading)) + (let* ((wire (event-fold-ir-json + 'future_named event-lines-language-grammar + 'Document '() forms '())) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t))) + (outer (vector-ref (hash-ref ir "line") 0)) + (inner (vector-ref (hash-ref outer "consequent") 0))) + (check (hash-ref (hash-ref inner "condition") "kind") + => "future_named_line_marker_before_boundary")))) + (test-case "named future marker stops at a source-named parent closer" + (let* ((name-start '(line-prefix-end "#+BEGIN_")) + (name-end `(line-scan-key ,name-start)) + (future + `(future-named-line-marker-before-boundary? + ,name-start ,name-end "#+END_" "" "" + "*" " " #t #t #t + (state-offset parent-start) (state-offset parent-end) + "#+CLOSE_" "" #t)) + (initial '((parent-start 0) (parent-end 0))) + (forms + `((if (line-starts-with "#+BEGIN_OUTER") + ((set-uint parent-start (offset ,name-start)) + (set-uint parent-end (offset ,name-end)) + (token Line start end)) + ((if (line-starts-with "#+BEGIN_INNER") + ((if ,future + ((start-node Heading) (token Line start end) + (finish-node)) + ((start-node Text) (token Line start end) + (finish-node)))) + ((token Line start end)))))))) + (check (map cadr + (filter (lambda (event) (eq? (car event) 'start)) + (run-event-fold + "#+BEGIN_OUTER\n#+BEGIN_INNER\n#+CLOSE_outer\n#+END_inner\n" + 'Document initial forms '()))) + => '(Document Text)) + (check (map cadr + (filter (lambda (event) (eq? (car event) 'start)) + (run-event-fold + "#+BEGIN_OUTER\n#+BEGIN_INNER\n#+END_inner\n#+CLOSE_outer\n" + 'Document initial forms '()))) + => '(Document Heading)) + (let* ((wire (event-fold-ir-json + 'future_named_parent event-lines-language-grammar + 'Document initial forms '())) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t))) + (outer (vector-ref (hash-ref ir "line") 0)) + (inner (vector-ref (hash-ref outer "alternate") 0)) + (future-ir (hash-ref (vector-ref (hash-ref inner "consequent") 0) + "condition"))) + (check (hash-ref future-ir "stop_prefix") => "#+CLOSE_") + (check (hash-ref future-ir "stop_ascii_case_insensitive") => #t)))) + (test-case "named future marker keeps parent case sensitivity independent" + (let* ((name-start '(line-prefix-end "#+BEGIN_")) + (name-end `(line-scan-key ,name-start)) + (future `(future-named-line-marker-before-boundary? + ,name-start ,name-end "#+END_" "" "" + "*" " " #t #t #t + (state-offset parent-start) (state-offset parent-end) + "#+END_" "" #f)) + (initial '((parent-start 0) (parent-end 0))) + (forms + `((if (line-starts-with "#+BEGIN_OUTER") + ((set-uint parent-start (offset ,name-start)) + (set-uint parent-end (offset ,name-end)) + (token Line start end)) + ((if (line-starts-with "#+BEGIN_INNER") + ((if ,future + ((start-node Heading) (finish-node)) + ((start-node Text) (finish-node)))) + ((token Line start end)))))))) + (check (map cadr + (filter (lambda (event) (eq? (car event) 'start)) + (run-event-fold + "#+BEGIN_OUTER\n#+BEGIN_INNER\n#+END_outer\n#+END_inner\n" + 'Document initial forms '()))) + => '(Document Heading)) + (check (map cadr + (filter (lambda (event) (eq? (car event) 'start)) + (run-event-fold + "#+BEGIN_OUTER\n#+BEGIN_INNER\n#+END_OUTER\n#+END_inner\n" + 'Document initial forms '()))) + => '(Document Text)))) + (test-case "state-only frame pop does not close syntax nodes" + (let ((initial '((saved (uint-stack)))) + (forms '((push-frame saved (uint 7)) + (pop-frame saved) + (token Line start end)))) + (check (run-event-fold "a\n" 'Document initial forms '()) + => '((start Document) (token Line 0 2) (finish))) + (let* ((wire (event-fold-ir-json + 'state_pop event-lines-language-grammar + 'Document initial forms '())) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (hash-ref (vector-ref (hash-ref ir "line") 1) "kind") + => "pop_frame")))) + (test-case "single frame close preserves nested equal frame scopes" + (let ((initial '((frames (uint-stack)))) + (forms '((start-node Heading) + (push-frame frames (uint 7)) + (start-node Text) + (push-frame frames (uint 7)) + (close-frame frames 1) + (close-frame frames 1)))) + (check (run-event-fold "a\n" 'Document initial forms '()) + => '((start Document) (start Heading) (start Text) + (finish) (finish) (finish))))) (test-case "byte-set membership is a boolean state value" (let (forms '((set-bool match (line-bytes-any-in? start (line-step start) @@ -103,6 +589,133 @@ 'byte_set_bool event-lines-language-grammar 'Document '((match #f)) forms '())) => #t))) + (test-case "bounded static name set executes and lowers as one predicate" + (let* ((forms '((if (line-bytes-in-set? start (line-content-end) + ("alpha" "beta")) + ((start-node Heading) (token Line start end) + (finish-node)) + ((start-node Text) (token Line start end) + (finish-node))))) + (wire (event-fold-ir-json + 'static_name_set event-lines-language-grammar + 'Document '() forms '())) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (run-event-fold "alpha\nbeta\nother\n" 'Document '() forms '()) + => '((start Document) + (start Heading) (token Line 0 6) (finish) + (start Heading) (token Line 6 11) (finish) + (start Text) (token Line 11 17) (finish) + (finish))) + (check (hash-ref (hash-ref (vector-ref (hash-ref ir "line") 0) + "condition") "kind") + => "line_bytes_in_set"))) + (test-case "saved source bounds scan one UTF-8 span across physical lines" + (let* ((initial '((span-start 0) (span-end 0))) + (line '((set-uint span-end (offset end)))) + (finish + '((with-source-bounds (state-offset span-start) + (state-offset span-end) + ((if (line-bytes-any-in? start end (10)) + ((start-node Text) (token Line start end) + (finish-node)) ()))))) + (wire (event-fold-ir-json + 'source_span event-lines-language-grammar + 'Document initial line finish)) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (run-event-fold "ab\ncd\n" 'Document initial line finish) + => '((start Document) (start Text) (token Line 0 6) + (finish) (finish))) + (check (hash-ref (vector-ref (hash-ref ir "finish") 0) "kind") + => "with_source_bounds"))) + (test-case "source-local helper executes once and stays typed in event IR" + (let* ((initial '((span-start 0) (span-end 0))) + (line '((set-uint span-end (offset end)))) + (finish '((call-source-helper inline-span + (state-offset span-start) + (state-offset span-end)))) + (helpers + '((inline-span + ((cursor 0)) + ((set-uint cursor (offset start)) + (if (line-bytes-any-in? start end (10)) + ((start-node Text) + (token Line (state-offset cursor) end) + (finish-node)) ()))))) + (wire (event-fold-ir-json + 'source_helper event-lines-language-grammar + 'Document initial line finish helpers)) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (run-event-fold "ab\ncd\n" 'Document initial line finish helpers) + => '((start Document) (start Text) (token Line 0 6) + (finish) (finish))) + (check (hash-ref (vector-ref (hash-ref ir "finish") 0) "kind") + => "call_source_helper") + (check (vector-length (hash-ref ir "helpers")) => 1))) + (test-case "typed helper arguments flow through Scheme execution and IR" + (let* ((initial '((threshold 2))) + (line '((call-source-helper local-span start end + ((state threshold))))) + (helpers + '((local-span ((threshold 2)) + ((if (uint-equal? (state threshold) (uint 2)) + ((start-node Heading) (finish-node)) + ((start-node Text) (finish-node)))) + (threshold)))) + (wire (event-fold-ir-json + 'typed_helper event-lines-language-grammar + 'Document initial line '() helpers)) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (run-event-fold "x\n" 'Document initial line '() helpers) + => '((start Document) (start Heading) (finish) (finish))) + (check (run-event-fold "x\n" 'Document initial line '() helpers + '((threshold . 5))) + => '((start Document) (start Text) (finish) (finish))) + (check (vector-length (hash-ref + (vector-ref (hash-ref ir "line") 0) + "arguments")) + => 1) + (check (vector-ref + (hash-ref (vector-ref (hash-ref ir "helpers") 0) + "parameters") 0) + => "threshold") + (check-exception + (event-fold-ir-json + 'bad_helper_arity event-lines-language-grammar 'Document initial + '((call-source-helper local-span start end)) '() helpers) + true))) + (test-case "source helpers compose without recursive execution" + (let* ((line '((call-source-helper outer start end))) + (helpers + '((outer () ((call-source-helper inner start end))) + (inner () ((start-node Text) + (token Line start end) + (finish-node))))) + (wire (event-fold-ir-json + 'nested_helpers event-lines-language-grammar + 'Document '() line '() helpers)) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (run-event-fold "ab\n" 'Document '() line '() helpers) + => '((start Document) (start Text) (token Line 0 3) + (finish) (finish))) + (check (hash-ref (vector-ref + (hash-ref (vector-ref (hash-ref ir "helpers") 0) + "body") 0) "kind") + => "call_source_helper") + (check-exception + (run-event-fold "ab\n" 'Document '() line '() + '((outer () ((call-source-helper inner start end))) + (inner () ((call-source-helper outer start end))))) + true))) (test-case "future marker search stops at heading or parent boundary" (let* ((condition '(and (line-starts-with-ascii-ci "#+BEGIN_QUOTE") (future-line-marker-before-boundary? @@ -134,6 +747,24 @@ (vector-ref (hash-ref ir "line") 0) "condition") "left") "kind") => "future_line_marker_before_boundary"))) + (test-case "future marker answers stay within one source" + (let* ((condition '(future-line-marker-before-boundary? + "END" "STOP" "*" " " #t #t "")) + (forms `((if ,condition + ((start-node Heading) (finish-node)) + ((start-node Text) (finish-node)))))) + (check (run-event-fold "x\nx\nEND\n" 'Document '() forms '()) + => '((start Document) + (start Heading) (finish) + (start Heading) (finish) + (start Text) (finish) + (finish))) + (check (run-event-fold "x\nSTOP\nEND\n" 'Document '() forms '()) + => '((start Document) + (start Text) (finish) + (start Heading) (finish) + (start Text) (finish) + (finish))))) (test-case "future marker can require a key-value body" (let* ((condition '(future-line-marker-before-boundary? ":END:" "" "*" " " #t #t ":")) @@ -151,14 +782,93 @@ => '((start Document) (start Heading) (finish) (start Heading) (finish) (start Text) (finish) (finish))) + (check (run-event-fold ":PROPERTIES:\n:header-args:python: :session local\n:END:\n" + 'Document '() forms '()) + => '((start Document) (start Heading) (finish) + (start Heading) (finish) (start Text) (finish) + (finish))) + (check (run-event-fold ":PROPERTIES:\n:ID: value:more\n:END:\n" + 'Document '() forms '()) + => '((start Document) (start Heading) (finish) + (start Heading) (finish) (start Text) (finish) + (finish))) (check (run-event-fold ":PROPERTIES:\nmalformed\n:END:\n" 'Document '() forms '()) => '((start Document) (start Text) (finish) (start Heading) (finish) (start Text) (finish) (finish))) + (check (run-event-fold ":PROPERTIES:\n:header args:python: x\n:END:\n" + 'Document '() forms '()) + => '((start Document) (start Text) (finish) + (start Heading) (finish) (start Text) (finish) + (finish))) (check (hash-ref (hash-ref (vector-ref (hash-ref ir "line") 0) "condition") "body_key_marker") => 58))) + (test-case "future heading title respects level and exact suffix" + (let* ((condition '(future-heading-title? "*" " " 4 "END")) + (forms `((if ,condition + ((start-node Heading) (finish-node)) + ((start-node Text) (finish-node))))) + (wire (event-fold-ir-json + 'future_heading event-lines-language-grammar + 'Document '() forms '())) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (run-event-fold "**** Task\n* Outline\n***** END \r\n" + 'Document '() forms '()) + => '((start Document) (start Heading) (finish) + (start Heading) (finish) (start Text) (finish) (finish))) + (check (run-event-fold "**** Task\n*** END\n" + 'Document '() forms '()) + => '((start Document) (start Text) (finish) + (start Text) (finish) (finish))) + (check (run-event-fold "**** Task\n**** END extra\n" + 'Document '() forms '()) + => '((start Document) (start Text) (finish) + (start Text) (finish) (finish))) + (check (hash-ref (hash-ref (vector-ref (hash-ref ir "line") 0) + "condition") "kind") + => "future_heading_title") + (check-exception + (event-fold-ir-json + 'bad_heading event-lines-language-grammar 'Document '() + '((if (future-heading-title? "*" " " 0 "END") () ())) '()) + true) + (let ((dynamic-forms + '((if (future-heading-title? "*" " " (state threshold) "END") + ((start-node Heading) (finish-node)) + ((start-node Text) (finish-node)))))) + (check (run-event-fold "x\n END\n" 'Document '((threshold 1)) + dynamic-forms '() '() '((threshold . 0))) + => '((start Document) (start Text) (finish) + (start Text) (finish) (finish)))))) + (test-case "typed parameter has one Scheme fold and AOT declaration" + (let* ((initial '((threshold 4))) + (forms '((if (uint-equal? (state threshold) (uint 7)) + ((start-node Heading) (finish-node)) + ((start-node Text) (finish-node))))) + (wire (event-fold-ir-json + 'parameterized event-lines-language-grammar + 'Document initial forms '() '() + '((configured_threshold threshold 4)))) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (run-event-fold "x\n" 'Document initial forms '()) + => '((start Document) (start Text) (finish) (finish))) + (check (run-event-fold "x\n" 'Document initial forms '() '() + '((threshold . 7))) + => '((start Document) (start Heading) (finish) (finish))) + (check (hash-ref (vector-ref (hash-ref ir "parameters") 0) "state") + => "threshold") + (check-exception + (event-fold-ir-json + 'parameterized event-lines-language-grammar + 'Document initial forms '() '() + '((configured_threshold threshold 5))) + true))) (test-case "n-ary boolean predicates evaluate every operand and lower to IR" (let* ((forms '((if (and (bool #t) (bool #t) (bool #f)) ((start-node Heading) (finish-node)) @@ -441,6 +1151,79 @@ => "state_offset") (check (hash-ref (hash-ref token "end") "kind") => "state_offset"))) + (test-case "final transition may flush a saved source-backed token" + (let* ((initial '((pending #f) (pending-start 0) (pending-end 0))) + (line '((if (line-blank?) + ((set-bool pending (bool #t)) + (set-uint pending-start (offset start)) + (set-uint pending-end (offset end))) ()))) + (finish '((if (state pending) + ((token Line (state-offset pending-start) + (state-offset pending-end))) ()))) + (wire (event-fold-ir-json + 'flush_pending event-lines-language-grammar + 'Document initial line finish)) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t))) + (flush (vector-ref + (hash-ref (vector-ref (hash-ref ir "finish") 0) + "consequent") 0))) + (check (run-event-fold "\n" 'Document initial line finish) + => '((start Document) (token Line 0 1) (finish))) + (check (hash-ref (hash-ref flush "start") "kind") + => "state_offset") + (check-exception + (event-fold-ir-json 'invalid event-lines-language-grammar + 'Document initial '() + '((token Line start end))) true) + (check-exception + (event-fold-ir-json 'invalid event-lines-language-grammar + 'Document initial '() + '((token Line + (line-step (state-offset pending-start)) + (state-offset pending-end)))) true))) + (test-case "join-once converges branches before the shared continuation" + (let* ((initial '()) + (forms '((join-once handled + ((if (line-starts-with "#") + ((start-node Heading) (token Line start end) + (finish-node) + (set-bool handled (bool #t))) ())) + ((start-node Text) (token Line start end) + (finish-node))))) + (wire (event-fold-ir-json + 'joined event-lines-language-grammar 'Document + initial forms '())) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t))) + (join (vector-ref (hash-ref ir "line") 0))) + (check (run-event-fold "# H\nbody\n" 'Document initial forms '()) + => '((start Document) + (start Heading) (token Line 0 4) (finish) + (start Text) (token Line 4 9) (finish) + (finish))) + (check (hash-ref join "kind") => "join_once") + (check (vector-length (hash-ref join "continuation")) => 3) + (check-exception + (event-fold-ir-json 'invalid event-lines-language-grammar 'Document + initial '((join-once handled ((finish-node)) ())) '()) + true) + (check-exception + (event-fold-ir-json 'invalid event-lines-language-grammar 'Document + initial '((join-once handled ((finish-node)) + ((token Line start end)))) '()) + true) + (check-exception + (run-event-fold "text\n" 'Document initial + '((join-once handled ((finish-node)) + ((token Line start end)))) '()) + true) + (check-exception + (event-fold-ir-json 'invalid event-lines-language-grammar 'Document + '((handled #f)) forms '()) + true))) (test-case "undeclared state and unsupported effects fail closed" (check-exception (event-fold-ir-json 'invalid event-lines-language-grammar 'Document diff --git a/t/event-program-test.ss b/t/event-program-test.ss new file mode 100644 index 00000000..3ae27964 --- /dev/null +++ b/t/event-program-test.ss @@ -0,0 +1,350 @@ +#!/usr/bin/env gxi +;;; Direct event lowering must match the independent original-production replay. +(import :std/test + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-parser parse-arithmetic) + (only-in :gerbil-parser/languages/hcl/parser hcl-parser parse-hcl) + :gerbil-parser/src/runtime/event-program + :gerbil-parser/src/runtime/event-reduce + :gerbil-parser/src/runtime/recognition + (only-in :gerbil-parser/src/runtime/reduce recognition-children-alias) + :gerbil-parser/src/runtime/funcs + (only-in :gerbil-parser/src/runtime/token make-token token-kind) + (only-in :gerbil-parser/src/runtime/artifact make-success-parse-artifact + parse-artifact-valid? parse-artifact-events sha256-text) + (only-in :gerbil-parser/src/runtime/lr-parser + current-lr-event-program-enabled? lr-runtime-event-program? + lr-recognition-fragment-runtime lr-recognition-fragment-value + lr-recognition-view? lr-recognition-view-base lr-recognition-fragment? + lr-recognition-fragment-children lr-recognition-fragment-end + lr-recognition-fragment-token-count lr-recognition-fragment-executor) + (only-in :gerbil-parser/src/compiler/hcl-reductions direct-event-step) + (only-in :gerbil-parser/src/runtime/incremental + make-incremental-session incremental-session-artifact + incremental-session-recognition-root incremental-session-project-artifact + incremental-session-publication-comparison + parse-incremental-session make-edit apply-edit) + "./scenarios/performance/selective-glr/scenario") + +(def (base-root root) + (if (lr-recognition-view? root) (base-root (lr-recognition-view-base root)) root)) +(def (program-root session) + (recognition-child-value + (car (recognition-sequence->list + (lr-recognition-fragment-value (base-root (incremental-session-recognition-root session))))))) +(def (find-program-piece root bound) + (let loop ((pending (list root))) + (and (pair? pending) + (let (piece (base-root (car pending))) + (if (lr-recognition-fragment? piece) + (let (values (recognition-sequence->list (lr-recognition-fragment-value piece))) + (if (and (< (lr-recognition-fragment-end piece) bound) + (> (lr-recognition-fragment-token-count piece) 2) + (pair? values) (null? (cdr values)) + (event-program-value? (recognition-child-value (car values)))) + piece + (loop (append (lr-recognition-fragment-children piece) (cdr pending))))) + (loop (cdr pending))))))) +(def (contains-piece? root expected) + (let loop ((pending (list root))) + (and (pair? pending) + (let (piece (base-root (car pending))) + (or (eq? piece expected) + (loop (append (if (lr-recognition-fragment? piece) + (lr-recognition-fragment-children piece) '()) + (cdr pending)))))))) +(def (artifact-for source tokens root) + (make-success-parse-artifact "event-test" source tokens root + (lambda (token) #f))) +(def event-program-test + (test-suite "persistent LR event programs" + (test-case "deep programs walk in source order and compose views without recursion" + (let* ((code (let loop ((n 0) (code #f)) + (if (= n 10000) code + (loop (+ n 1) (event-program-append code + (event-program-token (make-token 'number "1" n (+ n 1)))))))) + (moved (event-program-relocate (event-program-relocate code 13) -5)) + (count 0) (valid? #t)) + (event-program-walk + (lambda (operation token offset delta moved?) + (set! valid? (and valid? (eq? operation 'token) + (= (+ offset delta) (+ count 8)) moved?)) + (set! count (+ count 1))) moved) + (check count => 10000) + (check valid? => #t))) + (test-case "LR aliases retain the sole semantic sequence with cached boundaries" + (let* ((sequence (let loop ((n 0) (sequence #f)) + (if (= n 10000) sequence + (loop (+ n 1) (event-program-append sequence + (make-token 'number "1" n (+ n 1))))))) + (program (event-children-alias 'Root sequence 0)) + (view (recognition-sequence-relocate + (recognition-sequence-relocate sequence 17) -17))) + (check (eq? (event-program-value-body program) sequence) => #t) + (check (event-program-sequence-arity sequence) => 2) + (check (<= (event-program-sequence-height sequence) 12) => #t) + (check (recognition-sequence-start sequence 99) => 0) + (check (recognition-sequence-end sequence 99) => 10000) + (check (recognition-sequence-start view 99) => 0) + (check (recognition-sequence-end view 99) => 10000) + (check (length (recognition-sequence->list view)) => 10000))) + (test-case "bounded block append preserves frozen prefixes and composed moves" + (let* ((prefix (let loop ((n 0) (value #f)) + (if (= n 33) value + (loop (+ n 1) (event-program-append value + (make-token 'number "1" n (+ n 1))))))) + (prefix-list (recognition-sequence->list prefix)) + (next (event-program-append prefix (make-token 'number "1" 33 34))) + (moved (event-program-append + (event-program-relocate prefix 10) (make-token 'number "1" 43 44))) + (count 0) (valid? #t)) + (check (recognition-sequence->list prefix) => prefix-list) + (check (length (recognition-sequence->list next)) => 34) + (check (recognition-sequence-start moved 99) => 10) + (check (recognition-sequence-end moved 99) => 44) + (event-program-walk + (lambda (operation token offset delta moved?) + (set! valid? (and valid? (eq? operation 'token) + (= (+ offset delta) (+ count 10)))) + (set! count (+ count 1))) moved) + (check count => 34) + (check valid? => #t))) + (test-case "deep node publication retains opening IDs through the bounded fallback" + (let* ((token (make-token 'number "1" 0 1)) + (program (let loop ((n 0) (body token)) + (if (= n 10000) body + (loop (+ n 1) (event-program-node-value 'Node 0 1 body))))) + (artifact (make-success-parse-artifact (sha256-text "event-test") "1" + (list token) program (lambda (token) #f)))) + (check (parse-artifact-valid? artifact) => #t) + (check (length (parse-artifact-events artifact)) => 20001) + (check (car (parse-artifact-events artifact)) => (vector 'start-node 0 'Node 0)) + (check (last (parse-artifact-events artifact)) => (vector 'finish-node 0 'Node 1)))) + (test-case "asymmetric balanced joins preserve complete source order and old operands" + (def (range start count) + (let loop ((n 0) (value #f)) + (if (= n count) value + (loop (+ n 1) (event-program-append value + (make-token 'number "1" (+ start n) (+ start n 1))))))) + (for-each + (lambda (sizes) + (let* ((left-size (car sizes)) (right-size (cadr sizes)) + (left (range 0 left-size)) (right (range left-size right-size)) + (joined (event-program-append left right)) (seen 0) (valid? #t)) + (event-program-walk + (lambda (operation token offset delta moved?) + (set! valid? (and valid? (eq? operation 'token) (= offset seen))) + (set! seen (+ seen 1))) joined) + (check valid? => #t) + (check seen => (+ left-size right-size)) + (check (<= (event-program-sequence-height joined) 12) => #t) + (check (length (recognition-sequence->list left)) => left-size) + (check (length (recognition-sequence->list right)) => right-size))) + '((1 5000) (5000 1) (257 4096) (4096 257) (33 33)))) + (test-case "binary block carries preserve branching roots and deep fallback order" + (let* ((tokens (list->vector (map (lambda (i) (make-token 'number "1" i (+ i 1))) (iota 1025)))) + (retained '()) + (sequence (let loop ((i 0) (value #f)) + (when (memv i '(16 17 32 33 255 256 257 1024)) + (set! retained (cons (cons i value) retained))) + (if (= i 1025) value + (loop (+ i 1) (event-program-append value (vector-ref tokens i)))))) + (control (let loop ((i 0) (value #f)) + (if (= i 1025) value + (loop (+ i 1) (event-program-append/avl value (vector-ref tokens i)))))) + (root (let loop ((i 0) (value sequence)) + (if (= i 100) value + (loop (+ i 1) (event-program-node-value 'Root 0 1025 value))))) + (count 0) (opens 0) (closes 0)) + (check (recognition-sequence->list sequence) => (recognition-sequence->list control)) + (for-each + (lambda (entry) + (let* ((n (car entry)) (old (cdr entry)) + (a (event-program-append old (vector-ref tokens n))) + (b (event-program-append old (vector-ref tokens n)))) + (check (length (recognition-sequence->list old)) => n) + (check (recognition-sequence->list a) => (recognition-sequence->list b)) + (check (recognition-sequence-end a 0) => (+ n 1)))) retained) + (event-program-walk + (lambda (operation value offset delta moved?) + (case operation + ((token) (check offset => count) (set! count (+ count 1))) + ((open-node) (set! opens (+ opens 1))) + ((close-node) (set! closes (+ closes 1))))) root) + (check count => 1025) (check opens => 100) (check closes => 100))) + (test-case "alternating singleton and multi-sequence joins retain bounded rope height" + (let* ((tokens (list->vector (map (lambda (i) (make-token 'number "1" i (+ i 1))) (iota 3000)))) + (sequence (let loop ((i 0) (value #f)) + (if (= i 3000) value + (let* ((one (event-program-append value (vector-ref tokens i))) + (two (event-program-append (vector-ref tokens (+ i 1)) + (vector-ref tokens (+ i 2))))) + (loop (+ i 3) (event-program-append one two)))))) + (count 0)) + (check (<= (event-program-sequence-height sequence) 12) => #t) + (event-program-sequence-for-each + (lambda (field value delta moved?) + (check (eq? value (vector-ref tokens count)) => #t) + (set! count (+ count 1))) sequence) + (check count => 3000))) + (test-case "lowering snapshots field names rather than caching mutable children" + (let* ((token (make-token 'number "1" 0 1)) + (child (make-recognition-child #f token)) + (program (recognition-child-value (car (recognition-sequence->list (event-children-alias 'Root (list child) 0))))) + (expected (artifact-for "1" (list token) + (recognition-child-value + (car (recognition-children-alias 'Root (list (make-recognition-child #f token)) 0)))))) + (recognition-child-field-set! child 'mutated) + (check (artifact-for "1" (list token) program) => expected))) + (test-case "named singleton fields and cancelled moves preserve canonical output and token proof" + (let* ((old (make-token 'number "1" 0 1)) + (fresh (make-token 'number "1" 0 1)) + (children (list (make-recognition-child 'inner old))) + (program-children (event-children-field 'outer children 0)) + (ordinary-children + (list (make-recognition-child 'outer + (make-recognition-fragment 0 1 children)))) + (program-root (recognition-child-value + (car (recognition-sequence->list (event-children-alias 'Root program-children 0))))) + (moved-program (recognition-child-value + (car (recognition-sequence->list (event-children-alias 'Root + (recognition-sequence-relocate + (recognition-sequence-relocate program-children 7) -7) 0))))) + (moved-ordinary (recognition-child-value + (car (recognition-children-alias 'Root + (recognition-sequence->list + (recognition-sequence-relocate + (recognition-sequence-relocate ordinary-children 7) -7)) 0))))) + (check (artifact-for "1" (list old) program-root) + => (artifact-for "1" (list old) + (recognition-child-value (car (recognition-children-alias 'Root ordinary-children 0))))) + (check (artifact-for "1" (list fresh) moved-program) + => (artifact-for "1" (list fresh) moved-ordinary)) + (check-exception (artifact-for "1" (list fresh) program-root) true))) + (test-case "relocated program roots retain token proof and root clamps" + (let* ((old (make-token 'number "1" 0 1)) + (fresh (make-token 'number "1" 1 2)) + (tokens (list (make-token 'ws " " 0 1) fresh)) + (children (list (make-recognition-child #f old))) + (program (recognition-child-value (car (recognition-sequence->list (event-children-alias 'Root children 0))))) + (ordinary (recognition-child-value (car (recognition-children-alias 'Root children 0)))) + (trivia? (lambda (token) (eq? (token-kind token) 'ws)))) + (check (make-success-parse-artifact "event-test" " 1" tokens + (relocate-recognition-value program 1) trivia?) + => (make-success-parse-artifact "event-test" " 1" tokens + (relocate-recognition-value ordinary 1) trivia?)) + (check (make-success-parse-artifact "event-test" " 1" tokens + (relocate-recognition-value (relocate-recognition-value program 8) -7) trivia?) + => (make-success-parse-artifact "event-test" " 1" tokens + (relocate-recognition-value ordinary 1) trivia?)))) + (test-case "HCL events actually execute generated steps across topology Unicode and history edits" + (parameterize ((current-lr-event-program-enabled? #t)) + (let* ((source (string-append "# 前置\n" (string-join (make-list 100 "a = 1 # 中\n") "") "# 尾部\n")) + (session (make-incremental-session hcl-parser source #t))) + (check (event-program-value? (program-root session)) => #t) + (check + (let loop ((pending (list (incremental-session-recognition-root session)))) + (if (null? pending) #t + (let (piece (base-root (car pending))) + (and (or (not (lr-recognition-fragment? piece)) + (null? (lr-recognition-fragment-value piece)) + (event-program-sequence? (lr-recognition-fragment-value piece))) + (loop (append (if (lr-recognition-fragment? piece) + (lr-recognition-fragment-children piece) '()) + (cdr pending))))))) => #t) + (check (lr-runtime-event-program? + (lr-recognition-fragment-runtime (base-root (incremental-session-recognition-root session)))) => #t) + (check (incremental-session-artifact session) => (parse-hcl source)) + (check (eq? (lr-recognition-fragment-executor + (base-root (incremental-session-recognition-root session))) direct-event-step) => #t) + (let* ((piece (find-program-piece (incremental-session-recognition-root session) 500)) + (code (and piece (event-program-value-code + (recognition-child-value (car (recognition-sequence->list + (lr-recognition-fragment-value piece))))))) + (edit (make-edit (u8vector-length (string->utf8 source)) 0 "last = 2\n"))) + (check (not (not piece)) => #t) + (let-values (((next receipt) (parse-incremental-session session edit))) + (set! source (apply-edit source edit)) (set! session next) + (check (contains-piece? (incremental-session-recognition-root session) piece) => #t) + (check (eq? code (event-program-value-code + (recognition-child-value (car (recognition-sequence->list + (lr-recognition-fragment-value piece)))))) => #t) + (check (incremental-session-artifact session) => (parse-hcl source)) + (check (incremental-session-project-artifact session) => (incremental-session-artifact session)))) + (for-each + (lambda (edit) + (let-values (((next receipt) (parse-incremental-session session edit))) + (set! source (apply-edit source edit)) (set! session next) + (check (event-program-value? (program-root session)) => #t) + (check (incremental-session-artifact session) => (parse-hcl source)) + (check (incremental-session-project-artifact session) => (incremental-session-artifact session)))) + (list (make-edit 0 0 "x = 2\n") (make-edit 0 6 "") + (make-edit 9 0 "y = 3\n") (make-edit 9 6 "") + (make-edit 0 0 "# é\n") (make-edit 0 5 "")))))) + (test-case "selected machine preserves certified transfers and backend ownership across flag changes" + (let* ((source (string-join (make-list 100 "a = 1\n") "")) + (control (parameterize ((current-lr-event-program-enabled? #f)) + (make-incremental-session hcl-parser source #t))) + (candidate (parameterize ((current-lr-event-program-enabled? #t)) + (make-incremental-session hcl-parser source #t))) + (edit (make-edit 0 0 "b = 2\n"))) + (let-values (((control-next control-receipt) + (parameterize ((current-lr-event-program-enabled? #f)) + (parse-incremental-session control edit))) + ((candidate-next candidate-receipt) + (parameterize ((current-lr-event-program-enabled? #t)) + (parse-incremental-session candidate edit)))) + (check (> (cdr (assq 'reusedRecognitionFragmentCount candidate-receipt)) 0) => #t) + (for-each + (lambda (key) + (check (assq key candidate-receipt) => (assq key control-receipt))) + '(reusedRecognitionFragmentCount fragmentCertificateProbeByteCount + fragmentRejectedProbeCount fragmentControlProbeCount + fragmentProbeReuseTokenCount fragmentProbeReuseByteCount fragmentCursorVisitCount + remainingSignificantTokenCount relexedByteCount reusedSignificantTokenCount)) + (check (incremental-session-artifact candidate-next) + => (incremental-session-artifact control-next)) + (check (incremental-session-project-artifact candidate-next) + => (incremental-session-artifact candidate-next)) + (let-values (((restored receipt) + (parameterize ((current-lr-event-program-enabled? #f)) + (parse-incremental-session candidate-next (make-edit 0 6 ""))))) + (check (event-program-value? (program-root restored)) => #t) + (check (lr-runtime-event-program? (lr-recognition-fragment-runtime + (base-root (incremental-session-recognition-root restored)))) => #t) + (check (incremental-session-artifact restored) => (parse-hcl source)) + (check (incremental-session-project-artifact restored) + => (incremental-session-artifact restored)))))) + (test-case "publication closures retain edited source and token binding across inverse edits" + (let* ((source (string-join (make-list 100 "a = 1\n") "")) + (session (parameterize ((current-lr-event-program-enabled? #t)) + (make-incremental-session hcl-parser source #t)))) + (let-values (((next receipt) (parse-incremental-session session (make-edit 0 0 "b = 2\n")))) + (let-values (((control candidate code) (incremental-session-publication-comparison next))) + (let ((expected (parse-hcl (string-append "b = 2\n" source))) (moved 0)) + (event-program-walk + (lambda (op value offset delta moved?) + (when (and (eq? op 'token) moved?) (set! moved (+ moved 1)))) code) + (check (> moved 0) => #t) + (parameterize ((current-lr-event-program-enabled? #f)) + (let-values (((restored receipt) (parse-incremental-session next (make-edit 0 6 "")))) + (check (incremental-session-artifact restored) => (parse-hcl source)) + (check (control) => expected) + (check (candidate) => expected) + (check (incremental-session-artifact session) => (parse-hcl source))))))))) + (test-case "generic deterministic LR lowers events without a generated executor" + (parameterize ((current-lr-event-program-enabled? #t)) + (let* ((source "1+2*3") + (session (make-incremental-session arithmetic-parser source #t)) + (root (base-root (incremental-session-recognition-root session)))) + (check (event-program-value? (program-root session)) => #t) + (check (lr-runtime-event-program? (lr-recognition-fragment-runtime root)) => #t) + (check (lr-recognition-fragment-executor root) => #f) + (check (incremental-session-artifact session) => (parse-arithmetic source)) + (let-values (((next receipt) (parse-incremental-session session (make-edit 0 0 "0+")))) + (check (event-program-value? (program-root next)) => #t) + (check (incremental-session-artifact next) => (parse-arithmetic "0+1+2*3")) + (check (incremental-session-project-artifact next) => (incremental-session-artifact next)))))) + (test-case "unsupported selective GLR remains on the canonical backend" + (parameterize ((current-lr-event-program-enabled? #t)) + (check (selective-glr-scenario-pass? (selective-glr-scenario)) => #t))))) +(export event-program-test) diff --git a/t/event-strategy-fixture.ss b/t/event-strategy-fixture.ss index b28b1a5b..61d836f0 100644 --- a/t/event-strategy-fixture.ss +++ b/t/event-strategy-fixture.ss @@ -1,13 +1,13 @@ ;;; -*- Gerbil -*- -;;; One Scheme algorithm is executed in tests and compiled for Rowan. +;;; One Scheme algorithm is executed in tests and compiled for Rust. -(import (only-in :gerbil-parser/language-support deflanguage-grammar) +(import (only-in :gerbil-parser/src/compiler/language-expander compile-language) (only-in :gerbil-parser/src/compiler/event-strategy-aot define-line-event-parser event-node event-token line-starts-with?)) (export event-lines-language-grammar parse-event-lines parse_event_lines) -(deflanguage-grammar event-lines +(compile-language event-lines (identity "event-lines" "v1" "event-lines.v1") (syntax-kinds (Document node (line)) diff --git a/t/event-strategy-test.ss b/t/event-strategy-test.ss index 445e2c39..1d9e2c69 100644 --- a/t/event-strategy-test.ss +++ b/t/event-strategy-test.ss @@ -1,11 +1,12 @@ ;;; -*- Gerbil -*- ;;; Parser-event algorithms execute in Scheme before AOT and stay AST-backed. -(import (only-in :std/test check check-exception test-case test-suite) +(import (only-in :std/io/tempfile make-temporary-file-name) + (only-in :std/test check check-exception test-case test-suite) (only-in "event-strategy-fixture.ss" event-lines-language-grammar parse-event-lines parse_event_lines) (only-in :gerbil-parser/src/compiler/event-strategy-aot - compile-line-event-parser) + compile-line-event-parser generate-line-event-module) (only-in :gerbil-parser/src/compiler/rust-syntax rust-line-event-function-form-digest) (only-in "rust-aot-test-syntax.ss" check-rust-event-strategy)) @@ -26,6 +27,15 @@ (start Text) (token Line 0 6) (finish) (start Heading) (token Line 6 10) (finish) (finish)))) + (test-case "line extents retain NUL and all UTF-8 widths across CRLF and CR" + (let (prefix (list->string + (map integer->char '(0 127 128 2047 2048 55295 57344 65535 65536 1114111)))) + (check (parse-event-lines (string-append prefix "\r\nx\rY")) + => '((start Document) + (start Text) (token Line 0 28) (finish) + (start Text) (token Line 28 30) (finish) + (start Text) (token Line 30 31) (finish) + (finish))))) (test-case "AOT is a structured parser function, not source assembly" (check-rust-event-strategy parse_event_lines 'parse_event_lines 0) @@ -40,6 +50,14 @@ 'Document '(line start end) '(event-node Text (event-token Line start end))))) => #f)) + (test-case "compiled event strategy materializes its Rust module" + (let (path (make-temporary-file-name "gerbil-parser-event-strategy")) + (try + (begin + (generate-line-event-module path parse_event_lines) + (check (file-exists? path) => #t)) + (finally + (when (file-exists? path) (delete-file path)))))) (test-case "AOT admits only declared event kinds and pure expressions" (check-exception (compile-line-event-parser diff --git a/t/ffi-header-smoke.c b/t/ffi-header-smoke.c new file mode 100644 index 00000000..265f5025 --- /dev/null +++ b/t/ffi-header-smoke.c @@ -0,0 +1,22 @@ +#include +/* + * SPDX-FileCopyrightText: 2026 tao3k team and Contributors + * SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later + */ + +#include +#include + +int main(void) { + gerbil_parser_runtime_result runtime_result; + gerbil_parser_runtime_result_init(&runtime_result); + (void)gerbil_parser_runtime_compile("grammar.ss", &runtime_result); + gerbil_parser_runtime_result_release(&runtime_result); + gerbil_parser_result language_result; + gerbil_parser_result_init(&language_result); + (void)gerbil_parser_language_descriptor(1, &language_result); + (void)gerbil_parser_language_parse(1, (const uint8_t *)"a=1", 3, &language_result); + (void)gerbil_parser_language_release(1); + gerbil_parser_result_release(&language_result); + return 0; +} diff --git a/t/ffi-test.ss b/t/ffi-test.ss new file mode 100644 index 00000000..64af44c6 --- /dev/null +++ b/t/ffi-test.ss @@ -0,0 +1,83 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- + +;; Admit the negative-case parser fixture through the normal native loader +;; before the timed test dynamically inspects its exported descriptor kinds. +(import (only-in :gerbil-parser/t/abi-datum-support abi-datum-read) + :std/test + (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/parser + parse-records) + (only-in :std/vector/u8vector little u8vector-u32-ref) + + (only-in :gerbil-parser/src/ffi/language-artifact-codec + bind-language-abi artifact-abi-version + abi-descriptor-payload + abi-parse-binary-payload) + (only-in ../src/ffi/rust-runtime-aot + rust-aot-abi-version + rust-runtime-source)) + +(import (only-in :gerbil-parser/languages/gql/parser gql-language-grammar) + (only-in :gerbil-parser/languages/cypher/parser opencypher-language-grammar)) +(def gql-native (bind-language-abi gql-language-grammar)) +(def cypher-native (bind-language-abi opencypher-language-grammar)) + +(def ffi-tests + (test-suite "parser-owned native ParseArtifact v1 ABI" + (test-case "descriptor publishes the parser-owned grammar surface" + (check (artifact-abi-version) => 1) + (let (descriptor (abi-datum-read (abi-descriptor-payload gql-native) + )) + (check (hash-get descriptor "schema") + => "gerbil-parser.native-descriptor.v1") + (check (hash-get descriptor "language") => "gql") + (check (positive? (length (hash-get descriptor "syntaxKinds"))) + => #t) + (let (fields (hash-get descriptor "fields")) + (check (not (not (member "operator" fields))) => #t) + (check (not (not (member "sign" fields))) => #t)))) + (test-case "accepted source produces typed binary ParseArtifact v1" + (let (artifact + (abi-parse-binary-payload gql-native "MATCH (n) RETURN n")) + (check (subu8vector artifact 0 4) => #u8(71 80 65 49)) + (check (u8vector-u32-ref artifact 4 little) => 1) + (check (u8vector-u32-ref artifact 8 little) => 0) + (check (positive? (u8vector-u32-ref artifact 12 little)) => #t) + (check (u8vector-length artifact) + => (+ 80 (* 24 (u8vector-u32-ref artifact 12 little)))))) + (test-case "openCypher uses its own descriptor and parser" + (let* ((descriptor + (abi-datum-read (abi-descriptor-payload cypher-native) + )) + (artifact + (abi-parse-binary-payload + cypher-native "MATCH (n:Person) RETURN n\n"))) + (check (hash-get descriptor "language") => "opencypher") + (check (positive? (length (hash-get descriptor "syntaxKinds"))) + => #t) + (check (u8vector-u32-ref artifact 8 little) => 0) + (check (positive? (u8vector-u32-ref artifact 12 little)) => #t))) + (test-case "invalid descriptor fails closed" + (check-exception (bind-language-abi "unknown") true)) + (test-case "grammar path compiles through the Scheme Rust backend" + (check (rust-aot-abi-version) => 1) + (let (source + (rust-runtime-source + "t/fixtures/runtime-record-assignments/languages/records/grammar.ss")) + (check (not (not (string-contains + source "language: \"record-assignments\""))) => #t) + (check (not (not (string-contains + source "grammar_digest: \"sha256:9c7e8b0f"))) => #t) + (check (not (not (string-contains + source "pub static LANGUAGE: LanguageSpec"))) => #t))) + (test-case "module without a language descriptor fails closed" + (check-exception + (rust-runtime-source + "t/fixtures/runtime-record-assignments/languages/records/parser.ss") + true)))) + +(export ffi-tests) + +;; gxtest discovers only exported names ending in -test. +(def ffi-test ffi-tests) +(export ffi-test) diff --git a/t/fixtures/arithmetic-reductions.ss b/t/fixtures/arithmetic-reductions.ss new file mode 100644 index 00000000..8353d60c --- /dev/null +++ b/t/fixtures/arithmetic-reductions.ss @@ -0,0 +1,2 @@ +;;; Generated by the engine FusedReductionStrategy from Parser IR. +(begin (import (only-in :gerbil-parser/src/runtime/event-program event-program-append) (only-in :gerbil-parser/src/runtime/recognition make-recognition-fragment) (only-in :gerbil-parser/src/runtime/reduce recognition-children-field recognition-children-alias) (only-in :gerbil-parser/src/runtime/event-reduce event-children-field event-children-alias) (only-in :gerbil-parser/src/runtime/funcs association-row-index-ref recognition-sequence->list recognition-sequence-for-action recognition-sequence-append recognition-sequence-start) (only-in :gerbil-parser/src/runtime/token token-start)) (export direct-step direct-event-step direct-grammar-digest) (def direct-grammar-digest "sha256:fed71c7da68ff163c284a2f0e9d3b5e192649ea124862eb20d06961d17712fe0") (def (direct-step production-id states semantic-values rest input-end-offset goto-index) (case production-id ((0) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$accept))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((1) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'SourceFile (recognition-sequence-for-action (recognition-children-field 'expression (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'source-file))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((2) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.1))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((3) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.1))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((4) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.0))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((5) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((6) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.3))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((7) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.3))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((8) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'left (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operator (recognition-sequence-for-action v1) offset make-recognition-fragment)) (recognition-children-field 'right (recognition-sequence-for-action v2) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.2))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((9) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'Expression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((10) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((11) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((12) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((13) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((14) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append (recognition-sequence-append '() v0) (recognition-children-field 'expression (recognition-sequence-for-action v1) offset make-recognition-fragment)) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$grouped-expression.4))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((15) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'GroupedExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'grouped-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((16) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$prefix-expression.6))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((17) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$prefix-expression.6))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((18) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-sequence-append (recognition-sequence-append '() (recognition-children-field 'operator (recognition-sequence-for-action v0) offset make-recognition-fragment)) (recognition-children-field 'operand (recognition-sequence-for-action v1) offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$prefix-expression.5))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((19) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'PrefixExpression (recognition-sequence-for-action v0) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'prefix-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((20) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NameExpression (recognition-sequence-for-action (recognition-children-field 'name (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'name-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((21) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (recognition-children-alias 'NumberExpression (recognition-sequence-for-action (recognition-children-field 'value (recognition-sequence-for-action v0) offset make-recognition-fragment)) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'number-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) (else (error "unknown generated reduction" production-id)))) (def (direct-event-step production-id states semantic-values rest input-end-offset goto-index) (case production-id ((0) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$accept))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((1) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'SourceFile (event-children-field 'expression v0 offset make-recognition-fragment) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'source-file))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((2) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.1))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((3) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.1))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((4) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() (event-children-field 'left v0 offset make-recognition-fragment)) (event-children-field 'operator v1 offset make-recognition-fragment)) (event-children-field 'right v2 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.0))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((5) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'Expression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((6) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.3))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((7) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.3))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((8) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() (event-children-field 'left v0 offset make-recognition-fragment)) (event-children-field 'operator v1 offset make-recognition-fragment)) (event-children-field 'right v2 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$expression.2))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((9) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'Expression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((10) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((11) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((12) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((13) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((14) (let* ((remaining-states (cdr (cdr (cdr states)))) (v2 (car semantic-values)) (tail2 (cdr semantic-values)) (v1 (car tail2)) (tail1 (cdr tail2)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append (event-program-append '() v0) (event-children-field 'expression v1 offset make-recognition-fragment)) v2)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$grouped-expression.4))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((15) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'GroupedExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'grouped-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((16) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$prefix-expression.6))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((17) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value v0) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$prefix-expression.6))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((18) (let* ((remaining-states (cdr (cdr states))) (v1 (car semantic-values)) (tail1 (cdr semantic-values)) (v0 (car tail1)) (remaining-values (cdr tail1)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-program-append (event-program-append '() (event-children-field 'operator v0 offset make-recognition-fragment)) (event-children-field 'operand v1 offset make-recognition-fragment))) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) '$prefix-expression.5))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((19) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'PrefixExpression v0 offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'prefix-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((20) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'NameExpression (event-children-field 'name v0 offset make-recognition-fragment) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'name-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) ((21) (let* ((remaining-states (cdr states)) (v0 (car semantic-values)) (remaining-values (cdr semantic-values)) (offset (if (pair? rest) (token-start (car rest)) input-end-offset)) (value (event-children-alias 'NumberExpression (event-children-field 'value v0 offset make-recognition-fragment) offset)) (entry (and (pair? remaining-states) (association-row-index-ref goto-index (car remaining-states) 'number-expression))) (target (and entry (cdr entry)))) (if target (values target (cons target remaining-states) (cons value remaining-values)) (values #f #f #f)))) (else (error "unknown generated reduction" production-id))))) diff --git a/t/fixtures/artifact-window.ss b/t/fixtures/artifact-window.ss new file mode 100644 index 00000000..6dbbfa1c --- /dev/null +++ b/t/fixtures/artifact-window.ss @@ -0,0 +1,23 @@ +;;; Complete products for shared publication, independent of a language grammar. +(import (only-in :std/list/list-builder with-list-builder) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact sha256-text) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/recognition + make-recognition-node make-recognition-child)) +(export window-tokens window-artifact) + +(def (window-tokens source) + (with-list-builder (emit) + (let loop ((index 0) (offset 0)) + (unless (= index (string-length source)) + (let* ((lexeme (string (string-ref source index))) + (end (+ offset (u8vector-length (string->utf8 lexeme))))) + (emit (make-token 'word lexeme offset end)) + (loop (+ index 1) end)))))) + +(def (window-artifact source tokens) + (make-success-parse-artifact (sha256-text "shared-window-publication") source tokens + (make-recognition-node 'Root 0 (u8vector-length (string->utf8 source)) + (map (lambda (token) (make-recognition-child #f token)) tokens)) + (lambda (token) #f))) diff --git a/t/fixtures/bash-products.ss b/t/fixtures/bash-products.ss new file mode 100644 index 00000000..101041a2 --- /dev/null +++ b/t/fixtures/bash-products.ss @@ -0,0 +1,17 @@ +;;; Internal compiler-product inspection for engine tests and Rust generators. +;;; These are derived from the admitted source vocabulary, never author profiles. +(import (only-in :clan/poo/object .ref) + (only-in :gerbil-parser/languages/bash/grammar bash-syntax) + (only-in :gerbil-parser/languages/bash/parser bash-source-language) + (only-in :gerbil-parser/src/language/source source-syntax-strategy)) +(export bash-source-language bash-word-regions bash-command-scanner bash-results + bash-parts bash-commands bash-simple-binding bash-parameter-binding bash-assignment-binding) +(def products (source-syntax-strategy bash-syntax)) +(def bash-word-regions (.ref products 'regions)) +(def bash-command-scanner (.ref products 'scanner)) +(def bash-results (.ref products 'results)) +(def bash-parts (.ref products 'parts)) +(def bash-commands (.ref products 'commands)) +(def bash-simple-binding (cdr (assq 'simple (.ref bash-parts 'bindings)))) +(def bash-parameter-binding (cdr (assq 'parameter (.ref bash-parts 'bindings)))) +(def bash-assignment-binding (cdr (assq 'assignment (.ref bash-parts 'bindings)))) diff --git a/t/fixtures/bound-source-catalog.ss b/t/fixtures/bound-source-catalog.ss new file mode 100644 index 00000000..2bcd0999 --- /dev/null +++ b/t/fixtures/bound-source-catalog.ss @@ -0,0 +1,19 @@ +;;; Closed canonical declarations and their independent source locations. +(export bound-source-catalog) + +(def (bound-source-catalog width) + (let* ((names (map (lambda (index) (string->symbol (string-append "Kind" (number->string index)))) + (iota width))) + (source (lambda (name) (list (cons 'path (symbol->string name)) + (cons 'generated? #f)))) + (source-order (reverse names)) + (grammar + (list (cons 'schema "gerbil-parser.grammar-ir.v1") + (cons 'grammar 'bound-source-catalog) + (cons 'syntax-kinds (map (lambda (name) (list name 'node '(value))) names)) + (cons 'terminals '()) (cons 'lexical-rules '()) (cons 'rules '()))) + (sources + (list (cons 'syntax-kind (map (lambda (name) (cons name (source name))) source-order)) + (cons 'field (map (lambda (name) (cons (list name 'value) (source name))) + source-order))))) + (values grammar sources))) diff --git a/t/fixtures/contextual-result-selection.ss b/t/fixtures/contextual-result-selection.ss new file mode 100644 index 00000000..56f5ab76 --- /dev/null +++ b/t/fixtures/contextual-result-selection.ss @@ -0,0 +1,60 @@ +;;; Test-owned former dispatch, applied only to an independent prepared handle. +;;; Private access is resolved once during preparation, never inside timing. +(import (only-in :gerbil-parser/src/ffi/language-abi-context language-abi-context-plan)) +(import (only-in :gerbil-parser/src/runtime/region-scanner source-prefix-at?)) +(export install-reference-contextual-results!) +(def (private namespace name) (eval (string->symbol (string-append namespace "#" name)))) +(def (install-reference-contextual-results! context) + (let* ((ns "gerbil-parser/src/runtime/contextual-scanner") + (get (lambda (name) (private ns name))) + (plan ((private "gerbil-parser/src/runtime/parser" "contextual-parser-plan-scanner") + (language-abi-context-plan context))) + (seen (make-table test: eq?)) + (mode (get "runtime-scan-rule-mode")) (form (get "runtime-scan-rule-form")) + (set-result! (get "runtime-scan-rule-result-set!")) + (cells (get "contextual-scanner-cells")) + (modes (get "grouped-dispatch-modes")) + (matcher (get "runtime-scan-rule-matcher")) + (apply-action (get "apply-scan-action")) + (set-action! (get "runtime-scan-rule-action-step-set!")) + (set-executor! (get "runtime-scan-rule-executor-set!")) + (set-width! (get "runtime-scan-rule-byte-width-set!"))) + (def (install! rule) + (unless (table-ref seen rule #f) + (table-set! seen rule #t) + (let ((m (mode rule)) (f (form rule))) + (set-result! rule + (lambda (scanner position) + (let* ((index (cells scanner)) + (kind (if (table? index) + (table-ref index (list m position f) #f) + (let* ((positions (table-ref (modes index) m #f)) + (row (and positions (table-ref positions position '()))) + (cell (and row (assq f row)))) + (and cell (cdr cell)))))) + (or kind (error "missing contextual scanner dispatch" m position f))))) + (set-action! rule apply-action) + (set-width! rule #f) + (match (matcher rule) + (['literal value] + (let (width (string-length value)) + (set-executor! rule (lambda (source start _state) + (and (source-prefix-at? source start value) (+ start width)))))) + (else (void)))))) + (for-each + (lambda (index) + (for-each + (lambda (entry) + (let (rules (cdr entry)) + (if ((get "first-character-rules?") rules) + (begin + (for-each (lambda (row) (for-each install! row)) + (vector->list ((get "first-character-rules-ascii") rules))) + (for-each (lambda (row) (for-each install! (cdr row))) + (table->list ((get "first-character-rules-wide") rules))) + (for-each install! ((get "first-character-rules-alpha") rules)) + (for-each install! ((get "first-character-rules-other") rules))) + (for-each install! rules)))) (table->list index))) + (list ((get "contextual-scanner-plan-short-rules") plan) + ((get "contextual-scanner-plan-long-rules") plan))) + context)) diff --git a/t/fixtures/ffi/abi-probe.ss b/t/fixtures/ffi/abi-probe.ss new file mode 100644 index 00000000..ac30f270 --- /dev/null +++ b/t/fixtures/ffi/abi-probe.ss @@ -0,0 +1,94 @@ +;;; -*- Gerbil -*- +;;; Exercise the public C ABI after the normal Gerbil runtime initializes. +(import :gerbil-parser/src/ffi/language-abi + :gerbil-parser/src/ffi/rust-aot-abi + (only-in :gerbil-parser/src/ffi/rust-runtime-aot rust-runtime-source)) +(import (only-in :gerbil-parser/languages/gql/parser gql-language-grammar) + (only-in :gerbil-parser/src/ffi/language-handles register-language! release-language!)) +(export main) +(extern probe-reuse) + +(begin-foreign + (namespace ("gerbil-parser/t/fixtures/ffi/abi-probe#" probe-reuse)) + (c-declare #< +#include +#include +#include + +static int parse_source(uint64_t language, const char *source, gerbil_parser_result *result) { + return gerbil_parser_language_parse(language, (const uint8_t *)source, source ? strlen(source) : 1, result); +} +static int probe_result_reuse(uint64_t language, const char *expected) { + gerbil_parser_result result; + int i; + if (gerbil_parser_language_abi_version() != 2) return 3; + gerbil_parser_result_init(NULL); + gerbil_parser_result_release(NULL); + gerbil_parser_result_init(&result); + for (i = 0; i < 1000; ++i) { + if (parse_source(language, "RETURN 1", &result) != 0 || + result.status != 0 || result.payload == NULL || result.length < 80) { + gerbil_parser_result_release(&result); + return 1; + } + if ((i + 1) % 100 == 0) { + printf("ABI-PARSE-BATCH-OK calls=%d\n", i + 1); fflush(stdout); + } + } + gerbil_parser_result_release(&result); + if (result.payload != NULL || result.length != 0 || result.status != 0) return 2; + gerbil_parser_result_release(&result); + if (parse_source(language, "RETURN 1", NULL) != -1) return 4; + if (gerbil_parser_language_descriptor(language, &result) != 0 || + result.payload == NULL || result.length == 0) return 5; + if (parse_source(0, "RETURN 1", &result) != -1 || + result.status != -1 || result.payload == NULL || result.length == 0) return 6; + if (parse_source(language, NULL, &result) != -1 || + result.status != -1 || result.payload != NULL || result.length != 0) return 7; + if (gerbil_parser_language_descriptor(0, &result) != -1) return 8; + if (parse_source(language, "RETURN '你好'", &result) != 0 || + result.status != 0 || result.payload == NULL) return 9; + gerbil_parser_result_release(&result); + puts("ABI-PARSE-ERROR-UTF8-REUSE-OK"); fflush(stdout); + + gerbil_parser_runtime_result runtime; + gerbil_parser_runtime_result_init(&runtime); + gerbil_parser_runtime_result_init(NULL); + gerbil_parser_runtime_result_release(NULL); + if (gerbil_parser_runtime_aot_abi_version() != 1 || + gerbil_parser_runtime_compile("missing.ss", NULL) != -1) return 10; + for (i = 0; i < 3; ++i) { + if (gerbil_parser_runtime_compile( + "t/fixtures/runtime-record-assignments/languages/records/grammar.ss", + &runtime) != 0 || runtime.status != 0 || runtime.payload == NULL || + runtime.length == 0) return 11; + if (runtime.length != strlen(expected) || + memcmp(runtime.payload, expected, runtime.length) != 0) return 12; + printf("ABI-RUNTIME-GENERATED-MATCH calls=%d\n", i+1); fflush(stdout); + } + if (gerbil_parser_runtime_compile("missing.ss", &runtime) != -1 || + runtime.status != -1 || runtime.payload == NULL) return 15; + if (gerbil_parser_runtime_compile(NULL, &runtime) != -1 || + runtime.status != -1 || runtime.payload != NULL || runtime.length != 0) return 16; + gerbil_parser_runtime_result_release(&runtime); + gerbil_parser_runtime_result_release(&runtime); + puts("ABI-RUNTIME-ERROR-REUSE-OK"); fflush(stdout); + return 0; +} +END-C + ) + (define probe-reuse (c-lambda (unsigned-int64 UTF-8-string) int "probe_result_reuse"))) + +(def (main . _) + (displayln "NATIVE-ABI-MODULES-LOADED") (force-output) + (let (expected (rust-runtime-source + "t/fixtures/runtime-record-assignments/languages/records/grammar.ss")) + (displayln "NATIVE-ABI-EXPECTED-GENERATED") (force-output) + (let (handle (register-language! gql-language-grammar)) + (dynamic-wind void + (lambda () + (let (status (probe-reuse handle expected)) + (unless (zero? status) (error "public ABI regression failed" status)))) + (lambda () (release-language! handle))))) + (displayln "NATIVE-ABI-OK") (force-output)) diff --git a/t/fixtures/ffi/language-probe.ss b/t/fixtures/ffi/language-probe.ss new file mode 100644 index 00000000..1799a0fe --- /dev/null +++ b/t/fixtures/ffi/language-probe.ss @@ -0,0 +1,140 @@ +;;; Public byte-length C API with an independently compiled language pack. +(import :gerbil-parser/src/ffi/language-abi + :gerbil-parser/t/fixtures/shared-scanner/records-abi + (only-in :gerbil-parser/src/ffi/language-handles + release-language! language-handle-parse) + (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(export probe-source-language main parse-native-batch parse-native-sized-batch create-native-handle) +(extern probe-source-language probe-language parse-native-batch parse-native-sized-batch create-native-handle) +(begin-foreign + (namespace ("gerbil-parser/t/fixtures/ffi/language-probe#" probe-source-language probe-language parse-native-batch parse-native-sized-batch create-native-handle)) + (c-declare #< +#include +#include +#include +#include +#include "t/fixtures/shared-scanner/records-abi.h" +#include +static double monotonic_seconds(void); +static int probe_source_language(uint64_t handle) { + gerbil_parser_result r; gerbil_parser_result_init(&r); + const uint8_t source[] = "echo \"α${x:-中}\"\n"; + const uint8_t invalid[] = {0xff}; + int status = 0; + if (gerbil_parser_language_descriptor(handle,&r) || !r.payload || !r.length) { status=1; goto done; } + double started=monotonic_seconds(); clock_t cpu_started=clock(); + for (int i=0; i<100; ++i) { + if (gerbil_parser_language_parse(handle,source,sizeof(source)-1,&r) || + r.length<80 || memcmp(r.payload,"GPA1",4) || r.payload[8]) { status=2; goto done; } + if ((i+1)%25==0) { printf("SOURCE-ABI-PARSE-OK calls=%d\n",i+1); fflush(stdout); } + } + printf("SOURCE-ABI-100-CALLS wall-ms=%.3f cpu-ms=%.3f\n",1000*(monotonic_seconds()-started),1000.0*(clock()-cpu_started)/CLOCKS_PER_SEC); fflush(stdout); + if (gerbil_parser_language_parse(handle,invalid,sizeof invalid,&r)!=-1 || r.status!=-1) { status=3; goto done; } + if (gerbil_parser_language_parse(handle,NULL,0,&r) || r.length<80 || r.payload[8]) { status=4; goto done; } + if (gerbil_parser_language_release(handle)) { status=5; goto done; } + if (gerbil_parser_language_parse(handle,source,sizeof(source)-1,&r)!=-1 || r.status!=-1) { status=6; goto done; } + done: + gerbil_parser_language_release(handle); + gerbil_parser_result_release(&r); gerbil_parser_result_release(&r); + if (r.payload || r.length || r.status) return 7; + return status; +} +static int parse_native_batch(uint64_t handle) { + const uint8_t source[] = "α=1\n"; + gerbil_parser_result result; gerbil_parser_result_init(&result); + for (int i=0;i<100;i++) { + if (gerbil_parser_language_parse(handle,source,sizeof(source)-1,&result) || result.length<80 || result.payload[8]) { + gerbil_parser_result_release(&result);return -1; + } + } + gerbil_parser_result_release(&result);return 0; +} +static int parse_native_sized_batch(uint64_t handle, int rows, int unicode, int calls) { + const char *row = unicode ? "α=1\r\n" : "a=1\n"; + size_t width = strlen(row); + if (rows < 1 || rows > 16384 || calls < 1 || calls > 100) return -1; + size_t length = width * (size_t)rows; + uint8_t *source = malloc(length); + if (!source) return -1; + for (int i=0;i\nint gerbil_parser_rust_native_probe(void);\nint gerbil_parser_rust_native_source_probe(uint64_t, uint64_t);") + (define rust-native-probe (c-lambda () int "gerbil_parser_rust_native_probe")) + (define rust-native-source-probe (c-lambda (unsigned-int64 unsigned-int64) int "gerbil_parser_rust_native_source_probe"))) +(def (main . _) + (displayln "RUST-NATIVE-START") (force-output) + (unless (zero? (rust-native-probe)) (error "Rust native ownership probe failed")) + (let* ((baseline (language-handle-count)) + (first (register-language! bash-source-language)) + (second (register-language! bash-source-language))) + (unless (zero? (rust-native-source-probe first second)) (error "Rust Source edit probe failed")) + (unless (= baseline (language-handle-count)) (error "Rust Source handle leak"))) + (displayln "RUST-NATIVE-OK") (force-output) + (test-child-process-exit! 0)) diff --git a/t/fixtures/fixture-release.ss b/t/fixtures/fixture-release.ss new file mode 100644 index 00000000..9e41cb94 --- /dev/null +++ b/t/fixtures/fixture-release.ss @@ -0,0 +1,16 @@ +;;; -*- Gerbil -*- +;;; Test release identity belongs outside the author grammar declaration. +(import (only-in :gerbil-parser/src/language/descriptor language-grammar-with-identity)) +(export bind-fixture-grammar-release) + +(defsyntax (bind-fixture-grammar-release stx) + (syntax-case stx () + ((_ prefix language version contract) + (identifier? #'prefix) + (with-syntax + ((definition (datum->syntax #'prefix + (string->symbol (string-append (symbol->string (syntax->datum #'prefix)) "-syntax")))) + (released (datum->syntax #'prefix + (string->symbol (string-append (symbol->string (syntax->datum #'prefix)) "-language-grammar"))))) + #'(def released (language-grammar-with-identity definition language version contract)))) + (_ (raise-syntax-error #f "expected fixture prefix and release identity" stx)))) diff --git a/t/fixtures/grammar-composition-models.ss b/t/fixtures/grammar-composition-models.ss new file mode 100644 index 00000000..3e962ba1 --- /dev/null +++ b/t/fixtures/grammar-composition-models.ss @@ -0,0 +1,311 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- + +(import :std/test + (only-in :clan/poo/object .o) + :gerbil-parser/src/grammar/algebra + :gerbil-parser/src/grammar/lexical-algebra + :gerbil-parser/src/compiler/normalize + :gerbil-parser/src/compiler/parser-ir + :gerbil-parser/src/compiler/lr + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + :gerbil-parser/src/language/grammar + :gerbil-parser/src/modules/parser/interface + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-roundtrip parse-artifact-success?) + (only-in :gerbil-parser/src/runtime/funcs vector-intern-map) + (only-in :gerbil-parser/src/runtime/incremental + apply-edit make-edit make-incremental-session + incremental-session-artifact parse-incremental-session + parse-source/incremental) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + :gerbil-parser/src/runtime/recognition + (only-in :gerbil-parser/src/runtime/lr-parser + lr-checkpoint? lr-checkpoint-deterministic-actions + lr-checkpoint-deterministic-shifts + lr-checkpoint-remaining-token-count + lr-checkpoint-advance lr-checkpoint-resume + lr-initial-checkpoint lr-parse lr-parse/receipt lr-prepare + lr-runtime-lexical-mode-catalog) + (only-in :gerbil-parser/src/runtime/parser parse-source) + :gerbil-parser/src/runtime/token + :gerbil-parser/src/compiler/machine + :gerbil-parser/languages/arithmetic/parser) + +;;; Both rules recognize the same spelling. Only the LR state can select the +;;; correct token identity for each position; global longest-match necessarily +;;; selects the declaration-first identity twice. +(begin + (deflanguage directed-lexical-mode + (syntax + (lexical + (root source-file) + (lex + (first FirstToken (literals "x" "y")) + (second SecondToken (literals "x" "y"))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile + (repeat + (seq (field first first) (field second second))))))) + (bind-fixture-grammar-release directed-lexical-mode "directed-lexical-mode" "v1" "directed-lexical-mode.v1") ) + +;;; The mode-local regular DFA must admit whole rules by LR state. Both +;;; identifiers have equal source behavior but distinct token identities. +(begin + (deflanguage directed-regular-mode + (syntax + (lexical + (root source-file) + (lex + (first FirstToken (identifier)) + (second SecondToken (identifier)) + (space Space (whitespace+))) + (extras space) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile + (seq (field first first) (field second second)))))) + (bind-fixture-grammar-release directed-regular-mode "directed-regular-mode" "v1" "directed-regular-mode.v1") ) + +(defgrammar-role base-lexical-role + (syntax-kinds + (Punctuation token (text))) + (terminals + (punctuation Punctuation)) + (lexical-rules + (punctuation (literals "+" "-" "*" "**" "/" "(" ")"))) + (rules) + (extras) + (keywords) + (parser-entrypoints) + (recoveries) + (flow)) + +(defgrammar base-object-grammar + (supers) + (roles base-lexical-role)) + +(defgrammar-role appended-identifier-role + (syntax-kinds (Identifier token (text))) + (terminals (identifier Identifier)) + (lexical-rules (identifier (identifier))) + (rules) + (extras) + (keywords) + (parser-entrypoints) + (recoveries) + (flow)) + +(defgrammar-compose explicit-composed-grammar + (supers base-object-grammar) + (compose (append appended-identifier-role))) + +(defgrammar-role conflicting-lexical-role + (syntax-kinds + (Punctuation token (text))) + (terminals + (punctuation Punctuation)) + (lexical-rules + (punctuation (literals "+" "-" "*" "/" "(" ")"))) + (rules) + (extras) + (keywords) + (parser-entrypoints) + (recoveries) + (flow)) + +(defgrammar conflicting-arithmetic-grammar + (supers base-object-grammar) + (roles conflicting-lexical-role)) + +(defgrammar-role unresolved-reference-role + (syntax-kinds + (SourceFile node ()) + (Unknown token (text))) + (terminals + (unknown Unknown)) + (lexical-rules + (unknown (fallback))) + (rules + (source-file (reference missing-rule))) + (extras) + (keywords) + (parser-entrypoints + (source-file parse pure)) + (recoveries) + (flow + (source lexical) + (lexical cst))) + +(defgrammar unresolved-reference-grammar + (supers) + (roles unresolved-reference-role)) + +(defgrammar-role invalid-terminal-kind-role + (syntax-kinds + (SourceFile node (value))) + (terminals + (value SourceFile)) + (lexical-rules + (value (identifier))) + (rules + (source-file + (alias SourceFile (field value (token value))))) + (extras) + (keywords) + (parser-entrypoints + (source-file parse pure)) + (recoveries) + (flow + (source lexical) + (lexical cst))) + +(defgrammar invalid-terminal-kind-grammar + (supers) + (roles invalid-terminal-kind-role)) + +(defgrammar-role missing-lexical-rule-role + (syntax-kinds + (SourceFile node (value)) + (Identifier token (text))) + (terminals + (identifier Identifier)) + (lexical-rules) + (rules + (source-file + (alias SourceFile (field value (token identifier))))) + (extras) + (keywords) + (parser-entrypoints + (source-file parse pure)) + (recoveries) + (flow + (source lexical) + (lexical cst))) + +(defgrammar missing-lexical-rule-grammar + (supers) + (roles missing-lexical-rule-role)) + +(defgrammar-role unsupported-precedence-role + (syntax-kinds + (SourceFile node (value)) + (Identifier token (text))) + (terminals + (identifier Identifier)) + (lexical-rules + (identifier (identifier))) + (rules + (source-file + (alias SourceFile + (field value + (prec left 10 (token identifier)))))) + (extras) + (keywords) + (parser-entrypoints + (source-file parse pure)) + (recoveries) + (flow + (source lexical) + (lexical cst))) + +(defgrammar unsupported-precedence-grammar + (supers) + (roles unsupported-precedence-role)) + +(def (condition-message thunk) + (with-catch (lambda (condition) (error-message condition)) thunk)) + +(def (row-ref row key) + (let (entry (assq key row)) (and entry (cdr entry)))) + +(def (compile-composed grammar) + (compile-parser (compile-grammar grammar))) + +(def dynamic-precedence-rules + '((source-file + (precedence dynamic 1 + (alias SourceFile (field value (token identifier))))))) + +(def competing-dynamic-precedence-rules + '((source-file + (choice (reference low-path) (reference high-path))) + (low-path + (precedence dynamic 1 + (alias LowPath (field value (token identifier))))) + (high-path + (precedence dynamic 2 + (alias HighPath (field value (token identifier))))))) + +;;; The two reductions accept the same token and publish the same tree. A +;;; correct selective-GLR runtime must execute both and merge their completed +;;; candidates even when neither production carries dynamic precedence. +(def static-equivalent-ambiguity-rules + '((source-file + (choice (reference first-path) (reference second-path))) + (first-path + (alias SourceFile (field value (token identifier)))) + (second-path + (alias SourceFile (field value (token identifier)))))) + +;;; The same reduce/reduce shape deliberately publishes distinct roots. The +;;; runtime must not turn declaration order into an implicit ambiguity policy. +(def static-distinct-ambiguity-rules + '((source-file + (choice (reference first-path) (reference second-path))) + (first-path + (alias FirstPath (field value (token identifier)))) + (second-path + (alias SecondPath (field value (token identifier)))))) + +(def ambiguous-left-recursive-rules + '((source-file + (choice + (sequence (reference source-file) (reference source-file)) + (alias SourceFile (field value (token identifier))))))) + +(def selective-session-rules + '((source-file + (alias SourceFile + (sequence (reference session-activity) + (optional (reference session-close))))) + (session-activity (repeat1 (reference session-set))) + (session-set (sequence (literal "SESSION") (literal "SET"))) + (session-close (sequence (literal "SESSION") (literal "CLOSE"))))) + +(def nonassociative-rules + '((source-file + (alias SourceFile (field expression (reference expression)))) + (expression + (choice + (precedence none 10 + (alias ComparisonExpression + (sequence + (field left (reference expression)) + (field operator (literal "<")) + (field right (reference expression))))) + (alias NumberExpression (field value (token number))))))) + +;;; Shared fixture declarations have a separate native compiler owner. +(export directed-lexical-mode-parser directed-regular-mode-parser + base-lexical-role base-object-grammar appended-identifier-role + explicit-composed-grammar conflicting-lexical-role + conflicting-arithmetic-grammar unresolved-reference-role + unresolved-reference-grammar invalid-terminal-kind-role + invalid-terminal-kind-grammar missing-lexical-rule-role + missing-lexical-rule-grammar unsupported-precedence-role + unsupported-precedence-grammar condition-message row-ref compile-composed + dynamic-precedence-rules competing-dynamic-precedence-rules + static-equivalent-ambiguity-rules static-distinct-ambiguity-rules + ambiguous-left-recursive-rules selective-session-rules nonassociative-rules) diff --git a/t/fixtures/language-diagnostics-vocabulary.ss b/t/fixtures/language-diagnostics-vocabulary.ss new file mode 100644 index 00000000..bded88f2 --- /dev/null +++ b/t/fixtures/language-diagnostics-vocabulary.ss @@ -0,0 +1,10 @@ +;;; -*- Gerbil -*- +(import (only-in ../../src/language/grammar defgrammar-syntax)) +(export required-items broken-wrapper) +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +;; A library template with an invalid constructor must blame its author call. +(defgrammar-syntax (broken-wrapper item) + (misspelled item)) diff --git a/t/fixtures/language-entries.ss b/t/fixtures/language-entries.ss new file mode 100644 index 00000000..08f7ca3a --- /dev/null +++ b/t/fixtures/language-entries.ss @@ -0,0 +1,32 @@ +;;; Shared production/development entries for independent language controls. +(import + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-language) + (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-test-language) + (only-in :gerbil-parser/languages/bash/parser bash-language) + (only-in :gerbil-parser/languages/bash/parser-test bash-test-language) + (only-in :gerbil-parser/languages/cypher/parser opencypher-language) + (only-in :gerbil-parser/languages/cypher/parser-test opencypher-test-language) + (only-in :gerbil-parser/languages/fhirpath/parser fhirpath-language) + (only-in :gerbil-parser/languages/fhirpath/parser-test fhirpath-test-language) + (only-in :gerbil-parser/languages/gql/parser gql-language) + (only-in :gerbil-parser/languages/gql/parser-test gql-test-language) + (only-in :gerbil-parser/languages/hcl/parser hcl-language) + (only-in :gerbil-parser/languages/hcl/parser-test hcl-test-language) + (only-in :gerbil-parser/languages/hl7/parser hl7-language) + (only-in :gerbil-parser/languages/hl7/parser-test hl7-test-language) + (only-in :gerbil-parser/languages/tla-plus/parser + tla-plus-core-language tla-plus-layout-language tla-plus-sany-candidate-language) + (only-in :gerbil-parser/languages/tla-plus/parser-test + tla-plus-core-test-language tla-plus-layout-test-language tla-plus-sany-candidate-test-language)) +(export language-entry-pairs) +(def language-entry-pairs + (list (list arithmetic-language arithmetic-test-language 2) + (list bash-language bash-test-language 2) + (list opencypher-language opencypher-test-language 5) + (list fhirpath-language fhirpath-test-language 4) + (list gql-language gql-test-language 15) + (list hcl-language hcl-test-language 15) + (list hl7-language hl7-test-language 2) + (list tla-plus-core-language tla-plus-core-test-language 6) + (list tla-plus-layout-language tla-plus-layout-test-language 2) + (list tla-plus-sany-candidate-language tla-plus-sany-candidate-test-language 2))) diff --git a/t/fixtures/language-pack-research/author-context.ss b/t/fixtures/language-pack-research/author-context.ss new file mode 100644 index 00000000..28411e6f --- /dev/null +++ b/t/fixtures/language-pack-research/author-context.ss @@ -0,0 +1,4 @@ +;;; -*- Gerbil -*- +(import :std/stxparam) +(export @@author-declaration) +(defsyntax-parameter @@author-declaration #f) diff --git a/t/fixtures/language-pack-research/author-diagnostics.ss b/t/fixtures/language-pack-research/author-diagnostics.ss new file mode 100644 index 00000000..147e7301 --- /dev/null +++ b/t/fixtures/language-pack-research/author-diagnostics.ss @@ -0,0 +1,95 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-component" "author-list-vocabulary" + (rename-in "author-list-vocabulary" (author-arguments imported-author-arguments)) + (for-syntax (only-in :gerbil/expander core-expand core-expand1))) + +(defsyntax (collect-author-errors stx) + (def (location syntax-value) + (let (loc (stx-source syntax-value)) + (and loc + (let (start (##position->filepos (##locat-start-position loc))) + (list (##container->path (##locat-container loc)) + (+ 1 (##filepos-line start)) (+ 1 (##filepos-col start))))))) + (def (capture form) + (with-catch + (lambda (condition) + (let* ((irritants (slot-ref condition 'irritants)) (blame (car irritants))) + (list (syntax-error? condition) (error-message condition) + (syntax->datum blame) (location blame) + (syntax->datum (cadr irritants))))) + (lambda () (core-expand form) #f))) + (syntax-case stx () + ((_ form ...) + (datum->syntax #'collect-author-errors + (list 'quote (stx-map capture #'(form ...))))))) + +(defsyntax (valid-author-expansion stx) + (syntax-case stx () + ((_ outer explicit) + (let (expanded (core-expand1 #'outer)) + (syntax-case expanded (field) + ((target binding owner entry item separator (field label)) + (datum->syntax #'valid-author-expansion + (list 'quote + (list (free-identifier=? #'target #'deflocated-list) + (equal? (syntax->datum expanded) (syntax->datum #'explicit)) + (equal? (stx-source #'label) + (stx-source (stx-list-ref #'outer 6)))))))))))) + +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "author diagnostic mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "AUTHOR-DIAGNOSTIC-CASE-OK: ") (display label) (newline) (force-output)) +(def (line-at path number) + (call-with-input-file path + (lambda (port) + (let loop ((index 1)) + (let (line (read-line port)) + (when (eof-object? line) (error "nested source line absent" path number)) + (if (= index number) line (loop (+ index 1)))))))) +(for-each + (lambda (result token message context-head) + (probe (list 'typed token) (car result) #t) + (probe (list 'message token) (cadr result) message) + (probe (list 'blame token) (caddr result) token) + (let* ((loc (list-ref result 3)) + (line (line-at (car loc) (cadr loc))) (offset (- (caddr loc) 1)) + (text (number->string token))) + (probe (list 'author-token token) + (substring line offset (+ offset (string-length text))) text)) + (probe (list 'declaration-context token) (car (list-ref result 4)) context-head)) + (collect-author-errors + (author-arguments component call-arguments arguments + (reference name) (literal ",") 81) + (imported-author-arguments component call-arguments arguments + (reference name) (literal ",") 82) + ;; Outer label validation wins even though owner is also invalid. + (author-arguments component 83 arguments + (reference name) (literal ",") 84) + ;; Valid outer label delegates invalid owner to the existing component. + (author-arguments component 85 arguments + (reference name) (literal ",") argument) + ;; The direct component uses its existing owner-before-label order. + (deflocated-list component 86 arguments + (reference name) (literal ",") (field 87))) + '(81 82 84 85 86) + '("author-arguments: label must be an identifier" + "author-arguments: label must be an identifier" + "author-arguments: label must be an identifier" + "deflocated-list: owner must be an identifier" + "deflocated-list: owner must be an identifier") + '(author-arguments imported-author-arguments author-arguments deflocated-list deflocated-list)) +(let (results + (valid-author-expansion + (author-arguments valid-component call-arguments arguments + (reference name) (literal ",") argument) + (deflocated-list valid-component call-arguments arguments + (reference name) (literal ",") (field argument)))) + (probe 'valid-target-binding (car results) #t) + (probe 'valid-expansion-datum (cadr results) #t) + (probe 'valid-label-reader-source (caddr results) #t)) +(display "AUTHOR-DIAGNOSTICS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/author-list-vocabulary.ss b/t/fixtures/language-pack-research/author-list-vocabulary.ss new file mode 100644 index 00000000..5bfa1b81 --- /dev/null +++ b/t/fixtures/language-pack-research/author-list-vocabulary.ss @@ -0,0 +1,13 @@ +;;; -*- Gerbil -*- +(import "located-component" + (for-syntax "declaration-contracts")) +(export author-arguments) +(defsyntax (author-arguments stx) + (syntax-case stx () + ((_ binding owner entry item separator label) + (begin + ;; This wrapper owns its positional label contract before delegation. + (require-declaration-identifier #'label stx "author-arguments" "label") + #'(deflocated-list binding owner entry item separator (field label)))) + (_ (raise-syntax-error #f + "author-arguments: expected binding owner entry item separator label" stx)))) diff --git a/t/fixtures/language-pack-research/binding.ss b/t/fixtures/language-pack-research/binding.ss new file mode 100644 index 00000000..4eaa386a --- /dev/null +++ b/t/fixtures/language-pack-research/binding.ss @@ -0,0 +1,4 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "consumer") +(run-binding-research) diff --git a/t/fixtures/language-pack-research/composition-history.ss b/t/fixtures/language-pack-research/composition-history.ss new file mode 100644 index 00000000..e9d03820 --- /dev/null +++ b/t/fixtures/language-pack-research/composition-history.ss @@ -0,0 +1,321 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; The existing normalizer remains the sole owner of composition behavior. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + "list-roles" "list-origins" + (only-in "../../../src/modules/parser/objects" make-grammar-role make-grammar grammar-role-ref) + (only-in "../../../src/compiler/normalize" compile-grammar compile-grammar/receipt grammar-ir-ref + compile-grammar/context normalized-grammar-ir normalized-grammar-receipt + normalized-grammar-source-map) + (only-in "../../../src/compiler/parser-ir" compile-parser parser-ir-ref) + (only-in "../../../src/compiler/language-artifact" project-language-catalog) + (only-in "../../../src/compiler/bound-ir" bind-grammar-ir bound-grammar-ir-binding)) +(export main) +(def (main) + + (def checked 0) + (def (probe label actual expected) + (unless (equal? actual expected) (error "composition history mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "HISTORY-CASE-OK: ") (display label) (newline) (force-output)) + (def (ref value key) (let (row (assq key value)) (and row (cdr row)))) + (def (rule-role owner rows) + (make-grammar-role owner '() '() '() rows '() '() '() '() '())) + (def base (make-list-study-grammar)) + (def replacement '(field argument (reference name))) + (def override-role (rule-role 'single-argument-extension (list (list 'arguments replacement)))) + (def remove-role (rule-role 'retired-tail '((call-arguments/tail (empty))))) + (def (error-receipt thunk) + (with-catch (lambda (condition) (list (error-message condition) (error-irritants condition))) thunk)) + + (let-values (((grammar receipt) + (compile-grammar/receipt + (make-grammar 'overridden (list base) '() + (list (cons 'override override-role)))))) + (probe 'override-replaces-rule (assq 'arguments (grammar-ir-ref grammar 'rules)) + (list 'arguments replacement)) + (probe 'override-retains-independent-helper + (and (assq 'call-arguments/tail (grammar-ir-ref grammar 'rules)) #t) #t) + (probe 'receipt-retains-operation-order + (map (lambda (step) (ref step 'operation)) (ref receipt 'steps)) + '(merge append append override)) + (probe 'receipt-retains-owner-history + (map (lambda (step) (ref step 'role)) (ref receipt 'steps)) + '(list-study-base-role call-arguments array-elements single-argument-extension)) + (let* ((append-only-map (list-study-source-map receipt)) + (bound (bind-grammar-ir grammar "history-fixture" '(composition-history) append-only-map))) + ;; Retain the negative control: this previous fixture only supports unique append. + (probe 'append-only-origin-is-stale-after-override + (ref (ref (bound-grammar-ir-binding bound 'rule 'arguments) 'source) 'componentOwner) + 'call-arguments)) + ;; Explicit selected provenance for this one row; not an automatic history resolver. + (let* ((source + '((path . "t/fixtures/language-pack-research/composition-history.ss") + (location . "single-argument-extension: arguments override") + (generated? . #t) + (componentOwner . single-argument-extension) + (contributionHistory . ((1 append call-arguments) (3 override single-argument-extension))))) + (bound (bind-grammar-ir grammar "history-fixture" '(composition-history) + (list (cons 'rule (list (cons 'arguments source))))))) + (probe 'selected-override-owner + (ref (ref (bound-grammar-ir-binding bound 'rule 'arguments) 'source) 'componentOwner) + 'single-argument-extension) + (probe 'retained-explicit-history + (ref (ref (bound-grammar-ir-binding bound 'rule 'arguments) 'source) 'contributionHistory) + '((1 append call-arguments) (3 override single-argument-extension)))) + (probe 'overridden-parser-admitted (parser-ir-ref (compile-parser grammar) 'root-rule) 'source-file)) + + (let ((dangling + (compile-grammar + (make-grammar 'dangling (list base) '() (list (cons 'remove remove-role)))))) + (probe 'normalizer-removes-helper + (assq 'call-arguments/tail (grammar-ir-ref dangling 'rules)) #f) + (probe 'parser-rejects-dangling-helper + (error-receipt (lambda () (compile-parser dangling))) + '("unresolved grammar identity" (arguments call-arguments/tail))) + (let (condition (error-receipt + (lambda () (bind-grammar-ir dangling "history-fixture" '(composition-history))))) + (probe 'bound-ir-rejects-dangling-helper (car condition) "unresolved bound grammar reference") + (probe 'bound-diagnostic-target + (ref (car (cadr condition)) 'name) 'call-arguments/tail))) + + (let-values (((closed receipt) + (compile-grammar/receipt + (make-grammar 'closed (list base) '() + (list (cons 'override override-role) (cons 'remove remove-role)))))) + (probe 'closed-replacement (assq 'arguments (grammar-ir-ref closed 'rules)) + (list 'arguments replacement)) + (probe 'closed-helper-absent (assq 'call-arguments/tail (grammar-ir-ref closed 'rules)) #f) + (probe 'closed-parser-admitted (parser-ir-ref (compile-parser closed) 'root-rule) 'source-file) + (probe 'closed-bound-helper-absent + (bound-grammar-ir-binding + (bind-grammar-ir closed "history-fixture" '(composition-history)) 'rule 'call-arguments/tail) #f) + (probe 'receipt-retains-removal-after-override + (map (lambda (step) (ref step 'operation)) (ref receipt 'steps)) + '(merge append append override remove))) + + (probe 'override-missing-target + (car (error-receipt + (lambda () (compile-grammar + (make-grammar 'missing '() '() (list (cons 'override override-role))))))) + "grammar override target does not exist") + (probe 'remove-missing-target + (car (error-receipt + (lambda () (compile-grammar + (make-grammar 'missing '() '() (list (cons 'remove remove-role))))))) + "grammar remove target does not exist") +;;; Engine study: canonical POO catalogs are explicit role data, unlike concise DSL inference. + (let* ((parameters (make-nonempty-list-role 'call-arguments 'arguments + '(reference name) '(literal ",") 'parameter)) + (rule-only (make-grammar-role 'parameter-extension '() '() '() + (grammar-role-ref parameters 'rules) '() '() '() '() '())) + (catalog-and-rules + (make-grammar-role 'parameter-extension + '((Call node (argument callee parameter))) '() '() + (grammar-role-ref parameters 'rules) '() '() '() '() '())) + (rules-grammar + (compile-grammar + (make-grammar 'field-contract (list base) '() + (list (cons 'override rule-only))))) + (complete-grammar + (compile-grammar + (make-grammar 'field-contract (list base) '() + (list (cons 'override catalog-and-rules))))) + (plain-bound (bind-grammar-ir rules-grammar "field-study" '(composition-history))) + (source + '((path . "t/fixtures/language-pack-research/composition-history.ss") + (location . "parameter-extension: selected rows") + (generated? . #t) (componentOwner . parameter-extension))) + (selected-map + (list (cons 'rule (list (cons 'arguments source) + (cons 'call-arguments/tail source))))) + (complete-bound + (bind-grammar-ir complete-grammar "field-study" '(composition-history) selected-map))) + (probe 'rule-only-override-preserves-explicit-catalog + (assq 'Call (grammar-ir-ref rules-grammar 'syntax-kinds)) + '(Call node (callee argument))) + (probe 'rule-only-entry-emits-parameter + (assq 'arguments (grammar-ir-ref rules-grammar 'rules)) + '(arguments (sequence (field parameter (reference name)) (reference call-arguments/tail)))) + (probe 'rule-only-tail-emits-parameter + (assq 'call-arguments/tail (grammar-ir-ref rules-grammar 'rules)) + '(call-arguments/tail (repeat (sequence (literal ",") (field parameter (reference name)))))) + (probe 'canonical-compiler-keeps-supplied-catalog + (assq 'Call (parser-ir-ref (compile-parser rules-grammar) 'syntax-kinds)) + '(Call node (callee argument))) + (probe 'bound-fields-follow-supplied-catalog + (bound-grammar-ir-binding plain-bound 'field 'parameter 'Call) #f) + (probe 'joint-override-retains-declared-optional-field + (assq 'Call (grammar-ir-ref complete-grammar 'syntax-kinds)) + '(Call node (argument callee parameter))) + (probe 'joint-override-keeps-same-effective-rules + (grammar-ir-ref complete-grammar 'rules) (grammar-ir-ref rules-grammar 'rules)) + (probe 'joint-override-binds-parameter-field + (and (bound-grammar-ir-binding complete-bound 'field 'parameter 'Call) #t) #t) + (probe 'joint-override-binds-retained-argument-field + (and (bound-grammar-ir-binding complete-bound 'field 'argument 'Call) #t) #t) + (probe 'joint-override-parser-retains-effective-catalog + (assq 'Call (parser-ir-ref (compile-parser complete-grammar) 'syntax-kinds)) + '(Call node (argument callee parameter))) + ;; Projection controls use explicit known catalogs, not a new field inference pass. + (let* ((retained-kinds (grammar-ir-ref complete-grammar 'syntax-kinds)) + (inferred-kinds + (map (lambda (row) + (if (eq? (car row) 'Call) '(Call node (callee parameter)) row)) + retained-kinds)) + (terminals (grammar-ir-ref complete-grammar 'terminals)) + (retained-catalog + (list 'catalog (cons 'syntax-kinds retained-kinds) (cons 'terminals terminals))) + (minimal-catalog + (list 'catalog (cons 'syntax-kinds inferred-kinds) (cons 'terminals terminals))) + (stale-catalog + (list 'catalog (cons 'syntax-kinds (grammar-ir-ref rules-grammar 'syntax-kinds)) + (cons 'terminals terminals)))) + (let-values (((kinds mapped) (project-language-catalog inferred-kinds terminals retained-catalog))) + (probe 'projection-admits-retained-optional-field + (assq 'Call kinds) '(Call node (argument callee parameter)))) + (probe 'projection-rejects-stale-emitted-field-catalog + (car (error-receipt + (lambda () (project-language-catalog inferred-kinds terminals stale-catalog)))) + "catalog must retain every inferred kind, category and field") + (probe 'projection-preserves-explicit-retained-field-obligation + (car (error-receipt + (lambda () (project-language-catalog retained-kinds terminals minimal-catalog)))) + "catalog must retain every inferred kind, category and field") + (probe 'projection-rejects-terminal-remapping + (car (error-receipt + (lambda () + (project-language-catalog inferred-kinds terminals + (list 'catalog (cons 'syntax-kinds retained-kinds) + (cons 'terminals + (map (lambda (row) + (if (eq? (car row) 'identifier) + '(identifier Punctuation) row)) terminals))))))) + "catalog must retain the exact inferred terminal mapping") + (probe 'projection-rejects-duplicate-fields + (car (error-receipt + (lambda () + (project-language-catalog inferred-kinds terminals + (list 'catalog + (cons 'syntax-kinds + (map (lambda (row) + (if (eq? (car row) 'Call) + '(Call node (callee parameter parameter)) row)) retained-kinds)) + (cons 'terminals terminals)))))) + "catalog has an invalid syntax-kind row") + (let-values (((kinds mapped) (project-language-catalog inferred-kinds terminals #f))) + (probe 'projection-without-explicit-catalog-preserves-inputs + (list kinds mapped) (list inferred-kinds terminals)))) + (for-each + (lambda (name) + (probe (list 'selected-source-for-effective-row name) + (ref (ref (bound-grammar-ir-binding complete-bound 'rule name) 'source) 'componentOwner) + 'parameter-extension)) + '(arguments call-arguments/tail))) +;;; Engine selection controls retain the manual and stale-map controls above. + (def (occurrence-source role index section row) + (list (cons 'path "composition-occurrence-fixture") + (cons 'location (number->string index)) + (cons 'generated? #t) + ;; An accidental stale owner in supplied metadata cannot win selection. + (cons 'componentOwner 'stale-supplied-owner))) + (def (selected-source normalized namespace name) + (ref (ref (normalized-grammar-source-map normalized "fallback-origin") namespace) name)) + (let* ((composed + (make-grammar 'closed (list base) '() + (list (cons 'override override-role) (cons 'remove remove-role)))) + (normalized (compile-grammar/context composed occurrence-source)) + (grammar (normalized-grammar-ir normalized)) + (sources (normalized-grammar-source-map normalized "fallback-origin")) + (bound (bind-grammar-ir grammar "fallback-origin" '(composition-history) sources)) + (selected (selected-source normalized 'rule 'arguments))) + (let-values (((plain receipt) (compile-grammar/receipt composed))) + (probe 'context-capture-preserves-complete-ir grammar plain) + (probe 'context-capture-preserves-complete-receipt + (normalized-grammar-receipt normalized) receipt)) + (probe 'automatic-override-owner (ref selected 'componentOwner) 'single-argument-extension) + (probe 'automatic-override-step (ref selected 'compositionStep) 3) + (probe 'automatic-override-coordinates (ref selected 'location) "3") + (probe 'automatic-override-history (ref selected 'contributionHistory) + '((1 append call-arguments) (3 override single-argument-extension))) + (probe 'automatic-remove-has-no-effective-source + (selected-source normalized 'rule 'call-arguments/tail) #f) + (probe 'automatic-remove-has-no-bound-binding + (bound-grammar-ir-binding bound 'rule 'call-arguments/tail) #f) + (probe 'automatic-bound-source-equals-selected-source + (ref (bound-grammar-ir-binding bound 'rule 'arguments) 'source) selected) + (probe 'unmodified-field-follows-effective-kind + (ref (ref (bound-grammar-ir-binding bound 'field 'argument 'Call) 'source) 'componentOwner) + 'list-study-base-role)) + (let* ((first (rule-role 'same-owner '((x (empty))))) + (duplicate (rule-role 'same-owner '((x (empty))))) + (replacement (rule-role 'same-owner '((x (literal "x"))))) + (removed (rule-role 'same-owner '((x (empty))))) + (merged (compile-grammar/context + (make-grammar 'same-name '() (list first duplicate)) occurrence-source)) + (overridden (compile-grammar/context + (make-grammar 'same-name '() (list first duplicate) + (list (cons 'override replacement))) occurrence-source)) + (reappended (compile-grammar/context + (make-grammar 'same-name '() (list first) + (list (cons 'remove removed) (cons 'append replacement))) occurrence-source))) + (probe 'equal-merge-keeps-first-step + (ref (selected-source merged 'rule 'x) 'compositionStep) 0) + (probe 'equal-merge-keeps-first-source + (ref (selected-source merged 'rule 'x) 'location) "0") + (probe 'equal-merge-retains-accepted-history + (ref (selected-source merged 'rule 'x) 'contributionHistory) + '((0 merge same-owner) (1 merge same-owner))) + (probe 'same-name-override-selects-occurrence + (ref (selected-source overridden 'rule 'x) 'compositionStep) 2) + (probe 'same-name-override-selects-reader-association + (ref (selected-source overridden 'rule 'x) 'location) "2") + (probe 'remove-reappend-selects-new-occurrence + (ref (selected-source reappended 'rule 'x) 'compositionStep) 2) + (probe 'remove-reappend-starts-effective-history + (ref (selected-source reappended 'rule 'x) 'contributionHistory) '((2 append same-owner))) + (probe 'remove-reappend-retains-full-receipt-history + (map (lambda (step) (ref step 'operation)) + (ref (normalized-grammar-receipt reappended) 'steps)) '(merge remove append)) + (let* ((unknown (compile-grammar/context (make-grammar 'unknown '() (list first)))) + (source (selected-source unknown 'rule 'x))) + (probe 'missing-reader-coordinates-use-origin (ref source 'location) "fallback-origin") + (probe 'missing-reader-association-does-not-invent-generated-status (assq 'generated? source) #f))) + (let* ((parameters (make-nonempty-list-role 'call-arguments 'arguments + '(reference name) '(literal ",") 'parameter)) + (catalog-and-rules + (make-grammar-role 'parameter-extension '((Call node (argument callee parameter))) + '() '() (grammar-role-ref parameters 'rules) '() '() '() '() '())) + (normalized (compile-grammar/context + (make-grammar 'field-selection (list base) '() + (list (cons 'override catalog-and-rules))) occurrence-source)) + (sources (normalized-grammar-source-map normalized "field-selection")) + (bound (bind-grammar-ir (normalized-grammar-ir normalized) "field-selection" + '(composition-history) sources))) + (probe 'joint-override-field-source-follows-kind + (ref (ref (bound-grammar-ir-binding bound 'field 'parameter 'Call) 'source) 'componentOwner) + 'parameter-extension) + (probe 'retained-field-source-follows-selected-catalog + (ref (ref (bound-grammar-ir-binding bound 'field 'argument 'Call) 'source) 'compositionStep) 3) + (probe 'joint-override-kind-and-field-share-source + (ref (bound-grammar-ir-binding bound 'syntax-kind 'Call) 'source) + (ref (bound-grammar-ir-binding bound 'field 'parameter 'Call) 'source))) + + (let ((calls 0)) + (probe 'invalid-operation-still-rejected + (car (error-receipt + (lambda () + (compile-grammar/context + (make-grammar 'invalid '() '() (list (cons 'override override-role))) + (lambda args (set! calls (+ calls 1)) '()))))) + "grammar override target does not exist") + (probe 'rejected-occurrence-does-not-read-source calls 0)) + (probe 'malformed-source-rejected + (car (error-receipt + (lambda () (compile-grammar/context base (lambda args 'not-a-source-map))))) + "invalid grammar occurrence source") + + (display "COMPOSITION-HISTORY-OK: ") (display checked) + (display " checks passed") (newline) (force-output) + (test-child-process-exit! 0) + ) diff --git a/t/fixtures/language-pack-research/consumer.ss b/t/fixtures/language-pack-research/consumer.ss new file mode 100644 index 00000000..02a2aadb --- /dev/null +++ b/t/fixtures/language-pack-research/consumer.ss @@ -0,0 +1,28 @@ +;;; -*- Gerbil -*- +(import (rename-in ./fragments (preserve-host-reference renamed-reference))) +(export run-binding-research) + +(def private-value 'use-site) + +(def (research-check label actual expected) + (unless (equal? actual expected) + (error "binding research mismatch" label actual expected)) + (display "BINDING-CASE-OK: ") (display label) (newline) (force-output)) + +(def (run-binding-research) + (research-check 'renamed-definition-reference + (renamed-reference private-value) 'definition-site) + (research-check 'reconstructed-use-site-reference + (reconstruct-host-reference private-value) 'use-site) + (research-check 'caller-parameter + (parameter-reference private-value) 'use-site) + (research-check 'local-shadow-preserves-definition + (let ((private-value 'local-shadow)) + (renamed-reference private-value)) 'definition-site) + (research-check 'local-shadow-reconstruction + (let ((private-value 'local-shadow)) + (reconstruct-host-reference private-value)) 'local-shadow) + (research-check 'local-shadow-parameter + (let ((private-value 'local-shadow)) + (parameter-reference private-value)) 'local-shadow) + (display "BINDING-RESEARCH-OK: 6 checks passed") (newline) (force-output)) diff --git a/t/fixtures/language-pack-research/context-language.ss b/t/fixtures/language-pack-research/context-language.ss new file mode 100644 index 00000000..fd13ac79 --- /dev/null +++ b/t/fixtures/language-pack-research/context-language.ss @@ -0,0 +1,49 @@ +;;; -*- Gerbil -*- +;;; Research language only: explicit operand tiers, no numeric precedence. +(import (rename-in "context-vocabulary" (deflanguage/tiers define-contextual-language))) +(export contextual-study-parser context-host-values) + +;;; Host values deliberately share grammar-rule spellings. +(def sum 'host-sum) +(def product 'host-product) +(def power 'host-power) +(def context-host-values (list sum product power)) + +(define-contextual-language contextual-study + (identity "contextual-study" "v1" "contextual-study.local.v1") + (root source-file) + (lex + (whitespace Whitespace (whitespace+)) + (number Number (decimal-digit+)) + (identifier Identifier (identifier)) + (punctuation Punctuation (literals "+" "-" "*" "**" "/" "(" ")")) + (unknown Unknown (fallback))) + (rules + (source-file (node SourceFile (field expression (reference sum)))) + (sum + (left-tier (reference product) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right))) + (product + (left-tier (reference power) + (choice (literal "*") (literal "/")) + (node Expression) (fields left operator right))) + (power + (right-tier (reference prefix) (literal "**") + (node Expression) (fields left operator right))) + ;; Explicit study policy: prefix binds more tightly than exponentiation. + (prefix + (choice + (node PrefixExpression + (seq (field operator (choice (literal "+") (literal "-"))) + (field operand (reference prefix)))) + (reference atom))) + (atom + (choice + (node GroupedExpression + (seq (literal "(") (field expression (reference sum)) (literal ")"))) + (node NameExpression (field name (token identifier))) + (node NumberExpression (field value (token number)))))) + (extras whitespace) + (keywords) (recoveries) + (conflicts reject) (case-insensitive #f)) diff --git a/t/fixtures/language-pack-research/context-runtime.ss b/t/fixtures/language-pack-research/context-runtime.ss new file mode 100644 index 00000000..b3de4fc6 --- /dev/null +++ b/t/fixtures/language-pack-research/context-runtime.ss @@ -0,0 +1,54 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "context-language" "tier-language" + (rename-in "context-vocabulary" (deflanguage/tiers define-contextual-language)) + (for-syntax (only-in :gerbil/expander core-expand)) + (only-in "../../../src/runtime/parser" parse-source) + (only-in "../../../src/runtime/artifact" + parse-artifact-success? parse-artifact-roundtrip parse-artifact-events)) + +(defsyntax (invalid-context-rows stx) + (def (rejects form) + (with-catch (lambda (condition) (error-message condition)) (lambda () (core-expand form) #f))) + (datum->syntax #'invalid-context-rows + (list 'quote + (list + (rejects #'(define-contextual-language bad #f #f #f + (rules (sum (left-tier (reference atom)))))) + (rejects #'(define-contextual-language bad #f #f #f + (rules (42 (literal "+"))))) + (rejects #'(define-contextual-language bad #f #f #f + (rules (sum (literal "+") (literal "-"))))))))) + +(def checked 0) +(def (context-check label actual expected) + (unless (equal? actual expected) + (error "context vocabulary mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "CONTEXT-CASE-OK: ") (display label) (newline) (force-output)) + +(context-check 'renamed-macro-and-host-values + context-host-values '(host-sum host-product host-power)) +(context-check 'malformed-rows-diagnostics (invalid-context-rows) + '("left-tier requires an operand and operators" + "expected a named rule with one expression" + "expected a named rule with one expression")) +(for-each + (lambda (source) + (let ((candidate (parse-source contextual-study-parser source)) + (control (parse-source tier-study-parser source))) + (context-check (list 'accepted source) (parse-artifact-success? candidate) #t) + (context-check (list 'control-accepted source) (parse-artifact-success? control) #t) + (context-check (list 'roundtrip source) (parse-artifact-roundtrip candidate) source) + (context-check (list 'complete-event-parity source) + (parse-artifact-events candidate) (parse-artifact-events control)))) + '("a" "42" "a-b-c" "1-2-3" "a - b - c" "a+b*c" "(a+b)*c" + "-a*b" "a+-b" "--a" "a / b - c" " \n a + b \t" "a**b**c")) +(for-each + (lambda (source) + (context-check (list 'rejected source) + (parse-artifact-success? (parse-source contextual-study-parser source)) #f)) + '("" "a+" "*a" "a b" "(a+b")) +(display "CONTEXT-RUNTIME-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/context-vocabulary.ss b/t/fixtures/language-pack-research/context-vocabulary.ss new file mode 100644 index 00000000..667c2455 --- /dev/null +++ b/t/fixtures/language-pack-research/context-vocabulary.ss @@ -0,0 +1,36 @@ +;;; -*- Gerbil -*- +;;; Local declaration-boundary experiment; no change to public deflanguage. +(import (only-in "../../../src/language/grammar" deflanguage) + (only-in "expression-vocabulary" + left-binary-tier right-binary-tier recursive-right-chain)) +(export deflanguage/tiers) + +(defsyntax (deflanguage/tiers stx) + (def (lower-row row) + (syntax-case row (left-tier right-tier right-chain) + ((name (left-tier operand operators customization ...)) + (identifier? #'name) + #'(name (left-binary-tier name operand operators customization ...))) + ((name (right-tier operand operators customization ...)) + (identifier? #'name) + #'(name (right-binary-tier name operand operators customization ...))) + ((name (right-chain operand operators customization ...)) + (identifier? #'name) + #'(name (recursive-right-chain name operand operators customization ...))) + ((name (left-tier argument ...)) + (raise-syntax-error #f "left-tier requires an operand and operators" stx row)) + ((name (right-tier argument ...)) + (raise-syntax-error #f "right-tier requires an operand and operators" stx row)) + ((name (right-chain argument ...)) + (raise-syntax-error #f "right-chain requires an operand and operators" stx row)) + ((name expression) + (identifier? #'name) + row) + (_ (raise-syntax-error #f "expected a named rule with one expression" stx row)))) + (syntax-case stx (rules) + ((_ prefix identity-section root-section lexical-section + (rules row ...) tail ...) + (with-syntax (((lowered-row ...) (stx-map lower-row #'(row ...)))) + #'(deflanguage prefix identity-section root-section lexical-section + (rules lowered-row ...) tail ...))) + (_ (raise-syntax-error #f "expected language sections with a rules section" stx)))) diff --git a/t/fixtures/language-pack-research/declaration-contracts.ss b/t/fixtures/language-pack-research/declaration-contracts.ss new file mode 100644 index 00000000..707a278f --- /dev/null +++ b/t/fixtures/language-pack-research/declaration-contracts.ss @@ -0,0 +1,10 @@ +;;; -*- Gerbil -*- +;;; Ordinary Scheme validators, imported at the transformer's phase. +(import (only-in :gerbil/expander identifier? raise-syntax-error)) +(export require-declaration-identifier) +(def (require-declaration-identifier term declaration operation parameter (author #f)) + (unless (identifier? term) + (let (message (string-append operation ": " parameter " must be an identifier")) + (if author + (raise-syntax-error #f message term author declaration) + (raise-syntax-error #f message term declaration))))) diff --git a/t/fixtures/language-pack-research/declaration-diagnostics.ss b/t/fixtures/language-pack-research/declaration-diagnostics.ss new file mode 100644 index 00000000..ea1fb713 --- /dev/null +++ b/t/fixtures/language-pack-research/declaration-diagnostics.ss @@ -0,0 +1,116 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-component" + (for-syntax (only-in :gerbil/expander core-expand1))) + +(defsyntax (diagnostic-cases stx) + (def (position value) + (and value + (let (filepos (##position->filepos value)) + (list (+ 1 (##filepos-line filepos)) (+ 1 (##filepos-col filepos)))))) + (def (location value) + (let (loc (stx-source value)) + (and loc (list (##container->path (##locat-container loc)) + (position (##locat-start-position loc)))))) + (def (capture form) + (with-catch + (lambda (condition) + ;; SyntaxError extends Exception, not Error. Use its actual slots. + (let* ((irritants (slot-ref condition 'irritants)) + (blame (car irritants))) + (list (syntax-error? condition) (error-message condition) + (syntax->datum blame) (location blame) + (syntax->datum (cadr irritants)) (location (cadr irritants))))) + (lambda () (core-expand1 form) #f))) +(syntax-case stx () + ((_ form ...) + (datum->syntax #'diagnostic-cases + (list 'quote (stx-map capture #'(form ...))))))) + +(defsyntax (source-free-cases stx) + (def (source-record tree) + (cond + ((and (pair? tree) (eq? (car tree) 'quote) + (pair? (cdr tree)) (list? (cadr tree)) + (andmap pair? (cadr tree)) (assq 'authorOrigin (cadr tree))) (cadr tree)) + ((pair? tree) (or (source-record (car tree)) (source-record (cdr tree)))) + (else #f))) + (def (fresh form) + ;; Retain lexical context, deliberately omit reader source. + (datum->syntax #'source-free-cases (syntax->datum form) #f #f)) + (let* ((valid (fresh #'(deflocated-list component call-arguments arguments + (reference name) (literal ",") (field argument)))) + (source (source-record (syntax->datum (core-expand1 valid)))) + (invalid (fresh #'(deflocated-list component 46 arguments + (reference name) (literal ",") (field argument)))) + (failure + (with-catch + (lambda (condition) + (let (blame (car (slot-ref condition 'irritants))) + (list (syntax-error? condition) (error-message condition) + (syntax->datum blame) (stx-source blame)))) + (lambda () (core-expand1 invalid) #f)))) + (datum->syntax #'source-free-cases (list 'quote (list source failure))))) + +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "declaration diagnostic mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "DECLARATION-DIAGNOSTIC-CASE-OK: ") (display label) (newline) (force-output)) +(def (line-at path number) + (call-with-input-file path + (lambda (port) + (let loop ((index 1)) + (let (line (read-line port)) + (when (eof-object? line) (error "diagnostic source line absent" path number)) + (if (= index number) line (loop (+ index 1)))))))) +(def messages + '("deflocated-list: binding must be an identifier" + "deflocated-list: owner must be an identifier" + "deflocated-list: entry must be an identifier" + "deflocated-list: field label must be an identifier")) +(for-each + (lambda (result message token) + (probe (list 'typed token) (car result) #t) + (probe (list 'message token) (cadr result) message) + (probe (list 'blame token) (caddr result) token) + (let* ((primary (list-ref result 3)) + (loc (or primary (list-ref result 5))) (start (cadr loc)) + (line (line-at (car loc) (car start))) + (offset (- (cadr start) 1)) + (text (if (pair? token) + (string-append "(" (number->string (car token)) ")") + (number->string token)))) + (probe (list 'parameter-source-available token) (pair? primary) #t) + (probe (list 'effective-reader-coordinate token) + (substring line offset (+ offset (string-length text))) text)) + (probe (list 'declaration-context token) + (car (list-ref result 4)) 'deflocated-list)) + (diagnostic-cases + (deflocated-list 42 call-arguments arguments + (reference name) (literal ",") (field argument)) + (deflocated-list component 43 arguments + (reference name) (literal ",") (field argument)) + (deflocated-list component call-arguments 44 + (reference name) (literal ",") (field argument)) + (deflocated-list component call-arguments arguments + (reference name) (literal ",") (field 45)) + (deflocated-list (42) call-arguments arguments + (reference name) (literal ",") (field argument)) + (deflocated-list component (43) arguments + (reference name) (literal ",") (field argument)) + (deflocated-list component call-arguments (44) + (reference name) (literal ",") (field argument)) + (deflocated-list component call-arguments arguments + (reference name) (literal ",") (field (45)))) (append messages messages) '(42 43 44 45 (42) (43) (44) (45))) +(let* ((results (source-free-cases)) (source (car results)) (failure (cadr results))) + (probe 'source-free-author (cdr (assq 'authorOrigin source)) #f) + (probe 'source-free-path (cdr (assq 'path source)) #f) + (probe 'source-free-location (cdr (assq 'location source)) #f) + (probe 'source-free-owner (cdr (assq 'componentOwner source)) 'call-arguments) + (probe 'source-free-template-known (pair? (cdr (assq 'templateOrigin source))) #t) + (probe 'source-free-error failure + '(#t "deflocated-list: owner must be an identifier" 46 #f))) +(display "DECLARATION-DIAGNOSTICS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/expression-comparison.ss b/t/fixtures/language-pack-research/expression-comparison.ss new file mode 100644 index 00000000..79ea7b79 --- /dev/null +++ b/t/fixtures/language-pack-research/expression-comparison.ss @@ -0,0 +1,56 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "expression-vocabulary" + (only-in "../../../src/grammar/algebra" grammar-expression)) + +(def (comparison-check label actual expected) + (unless (equal? actual expected) + (error "expression vocabulary mismatch" label actual expected)) + (display "EXPRESSION-CASE-OK: ") (display label) (newline) (force-output)) + +(def arithmetic-binary + (vocabulary-value + (binary-expression left 10 (reference expression) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right)))) +(comparison-check 'arithmetic-core arithmetic-binary + (grammar-expression + (prec left 10 + (alias Expression + (seq (field left (reference expression)) + (field operator (choice (literal "+") (literal "-"))) + (field right (reference expression))))))) + +(def hcl-chain + (vocabulary-value + (expression-chain (reference unary-expression) (reference binary-operator) + (node BinaryExpression) (fields left operator right)))) +(comparison-check 'hcl-core hcl-chain + (grammar-expression + (alias BinaryExpression + (seq (field left (reference unary-expression)) + (repeat + (seq (field operator (reference binary-operator)) + (field right (reference unary-expression)))))))) + +(comparison-check 'binary-defaults + (vocabulary-value + (binary-expression left 10 (reference expression) (literal "+"))) + (vocabulary-value + (binary-expression left 10 (reference expression) (literal "+") + (node BinaryExpression) (fields left operator right)))) +(comparison-check 'chain-defaults + (vocabulary-value + (expression-chain (reference unary-expression) (reference binary-operator))) + hcl-chain) +(comparison-check 'explicit-fields + (vocabulary-value + (expression-chain (reference term) (literal "+") + (node Sum) (fields first operation rest))) + (grammar-expression + (alias Sum + (seq (field first (reference term)) + (repeat (seq (field operation (literal "+")) + (field rest (reference term)))))))) +(comparison-check 'binary-and-chain-distinct (equal? arithmetic-binary hcl-chain) #f) +(display "EXPRESSION-COMPARISON-OK: 6 checks passed") (newline) (force-output) diff --git a/t/fixtures/language-pack-research/expression-vocabulary.ss b/t/fixtures/language-pack-research/expression-vocabulary.ss new file mode 100644 index 00000000..6d484a60 --- /dev/null +++ b/t/fixtures/language-pack-research/expression-vocabulary.ss @@ -0,0 +1,80 @@ +;;; -*- Gerbil -*- +;;; Local design prototype: templates lower to the existing GrammarExpr algebra. +(import (only-in "../../../src/grammar/algebra" grammar-expression) + (for-syntax (only-in :gerbil/expander core-expand1))) +(export binary-expression expression-chain + left-binary-tier right-binary-tier recursive-right-chain vocabulary-value) + +(defrules binary-expression (node fields) + ((_ direction rank operand operators) + (binary-expression direction rank operand operators + (node BinaryExpression) (fields left operator right))) + ((_ direction rank operand operators + (node kind) (fields left-label operator-label right-label)) + (prec direction rank + (alias kind + (seq (field left-label operand) + (field operator-label operators) + (field right-label operand)))))) + +(defrules expression-chain (node fields) + ((_ operand operators) + (expression-chain operand operators + (node BinaryExpression) (fields left operator right))) + ((_ operand operators + (node kind) (fields left-label operator-label right-label)) + (alias kind + (seq (field left-label operand) + (repeat + (seq (field operator-label operators) + (field right-label operand))))))) + +;;; Numeric-free tier alternatives. The base operand passes through unchanged. +(defrules left-binary-tier (node fields) + ((_ self operand operators) + (left-binary-tier self operand operators + (node BinaryExpression) (fields left operator right))) + ((_ self operand operators (node kind) (fields lhs op rhs)) + (choice + (alias kind + (seq (field lhs (reference self)) + (field op operators) + (field rhs operand))) + operand))) + +(defrules right-binary-tier (node fields) + ((_ self operand operators) + (right-binary-tier self operand operators + (node BinaryExpression) (fields left operator right))) + ((_ self operand operators (node kind) (fields lhs op rhs)) + (choice + (alias kind + (seq (field lhs operand) + (field op operators) + (field rhs (reference self)))) + operand))) + +;;; FHIRPath-style chain: the alias also wraps the zero-operator case. +(defrules recursive-right-chain (node fields) + ((_ self operand operators) + (recursive-right-chain self operand operators + (node BinaryExpression) (fields left operator right))) + ((_ self operand operators (node kind) (fields lhs op rhs)) + (alias kind + (seq (field lhs operand) + (optional + (seq (field op operators) + (field rhs (reference self)))))))) + +;;; Only the design templates above expand here. Core expressions are interpreted +;;; by the repository's actual GrammarExpr constructors, not a research parser. +(defsyntax (vocabulary-value stx) + (syntax-case stx () + ((_ expression) + (let loop ((form #'expression) (steps 0)) + (when (> steps 4) (raise-syntax-error #f "unexpected template depth" stx)) + (syntax-case form (prec alias choice) + ((prec . _) (with-syntax ((core form)) #'(grammar-expression core))) + ((choice . _) (with-syntax ((core form)) #'(grammar-expression core))) + ((alias . _) (with-syntax ((core form)) #'(grammar-expression core))) + (_ (loop (core-expand1 form) (+ steps 1)))))))) diff --git a/t/fixtures/language-pack-research/fragments.ss b/t/fixtures/language-pack-research/fragments.ss new file mode 100644 index 00000000..68dd36c6 --- /dev/null +++ b/t/fixtures/language-pack-research/fragments.ss @@ -0,0 +1,16 @@ +;;; -*- Gerbil -*- +;;; Research controls: preserve syntax versus reconstruct from a caller anchor. +(export preserve-host-reference reconstruct-host-reference parameter-reference) + +(def private-value 'definition-site) + +(defrules preserve-host-reference () + ((_ anchor) private-value)) + +(defsyntax (reconstruct-host-reference stx) + (syntax-case stx () + ((_ anchor) + (datum->syntax #'anchor (syntax->datum #'private-value))))) + +(defrules parameter-reference () + ((_ expression) expression)) diff --git a/t/fixtures/language-pack-research/list-collisions.ss b/t/fixtures/language-pack-research/list-collisions.ss new file mode 100644 index 00000000..cd58edbc --- /dev/null +++ b/t/fixtures/language-pack-research/list-collisions.ss @@ -0,0 +1,33 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Composition-only controls do not load parser generation/runtime modules. +(import "list-roles" + (only-in "../../../src/modules/parser/objects" make-grammar) + (only-in "../../../src/compiler/normalize" compile-grammar)) +(def checked 0) +(def (list-check label actual expected) + (unless (equal? actual expected) (error "list collision mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "LIST-COLLISION-CASE-OK: ") (display label) (newline) (force-output)) +;;; Distinguish entry collisions from generated-helper collisions. +(def (collision-receipt first-owner first-entry second-owner second-entry) + (with-catch (lambda (condition) + (list (error-message condition) (error-irritants condition))) + (lambda () + (compile-grammar + (make-grammar 'collision '() '() + (list + (cons 'append (make-nonempty-list-role first-owner first-entry + '(reference name) '(literal ",") 'argument)) + (cons 'append (make-nonempty-list-role second-owner second-entry + '(reference name) '(literal ",") 'element)))))))) +(list-check 'public-entry-collision + (collision-receipt 'first 'shared 'second 'shared) + '("grammar append target already exists" (rules shared))) +(list-check 'generated-helper-collision + (collision-receipt 'shared-owner 'arguments 'shared-owner 'elements) + '("grammar append target already exists" (rules shared-owner/tail))) + +(display "LIST-COLLISIONS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/list-language.ss b/t/fixtures/language-pack-research/list-language.ss new file mode 100644 index 00000000..210dcc1b --- /dev/null +++ b/t/fixtures/language-pack-research/list-language.ss @@ -0,0 +1,46 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +(import "list-stage" (only-in "../../../src/language/grammar" deflanguage)) +(export list-study-language-grammar list-study-parser list-study-grammar list-study-bound-grammar-ir list-study-parser-ir list-composed-ir list-composition-receipt + list-control-parser + list-single-language-grammar list-single-grammar list-single-parser-ir + list-single-bound-grammar-ir list-single-composed-ir list-single-composition-receipt + native-list-language-grammar native-list-bound-grammar-ir + native-single-language-grammar native-single-bound-grammar-ir) +(deflist-study list-study list-composed-ir list-composition-receipt) +(defsingle-list-study list-single list-single-composed-ir list-single-composition-receipt) +(defnative-list-study native-list native-composed-ir native-composition-receipt) +(defnative-single-list-study native-single native-single-composed-ir native-single-composition-receipt) + +;;; Inline control has identical public structure, with no list helper rules. +(begin + (deflanguage list-control + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier)) + (punctuation Punctuation (literals "(" ")" "[" "]" ",")) + (whitespace Whitespace (whitespace+))) + (extras whitespace) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file (node SourceFile (field form (choice (reference call) (reference array))))) + (call + (node Call + (seq (field callee (token identifier)) (literal "(") + (optional + (seq (field argument (reference name)) + (repeat (seq (literal ",") (field argument (reference name)))))) + (literal ")")))) + (array + (node Array + (seq (literal "[") + (optional + (seq (field element (reference name)) + (repeat (seq (literal ",") (field element (reference name)))))) + (literal "]")))) + (name (node Name (field value (token identifier)))))) + (bind-fixture-grammar-release list-control "list-control" "v1" "list-control.local.v1") ) diff --git a/t/fixtures/language-pack-research/list-nullability-contract.ss b/t/fixtures/language-pack-research/list-nullability-contract.ss new file mode 100644 index 00000000..94ccab52 --- /dev/null +++ b/t/fixtures/language-pack-research/list-nullability-contract.ss @@ -0,0 +1,41 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import (only-in "list-roles" make-nonempty-list-role) + (only-in "../../../src/modules/parser/objects" grammar-role-ref) + (only-in "../../../src/compiler/lr" lower-rules compute-nullable compute-first)) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "list nullability contract mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "LIST-NULLABILITY-CASE-OK: ") (display label) (newline) (force-output)) +(for-each + (lambda (case) + (let* ((label (car case)) (item (cadr case)) (extra (caddr case)) + (nullable? (list-ref case 3)) (token-first? (list-ref case 4)) + (comma-first? (list-ref case 5)) + (role (make-nonempty-list-role 'call-arguments 'arguments + '(reference item) '(literal ",") 'argument)) + (rules (append (grammar-role-ref role 'rules) (list (list 'item item)) extra)) + (productions (lower-rules rules 'arguments))) + (let-values (((nullable nullable-index) (compute-nullable productions))) + (probe (list label 'item-nullable) (table-ref nullable-index 'item #f) nullable?) + (probe (list label 'tail-nullable) (table-ref nullable-index 'call-arguments/tail #f) #t) + (probe (list label 'entry-nullable) (table-ref nullable-index 'arguments #f) nullable?) + (let-values (((first first-index) (compute-first productions nullable-index))) + (let (entry-first (table-ref first-index 'arguments '())) + (probe (list label 'token-first) + (and (member '(terminal token identifier) entry-first) #t) token-first?) + (probe (list label 'comma-first) + (and (member '(terminal literal ",") entry-first) #t) comma-first?)))))) + '((token (token identifier) () #f #t #f) + (optional-token (optional (token identifier)) () #t #t #t) + (empty-item (empty) () #t #f #t) + (recursive-with-base (choice (reference item) (token identifier)) () #f #t #f) + (indirect-nullable (reference peer) ((peer (optional (token identifier)))) #t #t #t) + (closed-unproductive-cycle (reference item) () #f #f #f) + (token-prefix-with-dead-tail + (sequence (token identifier) (reference blocked)) + ((blocked (reference blocked))) #f #t #f))) +(display "LIST-NULLABILITY-CONTRACT-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/list-origins.ss b/t/fixtures/language-pack-research/list-origins.ss new file mode 100644 index 00000000..045d4416 --- /dev/null +++ b/t/fixtures/language-pack-research/list-origins.ss @@ -0,0 +1,38 @@ +;;; -*- Gerbil -*- +;;; Fixture-specific ownership map for unique merge/append contributions. +(import (only-in :clan/poo/object .ref .slot?) + (only-in "../../../src/modules/parser/objects" grammar-role-name)) +(export list-study-source-map list-study-row-source) +(def (ref value key) (let (row (assq key value)) (and row (cdr row)))) +(def (list-study-source-map receipt) + (list + (cons 'rule + (apply append + (map + (lambda (step) + (let* ((owner (ref step 'role)) + (rules (ref (ref step 'sections) 'rules)) + (generated? (not (eq? owner 'list-study-base-role)))) + (map + (lambda (name) + (cons name + (list + (cons 'path "t/fixtures/language-pack-research/list-roles.ss") + (cons 'location + (string-append "role " (symbol->string owner) + "; rule " (symbol->string name))) + (cons 'generated? generated?) + (cons 'componentOwner owner)))) + (or rules '())))) + (ref receipt 'steps)))))) + +;;; Metadata for a known fixture occurrence, not an engine source resolver. +(def (list-study-row-source role index section row) + (if (.slot? role 'row-source) + ((.ref role 'row-source) section row) + (let (owner (grammar-role-name role)) + (list (cons 'path "t/fixtures/language-pack-research/list-roles.ss") + (cons 'location (string-append "role " (symbol->string owner) + "; " (symbol->string section) + " " (symbol->string (car row)))) + (cons 'generated? (not (eq? owner 'list-study-base-role))))))) diff --git a/t/fixtures/language-pack-research/list-parser-test.ss b/t/fixtures/language-pack-research/list-parser-test.ss new file mode 100644 index 00000000..71803b75 --- /dev/null +++ b/t/fixtures/language-pack-research/list-parser-test.ss @@ -0,0 +1,36 @@ +;;; -*- Gerbil -*- +;;; Independent conformance declarations for the canonical composed package. +(import :gerbil-parser/language-test-support "list-parser") + +(deflanguage-parser-tests list-parser-test "canonical POO composition package DSL" + (loader list-composed-language) + (identity "package identity" + (language "list-study") (version "v1") (contract "list-study.local.v1") + (schema "gerbil-parser.language-entry.v2")) + (accepted "empty call" "f()" + (root SourceFile) (counts (Call 1) (Name 0)) + (field-counts Call argument (0))) + (accepted "empty array" "[]" + (root SourceFile) (counts (Array 1) (Name 0)) + (field-counts Array element (0))) + (accepted "single call argument" "f(a)" + (counts (Call 1) (Name 1)) (field-counts Call argument (1))) + (accepted "three call arguments" "f(a,b,c)" + (counts (Call 1) (Name 3)) (field-counts Call argument (3))) + (accepted "call whitespace" "f( a , b )" + (counts (Call 1) (Name 2)) (field-counts Call argument (2))) + (accepted "single array element" "[a]" + (counts (Array 1) (Name 1)) (field-counts Array element (1))) + (accepted "three array elements" "[a,b,c]" + (counts (Array 1) (Name 3)) (field-counts Array element (3))) + (accepted "array whitespace and newline" "[ a ,\n b ]" + (counts (Array 1) (Name 2)) (field-counts Array element (2))) + (rejected-many "separator and closing policies" + '("f(,)" "f(a,)" "f(a b)" "f(a" "[,]" "[a,]" "[a b]" "[a")) + (ast "exact empty call tree and spans" "f()" + (node SourceFile 0 3 + (field form 0 3 + (node Call 0 3 + (field callee 0 1 (token identifier "f" 0 1)) + (token punctuation "(" 1 2) + (token punctuation ")" 2 3)))))) diff --git a/t/fixtures/language-pack-research/list-parser.ss b/t/fixtures/language-pack-research/list-parser.ss new file mode 100644 index 00000000..5e9ac84a --- /dev/null +++ b/t/fixtures/language-pack-research/list-parser.ss @@ -0,0 +1,19 @@ +;;; -*- Gerbil -*- +;;; Checked public entry for the composed canonical language package. +(import "list-language" + (only-in "../../../language-support/development" deflanguage-development-loader LanguageDevelopmentLoader.)) +(export (import: "list-language") list-composed-language parse-list-composed + list-single-language parse-list-single + native-list-language parse-native-list native-single-language parse-native-single) +(deflanguage-development-loader (list-composed-language :: self LanguageDevelopmentLoader.) + (grammar list-study-language-grammar) + (parse parse-list-composed)) + +(deflanguage-development-loader (list-single-language :: self LanguageDevelopmentLoader.) + (grammar list-single-language-grammar) + (parse parse-list-single)) + +(deflanguage-development-loader (native-list-language :: self LanguageDevelopmentLoader.) + (grammar native-list-language-grammar) (parse parse-native-list)) +(deflanguage-development-loader (native-single-language :: self LanguageDevelopmentLoader.) + (grammar native-single-language-grammar) (parse parse-native-single)) diff --git a/t/fixtures/language-pack-research/list-repetition-contract.ss b/t/fixtures/language-pack-research/list-repetition-contract.ss new file mode 100644 index 00000000..6bbae9b0 --- /dev/null +++ b/t/fixtures/language-pack-research/list-repetition-contract.ss @@ -0,0 +1,46 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import (only-in "list-roles" make-nonempty-list-role) + (only-in "../../../src/modules/parser/objects" grammar-role-ref) + (only-in "../../../src/grammar/algebra" + grammar-expression-nullable? grammar-expression) + (only-in "../../../src/compiler/lr" lower-rules compute-nullable)) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "list repetition contract mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "LIST-REPETITION-CASE-OK: ") (display label) (newline) (force-output)) +(def (attempt thunk) + (with-catch (lambda (condition) (list (error-message condition) (error-irritants condition))) thunk)) +(for-each + (lambda (case) + (let* ((label (car case)) (item (cadr case)) (separator (caddr case)) + (item-nullable? (list-ref case 3)) (separator-nullable? (list-ref case 4)) + (body (list 'sequence '(reference separator) '(field argument (reference item)))) + (role (make-nonempty-list-role 'call-arguments 'arguments + '(reference item) '(reference separator) 'argument)) + (rules (append (grammar-role-ref role 'rules) + (list (list 'item item) (list 'separator separator) + (list 'iteration-body body))))) + (let-values (((nullable index) (compute-nullable (lower-rules rules 'arguments)))) + (probe (list label 'item) (table-ref index 'item #f) item-nullable?) + (probe (list label 'separator) (table-ref index 'separator #f) separator-nullable?) + (probe (list label 'body) + (table-ref index 'iteration-body #f) (and item-nullable? separator-nullable?)) + (probe (list label 'entry) (table-ref index 'arguments #f) item-nullable?) + (probe (list label 'tail) (table-ref index 'call-arguments/tail #f) #t) + ;; Local inspection has no environment to resolve either reference. + (probe (list label 'local-reference-summary) (grammar-expression-nullable? body) #f) + (probe (list label 'referenced-repeat-construction) + (grammar-expression (repeat (seq (reference separator) (field argument (reference item))))) + (list 'repeat body))))) + '((token-and-comma (token identifier) (literal ",") #f #f) + (empty-item-and-comma (empty) (literal ",") #t #f) + (token-and-empty-separator (token identifier) (empty) #f #t) + (both-empty (empty) (empty) #t #t))) +(probe 'explicit-empty-repeat-rejected + (attempt (lambda () (grammar-expression (repeat (seq (empty) (field argument (empty))))))) + '("grammar repeat operand must consume input" ((sequence (empty) (field argument (empty)))))) +(display "LIST-REPETITION-CONTRACT-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/list-roles.ss b/t/fixtures/language-pack-research/list-roles.ss new file mode 100644 index 00000000..0a94aeec --- /dev/null +++ b/t/fixtures/language-pack-research/list-roles.ss @@ -0,0 +1,117 @@ +;;; -*- Gerbil -*- +;;; Checked POO contributions built at the importing module's phase. +(import (only-in :clan/poo/object .o) + (only-in "../../../src/modules/parser/objects" make-grammar-role make-grammar grammar-role-ref) + (only-in "../../../src/modules/parser/syntax" defgrammar-role) + (only-in "../../../src/grammar/algebra" grammar-expression?)) +(export make-nonempty-list-role make-list-study-grammar make-single-argument-list-grammar list-study-base-role + make-native-list-grammar make-native-single-list-grammar) + +(def (make-nonempty-list-role owner entry item separator field-name) + (unless (and (symbol? owner) (symbol? entry) (symbol? field-name) + (grammar-expression? item) (grammar-expression? separator)) + (error "invalid nonempty list component" owner entry)) + ;; Stable domain identity; no fresh host binding is used as a grammar key. + (let* ((tail (string->symbol (string-append (symbol->string owner) "/tail"))) + (item-field (list 'field field-name item)) + (rows + (list + (list entry (list 'sequence item-field (list 'reference tail))) + (list tail (list 'repeat (list 'sequence separator item-field)))))) + (make-grammar-role owner '() '() '() rows '() '() '() '() '()))) + +(defgrammar-role list-study-base-role + (syntax-kinds + (SourceFile node (form)) (Call node (callee argument)) + (Array node (element)) (Name node (value)) + (Identifier token (text)) (Punctuation token (text)) (Whitespace token (text))) + (terminals (identifier Identifier) (punctuation Punctuation) (whitespace Whitespace)) + (lexical-rules + (identifier (identifier)) + (punctuation (literals "(" ")" "[" "]" ",")) + (whitespace (whitespace+))) + (rules + (source-file + (alias SourceFile (field form (choice (reference call) (reference array))))) + (call + (alias Call + (seq (field callee (token identifier)) (literal "(") + (optional (reference arguments)) (literal ")")))) + (array + (alias Array + (seq (literal "[") (optional (reference elements)) (literal "]")))) + (name (alias Name (field value (token identifier))))) + (extras whitespace) (keywords) + (parser-entrypoints (source-file parse pure)) + (recoveries) (flow (source lexical) (lexical cst))) + +(def (make-list-study-grammar) + (let ((arguments + (make-nonempty-list-role 'call-arguments 'arguments + '(reference name) '(literal ",") 'argument)) + (elements + (make-nonempty-list-role 'array-elements 'elements + '(reference name) '(literal ",") 'element))) + (make-grammar 'list-study '() (list list-study-base-role) + (list (cons 'append arguments) (cons 'append elements))))) + +;;; Independent policy variant exercises accepted override and helper removal. +(def (make-single-argument-list-grammar) + (let ((replacement + (make-grammar-role 'single-argument-extension '() '() '() + '((arguments (field argument (reference name)))) '() '() '() '() '())) + (removed + (make-grammar-role 'retired-tail '() '() '() + '((call-arguments/tail (empty))) '() '() '() '() '()))) + (make-grammar 'list-study-single (list (make-list-study-grammar)) '() + (list (cons 'override replacement) (cons 'remove removed))))) + +;;; Author control uses upstream POO slot syntax, not a composition-op DSL. +(def (native-row-source owner generated? section row) + (list (cons 'path "t/fixtures/language-pack-research/list-roles.ss") + (cons 'location (string-append "native prototype " (symbol->string owner) + "; " (symbol->string section) " " + (symbol->string (car row)))) + (cons 'generated? generated?) (cons 'declarationOwner owner))) + +(def (make-native-list-role) + (let ((arguments (make-nonempty-list-role 'call-arguments 'arguments + '(reference name) '(literal ",") 'argument)) + (elements (make-nonempty-list-role 'array-elements 'elements + '(reference name) '(literal ",") 'element))) + (.o (:: self list-study-base-role) + name: 'native-list-role + (rules => append (grammar-role-ref arguments 'rules) (grammar-role-ref elements 'rules)) + row-source: + (lambda (section row) + (let (owner (if (eq? section 'rules) + (case (car row) + ((arguments call-arguments/tail) 'call-arguments) + ((elements array-elements/tail) 'array-elements) + (else 'list-study-base-role)) + 'list-study-base-role)) + (native-row-source owner (not (eq? owner 'list-study-base-role)) section row)))))) + +(def (make-native-list-grammar) + (make-grammar 'native-list-study '() (list (make-native-list-role)))) + +(def (make-native-single-list-grammar) + (let (role + (.o (:: self (make-native-list-role)) + name: 'native-single-list-role + ;; Native inherited computation supplies the original table. Scheme + ;; values express the selected rule policy; there is no new keyword. + (rules (inherited) + (filter-map + (lambda (row) + (case (car row) + ((arguments) '(arguments (field argument (reference name)))) + ((call-arguments/tail) #f) + (else row))) + (inherited))) + (row-source (inherited) + (lambda (section row) + (if (and (eq? section 'rules) (eq? (car row) 'arguments)) + (native-row-source 'single-argument-extension #t section row) + ((inherited) section row)))))) + (make-grammar 'native-list-study-single '() (list role)))) diff --git a/t/fixtures/language-pack-research/list-runtime.ss b/t/fixtures/language-pack-research/list-runtime.ss new file mode 100644 index 00000000..a138abb7 --- /dev/null +++ b/t/fixtures/language-pack-research/list-runtime.ss @@ -0,0 +1,125 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + "list-parser" "list-roles" + (only-in "../../../src/modules/parser/objects" make-grammar) + (only-in "../../../src/compiler/normalize" compile-grammar grammar-ir-ref) + (only-in "../../../src/runtime/parser" parse-source) + (only-in "../../../src/compiler/parser-ir" parser-ir-ref) + (only-in "../../../src/compiler/bound-ir" bound-grammar-ir-binding) + (only-in "../../../src/runtime/artifact" + parse-artifact-success? parse-artifact-events parse-artifact-roundtrip)) +(export main) +(def (main) + + (def checked 0) + (def (list-check label actual expected) + (unless (equal? actual expected) (error "list component mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "LIST-CASE-OK: ") (display label) (newline) (force-output)) + (def (ref value key) (cdr (assq key value))) + (list-check 'composition-operations + (map (lambda (step) (ref step 'operation)) (ref list-composition-receipt 'steps)) + '(merge append append)) + (list-check 'composition-owners + (map (lambda (step) (ref step 'role)) (ref list-composition-receipt 'steps)) + '(list-study-base-role call-arguments array-elements)) + (list-check 'deterministic-composition + (compile-grammar (make-list-study-grammar)) list-composed-ir) + (list-check 'helper-rule-identities + (map car (filter (lambda (row) (memq (car row) '(call-arguments/tail array-elements/tail))) + (grammar-ir-ref list-composed-ir 'rules))) + '(call-arguments/tail array-elements/tail)) + (for-each + (lambda (section) + (list-check (list 'reified-section section) + (grammar-ir-ref list-study-grammar section) (grammar-ir-ref list-composed-ir section))) + '(syntax-kinds terminals lexical-rules rules extras keywords parser-entrypoints recoveries flow)) + (list-check 'duplicate-append-diagnostic + (with-catch (lambda (condition) (error-message condition)) + (lambda () + (compile-grammar + (make-grammar 'duplicate '() '() + (let (role (make-nonempty-list-role 'call-arguments 'arguments + '(reference name) '(literal ",") 'argument)) + (list (cons 'append role) (cons 'append role))))))) + "grammar append target already exists") + (for-each + (lambda (case) + (let* ((source (car case)) (field-name (cadr case)) (count (caddr case)) + (candidate (parse-list-composed source)) + (control (parse-source list-control-parser source))) + (list-check (list 'accepted source) (parse-artifact-success? candidate) #t) + (list-check (list 'control-accepted source) (parse-artifact-success? control) #t) + (list-check (list 'roundtrip source) (parse-artifact-roundtrip candidate) source) + (list-check (list 'complete-event-parity source) + (parse-artifact-events candidate) (parse-artifact-events control)) + (list-check (list 'per-item-fields source) + (length (filter (lambda (event) + (and (eq? (vector-ref event 0) 'start-field) + (eq? (vector-ref event 1) field-name))) + (parse-artifact-events candidate))) count))) + '(("f()" argument 0) ("f(a)" argument 1) ("f(a,b,c)" argument 3) + ("f( a , b )" argument 2) ("[]" element 0) ("[a]" element 1) + ("[a,b,c]" element 3) ("[ a ,\n b ]" element 2))) + (for-each + (lambda (source) + (list-check (list 'rejected source) + (parse-artifact-success? (parse-list-composed source)) #f)) + '("f(,)" "f(a,)" "f(a b)" "f(a" "[,]" "[a,]" "[a b]" "[a")) +;;; Both policy variants consume the exact canonical product via checked entry. + (list-check 'published-composed-grammar-is-complete-canonical-input list-study-grammar list-composed-ir) + (list-check 'published-parser-retains-composition + (parser-ir-ref list-study-parser-ir 'compositionDigest) (ref list-composition-receipt 'compositionDigest)) + (list-check 'single-policy-canonical-input list-single-grammar list-single-composed-ir) + (list-check 'single-policy-parser-composition + (parser-ir-ref list-single-parser-ir 'compositionDigest) (ref list-single-composition-receipt 'compositionDigest)) + (list-check 'single-policy-selected-bound-owner + (ref (ref (bound-grammar-ir-binding list-single-bound-grammar-ir 'rule 'arguments) 'source) 'componentOwner) + 'single-argument-extension) + (list-check 'single-policy-removed-helper-binding + (bound-grammar-ir-binding list-single-bound-grammar-ir 'rule 'call-arguments/tail) #f) + (for-each + (lambda (source) + (let ((candidate (parse-list-single source)) + (control (parse-list-composed source))) + (list-check (list 'single-policy-accepted source) (parse-artifact-success? candidate) #t) + (list-check (list 'single-policy-event-parity source) + (parse-artifact-events candidate) (parse-artifact-events control)) + (list-check (list 'single-policy-roundtrip source) (parse-artifact-roundtrip candidate) source))) + '("f()" "f(a)" "[a,b,c]")) + (list-check 'single-policy-rejects-second-argument + (parse-artifact-success? (parse-list-single "f(a,b)")) #f) + (list-check 'ordinary-policy-keeps-multiple-arguments + (parse-artifact-success? (parse-list-composed "f(a,b)")) #t) + +;;; Native upstream slot inheritance must preserve the accepted language policy. + (for-each + (lambda (source) + (for-each + (lambda (entries) + (let ((candidate ((car entries) source)) (control ((cadr entries) source))) + (list-check (list 'native-status source) + (parse-artifact-success? candidate) (parse-artifact-success? control)) + (list-check (list 'native-complete-events source) + (parse-artifact-events candidate) (parse-artifact-events control)) + (list-check (list 'native-roundtrip source) (parse-artifact-roundtrip candidate) source))) + (list (list parse-native-list parse-list-composed) + (list parse-native-single parse-list-single)))) + '("f()" "f(a)" "f(a,b,c)" "[a,b,c]" "f(a,)" "[a b]")) + (list-check 'native-append-helper-declaration-owner + (ref (ref (bound-grammar-ir-binding native-list-bound-grammar-ir 'rule 'arguments) 'source) + 'declarationOwner) 'call-arguments) + (list-check 'native-inherited-base-not-generated + (ref (ref (bound-grammar-ir-binding native-list-bound-grammar-ir 'rule 'source-file) 'source) + 'generated?) #f) + (list-check 'native-override-declaration-owner + (ref (ref (bound-grammar-ir-binding native-single-bound-grammar-ir 'rule 'arguments) 'source) + 'declarationOwner) 'single-argument-extension) + (list-check 'native-removed-helper-absent + (bound-grammar-ir-binding native-single-bound-grammar-ir 'rule 'call-arguments/tail) #f) + + (display "LIST-RUNTIME-OK: ") (display checked) + (display " checks passed") (newline) (force-output) + (test-child-process-exit! 0) + ) diff --git a/t/fixtures/language-pack-research/list-stage.ss b/t/fixtures/language-pack-research/list-stage.ss new file mode 100644 index 00000000..e0f701ae --- /dev/null +++ b/t/fixtures/language-pack-research/list-stage.ss @@ -0,0 +1,64 @@ +;;; -*- Gerbil -*- +;;; Phase-1 POO normalization enters canonical compilation without DSL replay. +(import (only-in "../../../src/language/assembly" assemble-language-parser) + (only-in "../../../src/language/descriptor" make-language-grammar) + (for-syntax "list-roles" "list-origins" + (only-in "../../../src/compiler/normalize" + compile-grammar/context normalized-grammar-ir + normalized-grammar-receipt normalized-grammar-source-map) + (only-in "../../../src/compiler/language-artifact" + make-language-declaration make-admitted-language-declaration + expand-admitted-language-declaration-syntax) + (only-in "../../../src/compiler/parser-ir" compile-parser) + (only-in "../../../src/compiler/bound-ir" bind-grammar-ir))) +(export deflist-study defsingle-list-study defnative-list-study defnative-single-list-study) + +(begin-syntax +(def (expand-list-study stx composed identities lineage) + (syntax-case stx () + ((_ prefix composed-binding receipt-binding) + (let* ((normalized (compile-grammar/context composed list-study-row-source)) + (ir (normalized-grammar-ir normalized)) + (receipt (normalized-grammar-receipt normalized)) + (origin (let (source (stx-source stx)) + (if source (call-with-output-string (lambda (port) (display source port))) + ""))) + (sources (normalized-grammar-source-map normalized origin)) + (declaration + (make-language-declaration + stx #'prefix (syntax->list (datum->syntax #'prefix identities)) + '() 'reject #f lineage sources))) + (with-syntax + ((composed-value (datum->syntax #'prefix (list 'quote ir))) + (receipt-value (datum->syntax #'prefix (list 'quote receipt))) + (compiled + (expand-admitted-language-declaration-syntax + declaration (make-admitted-language-declaration ir origin lineage sources) + compile-parser bind-grammar-ir + #'assemble-language-parser #'make-language-grammar #'begin #'def))) + #'(begin + (def composed-binding composed-value) + (def receipt-binding receipt-value) + compiled)))))) + +) + +(defsyntax (deflist-study stx) + (expand-list-study stx (make-list-study-grammar) + '("list-study" "v1" "list-study.local.v1") + '(deflist-study make-nonempty-list-role))) + +(defsyntax (defsingle-list-study stx) + (expand-list-study stx (make-single-argument-list-grammar) + '("list-study-single" "v1" "list-study-single.local.v1") + '(defsingle-list-study make-single-argument-list-grammar))) + +(defsyntax (defnative-list-study stx) + (expand-list-study stx (make-native-list-grammar) + '("native-list-study" "v1" "native-list-study.local.v1") + '(defnative-list-study native-poo-slot-composition))) + +(defsyntax (defnative-single-list-study stx) + (expand-list-study stx (make-native-single-list-grammar) + '("native-list-study-single" "v1" "native-list-study-single.local.v1") + '(defnative-single-list-study native-poo-inherited-computation))) diff --git a/t/fixtures/language-pack-research/located-component.ss b/t/fixtures/language-pack-research/located-component.ss new file mode 100644 index 00000000..78531ebc --- /dev/null +++ b/t/fixtures/language-pack-research/located-component.ss @@ -0,0 +1,44 @@ +;;; -*- Gerbil -*- +;;; Source locations come from reader syntax, not a hand-written line table. +(import :std/stxparam "author-context" "reader-origin" (for-syntax "declaration-contracts") (only-in "list-roles" make-nonempty-list-role) + (only-in "../../../src/grammar/algebra" grammar-expression)) +(export deflocated-list) + +(with-reader-origin template-definition-origin + (defsyntax (deflocated-list stx) + (def (position value) + (and value + (let (filepos (##position->filepos value)) + (list (cons 'line (+ 1 (##filepos-line filepos))) + (cons 'column (+ 1 (##filepos-col filepos))))))) + (def (origin syntax-value) + (let (location (stx-source syntax-value)) + (and location + (list (cons 'path (##container->path (##locat-container location))) + (cons 'start (position (##locat-start-position location))) + (cons 'end (position (##locat-end-position location))))))) + (def author-declaration (syntax-parameter-value #'@@author-declaration)) + (syntax-case stx (field) + ((_ binding owner entry item separator (field label)) + (begin + (require-declaration-identifier #'binding stx "deflocated-list" "binding" author-declaration) + (require-declaration-identifier #'owner stx "deflocated-list" "owner" author-declaration) + (require-declaration-identifier #'entry stx "deflocated-list" "entry" author-declaration) + (require-declaration-identifier #'label stx "deflocated-list" "field label" author-declaration) + (let* ((author (origin #'owner)) + (template template-definition-origin) + (source + (list (cons 'path (and author (cdr (assq 'path author)))) + (cons 'location author) + (cons 'generated? #t) + (cons 'componentOwner (syntax->datum #'owner)) + (cons 'authorOrigin author) + (cons 'templateOrigin template)))) + (with-syntax ((source-value (datum->syntax #'binding (list 'quote source)))) + #'(def binding + (list + (cons 'role + (make-nonempty-list-role 'owner 'entry + (grammar-expression item) (grammar-expression separator) 'label)) + (cons 'source source-value))))))) + (_ (raise-syntax-error #f "deflocated-list: expected binding owner entry item separator (field label)" stx))))) diff --git a/t/fixtures/language-pack-research/located-list-language.ss b/t/fixtures/language-pack-research/located-list-language.ss new file mode 100644 index 00000000..e198b7ca --- /dev/null +++ b/t/fixtures/language-pack-research/located-list-language.ss @@ -0,0 +1,4 @@ +;;; -*- Gerbil -*- +(import "located-list-stage") +(export located-study-parser located-study-bound-grammar-ir) +(deflocated-list-study located-study located-composed-ir located-receipt) diff --git a/t/fixtures/language-pack-research/located-list-roles.ss b/t/fixtures/language-pack-research/located-list-roles.ss new file mode 100644 index 00000000..28d20e18 --- /dev/null +++ b/t/fixtures/language-pack-research/located-list-roles.ss @@ -0,0 +1,29 @@ +;;; -*- Gerbil -*- +(import "located-component" "list-origins" + (only-in "list-roles" list-study-base-role) + (only-in "../../../src/modules/parser/objects" make-grammar)) +(export make-located-list-grammar located-list-source-map) + +(deflocated-list arguments-component call-arguments arguments + (reference name) (literal ",") (field argument)) +(deflocated-list elements-component array-elements elements + (reference name) (literal ",") (field element)) + +(def (ref value key) (let (row (assq key value)) (and row (cdr row)))) +(def (make-located-list-grammar) + (make-grammar 'list-study '() (list list-study-base-role) + (list (cons 'append (ref arguments-component 'role)) + (cons 'append (ref elements-component 'role))))) + +;;; Replace component source entries; retain base-rule descriptive sources. +(def (located-list-source-map receipt) + (let ((sources (cdr (assq 'rule (list-study-source-map receipt)))) + (origins (list (ref arguments-component 'source) (ref elements-component 'source)))) + (list + (cons 'rule + (map (lambda (row) + (let (origin (find (lambda (value) + (eq? (ref value 'componentOwner) + (ref (cdr row) 'componentOwner))) origins)) + (if origin (cons (car row) origin) row))) + sources))))) diff --git a/t/fixtures/language-pack-research/located-list-stage.ss b/t/fixtures/language-pack-research/located-list-stage.ss new file mode 100644 index 00000000..c6126c97 --- /dev/null +++ b/t/fixtures/language-pack-research/located-list-stage.ss @@ -0,0 +1,38 @@ +;;; -*- Gerbil -*- +;;; Research adapter: phase-1 POO composition -> existing verbose declaration. +(import (only-in "../../../src/compiler/language-expander" compile-language) + (for-syntax "located-list-roles" + (only-in "../../../src/compiler/normalize" + compile-grammar/receipt grammar-ir-ref))) +(export deflocated-list-study) + +(defsyntax (deflocated-list-study stx) + (syntax-case stx () + ((_ prefix composed-binding receipt-binding) + (let-values (((ir receipt) (compile-grammar/receipt (make-located-list-grammar)))) + (with-syntax + ((composed-value (datum->syntax #'prefix (list 'quote ir))) + (receipt-value (datum->syntax #'prefix (list 'quote receipt))) + (source-map (datum->syntax #'prefix (located-list-source-map receipt))) + ((kind-row ...) (datum->syntax #'prefix (grammar-ir-ref ir 'syntax-kinds))) + ((terminal-row ...) (datum->syntax #'prefix (grammar-ir-ref ir 'terminals))) + ((lexical-row ...) (datum->syntax #'prefix (grammar-ir-ref ir 'lexical-rules))) + ((rule-row ...) (datum->syntax #'prefix (grammar-ir-ref ir 'rules))) + ((extra ...) (datum->syntax #'prefix (map car (grammar-ir-ref ir 'extras)))) + ((entry-row ...) (datum->syntax #'prefix (grammar-ir-ref ir 'parser-entrypoints))) + ((flow-row ...) (datum->syntax #'prefix (grammar-ir-ref ir 'flow)))) + #'(begin + (def composed-binding composed-value) + (def receipt-binding receipt-value) + (compile-language prefix + (identity "list-study" "v1" "list-study.local.v1") + (syntax-kinds kind-row ...) + (terminals terminal-row ...) + (lexical-rules lexical-row ...) + (rules rule-row ...) + (extras extra ...) (keywords) + (parser-entrypoints entry-row ...) + (recoveries) + (source-ownership source-map) + (lineage deflocated-list-study deflocated-list make-nonempty-list-role) + (flow flow-row ...)))))))) diff --git a/t/fixtures/language-pack-research/nested-diagnostics.ss b/t/fixtures/language-pack-research/nested-diagnostics.ss new file mode 100644 index 00000000..b86bf13f --- /dev/null +++ b/t/fixtures/language-pack-research/nested-diagnostics.ss @@ -0,0 +1,68 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-component" "nested-list-vocabulary" + (rename-in "nested-list-vocabulary" (public-argument-list imported-arguments)) + (for-syntax (only-in :gerbil/expander core-expand))) + +(defsyntax (collect-nested-errors stx) + (def (location syntax-value) + (let (loc (stx-source syntax-value)) + (and loc + (let (start (##position->filepos (##locat-start-position loc))) + (list (##container->path (##locat-container loc)) + (+ 1 (##filepos-line start)) (+ 1 (##filepos-col start))))))) + (def (capture form) + (with-catch + (lambda (condition) + (let* ((irritants (slot-ref condition 'irritants)) (blame (car irritants))) + (list (syntax-error? condition) (error-message condition) + (syntax->datum blame) (location blame) + (syntax->datum (cadr irritants))))) + (lambda () (core-expand form) #f))) + (syntax-case stx () + ((_ form ...) + (datum->syntax #'collect-nested-errors + (list 'quote (stx-map capture #'(form ...))))))) + +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "nested diagnostic mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "NESTED-DIAGNOSTIC-CASE-OK: ") (display label) (newline) (force-output)) +(def (line-at path number) + (call-with-input-file path + (lambda (port) + (let loop ((index 1)) + (let (line (read-line port)) + (when (eof-object? line) (error "nested source line absent" path number)) + (if (= index number) line (loop (+ index 1)))))))) +(for-each + (lambda (result token source-present?) + (probe (list 'typed token) (car result) #t) + (probe (list 'message token) (cadr result) + "deflocated-list: field label must be an identifier") + (probe (list 'blame token) (caddr result) token) + (let (loc (list-ref result 3)) + (probe (list 'source-preserved token) (pair? loc) source-present?) + (if loc + (let* ((line (line-at (car loc) (cadr loc))) (offset (- (caddr loc) 1)) + (text (number->string token))) + (probe (list 'author-token token) + (substring line offset (+ offset (string-length text))) text)) + (probe (list 'unknown-source token) loc #f))) + ;; Retain this observation: the inner declaration, not the outer wrapper, + ;; is the context supplied by the current component transformer. + (probe (list 'inner-context token) (car (list-ref result 4)) 'deflocated-list)) + (collect-nested-errors + (deflocated-list component call-arguments arguments + (reference name) (literal ",") (field 61)) + (argument-list component call-arguments arguments + (reference name) (literal ",") 62) + (imported-arguments component call-arguments arguments + (reference name) (literal ",") 63) + (reconstructed-argument-list component call-arguments arguments + (reference name) (literal ",") 64)) + '(61 62 63 64) '(#t #t #t #f)) +(display "NESTED-DIAGNOSTICS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/nested-list-vocabulary.ss b/t/fixtures/language-pack-research/nested-list-vocabulary.ss new file mode 100644 index 00000000..59af4349 --- /dev/null +++ b/t/fixtures/language-pack-research/nested-list-vocabulary.ss @@ -0,0 +1,22 @@ +;;; -*- Gerbil -*- +(import "located-component") +(export argument-list public-argument-list reconstructed-argument-list) + +(defrules argument-list () + ((_ binding owner entry item separator label) + (deflocated-list binding owner entry item separator (field label)))) + +(defrules public-argument-list () + ((_ binding owner entry item separator label) + (argument-list binding owner entry item separator label))) + +(defsyntax (reconstructed-argument-list stx) + (syntax-case stx () + ((_ binding owner entry item separator label) + ;; Negative control: retain definition bindings, discard argument syntax. + (datum->syntax #'reconstructed-argument-list + (list 'deflocated-list + (syntax->datum #'binding) (syntax->datum #'owner) + (syntax->datum #'entry) (syntax->datum #'item) + (syntax->datum #'separator) (list 'field (syntax->datum #'label))) + #f #f)))) diff --git a/t/fixtures/language-pack-research/package-expression-grammar.ss b/t/fixtures/language-pack-research/package-expression-grammar.ss new file mode 100644 index 00000000..c06ee7b4 --- /dev/null +++ b/t/fixtures/language-pack-research/package-expression-grammar.ss @@ -0,0 +1,40 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +;; grammar.ss: candidate author module exercised by the local package control. +(import (only-in :gerbil-parser/language-support/grammar + deflanguage defgrammar-syntax)) +(export list-study-language-grammar) + +(defgrammar-syntax (required-items label item separator) + (seq (field label item) + (repeat (seq separator (field label item))))) + +(begin + (deflanguage list-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier)) + (punctuation Punctuation (literals "(" ")" "[" "]" ",")) + (whitespace Whitespace (whitespace+))) + (extras whitespace) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile + (field form (choice call array)))) + (call + (node Call + (field callee identifier) "(" + (optional (required-items argument name ",")) + ")")) + (array + (node Array + "[" + (optional (required-items element name ",")) + "]")) + (name (node Name (field value identifier))))) + (bind-fixture-grammar-release list-study "list-study" "v1" "list-study.local.v1") ) diff --git a/t/fixtures/language-pack-research/package-expression-parser-test.ss b/t/fixtures/language-pack-research/package-expression-parser-test.ss new file mode 100644 index 00000000..0dab38a5 --- /dev/null +++ b/t/fixtures/language-pack-research/package-expression-parser-test.ss @@ -0,0 +1,36 @@ +;;; -*- Gerbil -*- +;;; User-facing test declarations for the same expression-vocabulary package. +(import :gerbil-parser/language-test-support "package-expression-parser") + +(deflanguage-parser-tests package-expression-parser-test "expression vocabulary package DSL" + (loader list-study-language) + (identity "package identity" + (language "list-study") (version "v1") (contract "list-study.local.v1") + (schema "gerbil-parser.language-entry.v2")) + (accepted "empty call" "f()" + (root SourceFile) (counts (Call 1) (Name 0)) + (field-counts Call argument (0))) + (accepted "empty array" "[]" + (root SourceFile) (counts (Array 1) (Name 0)) + (field-counts Array element (0))) + (accepted "single call argument" "f(a)" + (counts (Call 1) (Name 1)) (field-counts Call argument (1))) + (accepted "three call arguments" "f(a,b,c)" + (counts (Call 1) (Name 3)) (field-counts Call argument (3))) + (accepted "call whitespace" "f( a , b )" + (counts (Call 1) (Name 2)) (field-counts Call argument (2))) + (accepted "single array element" "[a]" + (counts (Array 1) (Name 1)) (field-counts Array element (1))) + (accepted "three array elements" "[a,b,c]" + (counts (Array 1) (Name 3)) (field-counts Array element (3))) + (accepted "array whitespace and newline" "[ a ,\n b ]" + (counts (Array 1) (Name 2)) (field-counts Array element (2))) + (rejected-many "separator and closing policies" + '("f(,)" "f(a,)" "f(a b)" "f(a" "[,]" "[a,]" "[a b]" "[a")) + (ast "exact empty call tree and spans" "f()" + (node SourceFile 0 3 + (field form 0 3 + (node Call 0 3 + (field callee 0 1 (token identifier "f" 0 1)) + (token punctuation "(" 1 2) + (token punctuation ")" 2 3)))))) diff --git a/t/fixtures/language-pack-research/package-expression-parser.ss b/t/fixtures/language-pack-research/package-expression-parser.ss new file mode 100644 index 00000000..c49665e6 --- /dev/null +++ b/t/fixtures/language-pack-research/package-expression-parser.ss @@ -0,0 +1,10 @@ +;;; -*- Gerbil -*- +;;; Complete research package entry using the existing checked Loader. +(import (only-in :gerbil-parser/language-support/development + deflanguage-development-loader LanguageDevelopmentLoader.) + "package-expression-grammar") +(export list-study-language parse-list-study) + +(deflanguage-development-loader (list-study-language :: self LanguageDevelopmentLoader.) + (grammar list-study-language-grammar) + (parse parse-list-study)) diff --git a/t/fixtures/language-pack-research/package-expression-runtime.ss b/t/fixtures/language-pack-research/package-expression-runtime.ss new file mode 100644 index 00000000..df504abc --- /dev/null +++ b/t/fixtures/language-pack-research/package-expression-runtime.ss @@ -0,0 +1,58 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Whole expression-vocabulary package control, through its checked public entry. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + "package-expression-parser" + (only-in "list-language" list-control-parser) + (only-in :clan/poo/object .ref) + (only-in "../../../src/runtime/parser" parse-source) + (only-in "../../../src/runtime/artifact" + parse-artifact-success? parse-artifact-valid? + parse-artifact-events parse-artifact-roundtrip)) +(export main) +(def (main) + + (def checked 0) + (def (probe label actual expected) + (unless (equal? actual expected) + (error "expression package mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "PACKAGE-EXPRESSION-CASE-OK: ") (display label) + (newline) (force-output)) + (probe 'loader-schema (.ref list-study-language 'schema) "gerbil-parser.language-entry.v2") + (probe 'loader-language (.ref list-study-language 'language) "list-study") + (probe 'loader-version (.ref list-study-language 'version) "v1") + (probe 'loader-contract (.ref list-study-language 'contract) "list-study.local.v1") + (probe 'bound-parse-method (eq? parse-list-study (.ref list-study-language '.parse)) #t) + (for-each + (lambda (case) + (let* ((source (car case)) (field-name (cadr case)) (count (caddr case)) + (candidate (parse-list-study source)) + (control (parse-source list-control-parser source))) + (probe (list 'accepted source) (parse-artifact-success? candidate) #t) + (probe (list 'control-accepted source) (parse-artifact-success? control) #t) + (probe (list 'valid-artifact source) (parse-artifact-valid? candidate) #t) + (probe (list 'roundtrip source) (parse-artifact-roundtrip candidate) source) + (probe (list 'complete-event-parity source) + (parse-artifact-events candidate) (parse-artifact-events control)) + (probe (list 'per-item-fields source) + (length (filter (lambda (event) + (and (eq? (vector-ref event 0) 'start-field) + (eq? (vector-ref event 1) field-name))) + (parse-artifact-events candidate))) count))) + '(("f()" argument 0) ("f(a)" argument 1) ("f(a,b,c)" argument 3) + ("f( a , b )" argument 2) ("[]" element 0) ("[a]" element 1) + ("[a,b,c]" element 3) ("[ a ,\n b ]" element 2))) + (for-each + (lambda (source) + (let ((candidate (parse-list-study source)) + (control (parse-source list-control-parser source))) + (probe (list 'rejected source) (parse-artifact-success? candidate) #f) + (probe (list 'control-rejected source) (parse-artifact-success? control) #f) + (probe (list 'rejected-artifact-valid source) (parse-artifact-valid? candidate) #t) + (probe (list 'rejected-roundtrip source) (parse-artifact-roundtrip candidate) source))) + '("f(,)" "f(a,)" "f(a b)" "f(a" "[,]" "[a,]" "[a b]" "[a")) + (display "PACKAGE-EXPRESSION-OK: ") (display checked) + (display " checks passed") (newline) (force-output) + (test-child-process-exit! 0) + ) diff --git a/t/fixtures/language-pack-research/package-expression-test-runner.ss b/t/fixtures/language-pack-research/package-expression-test-runner.ss new file mode 100644 index 00000000..84982c28 --- /dev/null +++ b/t/fixtures/language-pack-research/package-expression-test-runner.ss @@ -0,0 +1,14 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Research infrastructure: run the declarative suite and propagate its result. +(import "package-expression-parser-test" + (only-in :std/test/base TestConfig current-test-config VERBOSITY-CASE + test-suite! test-result-ok?)) +(def result + (parameterize ((current-test-config (TestConfig verbosity: VERBOSITY-CASE capture-output?: #f))) + (test-suite! package-expression-parser-test))) +(unless (test-result-ok? result) + (display "PACKAGE-EXPRESSION-DSL-FAILED") (newline) (force-output) + (exit 1)) +(display "PACKAGE-EXPRESSION-DSL-OK") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/package-parameter-inferred-grammar.ss b/t/fixtures/language-pack-research/package-parameter-inferred-grammar.ss new file mode 100644 index 00000000..5febfcae --- /dev/null +++ b/t/fixtures/language-pack-research/package-parameter-inferred-grammar.ss @@ -0,0 +1,38 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +;;; Matched complete package variant; authoring still uses the three DSL interfaces. +(import (only-in :gerbil-parser/language-support deflanguage defgrammar-syntax)) +(export list-parameter-inferred-language-grammar) +(defgrammar-syntax (required-items label item separator) + (seq (field label item) (repeat (seq separator (field label item))))) +(begin + (deflanguage list-parameter-inferred + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier)) + (punctuation Punctuation (literals "(" ")" "[" "]" ",")) + (whitespace Whitespace (whitespace+))) + (extras whitespace) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile + (field form (choice (reference call) (reference array))))) + (call + (node Call + (seq (field callee (token identifier)) (literal "(") + (optional + (required-items parameter (reference name) (literal ","))) + (literal ")")))) + (array + (node Array + (seq (literal "[") + (optional + (required-items element (reference name) (literal ","))) + (literal "]")))) + (name (node Name (field value (token identifier)))))) + (bind-fixture-grammar-release list-parameter-inferred "list-parameter-inferred" "v1" "list-parameter-inferred.local.v1") ) diff --git a/t/fixtures/language-pack-research/package-parameter-inferred-parser-test.ss b/t/fixtures/language-pack-research/package-parameter-inferred-parser-test.ss new file mode 100644 index 00000000..b9264746 --- /dev/null +++ b/t/fixtures/language-pack-research/package-parameter-inferred-parser-test.ss @@ -0,0 +1,22 @@ +;;; -*- Gerbil -*- +(import :gerbil-parser/language-test-support "package-parameter-inferred-parser") +(deflanguage-parser-tests package-parameter-inferred-test "parameter fields inferred package DSL" + (loader list-parameter-inferred-language) + (syntax-kind "exact effective Call field catalog" Call node (callee parameter)) + (accepted "empty call emits no argument or parameter" "f()" + (field-counts Call argument (0)) (field-counts Call parameter (0))) + (accepted "three parameters without historical argument fields" "f(a,b,c)" + (counts (Call 1) (Name 3)) + (field-counts Call parameter (3)) (field-counts Call argument (0))) + (rejected-many "separator policies unchanged" + '("f(,)" "f(a,)" "f(a b)" "f(a")) + (ast "exact single parameter tree without fabricated optional fields" "f(a)" + (node SourceFile 0 4 + (field form 0 4 + (node Call 0 4 + (field callee 0 1 (token identifier "f" 0 1)) + (token punctuation "(" 1 2) + (field parameter 2 3 + (node Name 2 3 + (field value 2 3 (token identifier "a" 2 3)))) + (token punctuation ")" 3 4)))))) diff --git a/t/fixtures/language-pack-research/package-parameter-inferred-parser.ss b/t/fixtures/language-pack-research/package-parameter-inferred-parser.ss new file mode 100644 index 00000000..eff36968 --- /dev/null +++ b/t/fixtures/language-pack-research/package-parameter-inferred-parser.ss @@ -0,0 +1,7 @@ +;;; -*- Gerbil -*- +(import (only-in :gerbil-parser/language-support/development deflanguage-development-loader) + "package-parameter-inferred-grammar") +(export list-parameter-inferred-language parse-list-parameter-inferred) +(deflanguage-development-loader list-parameter-inferred-language + (grammar list-parameter-inferred-language-grammar) + (parse parse-list-parameter-inferred)) diff --git a/t/fixtures/language-pack-research/package-parameter-inferred-runner.ss b/t/fixtures/language-pack-research/package-parameter-inferred-runner.ss new file mode 100644 index 00000000..77896577 --- /dev/null +++ b/t/fixtures/language-pack-research/package-parameter-inferred-runner.ss @@ -0,0 +1,14 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Research infrastructure: run the declarative suite and propagate its result. +(import "package-parameter-inferred-parser-test" + (only-in :std/test/base TestConfig current-test-config VERBOSITY-CASE + test-suite! test-result-ok?)) +(def result + (parameterize ((current-test-config (TestConfig verbosity: VERBOSITY-CASE capture-output?: #f))) + (test-suite! package-parameter-inferred-test))) +(unless (test-result-ok? result) + (display "PACKAGE-PARAMETER-INFERRED-DSL-FAILED") (newline) (force-output) + (exit 1)) +(display "PACKAGE-PARAMETER-INFERRED-DSL-OK") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/package-parameter-retained-grammar.ss b/t/fixtures/language-pack-research/package-parameter-retained-grammar.ss new file mode 100644 index 00000000..add7a220 --- /dev/null +++ b/t/fixtures/language-pack-research/package-parameter-retained-grammar.ss @@ -0,0 +1,39 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +;;; Matched complete package variant; authoring still uses the three DSL interfaces. +(import (only-in :gerbil-parser/language-support deflanguage defgrammar-syntax)) +(export list-parameter-retained-language-grammar) +(defgrammar-syntax (required-items label item separator) + (seq (field label item) (repeat (seq separator (field label item))))) +(begin + (deflanguage list-parameter-retained + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier)) + (punctuation Punctuation (literals "(" ")" "[" "]" ",")) + (whitespace Whitespace (whitespace+))) + (extras whitespace) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f) + (node-fields (Call argument)))) + (rules + (source-file + (node SourceFile + (field form (choice (reference call) (reference array))))) + (call + (node Call + (seq (field callee (token identifier)) (literal "(") + (optional + (required-items parameter (reference name) (literal ","))) + (literal ")")))) + (array + (node Array + (seq (literal "[") + (optional + (required-items element (reference name) (literal ","))) + (literal "]")))) + (name (node Name (field value (token identifier)))))) + (bind-fixture-grammar-release list-parameter-retained "list-parameter-retained" "v1" "list-parameter-retained.local.v1") ) diff --git a/t/fixtures/language-pack-research/package-parameter-retained-parser-test.ss b/t/fixtures/language-pack-research/package-parameter-retained-parser-test.ss new file mode 100644 index 00000000..b204a264 --- /dev/null +++ b/t/fixtures/language-pack-research/package-parameter-retained-parser-test.ss @@ -0,0 +1,22 @@ +;;; -*- Gerbil -*- +(import :gerbil-parser/language-test-support "package-parameter-retained-parser") +(deflanguage-parser-tests package-parameter-retained-test "parameter fields retained package DSL" + (loader list-parameter-retained-language) + (syntax-kind "exact effective Call field catalog" Call node (argument callee parameter)) + (accepted "empty call emits no argument or parameter" "f()" + (field-counts Call argument (0)) (field-counts Call parameter (0))) + (accepted "three parameters without historical argument fields" "f(a,b,c)" + (counts (Call 1) (Name 3)) + (field-counts Call parameter (3)) (field-counts Call argument (0))) + (rejected-many "separator policies unchanged" + '("f(,)" "f(a,)" "f(a b)" "f(a")) + (ast "exact single parameter tree without fabricated optional fields" "f(a)" + (node SourceFile 0 4 + (field form 0 4 + (node Call 0 4 + (field callee 0 1 (token identifier "f" 0 1)) + (token punctuation "(" 1 2) + (field parameter 2 3 + (node Name 2 3 + (field value 2 3 (token identifier "a" 2 3)))) + (token punctuation ")" 3 4)))))) diff --git a/t/fixtures/language-pack-research/package-parameter-retained-parser.ss b/t/fixtures/language-pack-research/package-parameter-retained-parser.ss new file mode 100644 index 00000000..1ad4540e --- /dev/null +++ b/t/fixtures/language-pack-research/package-parameter-retained-parser.ss @@ -0,0 +1,7 @@ +;;; -*- Gerbil -*- +(import (only-in :gerbil-parser/language-support/development deflanguage-development-loader) + "package-parameter-retained-grammar") +(export list-parameter-retained-language parse-list-parameter-retained) +(deflanguage-development-loader list-parameter-retained-language + (grammar list-parameter-retained-language-grammar) + (parse parse-list-parameter-retained)) diff --git a/t/fixtures/language-pack-research/package-parameter-retained-runner.ss b/t/fixtures/language-pack-research/package-parameter-retained-runner.ss new file mode 100644 index 00000000..6672011d --- /dev/null +++ b/t/fixtures/language-pack-research/package-parameter-retained-runner.ss @@ -0,0 +1,14 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Research infrastructure: run the declarative suite and propagate its result. +(import "package-parameter-retained-parser-test" + (only-in :std/test/base TestConfig current-test-config VERBOSITY-CASE + test-suite! test-result-ok?)) +(def result + (parameterize ((current-test-config (TestConfig verbosity: VERBOSITY-CASE capture-output?: #f))) + (test-suite! package-parameter-retained-test))) +(unless (test-result-ok? result) + (display "PACKAGE-PARAMETER-RETAINED-DSL-FAILED") (newline) (force-output) + (exit 1)) +(display "PACKAGE-PARAMETER-RETAINED-DSL-OK") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/provenance-language.ss b/t/fixtures/language-pack-research/provenance-language.ss new file mode 100644 index 00000000..6cd80f39 --- /dev/null +++ b/t/fixtures/language-pack-research/provenance-language.ss @@ -0,0 +1,42 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + "list-language" + (only-in "../../../src/compiler/bound-ir" bound-grammar-ir-ref bound-grammar-ir-binding) + (only-in "../../../src/compiler/parser-ir" parser-ir-ref) + (only-in "../../../src/runtime/parser" parse-source) + (only-in "../../../src/runtime/artifact" + parse-artifact-success? parse-artifact-roundtrip parse-artifact-events)) +(export main) +(def (main) + + (def checked 0) + (def (probe label actual expected) + (unless (equal? actual expected) (error "language provenance mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "PROVENANCE-LANGUAGE-CASE-OK: ") (display label) (newline) (force-output)) + (def (ref value key) (let (row (assq key value)) (and row (cdr row)))) + (let* ((helper (bound-grammar-ir-binding list-study-bound-grammar-ir 'rule 'call-arguments/tail)) + (source (ref helper 'source))) + (probe 'admitted-component-owner (ref source 'componentOwner) 'call-arguments) + (probe 'admitted-generated-flag (ref source 'generated?) #t) + (probe 'admitted-provider-path (ref source 'path) + "t/fixtures/language-pack-research/list-roles.ss") + (probe 'admitted-expansion-lineage (ref helper 'expansionLineage) + '(deflist-study make-nonempty-list-role))) + (probe 'composition-propagates-to-published-parser + (parser-ir-ref list-study-parser-ir 'compositionDigest) (ref list-composition-receipt 'compositionDigest)) + (probe 'complete-canonical-input-retained list-study-grammar list-composed-ir) + (for-each + (lambda (source) + (let ((candidate (parse-source list-study-parser source)) + (control (parse-source list-control-parser source))) + (probe (list 'accepted source) (parse-artifact-success? candidate) #t) + (probe (list 'roundtrip source) (parse-artifact-roundtrip candidate) source) + (probe (list 'event-parity source) + (parse-artifact-events candidate) (parse-artifact-events control)))) + '("f(a,b)" "[a,b]" "f()")) + (display "PROVENANCE-LANGUAGE-OK: ") (display checked) + (display " checks passed") (newline) (force-output) + (test-child-process-exit! 0) + ) diff --git a/t/fixtures/language-pack-research/provenance-study.ss b/t/fixtures/language-pack-research/provenance-study.ss new file mode 100644 index 00000000..ea9a4866 --- /dev/null +++ b/t/fixtures/language-pack-research/provenance-study.ss @@ -0,0 +1,151 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "list-roles" "list-origins" + (only-in "../../../src/compiler/normalize" compile-grammar/receipt) + (only-in "../../../src/compiler/bound-ir" + bind-grammar-ir bound-grammar-ir-ref bound-grammar-ir-binding) + (only-in "../../../src/compiler/parser-ir" compile-parser parser-ir-ref) + (only-in "../../../src/compiler/language-artifact" + materialize-compiled-language-artifact/output-dirs + compile-language-declaration-artifacts/output-dirs + make-admitted-language-declaration compile-admitted-language-declaration + compiled-language-declaration-grammar + compiled-language-declaration-grammar-locator + compiled-language-declaration-bound-locator + compiled-language-declaration-parser-locator) + (only-in "../../../src/runtime/language-artifact" + load-compiled-language-artifact/roots)) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "provenance study mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "PROVENANCE-CASE-OK: ") (display label) (newline) (force-output)) +(def (ref value key) (let (row (assq key value)) (and row (cdr row)))) +(def (without value key) (filter (lambda (row) (not (eq? (car row) key))) value)) +(let-values (((grammar receipt) (compile-grammar/receipt (make-list-study-grammar)))) + (let* ((origin "list-study-provider") + (lineage '(deflist-study make-nonempty-list-role)) + (sources (list-study-source-map receipt)) + (ordinary (bind-grammar-ir grammar origin lineage)) + (owned (bind-grammar-ir grammar origin lineage sources)) + (entry (bound-grammar-ir-binding owned 'rule 'arguments)) + (helper (bound-grammar-ir-binding owned 'rule 'call-arguments/tail)) + (old-helper (bound-grammar-ir-binding ordinary 'rule 'call-arguments/tail)) + (old-entry (bound-grammar-ir-binding ordinary 'rule 'arguments))) + (probe 'binding-identity-stable (ref helper 'bindingId) (ref old-helper 'bindingId)) + (probe 'reference-identity-stable (ref owned 'referenceDigest) (ref ordinary 'referenceDigest)) + (probe 'grammar-digest-stable (ref owned 'grammarDigest) (ref ordinary 'grammarDigest)) + (probe 'source-sensitive-identity + (equal? (ref owned 'identityDigest) (ref ordinary 'identityDigest)) #f) + (probe 'source-sensitive-declaration + (equal? (ref entry 'declarationId) (ref old-entry 'declarationId)) #f) + (probe 'helper-generated (ref (ref helper 'source) 'generated?) #t) + (probe 'helper-component-owner (ref (ref helper 'source) 'componentOwner) 'call-arguments) + (probe 'entry-component-owner (ref (ref entry 'source) 'componentOwner) 'call-arguments) + (probe 'base-not-generated + (ref (ref (bound-grammar-ir-binding owned 'rule 'source-file) 'source) 'generated?) #f) + (probe 'lineage-preserved (ref helper 'expansionLineage) lineage) + (let* ((roots '("/private/tmp/gerbil-parser-provenance-study")) + (locator (materialize-compiled-language-artifact/output-dirs owned roots)) + (loaded (load-compiled-language-artifact/roots + "gerbil-parser.bound-grammar-ir.v1" locator roots))) + (probe 'materialized-bound-artifact loaded owned)) + (let* ((flattened (without grammar 'compositionDigest)) + (composed-parser (compile-parser grammar)) + (flattened-parser (compile-parser flattened)) + (flattened-bound (bind-grammar-ir flattened origin lineage sources))) + (probe 'parser-retains-composition + (parser-ir-ref composed-parser 'compositionDigest) (ref receipt 'compositionDigest)) + (probe 'flattening-loses-composition (parser-ir-ref flattened-parser 'compositionDigest) #f) + (probe 'lr-spec-stable-on-composition-metadata + (parser-ir-ref composed-parser 'lr-spec) (parser-ir-ref flattened-parser 'lr-spec)) + (probe 'parser-difference-is-composition-field + (without composed-parser 'compositionDigest) (without flattened-parser 'compositionDigest)) + (probe 'bound-identity-independent-of-composition-metadata + (ref owned 'identityDigest) (ref flattened-bound 'identityDigest)) + (probe 'grammar-digest-covers-composition-metadata + (equal? (ref owned 'grammarDigest) (ref flattened-bound 'grammarDigest)) #f)))) +;;; Engine-owned declaration caching for the same composed package. +;;; A fresh root is supplied by the bounded research runner; no timing claim. +(let-values (((grammar receipt) (compile-grammar/receipt (make-list-study-grammar)))) + (let* ((roots (list (getenv "PARSER_RESEARCH_ARTIFACT_ROOT"))) + (sources (list-study-source-map receipt)) + (lineage '(deflist-study make-nonempty-list-role)) + (bound-calls 0) (parser-calls 0)) + (def (publish origin) + (compile-language-declaration-artifacts/output-dirs + (list origin lineage sources) grammar + (lambda () + (set! bound-calls (+ bound-calls 1)) + (bind-grammar-ir grammar origin lineage sources)) + (lambda () + (set! parser-calls (+ parser-calls 1)) + (compile-parser grammar)) + roots)) + (let-values (((g1 b1 p1 status1) (publish "cache-owner-A"))) + (probe 'declaration-first-miss status1 'miss) + (probe 'first-compile-calls (list bound-calls parser-calls) '(1 1)) + (probe 'reload-exact-canonical-grammar + (load-compiled-language-artifact/roots "gerbil-parser.grammar-ir.v1" g1 roots) grammar) + (probe 'reload-parser-composition + (parser-ir-ref + (load-compiled-language-artifact/roots "gerbil-parser.parser-ir.v1" p1 roots) + 'compositionDigest) + (ref receipt 'compositionDigest)) + (let-values (((g2 b2 p2 status2) (publish "cache-owner-A"))) + (probe 'declaration-repeat-hit status2 'hit) + (probe 'repeat-skips-compilers (list bound-calls parser-calls) '(1 1)) + (probe 'repeat-identical-locators (list g2 b2 p2) (list g1 b1 p1))) + (let-values (((g3 b3 p3 status3) (publish "cache-owner-B"))) + (probe 'context-change-declaration-miss status3 'miss) + (probe 'context-change-rebinds-reuses-parser (list bound-calls parser-calls) '(2 1)) + (probe 'context-change-same-grammar-locator g3 g1) + (probe 'context-change-same-parser-locator p3 p1) + (probe 'context-change-different-bound-locator (equal? b3 b1) #f) + (let* ((bound (load-compiled-language-artifact/roots + "gerbil-parser.bound-grammar-ir.v1" b3 roots)) + (entry (bound-grammar-ir-binding bound 'rule 'arguments))) + (probe 'context-change-loaded-origin (ref entry 'originModule) "cache-owner-B") + (probe 'context-change-retains-component-owner + (ref (ref entry 'source) 'componentOwner) 'call-arguments)))))) +;;; The same publication stage accepts normalized POO data and its effective +;;; source context directly, without reconstructing a DSL declaration. +(let-values (((grammar receipt) (compile-grammar/receipt (make-list-study-grammar)))) + (let* ((roots (list (getenv "PARSER_RESEARCH_ARTIFACT_ROOT"))) + (origin "canonical-poo-stage") + (lineage '(deflist-study make-nonempty-list-role)) + (sources (list-study-source-map receipt)) + (compiled + (compile-admitted-language-declaration + (make-admitted-language-declaration grammar origin lineage sources) + compile-parser bind-grammar-ir output-dirs: roots)) + (loaded-grammar + (load-compiled-language-artifact/roots + "gerbil-parser.grammar-ir.v1" + (compiled-language-declaration-grammar-locator compiled) roots)) + (loaded-bound + (load-compiled-language-artifact/roots + "gerbil-parser.bound-grammar-ir.v1" + (compiled-language-declaration-bound-locator compiled) roots)) + (loaded-parser + (load-compiled-language-artifact/roots + "gerbil-parser.parser-ir.v1" + (compiled-language-declaration-parser-locator compiled) roots))) + (probe 'canonical-stage-retains-input-object + (eq? (compiled-language-declaration-grammar compiled) grammar) #t) + (probe 'canonical-stage-grammar-reload loaded-grammar grammar) + (probe 'canonical-stage-bound-reload loaded-bound + (bind-grammar-ir grammar origin lineage sources)) + (probe 'canonical-stage-parser-materialization + (parser-ir-ref loaded-parser 'materialization) 'aot-expansion) + (probe 'canonical-stage-parser-reload + (without loaded-parser 'materialization) (compile-parser grammar)) + (probe 'canonical-stage-retains-composition + (parser-ir-ref loaded-parser 'compositionDigest) (ref receipt 'compositionDigest)) + (probe 'canonical-stage-retains-effective-owner + (ref (ref (bound-grammar-ir-binding loaded-bound 'rule 'arguments) 'source) + 'componentOwner) + 'call-arguments))) +(display "PROVENANCE-STUDY-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/reader-diagnostics.ss b/t/fixtures/language-pack-research/reader-diagnostics.ss new file mode 100644 index 00000000..163d1c41 --- /dev/null +++ b/t/fixtures/language-pack-research/reader-diagnostics.ss @@ -0,0 +1,83 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-component" + (for-syntax (only-in :gerbil/expander core-expand1))) + +(defsyntax (diagnostic-cases stx) + (def (position value) + (and value + (let (filepos (##position->filepos value)) + (list (+ 1 (##filepos-line filepos)) (+ 1 (##filepos-col filepos)))))) + (def (location value) + (let (loc (stx-source value)) + (and loc (list (##container->path (##locat-container loc)) + (position (##locat-start-position loc)))))) + (def (capture form) + (with-catch + (lambda (condition) + ;; SyntaxError extends Exception, not Error. Use its actual slots. + (let* ((irritants (slot-ref condition 'irritants)) + (blame (car irritants))) + (list (syntax-error? condition) (error-message condition) + (syntax->datum blame) (location blame) + (syntax->datum (cadr irritants)) (location (cadr irritants))))) + (lambda () (core-expand1 form) #f))) +(syntax-case stx () + ((_ form ...) + (datum->syntax #'diagnostic-cases + (list 'quote (stx-map capture #'(form ...))))))) + +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "declaration diagnostic mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "DECLARATION-DIAGNOSTIC-CASE-OK: ") (display label) (newline) (force-output)) +(def (line-at path number) + (call-with-input-file path + (lambda (port) + (let loop ((index 1)) + (let (line (read-line port)) + (when (eof-object? line) (error "diagnostic source line absent" path number)) + (if (= index number) line (loop (+ index 1)))))))) +(def messages + '("deflocated-list: binding must be an identifier" + "deflocated-list: owner must be an identifier" + "deflocated-list: entry must be an identifier" + "deflocated-list: field label must be an identifier")) +(for-each + (lambda (result message token) + (probe (list 'typed token) (car result) #t) + (probe (list 'message token) (cadr result) message) + (probe (list 'blame token) (caddr result) token) + (let* ((primary (list-ref result 3)) + (loc (or primary (list-ref result 5))) (start (cadr loc)) + (line (line-at (car loc) (car start))) + (offset (- (cadr start) 1)) + (text (if (pair? token) + (string-append "(" (number->string (car token)) ")") + (number->string token)))) + (probe (list 'parameter-source-available token) (pair? primary) #t) + (probe (list 'effective-reader-coordinate token) + (substring line offset (+ offset (string-length text))) text)) + (probe (list 'declaration-context token) + (car (list-ref result 4)) 'deflocated-list)) + (diagnostic-cases + (deflocated-list 42 call-arguments arguments + (reference name) (literal ",") (field argument)) + (deflocated-list component 43 arguments + (reference name) (literal ",") (field argument)) + (deflocated-list component call-arguments 44 + (reference name) (literal ",") (field argument)) + (deflocated-list component call-arguments arguments + (reference name) (literal ",") (field 45)) + (deflocated-list (42) call-arguments arguments + (reference name) (literal ",") (field argument)) + (deflocated-list component (43) arguments + (reference name) (literal ",") (field argument)) + (deflocated-list component call-arguments (44) + (reference name) (literal ",") (field argument)) + (deflocated-list component call-arguments arguments + (reference name) (literal ",") (field (45)))) (append messages messages) '(42 43 44 45 (42) (43) (44) (45))) +(display "READER-DIAGNOSTICS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/reader-origin.ss b/t/fixtures/language-pack-research/reader-origin.ss new file mode 100644 index 00000000..2da1a0a3 --- /dev/null +++ b/t/fixtures/language-pack-research/reader-origin.ss @@ -0,0 +1,23 @@ +;;; -*- Gerbil -*- +;;; Capture definition syntax while its reader source is still available. +(export with-reader-origin) +(defsyntax (with-reader-origin stx) + (def (position value) + (and value + (let (filepos (##position->filepos value)) + (list (cons 'line (+ 1 (##filepos-line filepos))) + (cons 'column (+ 1 (##filepos-col filepos))))))) + (syntax-case stx () + ((_ origin-binding definition) + (identifier? #'origin-binding) + (let* ((location (stx-source #'definition)) + (origin + (and location + (list (cons 'path (##container->path (##locat-container location))) + (cons 'start (position (##locat-start-position location))) + (cons 'end (position (##locat-end-position location))))))) + (with-syntax ((source-value (datum->syntax #'origin-binding (list 'quote origin)))) + #'(begin + (begin-syntax (def origin-binding source-value)) + definition)))) + (_ (raise-syntax-error #f "expected an origin binding and a definition" stx)))) diff --git a/t/fixtures/language-pack-research/scoped-author-diagnostics.ss b/t/fixtures/language-pack-research/scoped-author-diagnostics.ss new file mode 100644 index 00000000..0d086b75 --- /dev/null +++ b/t/fixtures/language-pack-research/scoped-author-diagnostics.ss @@ -0,0 +1,65 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-component" "scoped-author-vocabulary" + (rename-in "scoped-author-vocabulary" (scoped-arguments imported-scoped-arguments)) + (for-syntax (only-in :gerbil/expander core-expand))) +(defsyntax (scoped-errors stx) + (def (capture form) + (display "SCOPED-EXPAND-BEGIN: ") (write (syntax->datum (stx-car form))) + (newline) (force-output) + (with-catch + (lambda (condition) + (display "SCOPED-EXPAND-CAUGHT: ") (write (error-message condition)) + (newline) (force-output) + (let* ((details (slot-ref condition 'irritants)) (blame (car details)) + (loc (stx-source blame)) + (start (and loc (##position->filepos (##locat-start-position loc))))) + (list (syntax-error? condition) (error-message condition) + (syntax->datum blame) + (and start (list (##container->path (##locat-container loc)) + (+ 1 (##filepos-line start)) (+ 1 (##filepos-col start)))) + (map syntax->datum (cdr details))))) + (lambda () (core-expand form) #f))) + (syntax-case stx () + ((_ form ...) + (datum->syntax #'scoped-errors (list 'quote (stx-map capture #'(form ...))))))) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "scoped author mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "SCOPED-AUTHOR-CASE-OK: ") (display label) (newline) (force-output)) +(def (line-at path number) + (call-with-input-file path + (lambda (port) + (let loop ((index 1)) + (let (line (read-line port)) + (when (eof-object? line) (error "scoped source line absent" path number)) + (if (= index number) line (loop (+ index 1)))))))) +(for-each + (lambda (result token parameter context-head depth) + (probe (list 'typed token) (car result) #t) + (probe (list 'message token) (cadr result) + (string-append "deflocated-list: " parameter " must be an identifier")) + (probe (list 'blame token) (caddr result) token) + (let* ((loc (list-ref result 3)) (line (line-at (car loc) (cadr loc))) + (offset (- (caddr loc) 1)) (text (number->string token))) + (probe (list 'source token) (substring line offset (+ offset (string-length text))) text)) + (let (contexts (list-ref result 4)) + (probe (list 'contexts-count token) (length contexts) depth) + (probe (list 'author-or-direct-context token) (caar contexts) context-head) + (probe (list 'inner-context token) (car (last contexts)) 'deflocated-list))) + (scoped-errors + (scoped-arguments component 91 arguments + (reference name) (literal ",") argument) + (imported-scoped-arguments component call-arguments 92 + (reference name) (literal ",") argument) + (scoped-arguments component call-arguments arguments + (reference name) (literal ",") 93) + ;; A prior failed scoped expansion must not change this direct invocation. + (deflocated-list component 94 arguments + (reference name) (literal ",") (field argument))) + '(91 92 93 94) '("owner" "entry" "field label" "owner") + '(scoped-arguments imported-scoped-arguments scoped-arguments deflocated-list) '(2 2 2 1)) +(display "SCOPED-AUTHOR-DIAGNOSTICS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/scoped-author-vocabulary.ss b/t/fixtures/language-pack-research/scoped-author-vocabulary.ss new file mode 100644 index 00000000..ace906bc --- /dev/null +++ b/t/fixtures/language-pack-research/scoped-author-vocabulary.ss @@ -0,0 +1,19 @@ +;;; -*- Gerbil -*- +(import :std/stxparam "author-context" "located-component" + (for-syntax "declaration-contracts")) +(export scoped-arguments) +(defsyntax (scoped-arguments stx) + (syntax-case stx () + ((_ binding owner entry item separator label) + (begin + (require-declaration-identifier #'binding stx "scoped-arguments" "binding") + (with-syntax ((author-form stx)) + ;; The outer declaration owns the public binding. The scoped body + ;; constructs and returns a value through a hygienic local binding. + #'(def binding + (syntax-parameterize + ((@@author-declaration (quote-syntax author-form))) + (deflocated-list component-value owner entry item separator (field label)) + component-value))))) + (_ (raise-syntax-error #f + "scoped-arguments: expected binding owner entry item separator label" stx)))) diff --git a/t/fixtures/language-pack-research/scoped-valid-components.ss b/t/fixtures/language-pack-research/scoped-valid-components.ss new file mode 100644 index 00000000..d6080b65 --- /dev/null +++ b/t/fixtures/language-pack-research/scoped-valid-components.ss @@ -0,0 +1,27 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-component" "scoped-author-vocabulary" + (only-in "../../../src/modules/parser/objects" + grammar-role? grammar-role-name grammar-role-ref)) +(def component-value 'host-component-value) +(scoped-arguments scoped-component call-arguments arguments + (reference name) (literal ",") argument) +(deflocated-list direct-component call-arguments arguments + (reference name) (literal ",") (field argument)) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "scoped valid component mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "SCOPED-VALID-CASE-OK: ") (display label) (newline) (force-output)) +(def scoped-role (cdr (assq 'role scoped-component))) +(def direct-role (cdr (assq 'role direct-component))) +(probe 'hygienic-local-binding component-value 'host-component-value) +(probe 'checked-role (grammar-role? scoped-role) #t) +(probe 'same-owner (grammar-role-name scoped-role) (grammar-role-name direct-role)) +(for-each + (lambda (section) + (probe section (grammar-role-ref scoped-role section) (grammar-role-ref direct-role section))) + '(syntax-kinds terminals lexical-rules rules extras keywords parser-entrypoints recoveries flow)) +(display "SCOPED-VALID-COMPONENTS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/source-bound-locations.ss b/t/fixtures/language-pack-research/source-bound-locations.ss new file mode 100644 index 00000000..89b494af --- /dev/null +++ b/t/fixtures/language-pack-research/source-bound-locations.ss @@ -0,0 +1,43 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-list-language" + (only-in "../../../src/compiler/bound-ir" bound-grammar-ir-binding)) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "source location mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "SOURCE-LOCATION-CASE-OK: ") (display label) (newline) (force-output)) +(def (ref value key) (let (row (assq key value)) (and row (cdr row)))) +(def (line-at path number) + (call-with-input-file path + (lambda (port) + (let loop ((index 1)) + (let (line (read-line port)) + (when (eof-object? line) (error "source line absent" path number)) + (if (= index number) line (loop (+ index 1)))))))) +(def (source-token origin expected) + (let* ((start (ref origin 'start)) + (line (line-at (ref origin 'path) (ref start 'line))) + (offset (- (ref start 'column) 1))) + (substring line offset (+ offset (string-length expected))))) +(for-each + (lambda (case) + (let* ((entry (car case)) (helper (cadr case)) (owner (caddr case)) + (binding (bound-grammar-ir-binding located-study-bound-grammar-ir 'rule helper)) + (source (ref binding 'source)) + (author (ref source 'authorOrigin)) (template (ref source 'templateOrigin))) + (probe (list 'owner helper) (ref source 'componentOwner) owner) + (probe (list 'generated helper) (ref source 'generated?) #t) + (probe (list 'author-token helper) (source-token author (symbol->string owner)) + (symbol->string owner)) + (probe (list 'template-token helper) (source-token template "(defsyntax") + "(defsyntax") + (probe (list 'author-and-template-distinct helper) + (equal? (ref author 'path) (ref template 'path)) #f) + (probe (list 'entry-and-helper-share-origin entry) + (ref (bound-grammar-ir-binding located-study-bound-grammar-ir 'rule entry) 'source) source))) + '((arguments call-arguments/tail call-arguments) + (elements array-elements/tail array-elements))) +(display "SOURCE-BOUND-LOCATIONS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/source-free-diagnostics.ss b/t/fixtures/language-pack-research/source-free-diagnostics.ss new file mode 100644 index 00000000..dcce2a1e --- /dev/null +++ b/t/fixtures/language-pack-research/source-free-diagnostics.ss @@ -0,0 +1,46 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-component" + (for-syntax (only-in :gerbil/expander core-expand1))) + +(defsyntax (source-free-cases stx) + (def (source-record tree) + (cond + ((and (pair? tree) (eq? (car tree) 'quote) + (pair? (cdr tree)) (list? (cadr tree)) + (andmap pair? (cadr tree)) (assq 'authorOrigin (cadr tree))) (cadr tree)) + ((pair? tree) (or (source-record (car tree)) (source-record (cdr tree)))) + (else #f))) + (def (fresh form) + ;; Retain lexical context, deliberately omit reader source. + (datum->syntax #'source-free-cases (syntax->datum form) #f #f)) + (let* ((valid (fresh #'(deflocated-list component call-arguments arguments + (reference name) (literal ",") (field argument)))) + (source (source-record (syntax->datum (core-expand1 valid)))) + (invalid (fresh #'(deflocated-list component 46 arguments + (reference name) (literal ",") (field argument)))) + (failure + (with-catch + (lambda (condition) + (let (blame (car (slot-ref condition 'irritants))) + (list (syntax-error? condition) (error-message condition) + (syntax->datum blame) (stx-source blame)))) + (lambda () (core-expand1 invalid) #f)))) + (datum->syntax #'source-free-cases (list 'quote (list source failure))))) + +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "declaration diagnostic mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "DECLARATION-DIAGNOSTIC-CASE-OK: ") (display label) (newline) (force-output)) +(let* ((results (source-free-cases)) (source (car results)) (failure (cadr results))) + (probe 'source-free-author (cdr (assq 'authorOrigin source)) #f) + (probe 'source-free-path (cdr (assq 'path source)) #f) + (probe 'source-free-location (cdr (assq 'location source)) #f) + (probe 'source-free-owner (cdr (assq 'componentOwner source)) 'call-arguments) + (probe 'source-free-template-known (pair? (cdr (assq 'templateOrigin source))) #t) + (probe 'source-free-error failure + '(#t "deflocated-list: owner must be an identifier" 46 #f))) +(display "SOURCE-FREE-DIAGNOSTICS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/source-locations.ss b/t/fixtures/language-pack-research/source-locations.ss new file mode 100644 index 00000000..5fbdbdb6 --- /dev/null +++ b/t/fixtures/language-pack-research/source-locations.ss @@ -0,0 +1,55 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-list-language" (only-in "list-language" list-control-parser) + (only-in "../../../src/compiler/bound-ir" bound-grammar-ir-binding) + (only-in "../../../src/runtime/parser" parse-source) + (only-in "../../../src/runtime/artifact" + parse-artifact-success? parse-artifact-events parse-artifact-roundtrip)) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "source location mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "SOURCE-LOCATION-CASE-OK: ") (display label) (newline) (force-output)) +(def (ref value key) (let (row (assq key value)) (and row (cdr row)))) +(def (line-at path number) + (call-with-input-file path + (lambda (port) + (let loop ((index 1)) + (let (line (read-line port)) + (when (eof-object? line) (error "source line absent" path number)) + (if (= index number) line (loop (+ index 1)))))))) +(def (source-token origin expected) + (let* ((start (ref origin 'start)) + (line (line-at (ref origin 'path) (ref start 'line))) + (offset (- (ref start 'column) 1))) + (substring line offset (+ offset (string-length expected))))) +(for-each + (lambda (case) + (let* ((entry (car case)) (helper (cadr case)) (owner (caddr case)) + (binding (bound-grammar-ir-binding located-study-bound-grammar-ir 'rule helper)) + (source (ref binding 'source)) + (author (ref source 'authorOrigin)) (template (ref source 'templateOrigin))) + (probe (list 'owner helper) (ref source 'componentOwner) owner) + (probe (list 'generated helper) (ref source 'generated?) #t) + (probe (list 'author-token helper) (source-token author (symbol->string owner)) + (symbol->string owner)) + (probe (list 'template-token helper) (source-token template "(defsyntax") + "(defsyntax") + (probe (list 'author-and-template-distinct helper) + (equal? (ref author 'path) (ref template 'path)) #f) + (probe (list 'entry-and-helper-share-origin entry) + (ref (bound-grammar-ir-binding located-study-bound-grammar-ir 'rule entry) 'source) source))) + '((arguments call-arguments/tail call-arguments) + (elements array-elements/tail array-elements))) +(for-each + (lambda (source) + (let ((candidate (parse-source located-study-parser source)) + (control (parse-source list-control-parser source))) + (probe (list 'accepted source) (parse-artifact-success? candidate) #t) + (probe (list 'roundtrip source) (parse-artifact-roundtrip candidate) source) + (probe (list 'event-parity source) + (parse-artifact-events candidate) (parse-artifact-events control)))) + '("f(a,b)" "[a,b]" "f()")) +(display "SOURCE-LOCATIONS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/source-origins.ss b/t/fixtures/language-pack-research/source-origins.ss new file mode 100644 index 00000000..44505031 --- /dev/null +++ b/t/fixtures/language-pack-research/source-origins.ss @@ -0,0 +1,48 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import "located-list-roles" + (only-in "../../../src/compiler/normalize" compile-grammar/receipt) + (only-in "../../../src/compiler/bound-ir" bind-grammar-ir bound-grammar-ir-binding)) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "source location mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "SOURCE-LOCATION-CASE-OK: ") (display label) (newline) (force-output)) +(def (ref value key) (let (row (assq key value)) (and row (cdr row)))) +(def (line-at path number) + (call-with-input-file path + (lambda (port) + (let loop ((index 1)) + (let (line (read-line port)) + (when (eof-object? line) (error "source line absent" path number)) + (if (= index number) line (loop (+ index 1)))))))) +(def (source-token origin expected) + (let* ((start (ref origin 'start)) + (line (line-at (ref origin 'path) (ref start 'line))) + (offset (- (ref start 'column) 1))) + (substring line offset (+ offset (string-length expected))))) +(let-values (((grammar receipt) (compile-grammar/receipt (make-located-list-grammar)))) + (def located-study-bound-grammar-ir + (bind-grammar-ir grammar "located-list-roles" '(deflocated-list) + (located-list-source-map receipt))) +(for-each + (lambda (case) + (let* ((entry (car case)) (helper (cadr case)) (owner (caddr case)) + (binding (bound-grammar-ir-binding located-study-bound-grammar-ir 'rule helper)) + (source (ref binding 'source)) + (author (ref source 'authorOrigin)) (template (ref source 'templateOrigin))) + (probe (list 'owner helper) (ref source 'componentOwner) owner) + (probe (list 'generated helper) (ref source 'generated?) #t) + (probe (list 'author-token helper) (source-token author (symbol->string owner)) + (symbol->string owner)) + (probe (list 'template-token helper) (source-token template "(defsyntax") + "(defsyntax") + (probe (list 'author-and-template-distinct helper) + (equal? (ref author 'path) (ref template 'path)) #f) + (probe (list 'entry-and-helper-share-origin entry) + (ref (bound-grammar-ir-binding located-study-bound-grammar-ir 'rule entry) 'source) source))) + '((arguments call-arguments/tail call-arguments) + (elements array-elements/tail array-elements)))) +(display "SOURCE-ORIGINS-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/syntax-parameter-scope.ss b/t/fixtures/language-pack-research/syntax-parameter-scope.ss new file mode 100644 index 00000000..806c2771 --- /dev/null +++ b/t/fixtures/language-pack-research/syntax-parameter-scope.ss @@ -0,0 +1,49 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import :std/stxparam "author-context" + (rename-in "author-context" (@@author-declaration @@renamed-author))) +(defsyntax (context-value stx) + (let (value (syntax-parameter-value #'@@author-declaration)) + (datum->syntax #'context-value (list 'quote (and value (syntax->datum value)))))) +;; Experimental first-context policy; not a published declaration protocol. +(defsyntax (retain-author-context stx) + (syntax-case stx () + ((_ candidate body ...) + (with-syntax ((selected (or (syntax-parameter-value #'@@author-declaration) + #'candidate))) + #'(syntax-parameterize ((@@author-declaration (quote-syntax selected))) + body ...))))) +(def checked 0) +(def (probe label actual expected) + (unless (equal? actual expected) (error "syntax parameter scope mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "SYNTAX-PARAMETER-CASE-OK: ") (display label) (newline) (force-output)) +(probe 'default (context-value) #f) +(probe 'scoped + (syntax-parameterize ((@@author-declaration (quote-syntax (author declaration)))) + (context-value)) + '(author declaration)) +(probe 'after-scope (context-value) #f) +(probe 'nested-nearest-and-restoration + (syntax-parameterize ((@@author-declaration (quote-syntax (outer invocation)))) + (list (context-value) + (syntax-parameterize ((@@author-declaration (quote-syntax (inner invocation)))) + (context-value)) + (context-value))) + '((outer invocation) (inner invocation) (outer invocation))) +(probe 'after-nested (context-value) #f) +(probe 'retain-existing-author + (syntax-parameterize ((@@author-declaration (quote-syntax (root invocation)))) + (retain-author-context (inner invocation) (context-value))) + '(root invocation)) +(probe 'retain-default-author + (retain-author-context (new invocation) (context-value)) + '(new invocation)) +(probe 'after-retention (context-value) #f) +(probe 'renamed-parameter-binding + (syntax-parameterize ((@@renamed-author (quote-syntax (renamed invocation)))) + (context-value)) + '(renamed invocation)) +(display "SYNTAX-PARAMETER-SCOPE-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +(exit 0) diff --git a/t/fixtures/language-pack-research/tier-comparison.ss b/t/fixtures/language-pack-research/tier-comparison.ss new file mode 100644 index 00000000..65572cee --- /dev/null +++ b/t/fixtures/language-pack-research/tier-comparison.ss @@ -0,0 +1,80 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Numeric-free templates versus explicit core grammar structures. +(import "expression-vocabulary" + (only-in "../../../src/grammar/algebra" grammar-expression)) + +(def (tier-check label actual expected) + (unless (equal? actual expected) + (error "tier vocabulary mismatch" label actual expected)) + (display "TIER-CASE-OK: ") (display label) (newline) (force-output)) + +(def sum-core + (vocabulary-value + (left-binary-tier sum (reference product) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right)))) +(tier-check 'left-binary-structure sum-core + (grammar-expression + (choice + (alias Expression + (seq (field left (reference sum)) + (field operator (choice (literal "+") (literal "-"))) + (field right (reference product)))) + (reference product)))) + +(def power-core + (vocabulary-value + (right-binary-tier power (reference atom) (literal "**")))) +(tier-check 'right-binary-structure power-core + (grammar-expression + (choice + (alias BinaryExpression + (seq (field left (reference atom)) + (field operator (literal "**")) + (field right (reference power)))) + (reference atom)))) + +(tier-check 'fhirpath-or-tier + (vocabulary-value + (expression-chain (reference and-expression) + (choice (literal "or") (literal "xor")) + (node Expression) (fields left operator right))) + (grammar-expression + (alias Expression + (seq (field left (reference and-expression)) + (repeat + (seq (field operator (choice (literal "or") (literal "xor"))) + (field right (reference and-expression)))))))) +(tier-check 'fhirpath-implies-tier + (vocabulary-value + (recursive-right-chain implies-expression (reference or-expression) + (literal "implies") (node Expression) (fields left operator right))) + (grammar-expression + (alias Expression + (seq (field left (reference or-expression)) + (optional + (seq (field operator (literal "implies")) + (field right (reference implies-expression)))))))) + +(tier-check 'explicit-node-and-fields + (vocabulary-value + (left-binary-tier sum (reference term) (literal "+") + (node Sum) (fields first operation rest))) + (grammar-expression + (choice + (alias Sum + (seq (field first (reference sum)) + (field operation (literal "+")) + (field rest (reference term)))) + (reference term)))) + +;;; Inspect the admitted values rather than evaluating an alternate parser. +(def (contains-precedence? value) + (and (pair? value) + (or (eq? (car value) 'precedence) + (contains-precedence? (car value)) + (contains-precedence? (cdr value))))) +(tier-check 'numeric-precedence-absent + (or (contains-precedence? sum-core) (contains-precedence? power-core)) #f) +(display "TIER-COMPARISON-OK: 6 checks passed") (newline) (force-output) diff --git a/t/fixtures/language-pack-research/tier-language.ss b/t/fixtures/language-pack-research/tier-language.ss new file mode 100644 index 00000000..71790a98 --- /dev/null +++ b/t/fixtures/language-pack-research/tier-language.ss @@ -0,0 +1,50 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +;;; Research language only: explicit operand tiers, no numeric precedence. +(import "expression-vocabulary" + (only-in "../../../src/language/grammar" deflanguage)) +(export tier-study-parser) + +(begin + (deflanguage tier-study + (syntax + (lexical + (root source-file) + (lex + (whitespace Whitespace (whitespace+)) + (number Number (decimal-digit+)) + (identifier Identifier (identifier)) + (punctuation Punctuation (literals "+" "-" "*" "**" "/" "(" ")")) + (unknown Unknown (fallback))) + (extras whitespace) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file (node SourceFile (field expression (reference sum)))) + (sum + (left-binary-tier sum (reference product) + (choice (literal "+") (literal "-")) + (node Expression) (fields left operator right))) + (product + (left-binary-tier product (reference power) + (choice (literal "*") (literal "/")) + (node Expression) (fields left operator right))) + (power + (right-binary-tier power (reference prefix) (literal "**") + (node Expression) (fields left operator right))) + ;; Explicit study policy: prefix binds more tightly than exponentiation. + (prefix + (choice + (node PrefixExpression + (seq (field operator (choice (literal "+") (literal "-"))) + (field operand (reference prefix)))) + (reference atom))) + (atom + (choice + (node GroupedExpression + (seq (literal "(") (field expression (reference sum)) (literal ")"))) + (node NameExpression (field name (token identifier))) + (node NumberExpression (field value (token number))))))) + (bind-fixture-grammar-release tier-study "tier-study" "v1" "tier-study.local.v1") ) diff --git a/t/fixtures/language-pack-research/tier-runtime.ss b/t/fixtures/language-pack-research/tier-runtime.ss new file mode 100644 index 00000000..0cfced7b --- /dev/null +++ b/t/fixtures/language-pack-research/tier-runtime.ss @@ -0,0 +1,62 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Runtime controls consume the repository's canonical lossless events. +(import "tier-language" + (only-in "../../../languages/arithmetic/grammar" arithmetic-parser) + (only-in "../../../src/runtime/parser" parse-source) + (only-in "../../../src/runtime/artifact" + parse-artifact-success? parse-artifact-roundtrip parse-artifact-events)) + +(def checked 0) +(def (runtime-check label actual expected) + (unless (equal? actual expected) + (error "tier runtime mismatch" label actual expected)) + (set! checked (+ checked 1)) + (display "TIER-RUNTIME-CASE-OK: ") (display label) (newline) (force-output)) + +(def (accepted source) + (let (artifact (parse-source tier-study-parser source)) + (runtime-check (list 'accepted source) (parse-artifact-success? artifact) #t) + (runtime-check (list 'roundtrip source) (parse-artifact-roundtrip artifact) source) + artifact)) + +;;; Compare complete events: node ids/kinds/order, fields, tokens, and spans. +(for-each + (lambda (source) + (let ((candidate (accepted source)) + (baseline (parse-source arithmetic-parser source))) + (runtime-check (list 'baseline-accepted source) + (parse-artifact-success? baseline) #t) + (runtime-check (list 'arithmetic-event-parity source) + (parse-artifact-events candidate) (parse-artifact-events baseline)))) + '("a" "42" "a-b-c" "1-2-3" "a - b - c" "a+b*c" "(a+b)*c" "-a*b" "a+-b" "--a" + "a / b - c" " \n a + b \t")) + +;;; Exact binary node boundaries distinguish left and right nesting. +(def (binary-boundaries artifact) + (map (lambda (event) + (list (vector-ref event 0) (vector-ref event 3))) + (filter (lambda (event) + (and (memq (vector-ref event 0) '(start-node finish-node)) + (eq? (vector-ref event 2) 'Expression))) + (parse-artifact-events artifact)))) +(runtime-check 'left-nesting-and-spans + (binary-boundaries (accepted "1-2-3")) + '((start-node 0) (start-node 0) (finish-node 3) (finish-node 5))) +(runtime-check 'right-nesting-and-spans + (binary-boundaries (accepted "a**b**c")) + '((start-node 0) (start-node 3) (finish-node 7) (finish-node 7))) +(runtime-check 'hyphenated-identifier-is-not-subtraction + (binary-boundaries (accepted "a-b-c")) '()) +(runtime-check 'base-has-no-binary-wrapper + (binary-boundaries (accepted "a")) '()) + +(for-each + (lambda (source) + (runtime-check (list 'rejected source) + (parse-artifact-success? (parse-source tier-study-parser source)) #f)) + '("" "a+" "*a" "a b" "(a+b")) +(display "TIER-RUNTIME-OK: ") (display checked) +(display " checks passed") (newline) (force-output) +;;; This is a terminal research driver; successful completion must terminate. +(exit 0) diff --git a/t/fixtures/layout-columns.ss b/t/fixtures/layout-columns.ss new file mode 100644 index 00000000..3a37d713 --- /dev/null +++ b/t/fixtures/layout-columns.ss @@ -0,0 +1,45 @@ +;;; Boxed-vector oracle, independent of packed storage and lookup dispatch. +(export reference-layout-columns reference-packed-layout-columns layout-column-sources) + +(def (reference-layout-columns source) + (let* ((bytes (string->utf8 source)) (size (u8vector-length bytes)) + (columns (make-vector (+ size 1) 0))) + (let loop ((offset 0) (column 0)) + (if (= offset size) + (begin (vector-set! columns offset column) columns) + (let* ((byte (u8vector-ref bytes offset)) + (width (cond ((< byte 128) 1) ((< byte 224) 2) + ((< byte 240) 3) (else 4))) + (end (+ offset width))) + (let fill ((cursor offset)) + (when (< cursor end) + (vector-set! columns cursor column) (fill (+ cursor 1)))) + (loop end (cond ((or (= byte 10) (= byte 13)) 0) + ((= byte 9) (* (+ (quotient column 8) 1) 8)) + (else (+ column 1))))))))) + +(def layout-column-sources + (list "" "ascii" "\t\t" "1234567\tX\t" "a\r\nb\rc\nd" + "éλ中😀\tZ\r\n名字\tend" + (list->string (map integer->char '(0 1 127 128 2047 2048 55295 57344 65535 65536 1114111))))) + +;;; Matched former packed producer, independent of the ASCII-specialized writer. +;;; Keep the same capacity proof and private native setter as production. +(def (reference-packed-layout-columns source) + (if (> (string-length source) (quotient #xffffffff 8)) + (reference-layout-columns source) + (let* ((bytes (string->utf8 source)) (size (u8vector-length bytes)) + (columns (make-u32vector (+ size 1) 0))) + (let loop ((offset 0) (column 0)) + (if (= offset size) + (begin (##u32vector-set! columns offset column) columns) + (let* ((byte (u8vector-ref bytes offset)) + (width (cond ((< byte 128) 1) ((< byte 224) 2) ((< byte 240) 3) (else 4))) + (end (+ offset width)) + (next-column (case byte ((10 13) 0) + ((9) (* (+ (quotient column 8) 1) 8)) + (else (+ column 1))))) + (let fill ((cursor offset)) + (when (< cursor end) + (##u32vector-set! columns cursor column) (fill (+ cursor 1)))) + (loop end next-column))))))) diff --git a/t/fixtures/lr-action-selection.ss b/t/fixtures/lr-action-selection.ss new file mode 100644 index 00000000..9452e1f0 --- /dev/null +++ b/t/fixtures/lr-action-selection.ss @@ -0,0 +1,70 @@ +;;; Source-owned former layout lookup and projection primitive. +(import (only-in :gerbil-parser/src/runtime/lr-action-index + lr-action-row-eof lr-action-row-tokens lookup-action-entry + lookup-literal-action-entry lookup-casefolded-literal-action-entry + lookup-layout-next-action-entry lookup-layout-start-action-entry) + (only-in :gerbil-parser/src/runtime/layout + current-layout-columns layout-marker-eligible? layout-shift-allowed?) + (only-in :gerbil-parser/src/runtime/token token-kind token-lexeme)) +(export reference-lr-action-selector reference-layout-current-action-row) +(def (reference-layout-ordinary-entry entry shift-allowed?) + (if (or (not entry) shift-allowed?) + (if (and entry (eq? (cadr entry) 'layout-guard)) + (cons (car entry) (caddr entry)) entry) + (let (action (cdr entry)) + (case (car action) + ((layout-guard) (cons (car entry) (caddr action))) + ((shift) #f) + ((fork) + (let (admitted + (filter (lambda (branch) (not (eq? (car branch) 'shift))) + (cdr action))) + (cond ((null? admitted) #f) + ((null? (cdr admitted)) + (cons (car entry) (car admitted))) + (else (cons (car entry) (cons 'fork admitted)))))) + (else entry))))) + +(def (reference-layout-mark-action action role) + (case (car action) + ((layout-guard) (reference-layout-mark-action (cadr action) role)) + ((shift) (list 'layout-shift role (cadr action))) + ((fork) + (cons 'fork + (map (lambda (branch) (reference-layout-mark-action branch role)) + (cdr action)))) + (else action))) + +(def (reference-layout-mark-entry entry role) + (and entry (cons (car entry) + (reference-layout-mark-action (cdr entry) role)))) + +(def (reference-layout-current-action-row row token case-insensitive?) + (let* ((lexeme (token-lexeme token)) + (next-entry (lookup-layout-next-action-entry row lexeme)) + (next (and next-entry (layout-marker-eligible? 'layout-next token) + (reference-layout-mark-entry next-entry 'layout-next))) + (start-entry (and (not next) (lookup-layout-start-action-entry row lexeme))) + (start (and start-entry (layout-marker-eligible? 'layout-start token) + (reference-layout-mark-entry start-entry 'layout-start))) + (ordinary + (reference-layout-ordinary-entry + (or (lookup-literal-action-entry row lexeme) + (and case-insensitive? + (lookup-casefolded-literal-action-entry row lexeme)) + (lookup-action-entry (lr-action-row-tokens row) (token-kind token))) + (layout-shift-allowed? token)))) + (or next start ordinary + (reference-layout-ordinary-entry next-entry #f) + (reference-layout-ordinary-entry start-entry #f)))) + +(def (reference-lr-action-selector _rows index case-insensitive? layout?) + (lambda (state input) + (let (row (vector-ref index state)) + (if (not input) (lr-action-row-eof row) + (let (token input) + (if (and layout? (current-layout-columns)) + (reference-layout-current-action-row row token case-insensitive?) + (or (lookup-literal-action-entry row (token-lexeme token)) + (and case-insensitive? (lookup-casefolded-literal-action-entry row (token-lexeme token))) + (lookup-action-entry (lr-action-row-tokens row) (token-kind token))))))))) diff --git a/t/fixtures/lr1-construction.ss b/t/fixtures/lr1-construction.ss new file mode 100644 index 00000000..e2210b78 --- /dev/null +++ b/t/fixtures/lr1-construction.ss @@ -0,0 +1,247 @@ +;;; -*- Gerbil -*- +;;; Shared grammar fixtures for the LR(1) construction test and benchmark. + +(export lr1-not-lalr-rules shared-lookahead-rules + state-local-candidate-family-rules + genuine-reduce-conflict-rules precedence-expression-rules + mixed-context-rules lr1-context-family-rules + mixed-context-family-rules acyclic-mixed-context-family-rules + split-shared-context-family-rules + unreachable-repeated-context-family-rules + inactive-core-conflict-rules skew-width-rules) + +;;; Declared short rules remain legal composition members. Put the reachable +;;; wide rule last to expose padding in every packed-item construction route. +(def (skew-width-rules short-count width) + (cons '(source-file (reference wide)) + (append + (map (lambda (index) + (list (string->symbol (string-append "short-" (number->string index))) + '(literal "y"))) + (iota short-count)) + (list (list 'wide (cons 'sequence (make-list width '(literal "x")))))))) + +;; The c-rule item after "c" has no raw action before its nullable optional +;; symbol. It shares an LR(0) state with completed a-rule/b-rule items that do +;; conflict on d/e. Its own initial follow block must not split for that +;; unrelated conflict; backward refinement may still split it if required. +(def inactive-core-conflict-rules + '((source-file + (alias SourceFile + (choice + (sequence (literal "a") (reference a-rule) (literal "d")) + (sequence (literal "b") (reference a-rule) (literal "e")) + (sequence (literal "a") (reference b-rule) (literal "e")) + (sequence (literal "b") (reference b-rule) (literal "d")) + (sequence (literal "a") (reference c-rule) (literal "d")) + (sequence (literal "b") (reference c-rule) (literal "e"))))) + (a-rule (literal "c")) + (b-rule (literal "c")) + (c-rule (sequence (literal "c") (optional (literal "x")))))) + +;; Replicate the LR(1)-but-not-LALR(1) context under distinct leading literals. +;; Each region adds one LALR merge conflict and four accepted strings. +(def (lr1-context-family-rules count) + (let ((alternatives '()) + (rules '())) + (let loop ((index 0)) + (when (< index count) + (let* ((prefix (string-append "region-" (number->string index) ":")) + (a-rule (string->symbol + (string-append "a-region-" (number->string index)))) + (b-rule (string->symbol + (string-append "b-region-" (number->string index))))) + (set! alternatives + (cons (list 'sequence (list 'literal prefix) + (list 'literal "a") (list 'reference a-rule) + (list 'literal "d")) alternatives)) + (set! alternatives + (cons (list 'sequence (list 'literal prefix) + (list 'literal "b") (list 'reference a-rule) + (list 'literal "e")) alternatives)) + (set! alternatives + (cons (list 'sequence (list 'literal prefix) + (list 'literal "a") (list 'reference b-rule) + (list 'literal "e")) alternatives)) + (set! alternatives + (cons (list 'sequence (list 'literal prefix) + (list 'literal "b") (list 'reference b-rule) + (list 'literal "d")) alternatives)) + (set! rules (cons (list a-rule (list 'literal "c")) rules)) + (set! rules (cons (list b-rule (list 'literal "c")) rules)) + (loop (+ index 1))))) + (cons (list 'source-file + (list 'alias 'SourceFile + (cons 'choice (reverse alternatives)))) + (reverse rules)))) + +;; Each region has a real A/B reduce conflict on "e" after "a x". The C +;; reduction in that LR(0) state has only "d" locally, but C also appears on a +;; separate "b" path followed by "e". Production-wide follows therefore mark +;; C as a conflict candidate unless the seed consults state-local lookaheads. +(def (state-local-candidate-family-rules count) + (def (region-name prefix index) + (string->symbol (string-append prefix (number->string index)))) + (let ((alternatives '()) + (rules '())) + (let loop ((index 0)) + (when (< index count) + (let* ((prefix (string-append "region-" (number->string index) ":")) + (a (region-name "a-" index)) + (b (region-name "b-" index)) + (c (region-name "c-" index))) + (set! alternatives + (append + alternatives + (list + (list 'sequence (list 'literal prefix) '(literal "a") + (list 'reference c) '(literal "d")) + (list 'sequence (list 'literal prefix) '(literal "a") + (list 'reference a) '(literal "e")) + (list 'sequence (list 'literal prefix) '(literal "a") + (list 'reference b) '(literal "e")) + (list 'sequence (list 'literal prefix) '(literal "b") + (list 'reference c) (list 'reference c) + '(literal "e"))))) + (set! rules + (append rules + (list (list a '(literal "x")) + (list b '(literal "x")) + (list c '(literal "x"))))) + (loop (+ index 1))))) + (cons (list 'source-file + (list 'alias 'SourceFile (cons 'choice alternatives))) + rules))) + +;; Add a shared recursive region to independent LR(1)-only contexts. Direct +;; follow construction compresses the shared region's states and blocks, so +;; this family exercises forward refinement rather than canonical reuse. +(def (mixed-context-family-rules count) + (let* ((base (lr1-context-family-rules count)) + (alternatives (cdr (caddr (cadr (car base))))) + (recursive + (let loop ((index 0) (found '())) + (if (= index count) + (reverse found) + (loop (+ index 1) + (cons (list 'sequence + (list 'literal + (string-append "region-" + (number->string index) ":")) + '(reference component) + '(reference component)) + found)))))) + (cons (list 'source-file + (list 'alias 'SourceFile + (cons 'choice (append alternatives recursive)))) + (append (cdr base) + '((component + (choice + (sequence (literal "c") (reference component)) + (literal "d")))))))) + +;; Share a finite component across all regions. This family has no recursive +;; nonterminal dependency, but its canonical LR(1) table still needs follow +;; compression; acyclicity alone cannot justify a canonical fast route. +(def (acyclic-mixed-context-family-rules count) + (map (lambda (rule) + (if (eq? (car rule) 'component) + '(component (choice (literal "c") (literal "d"))) + rule)) + (mixed-context-family-rules count))) + +;; Sharing a component across distinct productions does not by itself +;; imply that direct follow construction can compress canonical LR(1) states. +(def (split-shared-context-family-rules count) + (let* ((base (lr1-context-family-rules count)) + (alternatives (cdr (caddr (cadr (car base))))) + (shared + (let loop ((index 0) (found '())) + (if (= index count) + (reverse found) + (let (prefix (string-append "region-" + (number->string index) ":")) + (loop (+ index 1) + (cons (list 'sequence (list 'literal prefix) + '(reference component) '(literal "x")) + (cons (list 'sequence (list 'literal prefix) + '(reference component) '(literal "y")) + found)))))))) + (cons (list 'source-file + (list 'alias 'SourceFile + (cons 'choice (append alternatives shared)))) + (append (cdr base) + '((component + (choice + (sequence (literal "c") (reference component)) + (literal "d")))))))) + +;; An unreachable repeated helper must not divert an otherwise exact +;; canonical trial to direct follow construction. +(def (unreachable-repeated-context-family-rules count) + (append (lr1-context-family-rules count) + '((unreachable + (sequence (reference helper) (reference helper))) + (helper (literal "x"))))) + +;; Merging the two LR(1) contexts for A -> c and B -> c creates a spurious +;; reduce/reduce conflict. Canonical LR(1) must keep them distinct. +(def lr1-not-lalr-rules + '((source-file + (alias SourceFile + (choice + (sequence (literal "a") (reference a-rule) (literal "d")) + (sequence (literal "b") (reference a-rule) (literal "e")) + (sequence (literal "a") (reference b-rule) (literal "e")) + (sequence (literal "b") (reference b-rule) (literal "d"))))) + (a-rule (literal "c")) + (b-rule (literal "c")))) + +(def shared-lookahead-rules + '((source-file + (alias SourceFile + (sequence (reference component) (reference component)))) + (component + (choice + (sequence (literal "c") (reference component)) + (literal "d"))))) + +;; Both reductions are valid after the same prefix even in canonical LR(1). +(def genuine-reduce-conflict-rules + '((source-file + (choice (reference a-rule) (reference b-rule))) + (a-rule (literal "c")) + (b-rule (literal "c")))) + +(def precedence-expression-rules + '((source-file + (alias SourceFile (reference expression))) + (expression + (choice + (precedence left 10 + (sequence (reference expression) + (literal "+") + (reference expression))) + (precedence left 20 + (sequence (reference expression) + (literal "*") + (reference expression))) + (literal "a"))))) + +;; One LR(1)-only region forces the direct path, while the independent +;; recursive region has contexts that canonical LR(1) could otherwise duplicate. +(def mixed-context-rules + '((source-file + (alias SourceFile + (choice + (sequence (literal "a") (reference a-rule) (literal "d")) + (sequence (literal "b") (reference a-rule) (literal "e")) + (sequence (literal "a") (reference b-rule) (literal "e")) + (sequence (literal "b") (reference b-rule) (literal "d")) + (sequence (reference component) (reference component))))) + (a-rule (literal "c")) + (b-rule (literal "c")) + (component + (choice + (sequence (literal "c") (reference component)) + (literal "d"))))) diff --git a/t/fixtures/progress.ss b/t/fixtures/progress.ss new file mode 100644 index 00000000..19b422e1 --- /dev/null +++ b/t/fixtures/progress.ss @@ -0,0 +1,9 @@ +;;; -*- Gerbil -*- +;;; Real completed-work receipts for watchdogs, independent of assertions. +(export report-test-progress! report-parser-batch!) +(def (report-test-progress! . parts) + (for-each display parts) + (newline) + (force-output)) +(def (report-parser-batch! count) + (report-test-progress! "BENCHMARK-BATCH-OK parses=" count)) diff --git a/t/fixtures/qualification/output.ss b/t/fixtures/qualification/output.ss new file mode 100644 index 00000000..c0d22537 --- /dev/null +++ b/t/fixtures/qualification/output.ss @@ -0,0 +1,19 @@ +;;; Real child output for qualification framing and completed-log controls. +(export main) +(def (main mode) + (case (string->symbol mode) + ((ordinary) (display "ERROR_COUNT is a counter\n*** NOTE\nOK\n")) + ((error) (display "ERROR\n")) + ((punctuation) (display "ERROR!")) + ((gambit) (display "*** ERROR rejected")) + ((boundary) + (display (make-string 65533 #\x)) (display "\nER") (force-output) + (display "ROR")) + ((large) + (let (block (string->utf8 (string-append "NOTE " (make-string 2042 #\x) "\n"))) + (let loop ((left 4096)) + (when (positive? left) + (write-subu8vector block 0 (u8vector-length block)) + (loop (- left 1)))))) + (else (error "unknown output fixture" mode))) + (force-output) (exit 0)) diff --git a/t/fixtures/qualification/process-controls.ss b/t/fixtures/qualification/process-controls.ss new file mode 100644 index 00000000..0f3aae77 --- /dev/null +++ b/t/fixtures/qualification/process-controls.ss @@ -0,0 +1,117 @@ +;;; Controlled children qualify cancellation and isolation, never existing jobs. +(import :gerbil-parser/t/support/qualification/process :gerbil-parser/t/support/qualification/ownership :std/io/tempfile :std/misc/ports) +(def (check condition label . details) + (unless condition (apply error "process control failed" label details))) +(def (main fixture) + (let (directory (make-temporary-file-name "parser-process-controls-")) + (create-directory directory) + (try + (def (run mode (budget 1.5) (idle 0.8)) + (run-observed-process [fixture mode] (path-expand (string-append mode ".log") directory) + timeout: budget idle-timeout: idle)) + ;; Exercise the first child-module load concurrently, before sequential + ;; controls can warm its objects. Keep the ordinary identity deadlines. + (let (workers + (map (lambda (index) + (spawn (lambda () + (run-observed-process [fixture "identity"] + (path-expand (string-append "cold-" (number->string index) ".log") directory) + timeout: 3 idle-timeout: 1.5)))) + '(0 1 2 3 4 5))) + (for-each + (lambda (worker) + (let (result (thread-join! worker)) + (check (and (= (process-result-status result) 0) + (not (process-result-reason result)) + (receipt-present? "^NATIVE-EXEC-IDENTITY-OK$" + (utf8->string (process-result-output result)))) + 'concurrent-cold-identity (process-result-status result) + (process-result-reason result) (process-result-phase result) + (process-result-elapsed result)))) + workers)) + (displayln "PROCESS-CONTROL-OK concurrent-cold-identity count=6") (force-output) + ;; The process contract must not consume an unrelated caller notification. + (thread-send (current-thread) '(caller-notification)) + (let (result (run "identity" 3 1.5)) + (check (= (process-result-status result) 0) 'exec-session-leader + (process-result-status result) (process-result-reason result) + (process-result-elapsed result) (process-result-phase result) + (utf8->string (process-result-output result))) + (check (not (process-result-reason result)) 'exec-identity-reason) + (check (receipt-present? "^NATIVE-EXEC-IDENTITY-OK$" + (utf8->string (process-result-output result))) 'exec-identity)) + (check (equal? (thread-receive 0 #f) '(caller-notification)) 'caller-mailbox-preserved) + (let (result (run-observed-process [fixture "arguments" "λ a;$(not-a-command)"] + (path-expand "arguments.log" directory) timeout: 3 idle-timeout: 1.5)) + (check (= (process-result-status result) 0) 'exec-argv) + (check (not (process-result-reason result)) 'exec-argv-reason)) + (let (result (run-observed-process ["/nonexistent/gerbil-parser-controlled-missing"] + (path-expand "missing.log" directory) timeout: 3 idle-timeout: 1.5)) + (check (not (= (process-result-status result) 0)) 'exec-failure-status) + (check (eq? (process-result-reason result) 'error-output) 'exec-failure-reason)) + (displayln "PROCESS-CONTROL-OK exec-identity utf8-argv exec-failure") (force-output) + (for-each + (lambda (row) + (let (result (run (car row))) + (check (eq? (process-result-reason result) (cadr row)) (car row) + 'expected (cadr row) 'actual (process-result-reason result) + 'status (process-result-status result) 'elapsed (process-result-elapsed result) + 'phase (process-result-phase result)) + (displayln "PROCESS-CONTROL-OK " (car row)) (force-output))) + '(("silent" idle-timeout) ("continuous" total-timeout) + ("closed-output" idle-timeout) ("error" error-output))) + (let* ((escaped (path-expand "escaped-marker" directory)) + (unrelated (path-expand "unrelated-marker" directory)) + (other (open-process [path: fixture arguments: ["unrelated" unrelated]]))) + (try + (let (result (run-observed-process [fixture "escaped" escaped] + (path-expand "escaped.log" directory) timeout: 1.5 idle-timeout: 0.8)) + (check (eq? (process-result-reason result) 'idle-timeout) 'escaped-timeout) + (check (receipt-present? "^ESCAPED-SESSION " (utf8->string (process-result-output result))) 'escaped-started)) + (check (= (process-status other 4 -1) 0) 'unrelated-exit) + (thread-sleep! 1) + (check (file-exists? unrelated) 'unrelated-survived) + (check (not (file-exists? escaped)) 'escaped-stopped) + (finally (close-port other))) + (displayln "PROCESS-CONTROL-OK escaped-session-stopped unrelated-survived") (force-output)) + (let* ((ready (path-expand "flood-ready" directory)) + (log (path-expand "flood.log" directory)) + (pipe (open-process [path: "gxi" arguments: + ["qualify.ss" "run" "--timeout" "6" "--idle-timeout" "0.8" + "--log" log "--require" "^OK$" "--" fixture "flood" ready] + stderr-redirection: #t]))) + (close-output-port pipe) + (try + (let (started (##current-time-point)) + (let loop () + (unless (file-exists? ready) + (when (> (- (##current-time-point) started) 4) + (let (status (process-status pipe 0 #f)) + (error "flood child did not start" status (and status (utf8->string (read-all-as-u8vector pipe)))))) + (thread-sleep! 0.05) (loop)))) + ;; No reads while the consumer applies more pressure than the idle fence. + (thread-sleep! 1.2) + (let* ((bytes (read-all-as-u8vector pipe)) (text (utf8->string bytes)) + (saved (call-with-input-file log read-all-as-u8vector))) + (check (= (process-status pipe 2 -1) 0) 'backpressure-exit) + (check (= (length (filter (lambda (line) (string-prefix? "PIPE-PROGRESS " line)) + (string-split text #\newline))) 2000) 'console-byte-completeness) + (check (= (u8vector-length saved) (+ (* 2000 (+ 15 2000)) 3)) 'logged-byte-completeness) + (check (receipt-present? "^OK$" text) 'flood-final-receipt)) + (finally (close-port pipe))) + (displayln "PROCESS-CONTROL-OK unicode-backpressure blocking-native-descriptors") (force-output)) + (let (result (run "early-error" 3 1.5)) + (check (eq? (process-result-reason result) 'error-output) 'early-error) + (check (< (process-result-elapsed result) 2.5) 'early-error-stopped) + (displayln "PROCESS-CONTROL-OK immediate-error-cancellation") (force-output)) + (let (result (run "signal" 3 1.5)) + (check (eq? (process-result-reason result) 'signal-exit) 'signal-distinguished) + (check (= (process-result-status result) 143) 'signal-status)) + (check (= (process-result-status (run "failure" 3 1.5)) 42) 'nonzero-propagated) + (check (not (receipt-present? "^FINAL-OK$" "CASE-OK\nHARNESS-OK\n")) 'missing-final-receipt) + (let (plan (ownership-plan 100 '((100 10 100) (101 100 100) (102 101 102) + (103 102 102) (200 10 200) (201 200 200)) 100)) + (check (and (member 100 (cadr plan)) (member 102 (cadr plan)) (andmap (lambda (group) (member group '(100 102))) (cadr plan))) 'owned-groups-only)) + (displayln "PROCESS-OWNERSHIP-CONTROLS-OK") (force-output) + (finally (delete-file-or-directory directory #t)))) + (exit 0)) diff --git a/t/fixtures/qualification/process-fixture.c b/t/fixtures/qualification/process-fixture.c new file mode 100644 index 00000000..8c3d61ba --- /dev/null +++ b/t/fixtures/qualification/process-fixture.c @@ -0,0 +1,97 @@ +/* Native child behavior; Scheme owns deadlines, receipts and cancellation. */ +#define _XOPEN_SOURCE 700 +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void mark(const char *name) { + FILE *file = fopen(name, "w"); + if (!file) exit(3); + fputs("finished\n", file); + fclose(file); +} + +int main(int argc, char **argv) { + if (argc < 2) return 2; + if (strcmp(argv[1], "unrelated") && + ((fcntl(1, F_GETFL) & O_NONBLOCK) || + (fcntl(2, F_GETFL) & O_NONBLOCK))) return 43; + if (!strcmp(argv[1], "identity")) { + /* The target must replace the acknowledged session leader, not be its + child. The previous waiting-GXI topology fails this assertion. */ + if (getpid() != getsid(0)) return 45; + puts("NATIVE-EXEC-IDENTITY-OK"); + return 0; + } + if (!strcmp(argv[1], "arguments")) { + if (argc != 3 || strcmp(argv[2], "λ a;$(not-a-command)")) return 46; + puts("NATIVE-EXEC-ARGUMENTS-OK"); + return 0; + } + if (!strcmp(argv[1], "flood")) { + if (argc != 3) return 2; + mark(argv[2]); + for (int i = 0; i < 2000; i++) { + fputs("PIPE-PROGRESS ", stdout); + for (int j = 0; j < 1000; j++) fputs("\xce\xbb", stdout); + fputc('\n', stdout); + } + puts("OK"); + return fflush(stdout) ? 44 : 0; + } + if (!strcmp(argv[1], "early-error")) { + puts("*** ERROR controlled failure"); + fflush(stdout); + sleep(4); + return 0; + } + if (!strcmp(argv[1], "signal")) { raise(SIGTERM); return 5; } + if (!strcmp(argv[1], "silent")) { sleep(4); return 0; } + if (!strcmp(argv[1], "continuous")) { + for (int i = 0; i < 100; i++) { + puts("CHILD-PROGRESS"); + fflush(stdout); + struct timespec remaining = { .tv_sec = 0, .tv_nsec = 20000000 }; + while (nanosleep(&remaining, &remaining) < 0) { + if (errno != EINTR) return 47; + } + } + return 0; + } + if (!strcmp(argv[1], "unrelated")) { + if (argc != 3) return 2; + sleep(2); + mark(argv[2]); + return 0; + } + if (!strcmp(argv[1], "escaped")) { + if (argc != 3) return 2; + pid_t child = fork(); + if (child < 0) return 3; + if (!child) { + if (setsid() < 0) return 4; + printf("ESCAPED-SESSION %d\n", getpid()); + fflush(stdout); + sleep(2); + mark(argv[2]); + _exit(0); + } + int result; + return waitpid(child, &result, 0) < 0 ? 6 : 0; + } + if (!strcmp(argv[1], "closed-output")) { + close(1); + close(2); + sleep(4); + return 0; + } + if (!strcmp(argv[1], "failure")) { puts("OK"); return 42; } + if (!strcmp(argv[1], "error")) { puts("ERROR swallowed"); return 0; } + return 2; +} diff --git a/t/fixtures/qualification/startup-controls.ss b/t/fixtures/qualification/startup-controls.ss new file mode 100644 index 00000000..cf0591ce --- /dev/null +++ b/t/fixtures/qualification/startup-controls.ss @@ -0,0 +1,44 @@ +;;; Diagnose startup phases only; this is not a process-deadline admission. +(import :gerbil-parser/t/support/qualification/process + :std/io/tempfile :std/misc/ports + (only-in :std/os/signal kill SIGKILL)) +(export main) +(def (main fixture) + (let (directory (make-temporary-file-name "parser-startup-controls-")) + (create-directory directory) + (try + (def environment + (map (lambda (entry) + (string-append (car entry) "=" + (if (equal? (car entry) "GERBIL_LOADPATH") + (string-append qualification-library-root ":" (cdr entry)) + (cdr entry)))) + (if (getenv "GERBIL_LOADPATH" #f) (get-environment-variables) + (cons (cons "GERBIL_LOADPATH" "") (get-environment-variables))))) + (def (probe label arguments) + (let* ((started (##current-time-point)) + (child (open-process [path: "gxi" arguments: arguments + environment: environment stderr-redirection: #t])) + (status #f)) + (close-output-port child) + (try + (set! status (process-status child 3 #f)) + (unless status (error "startup diagnostic exceeded three seconds" label)) + (displayln "STARTUP-RESULT " label " elapsed=" + (- (##current-time-point) started) " raw-status=" status) + (display (utf8->string (read-all-as-u8vector child))) (force-output) + (unless (zero? status) (error "startup diagnostic failed" label status)) + (finally + (unless status + (kill (process-pid child) SIGKILL) + (process-status child 1 #f)) + (close-port child))))) + (def phase-gxi "(displayln \"STARTUP-PHASE gxi \" (##current-time-point)) (force-output)") + (def phase-module "(displayln \"STARTUP-PHASE module \" (##current-time-point)) (force-output)") + (probe "gxi-only" ["-e" phase-gxi "-e" "(exit 0)"]) + (let (args (process-child-arguments (path-expand "session" directory) + (getenv "GERBIL_LOADPATH" "") [fixture "identity"])) + (probe "native-child" + (append ["-e" phase-gxi] [(car args) (cadr args)] + ["-e" phase-module] (cddr args)))) + (finally (delete-file-or-directory directory #t))))) diff --git a/t/fixtures/ranked-publication.ss b/t/fixtures/ranked-publication.ss new file mode 100644 index 00000000..9dad82b3 --- /dev/null +++ b/t/fixtures/ranked-publication.ss @@ -0,0 +1,60 @@ +;;; Matched compact-trie primitive from before terminal staging/publication. +;;; Keep the parser, lexical plans and result protocol outside this comparator. +(import (only-in :gerbil-parser/src/runtime/scan make-literal-trie literal-trie-child make-ranked-regular-scanner horizontal-whitespace? newline? identifier-start?) + (only-in :gerbil-parser/src/compiler/lexical-expression prefer-generated-match)) +(export reference-ranked-publication-scanner reference-ranked-lexical-scanner-factory) +(def (reference-ranked-publication-scanner entries) + (def (best-admitted candidates admitted) + (let loop ((remaining candidates) (best #f)) + (if (null? remaining) best + (let (candidate (car remaining)) + (loop (cdr remaining) + (if (and (or (not admitted) (vector-ref admitted (caddr candidate))) + (or (not best) (> (cadr candidate) (cadr best)) + (and (= (cadr candidate) (cadr best)) + (< (caddr candidate) (caddr best))))) + candidate best)))))) + (let (root (make-literal-trie + (map (lambda (entry) (cons (car entry) (cdr entry))) entries) + '() (lambda (previous payload) (cons payload previous)))) + (lambda (source start (admitted #f)) + (let (limit (string-length source)) + (let scan ((node root) (offset start) (selected #f)) + (if (= offset limit) selected + (let (child (literal-trie-child node (string-ref source offset))) + (if child + (let* ((next (+ offset 1)) (terminal (best-admitted (vector-ref child 0) admitted))) + (scan child next + (if terminal (list (car terminal) next (cadr terminal) (caddr terminal)) selected))) + selected)))))))) + + +;;; Original separate traversals and complete ranked result competition. +(def (reference-ranked-lexical-scanner-factory entries) + (let ((literal (reference-ranked-publication-scanner entries)) + (starts (make-vector 128 #f))) + (for-each (lambda (entry) + (let (code (char->integer (string-ref (car entry) 0))) + (when (< code 128) (vector-set! starts code #t)))) entries) + (lambda (regular-entries admissions) + (let* ((regular (and (pair? regular-entries) (make-ranked-regular-scanner regular-entries))) + (admitted (and admissions (vector-copy admissions))) + (has-literals? (any (lambda (entry) (or (not admitted) (vector-ref admitted (cadddr entry)))) entries)) + (regular-starts + (list->vector (map (lambda (code) + (let (ch (integer->char code)) + (any (lambda (entry) + (case (car entry) + ((whitespace+) (char-whitespace? ch)) + ((horizontal-whitespace+) (horizontal-whitespace? ch)) + ((newline+) (newline? ch)) + ((decimal-digit+ number) (char-numeric? ch)) + ((identifier) (identifier-start? ch)))) regular-entries))) (iota 128))))) + (lambda (source start) + (and (not (= start (string-length source))) + (let (code (char->integer (string-ref source start))) + (prefer-generated-match + (and regular (or (>= code 128) (vector-ref regular-starts code)) + (regular source start)) + (and has-literals? (or (>= code 128) (vector-ref starts code)) + (literal source start admitted)))))))))) diff --git a/t/fixtures/runtime-ascii-digit-run/Cargo.lock b/t/fixtures/runtime-ascii-digit-run/Cargo.lock new file mode 100644 index 00000000..937bb240 --- /dev/null +++ b/t/fixtures/runtime-ascii-digit-run/Cargo.lock @@ -0,0 +1,215 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gerbil-parser-artifact" +version = "0.1.0" +dependencies = [ + "serde_json", + "sha2", +] + +[[package]] +name = "gerbil-parser-runtime" +version = "0.1.0" +dependencies = [ + "gerbil-parser-artifact", + "sha2", +] + +[[package]] +name = "gerbil-parser-runtime-external" +version = "0.1.0" +dependencies = [ + "gerbil-parser-artifact", + "gerbil-parser-runtime", + "sha2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "runtime-ascii-digit-run-fixture" +version = "0.0.0" +dependencies = [ + "gerbil-parser-runtime", + "gerbil-parser-runtime-external", + "sha2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/t/fixtures/runtime-ascii-digit-run/Cargo.toml b/t/fixtures/runtime-ascii-digit-run/Cargo.toml new file mode 100644 index 00000000..9fd5d9e7 --- /dev/null +++ b/t/fixtures/runtime-ascii-digit-run/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "runtime-ascii-digit-run-fixture" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +gerbil-parser-runtime = { path = "../../../rust/gerbil-parser-runtime" } +gerbil-parser-runtime-external = { path = "../../../rust/gerbil-parser-runtime-external" } + +[build-dependencies] +sha2 = "0.10" + +[workspace] diff --git a/t/fixtures/runtime-ascii-digit-run/build.rs b/t/fixtures/runtime-ascii-digit-run/build.rs new file mode 100644 index 00000000..9f317ecf --- /dev/null +++ b/t/fixtures/runtime-ascii-digit-run/build.rs @@ -0,0 +1,15 @@ +use sha2::{Digest, Sha256}; +use std::{env, fs, path::PathBuf}; + +fn main() { + let source = PathBuf::from("src/parser.table"); + println!("cargo:rerun-if-changed={}", source.display()); + let artifact = fs::read(&source).expect("external parser artifact"); + let digest = format!("sha256:{:x}", Sha256::digest(artifact)); + let output = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")); + fs::write( + output.join("provider_digest.rs"), + format!("pub const PROVIDER_DIGEST: &str = {digest:?};\n"), + ) + .expect("write build-bound provider digest"); +} diff --git a/t/fixtures/runtime-ascii-digit-run/src/lib.rs b/t/fixtures/runtime-ascii-digit-run/src/lib.rs new file mode 100644 index 00000000..1e526942 --- /dev/null +++ b/t/fixtures/runtime-ascii-digit-run/src/lib.rs @@ -0,0 +1,89 @@ +//! An independent ASCII digit parser with a build-bound artifact and external receipt. + +use gerbil_parser_runtime::{Diagnostic, EventCatalog, KindCategory, KindSpec, TreeEvent}; +use gerbil_parser_runtime_external::{ + ExternalLanguageSpec, ExternalParse, ExternalParseError, parse_external_events, +}; + +include!(concat!(env!("OUT_DIR"), "/provider_digest.rs")); + +const ARTIFACT: &[u8] = include_bytes!("parser.table"); +static KINDS: &[KindSpec] = &[ + KindSpec { + name: "Digits", + category: KindCategory::Node, + }, + KindSpec { + name: "DigitRun", + category: KindCategory::Token, + }, +]; +static LANGUAGE: ExternalLanguageSpec = ExternalLanguageSpec { + language: "ascii-digit-run", + version: "fixture", + contract: "ascii-digit-run.syntax", + provider: "ascii-digit-table", + provider_version: "fixture", + provider_digest: PROVIDER_DIGEST, + catalog: EventCatalog { + root_kind: 0, + kinds: KINDS, + }, +}; + +/// Parse an ASCII digit run using the build-bound provider table. +/// +/// # Errors +/// +/// Rejects source outside the digit grammar or invalid source events. +pub fn parse_digits(source: &str) -> Result { + // The fixture interprets the artifact's declared character range itself. + let parser_range = ARTIFACT + .strip_prefix(b"digit-token:ascii:") + .and_then(|bytes| bytes.strip_suffix(b"\n")); + if parser_range != Some(&b"0-9"[..]) + || source.is_empty() + || !source.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(ExternalParseError::from_provider( + &LANGUAGE, + source, + Diagnostic { + reason_kind: "provider-rejected", + byte_offset: 0, + message: "expected an ASCII digit run".into(), + }, + )); + } + parse_external_events( + &LANGUAGE, + source, + &[ + TreeEvent::StartNode(0), + TreeEvent::Token { + kind: 1, + start: 0, + end: source.len(), + }, + TreeEvent::FinishNode, + ], + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn independent_parser_emits_lossless_external_tree_and_bound_receipt() { + let parsed = parse_digits("0123").unwrap(); + assert_eq!(parsed.syntax().to_string(), "0123"); + assert_eq!(parsed.receipt().authority, "external-parser"); + assert_eq!(parsed.receipt().provider_digest, PROVIDER_DIGEST); + assert_eq!(parsed.receipt().language, "ascii-digit-run"); + assert_eq!( + parse_digits("12x").unwrap_err().diagnostic.reason_kind, + "provider-rejected" + ); + } +} diff --git a/t/fixtures/runtime-ascii-digit-run/src/parser.table b/t/fixtures/runtime-ascii-digit-run/src/parser.table new file mode 100644 index 00000000..d9cf5a59 --- /dev/null +++ b/t/fixtures/runtime-ascii-digit-run/src/parser.table @@ -0,0 +1 @@ +digit-token:ascii:0-9 diff --git a/t/fixtures/rust-rowan-downstream/Cargo.lock b/t/fixtures/runtime-record-assignments/Cargo.lock similarity index 53% rename from t/fixtures/rust-rowan-downstream/Cargo.lock rename to t/fixtures/runtime-record-assignments/Cargo.lock index 931e519d..71664d78 100644 --- a/t/fixtures/rust-rowan-downstream/Cargo.lock +++ b/t/fixtures/runtime-record-assignments/Cargo.lock @@ -17,12 +17,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" -[[package]] -name = "countme" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" - [[package]] name = "cpufeatures" version = "0.2.17" @@ -52,13 +46,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "downstream-rowan-language-fixture" -version = "0.0.0" -dependencies = [ - "gerbil-parser-rowan", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -70,42 +57,105 @@ dependencies = [ ] [[package]] -name = "gerbil-parser-rowan" +name = "gerbil-parser-artifact" +version = "0.1.0" +dependencies = [ + "serde_json", + "sha2", +] + +[[package]] +name = "gerbil-parser-runtime" version = "0.1.0" dependencies = [ - "rowan", + "gerbil-parser-artifact", "sha2", ] [[package]] -name = "hashbrown" -version = "0.14.5" +name = "itoa" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "runtime-record-assignments-fixture" +version = "0.0.0" +dependencies = [ + "gerbil-parser-runtime", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] [[package]] -name = "rowan" -version = "0.16.1" +name = "serde_derive" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "417a3a9f582e349834051b8a10c8d71ca88da4211e4093528e36b9845f6b5f21" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ - "countme", - "hashbrown", - "rustc-hash", - "text-size", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "rustc-hash" -version = "1.1.0" +name = "serde_json" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] [[package]] name = "sha2" @@ -119,10 +169,15 @@ dependencies = [ ] [[package]] -name = "text-size" -version = "1.1.1" +name = "syn" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] [[package]] name = "typenum" @@ -130,8 +185,20 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/t/fixtures/runtime-record-assignments/Cargo.toml b/t/fixtures/runtime-record-assignments/Cargo.toml new file mode 100644 index 00000000..9428314a --- /dev/null +++ b/t/fixtures/runtime-record-assignments/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "runtime-record-assignments-fixture" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +gerbil-parser-runtime = { path = "../../../rust/gerbil-parser-runtime" } + +[workspace] diff --git a/t/fixtures/runtime-record-assignments/README.org b/t/fixtures/runtime-record-assignments/README.org new file mode 100644 index 00000000..3d31125b --- /dev/null +++ b/t/fixtures/runtime-record-assignments/README.org @@ -0,0 +1,17 @@ +#+title: Record assignments language-pack fixture + +This fixture owns the same =grammar.ss=, =parser.ss=, =fixtures.ss=, +=parser-test.ss=, and native corpus layout as an independent DSL package. +It deliberately has no =gerbil.pkg= or Gerbil installation contract. Upstream +generation CI drives its public +=:gerbil-parser/rust-runtime-grammar-support= declaration through the standalone +=gerbil-parser-runtime-aot= product and verifies the committed Rust module. The +consumer acceptance step is then a standalone Cargo build: it needs only +=gerbil-parser-runtime= and the generated module, never Gerbil, a private engine +module, or a handwritten syntax sink. + +The production =parser.ss= imports only the grammar. =parser-test.ss= imports +the optional corpus and declares =records-test-language= through +=deflanguage-development-loader= over the exact production descriptor. Upstream +native conformance includes this independent sample; production declarations +reject development services even when those services come from a POO parent. diff --git a/t/fixtures/runtime-record-assignments/generate.ss b/t/fixtures/runtime-record-assignments/generate.ss new file mode 100644 index 00000000..7dbc7c23 --- /dev/null +++ b/t/fixtures/runtime-record-assignments/generate.ss @@ -0,0 +1,16 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Record assignments AOT entry using only the public Rust generator facade. + +(import (only-in :gerbil-parser/rust-runtime-support + generate-language-rust-runtime-module) + (only-in "languages/records/grammar.ss" + records-language-grammar)) + +(def arguments (command-line)) +(unless (> (length arguments) 2) + (error "usage: generate.ss OUTPUT.rs")) + +(generate-language-rust-runtime-module + (car (reverse arguments)) + records-language-grammar) diff --git a/t/fixtures/rust-rowan-downstream/languages/records/v1/corpus/accepted.records b/t/fixtures/runtime-record-assignments/languages/records/corpus/accepted.records similarity index 100% rename from t/fixtures/rust-rowan-downstream/languages/records/v1/corpus/accepted.records rename to t/fixtures/runtime-record-assignments/languages/records/corpus/accepted.records diff --git a/t/fixtures/rust-rowan-downstream/languages/records/v1/corpus/rejected.records b/t/fixtures/runtime-record-assignments/languages/records/corpus/rejected.records similarity index 100% rename from t/fixtures/rust-rowan-downstream/languages/records/v1/corpus/rejected.records rename to t/fixtures/runtime-record-assignments/languages/records/corpus/rejected.records diff --git a/t/fixtures/rust-rowan-downstream/languages/records/v1/fixtures.ss b/t/fixtures/runtime-record-assignments/languages/records/fixtures.ss similarity index 52% rename from t/fixtures/rust-rowan-downstream/languages/records/v1/fixtures.ss rename to t/fixtures/runtime-record-assignments/languages/records/fixtures.ss index defd44ea..795d8ef3 100644 --- a/t/fixtures/rust-rowan-downstream/languages/records/v1/fixtures.ss +++ b/t/fixtures/runtime-record-assignments/languages/records/fixtures.ss @@ -1,18 +1,18 @@ ;;; -*- Gerbil -*- -;;; Downstream corpus manifest embedded at macro expansion. +;;; Record assignments corpus manifest embedded at macro expansion. (import (only-in :gerbil-parser/language-support defsyntax-corpus) (only-in ./grammar +records-language-version+ +records-syntax-contract+)) -(export records-v1-fixtures) +(export records-fixtures) -(defsyntax-corpus records-v1-fixtures - (identity "downstream-records" +(defsyntax-corpus records-fixtures + (identity "record-assignments" +records-language-version+ +records-syntax-contract+) (accepted - ("records/accepted" records-v1-accepted "corpus/accepted.records" + ("records/accepted" records-accepted "corpus/accepted.records" Document (Assignment))) (rejected - ("records/rejected" records-v1-rejected "corpus/rejected.records"))) + ("records/rejected" records-rejected "corpus/rejected.records"))) diff --git a/t/fixtures/rust-rowan-downstream/languages/records/v1/grammar.ss b/t/fixtures/runtime-record-assignments/languages/records/grammar.ss similarity index 53% rename from t/fixtures/rust-rowan-downstream/languages/records/v1/grammar.ss rename to t/fixtures/runtime-record-assignments/languages/records/grammar.ss index 63c13883..da1c5fbb 100644 --- a/t/fixtures/rust-rowan-downstream/languages/records/v1/grammar.ss +++ b/t/fixtures/runtime-record-assignments/languages/records/grammar.ss @@ -1,7 +1,8 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) ;;; -*- Gerbil -*- -;;; Downstream-owned record DSL used to qualify the public language-pack API. +;;; Record assignments DSL used to qualify the public language-pack API. -(import (only-in :gerbil-parser/rust-rowan-grammar-support deflanguage)) +(import (only-in :gerbil-parser/rust-runtime-grammar-support deflanguage)) (export +records-language-version+ +records-syntax-contract+ records-language-grammar @@ -10,13 +11,14 @@ records-parser) (def +records-language-version+ "v1") -(def +records-syntax-contract+ "downstream-records.v1") +(def +records-syntax-contract+ "record-assignments.v1") -(deflanguage records - (identity "downstream-records" +records-language-version+ - +records-syntax-contract+) - (root document) - (lex +(begin + (deflanguage records + (syntax + (lexical + (root document) + (lex (whitespace Whitespace (horizontal-whitespace+)) (newline Newline (newline+)) (number Number (decimal-digit+)) @@ -24,6 +26,11 @@ (string String (quoted-string "\"")) (punctuation Punctuation (literals "=")) (unknown Unknown (fallback))) + (extras whitespace) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) (rules (document (node Document (repeat assignment))) @@ -33,9 +40,5 @@ (field name identifier) (literal "=") (field value (choice number string identifier)) - (optional newline))))) - (extras whitespace) - (keywords) - (recoveries) - (conflicts reject) - (case-insensitive #f)) + (optional newline)))))) + (bind-fixture-grammar-release records "record-assignments" +records-language-version+ +records-syntax-contract+) ) diff --git a/t/fixtures/runtime-record-assignments/languages/records/parser-test.ss b/t/fixtures/runtime-record-assignments/languages/records/parser-test.ss new file mode 100644 index 00000000..2a815010 --- /dev/null +++ b/t/fixtures/runtime-record-assignments/languages/records/parser-test.ss @@ -0,0 +1,19 @@ +;;; -*- Gerbil -*- +;;; Independent downstream conformance uses only installed public facades. +(import :gerbil-parser/language-test-support + (only-in :gerbil-parser/language-support/development deflanguage-development-loader) + (only-in :gerbil-parser/language-support/entry language-parser-entry-ref) + ./parser ./fixtures) +(deflanguage-development-loader records-test-language + (descriptor (language-parser-entry-ref records-language 'descriptor)) + (parse parse-records-test) + (slots fixtures: records-fixtures)) +(deflanguage-parser-tests parser-test "record assignments language pack" + (loader records-test-language) + (identity "immutable public identity" + (schema "gerbil-parser.language-entry.v2") (language "record-assignments") + (version "v1") (contract "record-assignments.v1")) + (fixtures "accepted and rejected declared corpus")) + +(def records-parser-tests parser-test) +(export records-parser-tests records-test-language) diff --git a/t/fixtures/runtime-record-assignments/languages/records/parser.ss b/t/fixtures/runtime-record-assignments/languages/records/parser.ss new file mode 100644 index 00000000..c989fb21 --- /dev/null +++ b/t/fixtures/runtime-record-assignments/languages/records/parser.ss @@ -0,0 +1,10 @@ +;;; -*- Gerbil -*- +;;; Record assignments parser entry built only from the installed facade. + +(import (only-in :gerbil-parser/language-support deflanguage-parser-loader) + (only-in ./grammar records-language-grammar)) +(export records-language parse-records) + +(deflanguage-parser-loader records-language + (grammar records-language-grammar) + (parse parse-records)) diff --git a/t/fixtures/rust-rowan-downstream/src/generated/.gitignore b/t/fixtures/runtime-record-assignments/src/generated/.gitignore similarity index 53% rename from t/fixtures/rust-rowan-downstream/src/generated/.gitignore rename to t/fixtures/runtime-record-assignments/src/generated/.gitignore index d8edf592..08b90f16 100644 --- a/t/fixtures/rust-rowan-downstream/src/generated/.gitignore +++ b/t/fixtures/runtime-record-assignments/src/generated/.gitignore @@ -1,3 +1,3 @@ *.rs !.gitignore -!records_v1.rs +!records.rs diff --git a/t/fixtures/rust-rowan-downstream/src/generated/records_v1.rs b/t/fixtures/runtime-record-assignments/src/generated/records.rs similarity index 98% rename from t/fixtures/rust-rowan-downstream/src/generated/records_v1.rs rename to t/fixtures/runtime-record-assignments/src/generated/records.rs index 3f68c2a6..ecc74e68 100644 --- a/t/fixtures/rust-rowan-downstream/src/generated/records_v1.rs +++ b/t/fixtures/runtime-record-assignments/src/generated/records.rs @@ -1,7 +1,7 @@ -// @generated by gerbil-parser/src/compiler/rust-rowan.ss +// @generated by gerbil-parser/src/compiler/rust-runtime.ss // Source authority: canonical Grammar IR v1; do not edit. -use gerbil_parser_rowan::{ +use gerbil_parser_runtime::{ ActionEntry, GotoEntry, KindCategory, KindSpec, LanguageSpec, LexicalExpr, LexicalRule, Operand, OperandAction, ParserAction, Production, Reduction, Symbol, Terminal, TerminalSpec, }; @@ -423,9 +423,9 @@ static PRODUCTIONS: &[Production] = &[ ]; pub static LANGUAGE: LanguageSpec = LanguageSpec { - language: "downstream-records", + language: "record-assignments", version: "v1", - contract: "downstream-records.v1", + contract: "record-assignments.v1", grammar_digest: "sha256:9c7e8b0f807fb1bf13198c417ce5b74b90f65d67f054b48188cf5268dba4d1d4", case_insensitive: false, root_kind: 0, diff --git a/t/fixtures/runtime-record-assignments/src/lib.rs b/t/fixtures/runtime-record-assignments/src/lib.rs new file mode 100644 index 00000000..69cd3075 --- /dev/null +++ b/t/fixtures/runtime-record-assignments/src/lib.rs @@ -0,0 +1,16 @@ +//! Independent consumer of the generated record assignments language module. + +#[path = "generated/records.rs"] +pub mod records; + +/// Parse source with the record assignments grammar. +/// +/// # Errors +/// +/// Returns the generic engine diagnostic when the generated grammar rejects +/// the source. +pub fn parse_records( + source: &str, +) -> Result { + gerbil_parser_runtime::parse(&records::LANGUAGE, source) +} diff --git a/t/fixtures/rust-rowan-downstream/tests/consumer.rs b/t/fixtures/runtime-record-assignments/tests/consumer.rs similarity index 68% rename from t/fixtures/rust-rowan-downstream/tests/consumer.rs rename to t/fixtures/runtime-record-assignments/tests/consumer.rs index a02e5417..18831d81 100644 --- a/t/fixtures/rust-rowan-downstream/tests/consumer.rs +++ b/t/fixtures/runtime-record-assignments/tests/consumer.rs @@ -1,15 +1,15 @@ -use downstream_rowan_language_fixture::parse_records; +use runtime_record_assignments_fixture::parse_records; -const ACCEPTED: &str = include_str!("../languages/records/v1/corpus/accepted.records"); -const REJECTED: &str = include_str!("../languages/records/v1/corpus/rejected.records"); +const ACCEPTED: &str = include_str!("../languages/records/corpus/accepted.records"); +const REJECTED: &str = include_str!("../languages/records/corpus/rejected.records"); #[test] fn custom_generated_language_is_lossless_and_structural() { let parsed = parse_records(ACCEPTED).expect("custom records grammar must accept its corpus"); assert_eq!(parsed.syntax().to_string(), ACCEPTED); - assert_eq!(parsed.receipt().language, "downstream-records"); + assert_eq!(parsed.receipt().language, "record-assignments"); assert_eq!(parsed.receipt().version, "v1"); - assert_eq!(parsed.receipt().contract, "downstream-records.v1"); + assert_eq!(parsed.receipt().contract, "record-assignments.v1"); assert!(parsed.receipt().grammar_digest.starts_with("sha256:")); let kinds = parsed @@ -30,8 +30,8 @@ fn custom_generated_language_is_lossless_and_structural() { #[test] fn custom_generated_language_rejects_incomplete_input_with_typed_evidence() { let error = parse_records(REJECTED).expect_err("incomplete assignment must reject"); - assert_eq!(error.receipt.language, "downstream-records"); - assert_eq!(error.receipt.contract, "downstream-records.v1"); + assert_eq!(error.receipt.language, "record-assignments"); + assert_eq!(error.receipt.contract, "record-assignments.v1"); assert!(!error.diagnostic.reason_kind.is_empty()); assert!(error.diagnostic.byte_offset <= REJECTED.len()); } diff --git a/t/fixtures/rust-language-abi/Cargo.lock b/t/fixtures/rust-language-abi/Cargo.lock new file mode 100644 index 00000000..4eff64de --- /dev/null +++ b/t/fixtures/rust-language-abi/Cargo.lock @@ -0,0 +1,220 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gerbil-parser-artifact" +version = "0.1.0" +dependencies = [ + "serde_json", + "sha2", +] + +[[package]] +name = "gerbil-parser-ffi" +version = "0.1.0" +dependencies = [ + "gerbil-parser-artifact", + "tokio", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rust-language-abi-fixture" +version = "0.0.0" +dependencies = [ + "gerbil-parser-ffi", + "tokio", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "pin-project-lite", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/t/fixtures/rust-language-abi/Cargo.toml b/t/fixtures/rust-language-abi/Cargo.toml new file mode 100644 index 00000000..ac1b1d47 --- /dev/null +++ b/t/fixtures/rust-language-abi/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "rust-language-abi-fixture" +version = "0.0.0" +edition = "2024" +publish = false + +[lib] +crate-type = ["staticlib"] + +[features] +standalone = ["gerbil-parser-ffi/standalone"] +tokio = ["gerbil-parser-ffi/tokio", "gerbil-parser-ffi/benchmark-controls", "dep:tokio"] + +[dependencies] +gerbil-parser-ffi = { path = "../../../rust/gerbil-parser-ffi" } +tokio = { version = "1.53.1", features = ["rt", "sync"], optional = true } + +[workspace] + +# Match the production workspace for complete-request CPU qualification. +[profile.release] +opt-level = 3 +lto = "thin" +overflow-checks = true diff --git a/t/fixtures/rust-language-abi/src/lib.rs b/t/fixtures/rust-language-abi/src/lib.rs new file mode 100644 index 00000000..ec8d4f25 --- /dev/null +++ b/t/fixtures/rust-language-abi/src/lib.rs @@ -0,0 +1,9 @@ +//! Linked into the Scheme native probe, with all checks owned by its test module. +#[path = "../tests/unit/probe.rs"] +mod probe; +#[path = "../tests/unit/source_probe.rs"] +mod source_probe; + +#[cfg(feature = "tokio")] +#[path = "../tests/unit/tokio_cpu.rs"] +mod tokio_cpu; diff --git a/t/fixtures/rust-language-abi/tests/unit/probe.rs b/t/fixtures/rust-language-abi/tests/unit/probe.rs new file mode 100644 index 00000000..f77f7ec8 --- /dev/null +++ b/t/fixtures/rust-language-abi/tests/unit/probe.rs @@ -0,0 +1,237 @@ +//! Exercise the Rust ownership boundary through actual compiled Gerbil callbacks. +use gerbil_parser_ffi::NativeElement; +use gerbil_parser_ffi::{LanguageApi, NativeSession, RawResult}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +unsafe extern "C" { + fn records_language_create() -> u64; + fn gerbil_parser_language_release(handle: u64) -> i32; + fn gerbil_parser_result_release(result: *mut RawResult); +} +static RELEASED: AtomicUsize = AtomicUsize::new(0); +static FREED: AtomicUsize = AtomicUsize::new(0); + +unsafe extern "C" fn release(handle: u64) -> i32 { + // SAFETY: the session supplies its own live handle on the owner thread. + let status = unsafe { gerbil_parser_language_release(handle) }; + if status == 0 { + RELEASED.fetch_add(1, Ordering::Relaxed); + } + status +} +unsafe extern "C" fn free(result: *mut RawResult) { + // SAFETY: the session supplies its initialized, exclusive owned result. + unsafe { gerbil_parser_result_release(result) }; + FREED.fetch_add(1, Ordering::Relaxed); +} +fn api() -> LanguageApi { + let mut api = LanguageApi::linked(records_language_create); + api.release = release; + api.result_release = free; + api +} + +fn check_tokens(view: &gerbil_parser_ffi::NativeArtifactView<'_>, source: &str) { + let mut offset = 0; + for (ordinal, token) in view.tokens().enumerate() { + assert_eq!(token.id(), u64::try_from(ordinal).unwrap()); + assert_eq!(token.range().start, offset); + assert_eq!(token.text(), &source[offset..token.range().end]); + assert_eq!(token.text().as_ptr(), source[offset..].as_ptr()); + assert!( + view.catalog() + .terminals() + .iter() + .any(|t| t.name == token.class()) + ); + assert_eq!(token.parent().is_some(), view.accepted()); + offset = token.range().end; + } + assert_eq!(offset, source.len()); +} + +fn check() { + RELEASED.store(0, Ordering::Relaxed); + FREED.store(0, Ordering::Relaxed); + // Admission on a foreign OS thread must fail before invoking Scheme. + std::thread::spawn(|| { + // SAFETY: the linked API is live; its owner guard is safe on any thread. + assert!(matches!( + unsafe { NativeSession::attach(api()) }, + Err(gerbil_parser_ffi::NativeError::OwnerThread) + )); + }) + .join() + .expect("foreign thread"); + // SAFETY: called synchronously from the initialized Gerbil module's owner; + // Scheme does not shut down or unload modules until this function returns. + let session = unsafe { NativeSession::attach(api()) }.expect("owner admission"); + { + let language = session.language().expect("language"); + let second = session.language().expect("independent handle"); + { + let descriptor = language.descriptor().expect("descriptor"); + assert!( + std::str::from_utf8(descriptor.bytes().unwrap()) + .unwrap() + .contains("grammarDigest") + ); + } + for source in ["", "a=1\n", "α=1\r\n", "a=1\n\0", "a=!"] { + let parsed = language.parse(source).expect("transport"); + let view = parsed.view().expect("bound view"); + // The view uses the original C buffer: no Rust payload copy. + let original = parsed.bytes().unwrap(); + assert!(original.starts_with(b"GPA1")); + check_tokens(&view, source); + let other = second.parse("b=1\n").expect("concurrent owned result"); + assert!(other.view().unwrap().accepted()); + assert_eq!(parsed.bytes().unwrap().as_ptr(), original.as_ptr()); + assert_eq!(view.accepted(), matches!(source, "" | "a=1\n" | "α=1\r\n")); + if let Some(root) = view.root() { + assert_eq!(root.kind(), "Document"); + assert_eq!(root.text(), source); + assert_eq!(root.text().as_ptr(), source.as_ptr()); + for child in root.children() { + if let NativeElement::Node(assignment) = child { + assert_eq!(assignment.kind(), "Assignment"); + assert_eq!(assignment.field("name").count(), 1); + assert_eq!(assignment.field("value").count(), 1); + assert_eq!(assignment.parent().unwrap().id(), root.id()); + } + } + } else { + assert!(!view.accepted()); + } + println!("RUST-NATIVE-SOURCE-OK bytes={}", source.len()); + } + let started = std::time::Instant::now(); + for i in 1..=100 { + let parsed = language.parse("α=1\n").expect("repeat parse"); + assert!(parsed.view().unwrap().accepted()); + if i % 25 == 0 { + println!("RUST-NATIVE-PARSE-OK calls={i}"); + } + } + println!( + "RUST-NATIVE-100-CALLS wall-ms={:.3}", + started.elapsed().as_secs_f64() * 1000.0 + ); + } + assert_eq!(RELEASED.load(Ordering::Relaxed), 2); + assert_eq!(FREED.load(Ordering::Relaxed), 113); + println!("RUST-NATIVE-OWNERSHIP-OK handles=2 results=113"); +} + +/// Called by the compiled Scheme test probe while the VM is alive. +#[unsafe(no_mangle)] +pub extern "C" fn gerbil_parser_rust_native_probe() -> i32 { + match std::panic::catch_unwind(check) { + Ok(()) => 0, + Err(_) => -1, + } +} + +/// Independent C entry: Rust owns setup, handle/result lifetimes and cleanup. +#[cfg(feature = "standalone")] +#[unsafe(no_mangle)] +pub extern "C" fn gerbil_parser_rust_native_host_probe() -> i32 { + let result = std::panic::catch_unwind(|| { + println!("RUST-HOST-SETUP"); + // SAFETY: a fresh independent process links exactly this standalone + // bundle, which stays loaded until process exit. No other VM is live. + let mut runtime = unsafe { + gerbil_parser_ffi::NativeRuntime::start(gerbil_parser_ffi::RuntimeApi::linked( + records_language_create, + )) + } + .expect("runtime setup"); + println!("RUST-HOST-READY"); + { + let language = runtime.language().expect("owned language"); + let parsed = language.parse("α=1\n").expect("owned parse"); + assert!(parsed.view().unwrap().accepted()); + } + check(); + runtime.close().expect("normal SDK cleanup"); + runtime.close().expect("Rust close is idempotent"); + assert!(matches!( + runtime.language(), + Err(gerbil_parser_ffi::NativeError::RuntimeClosed) + )); + // SAFETY: the linked library is still loaded; admission guards reject + // both a new session and VM restart without entering the cleaned VM. + assert!(matches!( + unsafe { NativeSession::attach(api()) }, + Err(gerbil_parser_ffi::NativeError::OwnerThread) + )); + assert!(matches!( + unsafe { + gerbil_parser_ffi::NativeRuntime::start(gerbil_parser_ffi::RuntimeApi::linked( + records_language_create, + )) + }, + Err(gerbil_parser_ffi::NativeError::Runtime(-3)) + )); + println!("RUST-HOST-CLEANUP-OK"); + }); + if result.is_ok() { 0 } else { -1 } +} + +/// Independent process: native owner affinity through the Rust async interface. +#[cfg(feature = "tokio")] +#[unsafe(no_mangle)] +pub extern "C" fn gerbil_parser_rust_tokio_host_probe() -> i32 { + let result = std::panic::catch_unwind(|| { + let executor = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + executor.block_on(async { + println!("RUST-TOKIO-HOST-SETUP"); + // SAFETY: this independent process links one bundle for its lifetime. + let parser = unsafe { + gerbil_parser_ffi::TokioParser::start( + gerbil_parser_ffi::RuntimeApi::linked(records_language_create), + 4, + ) + .await + } + .expect("owner startup"); + let mut requests = Vec::new(); + for i in 0..32 { + let client = parser.clone(); + requests.push(tokio::spawn(async move { + let source = if i % 2 == 0 { "a=1\n" } else { "α=1\n" }.repeat(64); + let parsed = client.parse(source).await.unwrap(); + let view = parsed.view().unwrap(); + assert!(view.accepted()); + assert_eq!(view.root().unwrap().children().count(), 64); + assert_eq!(view.events().len(), 642); + parsed + })); + } + let mut retained = Vec::new(); + for request in requests { + retained.push(request.await.unwrap()); + } + println!("RUST-TOKIO-HOST-PARSE-OK requests=32 records=64"); + let rejected = parser.parse("α=@\n".into()).await.unwrap(); + assert!(!rejected.view().unwrap().accepted()); + parser.shutdown().await.expect("normal owner cleanup"); + assert!(matches!( + parser.parse("a=1\n".into()).await, + Err(gerbil_parser_ffi::TokioParserError::OwnerStopped) + )); + for parsed in retained { + assert!(parsed.view().unwrap().accepted()); + assert_eq!( + parsed.view().unwrap().root().unwrap().children().count(), + 64 + ); + } + assert!(!rejected.view().unwrap().accepted()); + println!("RUST-TOKIO-HOST-OK"); + }); + }); + if result.is_ok() { 0 } else { -1 } +} diff --git a/t/fixtures/rust-language-abi/tests/unit/source_probe.rs b/t/fixtures/rust-language-abi/tests/unit/source_probe.rs new file mode 100644 index 00000000..7946a186 --- /dev/null +++ b/t/fixtures/rust-language-abi/tests/unit/source_probe.rs @@ -0,0 +1,85 @@ +//! Own two transferred Source handles and exercise actual Scheme/C edit calls. +use gerbil_parser_ffi::{LanguageApi, NativeSession}; +use std::cell::Cell; + +unsafe extern "C" { + fn records_language_create() -> u64; + fn gerbil_parser_language_release(handle: u64) -> i32; +} +thread_local! { + static TRANSFERRED: Cell<(u64, u64)> = const { Cell::new((0, 0)) }; +} +unsafe extern "C" fn take_handle() -> u64 { + TRANSFERRED.with(|handles| { + let (first, second) = handles.get(); + handles.set((second, 0)); + first + }) +} +fn check_source() { + let mut api = LanguageApi::linked(records_language_create); + api.create = take_handle; + // SAFETY: the Scheme probe transfers two live handles on the VM owner + // thread. The VM and C callbacks remain live until this function returns. + let session = unsafe { NativeSession::attach(api) }.expect("Source owner"); + let cached = session.language().expect("transferred Source handle"); + let fresh = session.language().expect("independent Source handle"); + let original = "cat < i32 { + TRANSFERRED.with(|handles| handles.set((first, second))); + let result = std::panic::catch_unwind(check_source); + // A panic before acquisition must also release transferred handles. + TRANSFERRED.with(|handles| { + let (first, second) = handles.replace((0, 0)); + for handle in [first, second] { + if handle != 0 { + // SAFETY: these unacquired transferred handles are live and + // exclusive, and this callback runs on the VM owner thread. + unsafe { gerbil_parser_language_release(handle) }; + } + } + }); + if result.is_ok() { 0 } else { -1 } +} diff --git a/t/fixtures/rust-language-abi/tests/unit/tokio_cpu.rs b/t/fixtures/rust-language-abi/tests/unit/tokio_cpu.rs new file mode 100644 index 00000000..c4b28260 --- /dev/null +++ b/t/fixtures/rust-language-abi/tests/unit/tokio_cpu.rs @@ -0,0 +1,217 @@ +//! Matched complete requests: shared Tokio/FFI, independent old engine kernels. +use gerbil_parser_ffi::{RawResult, RuntimeApi, TokioParser}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +unsafe extern "C" { + fn records_language_create() -> u64; + fn records_engine_control_create() -> u64; + fn gerbil_parser_language_parse( + handle: u64, + source: *const u8, + length: usize, + result: *mut RawResult, + ) -> i32; + fn gerbil_parser_language_release(handle: u64) -> i32; + fn gerbil_parser_benchmark_cpu_seconds() -> f64; + fn gerbil_parser_benchmark_prepare_thread() -> i32; +} +static CONTROL: AtomicU64 = AtomicU64::new(0); +static REFERENCE: AtomicBool = AtomicBool::new(false); + +unsafe extern "C" fn create_pair() -> u64 { + // SAFETY: test-only scheduling setup acts on this current OS thread. + if unsafe { gerbil_parser_benchmark_prepare_thread() } != 0 { + return 0; + } + // SAFETY: TokioParser initializes and calls the factory on the unique owner. + let candidate = unsafe { records_language_create() }; + if candidate == 0 { + return 0; + } + // SAFETY: the independent control belongs to this same initialized bundle. + let control = unsafe { records_engine_control_create() }; + if control == 0 { + // SAFETY: release the owned candidate after a failed paired admission. + unsafe { gerbil_parser_language_release(candidate) }; + return 0; + } + CONTROL.store(control, Ordering::Relaxed); + candidate +} +unsafe extern "C" fn parse_pair( + handle: u64, + source: *const u8, + length: usize, + result: *mut RawResult, +) -> i32 { + let selected = if REFERENCE.load(Ordering::Acquire) { + CONTROL.load(Ordering::Relaxed) + } else { + handle + }; + // SAFETY: the admitted owner passes exclusive result storage and live bytes. + unsafe { gerbil_parser_language_parse(selected, source, length, result) } +} +unsafe extern "C" fn release_pair(handle: u64) -> i32 { + // SAFETY: both unique handles have no surviving C result borrowers. + let control = unsafe { gerbil_parser_language_release(CONTROL.swap(0, Ordering::Relaxed)) }; + // SAFETY: this is the original handle created by this paired factory. + let candidate = unsafe { gerbil_parser_language_release(handle) }; + if control == 0 { candidate } else { control } +} +fn cpu() -> f64 { + // SAFETY: the C helper reads process statistics without entering Scheme. + let value = unsafe { gerbil_parser_benchmark_cpu_seconds() }; + assert!(value >= 0.0, "process CPU measurement failed"); + value +} +async fn parity(parser: &TokioParser, source: &str) { + REFERENCE.store(true, Ordering::Release); + let reference = parser.parse(source.into()).await.unwrap(); + REFERENCE.store(false, Ordering::Release); + let candidate = parser.parse(source.into()).await.unwrap(); + assert_eq!(reference.bytes(), candidate.bytes(), "complete GPA1 parity"); + assert_eq!( + reference.view().unwrap().accepted(), + candidate.view().unwrap().accepted() + ); +} +async fn batch( + parser: &TokioParser, + source: &str, + expected: &[u8], + reference: bool, + calls: usize, +) -> f64 { + REFERENCE.store(reference, Ordering::Release); + let before = cpu(); + assert_eq!(calls % 8, 0); + let mut clients = tokio::task::JoinSet::new(); + for _ in 0..8 { + let parser = parser.clone(); + let source = source.to_owned(); + let expected = expected.to_owned(); + clients.spawn(async move { + for _ in 0..calls / 8 { + let parsed = parser.parse(source.clone()).await.unwrap(); + assert_eq!(parsed.bytes(), expected); + let view = if reference { + parsed.reference_view() + } else { + parsed.view() + } + .unwrap(); + assert!(view.accepted()); + assert_eq!(view.events().len(), 642); + assert_eq!(view.root().unwrap().children().count(), 64); + std::hint::black_box(view); + } + }); + } + while let Some(result) = clients.join_next().await { + result.unwrap(); + } + let elapsed = cpu() - before; + assert!(elapsed >= 0.001, "unresolved CPU batch: {elapsed}"); + elapsed +} +fn percentiles(values: &[f64]) -> [f64; 3] { + let mut ordered = values.to_vec(); + ordered.sort_by(f64::total_cmp); + // Twenty groups: exact nearest-rank P10/P50/P90, no discarded groups. + [ordered[1], ordered[9], ordered[17]] +} +async fn phase( + parser: &TokioParser, + source: &str, + expected: &[u8], + name: &str, + right_reference: bool, +) -> [f64; 3] { + let mut ratios = Vec::new(); + for group in 0..20 { + // Same eight-leg, four-batch-per-side protocol as Scheme Benchmark. + let order = if group % 2 == 0 { + [false, true, true, false, true, false, false, true] + } else { + [true, false, false, true, false, true, true, false] + }; + let mut left = 0.0; + let mut right = 0.0; + let mut costs = [0.0; 8]; + for (leg, is_right) in order.into_iter().enumerate() { + let reference = !is_right || right_reference; + let cost = batch(parser, source, expected, reference, 256).await; + if is_right { + right += cost; + } else { + left += cost; + } + costs[leg] = cost; + } + println!( + "RUST-TOKIO-CPU-GROUP phase={name} group={group} cpu={costs:?} ratio={:.9}", + right / left + ); + ratios.push(right / left); + } + percentiles(&ratios) +} +async fn qualify(unicode: bool) -> bool { + let mut api = RuntimeApi::linked(create_pair); + api.language.parse = parse_pair; + api.language.release = release_pair; + // SAFETY: this independent host process owns one permanently loaded bundle. + let parser = unsafe { TokioParser::start(api, 4).await }.unwrap(); + for source in ["", "a=1\n", "α=1\r\n", "name=\"猫\"\n", "a=\n", "?"] { + parity(&parser, source).await; + } + let source = if unicode { "α=1\n" } else { "a=1\n" }.repeat(64); + parity(&parser, &source).await; + let expected = parser.parse(source.clone()).await.unwrap().bytes().to_vec(); + assert_eq!(expected.len(), 15488); + // Identical complete requests, actor, catalog, decoder and row writer. + println!( + "RUST-TOKIO-CPU-METHOD records=64 events=642 payload=15488 profile=release groups-per-phase=20 calls=256 legs=8 phases=3 queue=4 owners=1 clients=8 workload=closed-loop candidate=fused-admission control=former-engine+two-pass-admission cpu=process-user+system gc=included source-bytes={} unicode={unicode}", + source.len() + ); + // Fixed symmetric warmup for the one-shot VM, including normal GC cycles. + for (leg, reference) in [true, false, false, true, false, true, true, false] + .into_iter() + .enumerate() + { + batch(&parser, &source, &expected, reference, 256).await; + println!("RUST-TOKIO-CPU-WARMUP leg={leg} calls=256 reference={reference}"); + } + let before = phase(&parser, &source, &expected, "before", true).await; + let production = phase(&parser, &source, &expected, "production", false).await; + let after = phase(&parser, &source, &expected, "after", true).await; + println!( + "RUST-TOKIO-CPU-SUMMARY unicode={unicode} before={before:?} production={production:?} after={after:?}" + ); + parser.shutdown().await.unwrap(); + let admitted = before[0] >= 0.90 + && before[2] <= 1.10 + && after[0] >= 0.90 + && after[2] <= 1.10 + && production[2] < 0.95; + println!( + "RUST-TOKIO-CPU-{} unicode={unicode}", + if admitted { "PROOF-OK" } else { "REJECTED" } + ); + admitted +} + +/// Independent process entry; reports rejection through both status and marker. +#[unsafe(no_mangle)] +pub extern "C" fn gerbil_parser_rust_tokio_cpu_probe(unicode: i32) -> i32 { + let result = std::panic::catch_unwind(|| { + // SAFETY: test-only scheduling setup acts on this current OS thread. + assert_eq!(unsafe { gerbil_parser_benchmark_prepare_thread() }, 0); + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(qualify(unicode != 0)) + }); + if matches!(result, Ok(true)) { 0 } else { -1 } +} diff --git a/t/fixtures/rust-rowan-downstream/Cargo.toml b/t/fixtures/rust-rowan-downstream/Cargo.toml deleted file mode 100644 index 5b29f836..00000000 --- a/t/fixtures/rust-rowan-downstream/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "downstream-rowan-language-fixture" -version = "0.0.0" -edition = "2024" -publish = false - -[dependencies] -gerbil-parser-rowan = { path = "../../../rust/gerbil-parser-rowan" } - -[workspace] diff --git a/t/fixtures/rust-rowan-downstream/README.md b/t/fixtures/rust-rowan-downstream/README.md deleted file mode 100644 index e76770a0..00000000 --- a/t/fixtures/rust-rowan-downstream/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Downstream Rust/Rowan language-pack fixture - -This fixture owns the same `grammar.ss`, `parser.ss`, `fixtures.ss`, -`parser-test.ss`, and native corpus layout as a real downstream DSL package. -It deliberately has no `gerbil.pkg` or Gerbil installation contract. Upstream -generation CI drives its public -`:gerbil-parser/rust-rowan-grammar-support` declaration through the standalone -`gerbil-parser-rowan-aot` product and verifies the committed Rust module. The -downstream acceptance step is then a standalone Cargo build: it needs only -`gerbil-parser-rowan` and the generated module, never Gerbil, a private engine -module, or a handwritten Rowan sink. diff --git a/t/fixtures/rust-rowan-downstream/generate.ss b/t/fixtures/rust-rowan-downstream/generate.ss deleted file mode 100644 index 8c783543..00000000 --- a/t/fixtures/rust-rowan-downstream/generate.ss +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- -;;; Downstream AOT entry using only the public Rust/Rowan generator facade. - -(import (only-in :gerbil-parser/rust-rowan-support - generate-language-rust-rowan-module) - (only-in "languages/records/v1/grammar.ss" - records-language-grammar)) - -(def arguments (command-line)) -(unless (> (length arguments) 2) - (error "usage: generate.ss OUTPUT.rs")) - -(generate-language-rust-rowan-module - (car (reverse arguments)) - records-language-grammar) diff --git a/t/fixtures/rust-rowan-downstream/languages/records/v1/parser-test.ss b/t/fixtures/rust-rowan-downstream/languages/records/v1/parser-test.ss deleted file mode 100644 index fb924a41..00000000 --- a/t/fixtures/rust-rowan-downstream/languages/records/v1/parser-test.ss +++ /dev/null @@ -1,42 +0,0 @@ -;;; -*- Gerbil -*- -;;; Native conformance for a downstream-owned complete language pack. - -(import (only-in :std/test check test-case test-suite) - (only-in :gerbil-parser/language-support - +language-parser-entry-schema+ - language-parser-entry-ref - parse-artifact-ref - parse-artifact-roundtrip - parse-artifact-success? - parse-artifact-valid? - syntax-fixture-expected-status - syntax-fixture-source - syntax-fixture-source-digest) - (only-in ./fixtures records-v1-fixtures) - (only-in ./parser records-v1-language parse-records-v1)) -(export records-v1-parser-tests) - -(def records-v1-parser-tests - (test-suite "downstream records v1 language pack" - (test-case "the public entry owns immutable language identity" - (check (language-parser-entry-ref records-v1-language 'schema) - => +language-parser-entry-schema+) - (check (language-parser-entry-ref records-v1-language 'language) - => "downstream-records") - (check (language-parser-entry-ref records-v1-language 'version) => "v1") - (check (language-parser-entry-ref records-v1-language 'contract) - => "downstream-records.v1")) - (test-case "accepted and rejected fixtures share one declared corpus" - (for-each - (lambda (fixture) - (let* ((source (syntax-fixture-source fixture)) - (artifact (parse-records-v1 source)) - (accepted? (eq? (syntax-fixture-expected-status fixture) - 'accepted))) - (check (parse-artifact-success? artifact) => accepted?) - (check (parse-artifact-valid? artifact) => #t) - (check (parse-artifact-ref artifact 'sourceDigest) - => (syntax-fixture-source-digest fixture)) - (when accepted? - (check (parse-artifact-roundtrip artifact) => source)))) - records-v1-fixtures)))) diff --git a/t/fixtures/rust-rowan-downstream/languages/records/v1/parser.ss b/t/fixtures/rust-rowan-downstream/languages/records/v1/parser.ss deleted file mode 100644 index 7e636c30..00000000 --- a/t/fixtures/rust-rowan-downstream/languages/records/v1/parser.ss +++ /dev/null @@ -1,10 +0,0 @@ -;;; -*- Gerbil -*- -;;; Downstream public parser entry built only from the installed facade. - -(import (only-in :gerbil-parser/language-support deflanguage-parser) - (only-in ./grammar records-language-grammar)) -(export records-v1-language parse-records-v1) - -(deflanguage-parser records-v1-language - (grammar records-language-grammar) - (parse parse-records-v1)) diff --git a/t/fixtures/rust-rowan-downstream/src/lib.rs b/t/fixtures/rust-rowan-downstream/src/lib.rs deleted file mode 100644 index e84144be..00000000 --- a/t/fixtures/rust-rowan-downstream/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Independent downstream consumer of one custom generated language module. - -#[path = "generated/records_v1.rs"] -pub mod records_v1; - -/// Parse source with the downstream-owned records grammar. -/// -/// # Errors -/// -/// Returns the generic engine diagnostic when the generated grammar rejects -/// the source. -pub fn parse_records( - source: &str, -) -> Result { - gerbil_parser_rowan::parse(&records_v1::LANGUAGE, source) -} diff --git a/t/fixtures/shared-scanner/bash.ss b/t/fixtures/shared-scanner/bash.ss new file mode 100644 index 00000000..04fcb7c3 --- /dev/null +++ b/t/fixtures/shared-scanner/bash.ss @@ -0,0 +1,20 @@ +;;; Production Bash scanner profile and portable conformance source controls. +(import (only-in :gerbil-parser/t/fixtures/bash-products bash-command-scanner) + (only-in :gerbil-parser/src/language/scanner-profile compile-scanner-profile)) +(export bash-scanner-ir bash-scanner-sources) +(def bash-scanner-ir (compile-scanner-profile bash-command-scanner)) +(def bash-scanner-sources +(append + '( + "" "echo α" "# unterminated '\n" "# $(unterminated\n" + "echo \\\nα" "echo <(printf α) >(cat)" "echo \rα" "echo \r\n" "\rfoo" "\x85;α" + "cat <>" "<<<" "&&" "||" "|&" ";;" ";&" ">>" "<>" "<&" ">&" ">|" "&>" ";" "&" "|" "(" ")" "<" ">")))) diff --git a/t/fixtures/shared-scanner/declaration.ss b/t/fixtures/shared-scanner/declaration.ss new file mode 100644 index 00000000..12c754fe --- /dev/null +++ b/t/fixtures/shared-scanner/declaration.ss @@ -0,0 +1,37 @@ +;;; Shared IR fixture: Bash FIFO delimiters and contextual positions. +(import (only-in :gerbil-parser/t/fixtures/bash-products bash-word-regions) + (only-in :gerbil-parser/src/runtime/region-scanner region-plan-specification) + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-method make-contextual-role make-contextual-scan-rule) + (only-in :gerbil-parser/src/compiler/contextual-dispatch compile-contextual-dispatch) + (only-in :gerbil-parser/src/compiler/contextual-scanner-ir compile-contextual-scanner)) +(export shared-scanner-ir shared-scanner-sources) +(def shared-scanner-sources + '("cat <<'A' <<-B\r\n$x α\r\nA\r\n\tβ $x\n\tB\n" + "cat <<\\\\EOF\ntext\n\\EOF\n" + "cat <<\"a\\qb\"\na\\qb\n" + "cat <<''\n\n" + "printf %s \"${x:-$(printf '%s' '}')}\"\n" + "echo α\n\n" + "echo $((1+(2*3))) <(printf 'α') >(cat)\n" + "echo `opaque ${x}`\n" + "echo α$(printf 'β')$((2))\n")) +(def role + (make-contextual-role 'shared + (map (lambda (row) (make-contextual-method (car row) 'any 'any (car row) (cadr row))) + '((open here-open) (word word) (space space) (newline newline) (body body) (end end))))) +(def dispatch (compile-contextual-dispatch (list role) '(command body) '(command argument) '(open word space newline body end))) +(def (rule name mode form matcher rank (action 'keep)) + (make-contextual-scan-rule name mode form matcher rank action)) +(def shared-scanner-ir + (compile-contextual-scanner + (list (rule 'strip 'command 'open '(literal "<<-") 30 '(expect-marker #t)) + (rule 'plain 'command 'open '(literal "<<") 30 '(expect-marker #f)) + (rule 'word 'command 'word + (list 'region-word + (append (take (region-plan-specification bash-word-regions) 3) '(#f))) + 0 '(enqueue-if-expecting shell-quote-removal)) + (rule 'space 'command 'space '(horizontal-whitespace+) 0) + (rule 'newline 'command 'newline '(newline-one) 0 '(activate-next body)) + (rule 'marker 'body 'end '(marker-line) 10 '(finish-marker command body)) + (rule 'body 'body 'body '(body-line) 0)) dispatch 'command)) diff --git a/t/fixtures/shared-scanner/generate-abi.ss b/t/fixtures/shared-scanner/generate-abi.ss new file mode 100644 index 00000000..fadca1c2 --- /dev/null +++ b/t/fixtures/shared-scanner/generate-abi.ss @@ -0,0 +1,7 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :gerbil-parser/src/compiler/language-pack generate-language-pack)) +(def (main source header) + (generate-language-pack source header "gerbil-parser/t/fixtures/shared-scanner/records-abi" + "gerbil-parser/t/fixtures/shared-scanner/records" 'records-language-grammar "records" 'records-contextual-product) + (displayln "LANGUAGE-PACK-GENERATED") (force-output) (test-child-process-exit! 0)) diff --git a/t/fixtures/shared-scanner/generate-artifact.ss b/t/fixtures/shared-scanner/generate-artifact.ss new file mode 100644 index 00000000..9fa3dc77 --- /dev/null +++ b/t/fixtures/shared-scanner/generate-artifact.ss @@ -0,0 +1,62 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; Canonical Scheme producer supplies Rust ABI conformance bytes, not reports. +(import (only-in :gerbil-parser/language-support deflanguage) + (only-in :gerbil-parser/t/abi-datum-support abi-datum-read) + "records" + + (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :gerbil-parser/src/ffi/language-artifact-codec + bind-language-abi abi-parse-binary-payload abi-descriptor-payload)) +;;; Native field scopes include repeated and zero-width associations. +(begin + (deflanguage field-scopes + (syntax + (lexical + (root document) + (lex (item-token Identifier (literals "x")) (unknown Unknown (fallback))))) + (rules (document (node Document (repeat (field item item-token)) + (field marker (node Marker (empty))) (field absent (empty)))))) + (bind-fixture-grammar-release field-scopes "field-scopes" "v1" "field-scopes.v1") ) +(def (emit-bytes port bytes) + (display "&[" port) + (for-each (lambda (byte) (display byte port) (display "," port)) (u8vector->list bytes)) + (display "]" port)) +(def (main output) + (let* ((language (bind-language-abi records-language-grammar)) + (contextual (bind-language-abi records-language-grammar records-contextual-product)) + (scopes (bind-language-abi field-scopes-language-grammar)) + (descriptor (abi-datum-read (abi-descriptor-payload language))) + (sources (list "" "α=1\n" "a = 1\r\nb=\"β\"\n" "name=value" "a=" + (string-append "a=1" (string (integer->char 0)) "!")))) + (call-with-output-file output + (lambda (port) + (display "// @generated by the Scheme GPA1 producer; do not edit.\n" port) + (display "pub const SCOPES_DESCRIPTOR: &[u8] = " port) + (emit-bytes port (string->utf8 (abi-descriptor-payload scopes))) + (display ";\npub static SCOPES_CASES: &[(&[u8], &[u8])] = &[\n" port) + (for-each (lambda (source) + (display "(" port) (emit-bytes port (string->utf8 source)) + (display "," port) (emit-bytes port (abi-parse-binary-payload scopes source)) + (display "),\n" port)) '("" "xx")) + (display "];\n" port) + (display "pub const DESCRIPTOR: &[u8] = " port) + (emit-bytes port (string->utf8 (abi-descriptor-payload language))) + (display ";\npub const CONTEXTUAL_DESCRIPTOR: &[u8] = " port) + (emit-bytes port (string->utf8 (abi-descriptor-payload contextual))) + (display ";\n" port) + (display "pub const FIELD_COUNT: usize = " port) + (display (length (hash-get descriptor "fields")) port) + (display ";\npub static CASES: &[(&[u8], &[u8])] = &[\n" port) + (for-each (lambda (source) + (display "(" port) (emit-bytes port (string->utf8 source)) + (display "," port) (emit-bytes port (abi-parse-binary-payload language source)) + (display "),\n" port)) sources) + (display "];\npub static CONTEXTUAL_CASES: &[(&[u8], &[u8])] = &[\n" port) + (for-each (lambda (source) + (display "(" port) (emit-bytes port (string->utf8 source)) + (display "," port) (emit-bytes port (abi-parse-binary-payload contextual source)) + (display "),\n" port)) sources) + (display "];\n" port)))) + (displayln "NATIVE-ARTIFACT-GENERATED") (force-output) + (test-child-process-exit! 0)) diff --git a/t/fixtures/shared-scanner/generate-bash.ss b/t/fixtures/shared-scanner/generate-bash.ss new file mode 100644 index 00000000..6a4d3cdd --- /dev/null +++ b/t/fixtures/shared-scanner/generate-bash.ss @@ -0,0 +1,35 @@ +#!/usr/bin/env gxi +;;; Rust controls come from execution of the admitted production Scheme profile. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + "bash" + (only-in :gerbil-parser/src/compiler/rust-scanner rust-scanner-module-source) + (only-in :gerbil-parser/src/runtime/contextual-scanner prepare-contextual-scanner contextual-scanner-initial-state contextual-scanner-step) + (only-in :gerbil-parser/src/runtime/token token-kind token-start token-end)) +(def (scan scanner) + (let loop ((state (contextual-scanner-initial-state scanner)) (tokens '())) + (let-values (((token next) (contextual-scanner-step scanner state 'source))) + (if token (loop next (cons token tokens)) (reverse tokens))))) +(def (main output) + (let ((accepted '()) (rejected '())) + (for-each (lambda (source) + (displayln "BASH-SCANNER-CONTROL " (object->string source)) (force-output) + (with-catch (lambda (_) (set! rejected (cons source rejected))) + (lambda () (set! accepted (cons (cons source + (scan (prepare-contextual-scanner bash-scanner-ir source))) accepted))))) + bash-scanner-sources) + (call-with-output-file output (lambda (port) + (display (rust-scanner-module-source bash-scanner-ir) port) + (display "pub static TRACES: &[(&str, &[gerbil_parser_runtime::ScannedToken])] = &[\n" port) + (for-each (lambda (trace) + (display "(" port) (write (car trace) port) (display ", &[" port) + (for-each (lambda (token) + (display "gerbil_parser_runtime::ScannedToken { terminal: " port) + (write (symbol->string (token-kind token)) port) + (display ", start: " port) (display (token-start token) port) + (display ", end: " port) (display (token-end token) port) (display " }," port)) (cdr trace)) + (display "]),\n" port)) (reverse accepted)) + (display "];\npub static REJECTED: &[&str] = &[" port) + (for-each (lambda (source) (write source port) (display "," port)) (reverse rejected)) + (display "];\n" port))) + (displayln "BASH-SCANNER-RUST-GENERATED accepted=" (length accepted) " rejected=" (length rejected)) + (force-output) (test-child-process-exit! 0))) diff --git a/t/fixtures/shared-scanner/generate-contextual.ss b/t/fixtures/shared-scanner/generate-contextual.ss new file mode 100644 index 00000000..d28cd46d --- /dev/null +++ b/t/fixtures/shared-scanner/generate-contextual.ss @@ -0,0 +1,7 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + "records" + (only-in :gerbil-parser/src/compiler/rust-scanner generate-contextual-language-rust-runtime-module)) +(def (main output) + (generate-contextual-language-rust-runtime-module output records-language-grammar records-contextual-product) + (displayln "CONTEXTUAL-AOT-GENERATED") (force-output) (test-child-process-exit! 0)) diff --git a/t/fixtures/shared-scanner/generate.ss b/t/fixtures/shared-scanner/generate.ss new file mode 100644 index 00000000..2f705ce6 --- /dev/null +++ b/t/fixtures/shared-scanner/generate.ss @@ -0,0 +1,34 @@ +#!/usr/bin/env gxi +;;; Compile Scanner IR and Scheme-executed token traces for Rust conformance. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + "declaration" + (only-in :gerbil-parser/src/compiler/rust-scanner rust-scanner-module-source) + (only-in :gerbil-parser/src/runtime/contextual-scanner + prepare-contextual-scanner contextual-scanner-initial-state contextual-scanner-step) + (only-in :gerbil-parser/src/runtime/token token-kind token-start token-end)) +(def (main output) + (let ((generated (rust-scanner-module-source shared-scanner-ir)) + (traces + (map + (lambda (source) + (let (scanner (prepare-contextual-scanner shared-scanner-ir source)) + (let loop ((state (contextual-scanner-initial-state scanner)) (rows '())) + (let-values (((token next) (contextual-scanner-step scanner state 'command))) + (if token + (loop next (cons (list (symbol->string (token-kind token)) (token-start token) (token-end token)) rows)) + (cons source (reverse rows))))))) shared-scanner-sources))) + (call-with-output-file output + (lambda (port) + (display generated port) + (display "pub static TRACES: &[(&str, &[gerbil_parser_runtime::ScannedToken])] = &[\n" port) + (for-each + (lambda (trace) + (display "(" port) (write (car trace) port) (display ", &[" port) + (for-each (lambda (row) + (display "gerbil_parser_runtime::ScannedToken { terminal: " port) + (write (car row) port) (display ", start: " port) (display (cadr row) port) + (display ", end: " port) (display (caddr row) port) (display " }," port)) (cdr trace)) + (display "]),\n" port)) traces) + (display "];\n" port))) + (displayln "SHARED-SCANNER-GENERATED cases=" (length traces)) + (force-output) (test-child-process-exit! 0))) diff --git a/t/fixtures/shared-scanner/records-abi.h b/t/fixtures/shared-scanner/records-abi.h new file mode 100644 index 00000000..4398622e --- /dev/null +++ b/t/fixtures/shared-scanner/records-abi.h @@ -0,0 +1,11 @@ +/* @generated language entry; runtime must already be initialized on its owner thread. */ +#pragma once +#include +#ifdef __cplusplus +extern "C" { +#endif +/* Creates a distinct handle. Zero signals admission failure. Release with the generic API. */ +gerbil_parser_language records_language_create(void); +#ifdef __cplusplus +} +#endif diff --git a/t/fixtures/shared-scanner/records-abi.ss b/t/fixtures/shared-scanner/records-abi.ss new file mode 100644 index 00000000..3e7a92f3 --- /dev/null +++ b/t/fixtures/shared-scanner/records-abi.ss @@ -0,0 +1,11 @@ +;;; @generated language C entry; compile through the normal package builder. +(import :gerbil-parser/src/ffi/language-abi + (only-in :gerbil-parser/src/ffi/language-handles register-language!) + (only-in :gerbil-parser/t/fixtures/shared-scanner/records records-language-grammar records-contextual-product)) +(export create-language-handle) +(def (create-language-handle) (register-language! records-language-grammar records-contextual-product)) +(begin-foreign + (namespace ("gerbil-parser/t/fixtures/shared-scanner/records-abi#" create-callback)) + (c-declare "#include \n___U64 records_language_create_impl(void);\nuint64_t records_language_create(void) { return gerbil_parser_language_is_owner_thread() ? records_language_create_impl() : 0; }") + (c-define (create-callback) () unsigned-int64 "records_language_create_impl" "extern" + (with-exception-catcher (lambda (_) 0) (lambda () (gerbil-parser/t/fixtures/shared-scanner/records-abi#create-language-handle))))) diff --git a/t/fixtures/shared-scanner/records-engine-abi.ss b/t/fixtures/shared-scanner/records-engine-abi.ss new file mode 100644 index 00000000..0c700a8d --- /dev/null +++ b/t/fixtures/shared-scanner/records-engine-abi.ss @@ -0,0 +1,11 @@ +;;; Test-owned paired factory; production language entry remains generated. +(import :gerbil-parser/t/fixtures/shared-scanner/records-abi + (only-in :gerbil-parser/src/ffi/language-handles register-language-context!) + (only-in :gerbil-parser/t/fixtures/shared-scanner/records-engine-control make-records-engine-control)) +(export create-engine-control-handle) +(def (create-engine-control-handle) (register-language-context! (make-records-engine-control))) +(begin-foreign + (namespace ("gerbil-parser/t/fixtures/shared-scanner/records-engine-abi#" control-callback)) + (c-declare "#include \n___U64 records_engine_control_create_impl(void);\nuint64_t records_engine_control_create(void) { return gerbil_parser_language_is_owner_thread() ? records_engine_control_create_impl() : 0; }") + (c-define (control-callback) () unsigned-int64 "records_engine_control_create_impl" "extern" + (with-exception-catcher (lambda (_) 0) (lambda () (gerbil-parser/t/fixtures/shared-scanner/records-engine-abi#create-engine-control-handle))))) diff --git a/t/fixtures/shared-scanner/records-engine-control.ss b/t/fixtures/shared-scanner/records-engine-control.ss new file mode 100644 index 00000000..f48965eb --- /dev/null +++ b/t/fixtures/shared-scanner/records-engine-control.ss @@ -0,0 +1,29 @@ +;;; Test-owned former scanner/LR kernels; no production selection switch. +(import (only-in :gerbil-parser/t/fixtures/shared-scanner/records records-language-grammar records-contextual-product) + (only-in :gerbil-parser/t/fixtures/contextual-result-selection install-reference-contextual-results!) + (only-in :gerbil-parser/t/fixtures/lr-action-selection reference-lr-action-selector) + (only-in :gerbil-parser/src/ffi/language-artifact-codec bind-language-abi) + (only-in :gerbil-parser/src/compiler/machine parser-machine-with-lr-action-selector parser-machine-runtime + parser-machine-grammar-digest install-parser-machine-direct-step!) + (only-in :gerbil-parser/src/runtime/lr-parser lr-runtime-direct-step) + (only-in :gerbil-parser/src/language/descriptor make-language-grammar + +language-grammar-schema+ language-grammar-language language-grammar-version + language-grammar-contract language-grammar-grammar language-grammar-ir + language-grammar-machine language-grammar-observability)) +(export make-records-engine-control) +(def (make-records-engine-control) + (let* ((base (language-grammar-machine records-language-grammar)) + (machine (parser-machine-with-lr-action-selector base reference-lr-action-selector)) + (step (lr-runtime-direct-step (parser-machine-runtime base)))) + (when step + (install-parser-machine-direct-step! machine (parser-machine-grammar-digest base) step)) + (install-reference-contextual-results! + (bind-language-abi + (make-language-grammar +language-grammar-schema+ + (language-grammar-language records-language-grammar) + (language-grammar-version records-language-grammar) + (language-grammar-contract records-language-grammar) + (language-grammar-grammar records-language-grammar) + (language-grammar-ir records-language-grammar) machine + (language-grammar-observability records-language-grammar)) + records-contextual-product)))) diff --git a/t/fixtures/shared-scanner/records.ss b/t/fixtures/shared-scanner/records.ss new file mode 100644 index 00000000..8c31ebb1 --- /dev/null +++ b/t/fixtures/shared-scanner/records.ss @@ -0,0 +1,24 @@ +;;; Independent language pack: normal descriptor + declarative scanner rows. +(import (only-in ../runtime-record-assignments/languages/records/grammar records-language-grammar) + (for-syntax + (only-in ../runtime-record-assignments/languages/records/grammar records-language-grammar records-parser-ir) + (only-in :gerbil-parser/src/compiler/contextual-parser-ir compile-contextual-parser/declaration) + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine) + (only-in :gerbil-parser/src/compiler/machine parser-machine-grammar-digest))) +(export records-language-grammar records-contextual-product) +(defsyntax (records-contextual stx) + (datum->syntax #'records-contextual + (list 'quote (compile-contextual-parser/declaration + records-parser-ir (parser-machine-grammar-digest (language-grammar-machine records-language-grammar)) + '((records ((id any any id identifier) (one any any one number) + (eq any any eq punctuation) (nl any any nl newline) + (ws any any ws whitespace) (str any any str string)))) + '(main) '(name value) '(id one eq nl ws str) + '((id main id (identifier) 0 keep) + (one main one (literal "1") 0 keep) + (eq main eq (literal "=") 0 keep) + (nl main nl (newline-one) 0 keep) + (ws main ws (horizontal-whitespace+) 0 keep) + (str main str (quoted-string ("\"")) 0 keep)) + 'main '((name ((token identifier)) ()) (value () ())))))) +(def records-contextual-product (records-contextual)) diff --git a/t/fixtures/source-strategies.ss b/t/fixtures/source-strategies.ss new file mode 100644 index 00000000..14e4b7eb --- /dev/null +++ b/t/fixtures/source-strategies.ss @@ -0,0 +1,18 @@ +;;; Negative controls register faulty engines at the internal extension boundary. +(import (only-in :clan/poo/object .o) + (only-in :gerbil-parser/src/language/result-profile ResultProfile. compile-result-profile) + (only-in :gerbil-parser/src/language/source-strategy + SourceStrategy. declare-source-strategy-provider make-source-engine) + (only-in :gerbil-parser/src/runtime/source-engines LineSourceStrategy.) + (only-in :gerbil-parser/src/compiler/contextual-program compile-line-contextual-program contextual-program-ir contextual-program-results)) +(export test-source-strategy) +(def (test-source-strategy factory scanner parser) + (let (engine-provider + (declare-source-strategy-provider 'test-engine (lambda (_) #t) + (lambda (_) + (let (program (compile-line-contextual-program + (.o (:: self LineSourceStrategy.) root-kind: 'Test token-kind: 'TestToken))) + (values (contextual-program-ir program) + (make-source-engine scanner factory parser #f + (contextual-program-results program) 'Test program)))))) + (.o (:: self SourceStrategy.) provider: engine-provider))) diff --git a/t/fixtures/tla-sany-differential/AlignedQuantifiers.tla b/t/fixtures/tla-sany-differential/AlignedQuantifiers.tla new file mode 100644 index 00000000..846ffa46 --- /dev/null +++ b/t/fixtures/tla-sany-differential/AlignedQuantifiers.tla @@ -0,0 +1,21 @@ +---- MODULE AlignedQuantifiers ---- +CONSTANT D +VARIABLE v +P == (/\ (\A i \in D : i = i) + /\ (\A j \in D : j = j)) +Q == [][ /\ (\A i \in D : i = i) + /\ (\A j \in D : j = j) ]_<> +R == [x \in D |-> /\ x = x + /\ x = x] +S == {x \in D : /\ x = x + /\ x = x} +T == << /\ (\A i \in D : i = i) + /\ (\A j \in D : j = j) >> +U == << /\ (\A i \in D : i = i) + /\ (\A j \in D : j = j) >>_v +V == (\/ (\E i \in D : i = i) + \/ (\E j \in D : j = j)) +W == /\ (/\ (\A i \in D : i = i) + /\ (\A j \in D : j = j)) + /\ TRUE +==== diff --git a/t/fixtures/tla-sany-differential/ArithmeticPrecedence.tla b/t/fixtures/tla-sany-differential/ArithmeticPrecedence.tla new file mode 100644 index 00000000..4e42dce7 --- /dev/null +++ b/t/fixtures/tla-sany-differential/ArithmeticPrecedence.tla @@ -0,0 +1,15 @@ +---- MODULE ArithmeticPrecedence ---- +EXTENDS Reals +CONSTANTS A, B, C +P == A + B * C +Q == A - B + C +R == A + B - C +S == A /= B +T == A =< B +U == A + B /= C +V == A / (B / C) +W == (A / B) / C +X == A ^ (B ^ C) +Y == (A ^ B) ^ C +Z == A \div B +==== diff --git a/t/fixtures/tla-sany-differential/BranchBoundaryContexts.tla b/t/fixtures/tla-sany-differential/BranchBoundaryContexts.tla new file mode 100644 index 00000000..856e65da --- /dev/null +++ b/t/fixtures/tla-sany-differential/BranchBoundaryContexts.tla @@ -0,0 +1,17 @@ +---- MODULE BranchBoundaryContexts ---- +CONSTANTS D, A, B, Op(_, _) +I == IF /\ A + /\ B THEN A ELSE B +J == IF A THEN /\ A + /\ B ELSE A +K == LET F == /\ A + /\ B IN F +L == [ /\ A + /\ B -> D] +M == [x \in /\ A + /\ B |-> x] +N == \A x \in /\ A + /\ B : x = x +O == Op(/\ A + /\ B, D) +==== diff --git a/t/fixtures/tla-sany-differential/CaseScope.tla b/t/fixtures/tla-sany-differential/CaseScope.tla new file mode 100644 index 00000000..1401c68e --- /dev/null +++ b/t/fixtures/tla-sany-differential/CaseScope.tla @@ -0,0 +1,9 @@ +---- MODULE CaseScope ---- +EXTENDS Naturals +CONSTANTS A, B, C, F +P == CASE A -> B [] OTHER -> F[C] +Q == CASE A -> B [] OTHER -> A + C +R == CASE A -> B [] OTHER -> F.x +S == (CASE A -> B [] OTHER -> F)[C] +T == CASE A -> B [] C -> A +==== diff --git a/t/fixtures/tla-sany-differential/DomainBindings.tla b/t/fixtures/tla-sany-differential/DomainBindings.tla new file mode 100644 index 00000000..9b30ee34 --- /dev/null +++ b/t/fixtures/tla-sany-differential/DomainBindings.tla @@ -0,0 +1,14 @@ +---- MODULE DomainBindings ---- +CONSTANTS S, T +F[x,y \in S] == <> +G[<> \in S] == <> +Q == \A x,y \in S, z \in T: <> = <> +R == [x,y \in S |-> <>] +U == {<> : x \in S, y \in T} +V == {<> \in S : x = y} +W == {x \in S : \E y \in T : <> \in S} +X == \E x \in S : x \in T +Y == \A x \in S : \E y \in T : x = y +Z == {x \in S : <> \in T} +Map == {(x \in S) : x \in T} +==== diff --git a/t/fixtures/tla-sany-differential/FairnessBindings.tla b/t/fixtures/tla-sany-differential/FairnessBindings.tla new file mode 100644 index 00000000..895f0dc0 --- /dev/null +++ b/t/fixtures/tla-sany-differential/FairnessBindings.tla @@ -0,0 +1,7 @@ +---- MODULE FairnessBindings ---- +CONSTANTS A, x, y +P == WF_<>(A) +Q == SF_<>(A) +R == WF_x(A) +S == SF_x(A) +==== diff --git a/t/fixtures/tla-sany-differential/FunctionBindings.tla b/t/fixtures/tla-sany-differential/FunctionBindings.tla new file mode 100644 index 00000000..ee5a363c --- /dev/null +++ b/t/fixtures/tla-sany-differential/FunctionBindings.tla @@ -0,0 +1,7 @@ +---- MODULE FunctionBindings ---- +CONSTANTS S, T +F[x \in S] == x +G[x \in S, y \in T] == <> +L == LET H[x \in S] == x IN H +M == LET H[x \in S] == x K[y \in T] == y IN <> +==== diff --git a/t/fixtures/tla-sany-differential/InvalidActionSubscript.tla b/t/fixtures/tla-sany-differential/InvalidActionSubscript.tla new file mode 100644 index 00000000..f88daa1f --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidActionSubscript.tla @@ -0,0 +1,4 @@ +---- MODULE InvalidActionSubscript ---- +CONSTANTS A, x +P == [A]x +==== diff --git a/t/fixtures/tla-sany-differential/InvalidDivisionChain.tla b/t/fixtures/tla-sany-differential/InvalidDivisionChain.tla new file mode 100644 index 00000000..8613f964 --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidDivisionChain.tla @@ -0,0 +1,5 @@ +---- MODULE InvalidDivisionChain ---- +EXTENDS Reals +CONSTANTS A, B, C +P == A / B / C +==== diff --git a/t/fixtures/tla-sany-differential/InvalidExponentChain.tla b/t/fixtures/tla-sany-differential/InvalidExponentChain.tla new file mode 100644 index 00000000..906c4cdc --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidExponentChain.tla @@ -0,0 +1,5 @@ +---- MODULE InvalidExponentChain ---- +EXTENDS Reals +CONSTANTS A, B, C +P == A ^ B ^ C +==== diff --git a/t/fixtures/tla-sany-differential/InvalidFunctionPairChain.tla b/t/fixtures/tla-sany-differential/InvalidFunctionPairChain.tla new file mode 100644 index 00000000..7053ed4b --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidFunctionPairChain.tla @@ -0,0 +1,5 @@ +---- MODULE InvalidFunctionPairChain ---- +EXTENDS TLC +CONSTANTS A, B, C +P == A :> B :> C +==== diff --git a/t/fixtures/tla-sany-differential/InvalidIntegerDivisionChain.tla b/t/fixtures/tla-sany-differential/InvalidIntegerDivisionChain.tla new file mode 100644 index 00000000..7ae95061 --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidIntegerDivisionChain.tla @@ -0,0 +1,5 @@ +---- MODULE InvalidIntegerDivisionChain ---- +EXTENDS Integers +CONSTANTS A, B, C +P == A \div B \div C +==== diff --git a/t/fixtures/tla-sany-differential/InvalidJunctionIndentation.tla b/t/fixtures/tla-sany-differential/InvalidJunctionIndentation.tla new file mode 100644 index 00000000..9568f0aa --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidJunctionIndentation.tla @@ -0,0 +1,4 @@ +---- MODULE InvalidJunctionIndentation ---- +P == (/\ TRUE +FALSE) +==== diff --git a/t/fixtures/tla-sany-differential/InvalidProofLevel.tla b/t/fixtures/tla-sany-differential/InvalidProofLevel.tla new file mode 100644 index 00000000..076c1526 --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidProofLevel.tla @@ -0,0 +1,10 @@ +---- MODULE InvalidProofLevel ---- +THEOREM TRUE +<1>1. TRUE + <2>1. TRUE + OBVIOUS + <3>2. TRUE + OBVIOUS + <2>. QED OBVIOUS +<1>. QED OBVIOUS +==== diff --git a/t/fixtures/tla-sany-differential/InvalidProofMissingQED.tla b/t/fixtures/tla-sany-differential/InvalidProofMissingQED.tla new file mode 100644 index 00000000..a7c4c717 --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidProofMissingQED.tla @@ -0,0 +1,5 @@ +---- MODULE InvalidProofMissingQED ---- +THEOREM TRUE +<1>1. TRUE + OBVIOUS +==== diff --git a/t/fixtures/tla-sany-differential/InvalidProofOfHave.tla b/t/fixtures/tla-sany-differential/InvalidProofOfHave.tla new file mode 100644 index 00000000..93378254 --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidProofOfHave.tla @@ -0,0 +1,6 @@ +---- MODULE InvalidProofOfHave ---- +THEOREM TRUE +<1>1. HAVE TRUE + <2>. QED OBVIOUS +<1>. QED OBVIOUS +==== diff --git a/t/fixtures/tla-sany-differential/InvalidTemporalBound.tla b/t/fixtures/tla-sany-differential/InvalidTemporalBound.tla new file mode 100644 index 00000000..18b2a3d7 --- /dev/null +++ b/t/fixtures/tla-sany-differential/InvalidTemporalBound.tla @@ -0,0 +1,3 @@ +---- MODULE InvalidTemporalBound ---- +P == \AA x \in S: x = x +==== diff --git a/t/fixtures/tla-sany-differential/MissingImportBoundary.tla b/t/fixtures/tla-sany-differential/MissingImportBoundary.tla new file mode 100644 index 00000000..d497525d --- /dev/null +++ b/t/fixtures/tla-sany-differential/MissingImportBoundary.tla @@ -0,0 +1,4 @@ +---- MODULE MissingImportBoundary ---- +EXTENDS SanyUnavailableProbe +P == TRUE +==== diff --git a/t/fixtures/tla-sany-differential/SemanticErrorBoundary.tla b/t/fixtures/tla-sany-differential/SemanticErrorBoundary.tla new file mode 100644 index 00000000..a8b7a818 --- /dev/null +++ b/t/fixtures/tla-sany-differential/SemanticErrorBoundary.tla @@ -0,0 +1,3 @@ +---- MODULE SemanticErrorBoundary ---- +P == Undeclared +==== diff --git a/t/fixtures/tla-sany-differential/SequenceFunctionOperators.tla b/t/fixtures/tla-sany-differential/SequenceFunctionOperators.tla new file mode 100644 index 00000000..5b05439a --- /dev/null +++ b/t/fixtures/tla-sany-differential/SequenceFunctionOperators.tla @@ -0,0 +1,11 @@ +---- MODULE SequenceFunctionOperators ---- +EXTENDS Sequences, TLC +CONSTANTS A, B, C +P == A \o B \o C +Q == A \circ B +R == A @@ B @@ C +S == A :> B @@ C +T == A @@ B :> C +U == A :> (B @@ C) +V == (A @@ B) :> C +==== diff --git a/t/fixtures/tla-sany-differential/TemporalPrecedence.tla b/t/fixtures/tla-sany-differential/TemporalPrecedence.tla new file mode 100644 index 00000000..79eb24c4 --- /dev/null +++ b/t/fixtures/tla-sany-differential/TemporalPrecedence.tla @@ -0,0 +1,8 @@ +---- MODULE TemporalPrecedence ---- +CONSTANTS A, B, C +P == A ~> B +Q == (A /\ B) ~> C +R == A /\ B ~> C +S == A => B ~> C +T == A ~> (B \/ C) +==== diff --git a/t/fixtures/tla-sany-differential/TemporalTuple.tla b/t/fixtures/tla-sany-differential/TemporalTuple.tla new file mode 100644 index 00000000..6a8b0ab7 --- /dev/null +++ b/t/fixtures/tla-sany-differential/TemporalTuple.tla @@ -0,0 +1,9 @@ +---- MODULE TemporalTuple ---- +CONSTANTS A, x, y +P == [A]_<> +Q == [][A]_<> +R == [A]_x +S == [A]_(<>) +T == <>_<> +U == <><>_x +==== diff --git a/t/fixtures/tla-sany-differential/UnboundQuantifiers.tla b/t/fixtures/tla-sany-differential/UnboundQuantifiers.tla new file mode 100644 index 00000000..cdec865f --- /dev/null +++ b/t/fixtures/tla-sany-differential/UnboundQuantifiers.tla @@ -0,0 +1,6 @@ +---- MODULE UnboundQuantifiers ---- +P == \A x,y: x = y +Q == \E x: x = x +R == \AA x: x = x +S == \EE x: x = x +==== diff --git a/t/fixtures/tla-sany-differential/affinity-exclusive.ss b/t/fixtures/tla-sany-differential/affinity-exclusive.ss new file mode 100644 index 00000000..fbf00b34 --- /dev/null +++ b/t/fixtures/tla-sany-differential/affinity-exclusive.ss @@ -0,0 +1,8 @@ +(import :std/test ./worker-control) +(def test-worker-affinity 'runtime-owner) +(def test-worker-exclusive? #t) +(export test-worker-affinity test-worker-exclusive? affinity-exclusive-test) +(def affinity-exclusive-test + (test-suite "exclusive runtime affinity" + (test-case "resource isolation retains the same persistent runtime thread" + (check (worker-control-runtime-owner!) => #t)))) diff --git a/t/fixtures/tla-sany-differential/affinity-left.ss b/t/fixtures/tla-sany-differential/affinity-left.ss new file mode 100644 index 00000000..97f97bb0 --- /dev/null +++ b/t/fixtures/tla-sany-differential/affinity-left.ss @@ -0,0 +1,7 @@ +(import :std/test ./worker-control) +(def test-worker-affinity 'runtime-owner) +(export test-worker-affinity affinity-test) +(def affinity-test + (test-suite "persistent runtime owner" + (test-case "declared modules execute on the same Worker thread" + (check (worker-control-runtime-owner!) => #t)))) diff --git a/t/fixtures/tla-sany-differential/affinity-right.ss b/t/fixtures/tla-sany-differential/affinity-right.ss new file mode 100644 index 00000000..97f97bb0 --- /dev/null +++ b/t/fixtures/tla-sany-differential/affinity-right.ss @@ -0,0 +1,7 @@ +(import :std/test ./worker-control) +(def test-worker-affinity 'runtime-owner) +(export test-worker-affinity affinity-test) +(def affinity-test + (test-suite "persistent runtime owner" + (test-case "declared modules execute on the same Worker thread" + (check (worker-control-runtime-owner!) => #t)))) diff --git a/t/fixtures/tla-sany-differential/compare.ss b/t/fixtures/tla-sany-differential/compare.ss new file mode 100644 index 00000000..4e7d6491 --- /dev/null +++ b/t/fixtures/tla-sany-differential/compare.ss @@ -0,0 +1,468 @@ +;;; -*- Gerbil -*- +;;; Compare bounded TLA+ expression trees with the pinned SANY XML exporter. + +(import (only-in :std/misc/ports read-all-as-string read-all-as-u8vector) + (only-in :std/misc/process run-process) + (only-in :std/text/markup/xml read-xml) + :gerbil-parser/languages/tla-plus/parser + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-valid? + parse-artifact-roundtrip) + (only-in :gerbil-parser/src/runtime/identity sha256-bytes) + :gerbil-parser/src/runtime/cst + (only-in :gerbil-parser/src/runtime/token token? token-lexeme)) + +(def +jar-digest+ + "sha256:936a262061c914694dfd669a543be24573c45d5aa0ff20a8b96b23d01e050e88") +(def +cases+ + '(("CaseScope" "P" "Q" "R" "S" "T") + ("TemporalPrecedence" "P" "Q" "R" "S" "T") + ("ArithmeticPrecedence" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z") + ("TemporalTuple" "P" "Q" "R" "S" "T" "U") + ("FunctionBindings" "F" "G" "L" "M") + ("FairnessBindings" "P" "Q" "R" "S") + ("DomainBindings" "F" "G" "Q" "R" "U" "V" "W" "X" "Y" "Z" "Map") + ("UnboundQuantifiers" "P" "Q" "R" "S") + ("SequenceFunctionOperators" "P" "Q" "R" "S" "T" "U" "V") + ("AlignedQuantifiers" "P" "Q" "R" "S" "T" "U" "V" "W") + ("BranchBoundaryContexts" "I" "J" "K" "L" "M" "N" "O"))) +(def +negative-cases+ + '("InvalidProofMissingQED" "InvalidProofLevel" "InvalidProofOfHave" + "InvalidActionSubscript" "InvalidTemporalBound" "InvalidDivisionChain" + "InvalidExponentChain" "InvalidFunctionPairChain" "InvalidIntegerDivisionChain" + "InvalidJunctionIndentation")) +(def +oracle-boundaries+ + '(("MissingImportBoundary" . "Cannot find source file for module ") + ("SemanticErrorBoundary" . "Semantic errors:"))) + +(def (one-child node tag) + (let (matches + (filter (lambda (child) + (and (pair? child) (eq? (car child) tag))) + (cdr node))) + (unless (= (length matches) 1) + (error "SANY XML missing unique child" tag (car node))) + (car matches))) + +(def (children node tag) + (filter (lambda (child) + (and (pair? child) (eq? (car child) tag))) + (cdr node))) + +(def (xml-text node) + (let (body (cdr node)) + (unless (and (= (length body) 1) (string? (car body))) + (error "SANY XML element has no single text value" (car node))) + (car body))) + +(def (child-text node tag) + (xml-text (one-child node tag))) + +(def (context-entry entry) + (let* ((uid (child-text entry 'UID)) + (nodes (filter (lambda (child) + (and (pair? child) (not (eq? (car child) 'UID)))) + (cdr entry)))) + (unless (= (length nodes) 1) + (error "SANY XML malformed context entry" uid)) + (cons uid (car nodes)))) + +(def (context-ref context reference) + (let* ((uid (child-text reference 'UID)) + (found (assoc uid context))) + (unless found (error "SANY XML unresolved reference" uid)) + (cdr found))) + +(def (normalized-operator name) + (cond ((equal? name "\\land") "/\\") + ((equal? name "\\lor") "\\/") + ((equal? name "=<") "\\leq") + ((equal? name "\\circ") "\\o") + (else name))) + +(def (sany-expression node context) + (cond ((eq? (car node) 'StringNode) (list 'string (if (pair? (children node 'StringValue)) + (child-text node 'StringValue) (xml-text node)))) + ((eq? (car node) 'LetInNode) + (list "LET" + (map (lambda (reference) + (let* ((definition (context-ref context reference)) + (body (one-child definition 'body))) + (unless (null? (filter pair? (cdr (one-child definition 'params)))) + (error "parameterized LET definition needs explicit projection")) + (list (child-text definition 'uniquename) + (sany-expression (car (filter pair? (cdr body))) context)))) + (children (one-child node 'opDefs) 'UserDefinedOpKindRef)) + (sany-expression + (car (filter pair? (cdr (one-child node 'body)))) context))) + (else (sany-application node context)))) + +(def (sany-application node context) + (unless (eq? (car node) 'OpApplNode) + (error "SANY expression is not an application" (car node))) + (let* ((operator (one-child node 'operator)) + (references (filter pair? (cdr operator))) + (operands (one-child node 'operands))) + (unless (= (length references) 1) + (error "SANY application has no unique operator")) + (let* ((target (context-ref context (car references))) + (name (normalized-operator (child-text target 'uniquename))) + (arguments (filter pair? (cdr operands))) + (trees (map (lambda (child) (sany-expression child context)) arguments)) + (bounds (children node 'boundSymbols))) + (cond + ((pair? bounds) + (cons name + (cons + (sany-bindings (car bounds) context) + trees))) + ((null? arguments) name) + (else (cons name trees)))))) + +(def (sany-bindings node context) + (let ((bounded (children node 'bound)) (unbounded (children node 'unbound))) + (cond + ((and (pair? bounded) (null? unbounded)) + (list 'bindings + (map (lambda (bound) + (let* ((names (children bound 'FormalParamNodeRef)) + (tuple (children bound 'tuple)) + (domains + (filter (lambda (child) + (and (pair? child) + (not (memq (car child) '(FormalParamNodeRef tuple))))) + (cdr bound)))) + (unless (and (pair? names) (= (length domains) 1) + (<= (length tuple) 1)) + (error "unsupported SANY binding shape")) + (list (if (pair? tuple) 'tuple 'names) + (map (lambda (reference) + (child-text (context-ref context reference) 'uniquename)) names) + (sany-expression (car domains) context)))) + bounded))) + ((and (pair? unbounded) (null? bounded)) + (list 'names + (map (lambda (bound) + (child-text (context-ref context (one-child bound 'FormalParamNodeRef)) + 'uniquename)) unbounded))) + (else (error "mixed or empty SANY binding shape"))))) + +(def (sany-shapes xml module-name) + (let* ((document (read-xml xml)) + (module-root (one-child document 'modules)) + (root-name (child-text module-root 'RootModule)) + (context + (map context-entry + (children (one-child module-root 'context) 'entry)))) + (unless (equal? root-name module-name) + (error "SANY XML root module differs" root-name module-name)) + (let* ((modules + (filter (lambda (node) + (and (eq? (car node) 'ModuleNode) + (equal? (child-text node 'uniquename) module-name))) + (map cdr context)))) + (unless (= (length modules) 1) + (error "SANY XML missing unique root module" module-name)) + (let loop ((references (children (car modules) 'UserDefinedOpKindRef)) + (result [])) + (if (null? references) + result + (let* ((definition (context-ref context (car references))) + (location (one-child definition 'location))) + (unless (eq? (car definition) 'UserDefinedOpKind) + (error "SANY module has malformed definition reference")) + (if (equal? (child-text location 'filename) module-name) + (let (name (child-text definition 'uniquename)) + (when (assoc name result) + (error "SANY XML duplicate definition" name)) + (let* ((body (one-child definition 'body)) + (nodes (filter pair? (cdr body)))) + (unless (= (length nodes) 1) + (error "SANY XML malformed definition body" name)) + (loop (cdr references) + (cons (cons name (sany-expression (car nodes) context)) + result)))) + (loop (cdr references) result)))))))) + +(def (child-field node name) + (let loop ((rest (syntax-node-children node))) + (cond ((null? rest) (error "missing syntax field" (syntax-node-kind node) name)) + ((and (syntax-field? (car rest)) + (eq? (syntax-field-name (car rest)) name)) + (car rest)) + (else (loop (cdr rest)))))) + +(def (first-descendant value predicate) + (cond ((predicate value) value) + ((syntax-node? value) + (let loop ((rest (syntax-node-children value))) + (and (pair? rest) + (or (first-descendant (car rest) predicate) + (loop (cdr rest)))))) + ((syntax-field? value) + (let loop ((rest (syntax-field-children value))) + (and (pair? rest) + (or (first-descendant (car rest) predicate) + (loop (cdr rest)))))) + (else #f))) + +(def (field-node node name) + (or (first-descendant (child-field node name) syntax-node?) + (error "syntax field has no node" (syntax-node-kind node) name))) + +(def (field-text node name) + (let (token (first-descendant (child-field node name) token?)) + (if token (token-lexeme token) + (error "syntax field has no token" (syntax-node-kind node) name)))) + +(def (field-nodes node name) + (map (lambda (field) + (or (first-descendant field syntax-node?) + (error "syntax field has no node" (syntax-node-kind node) name))) + (filter (lambda (child) + (and (syntax-field? child) (eq? (syntax-field-name child) name))) + (syntax-node-children node)))) + +(def (field-texts node name) + (map (lambda (field) + (let (token (first-descendant field token?)) + (if token (token-lexeme token) + (error "syntax field has no token" name)))) + (filter (lambda (child) + (and (syntax-field? child) (eq? (syntax-field-name child) name))) + (syntax-node-children node)))) + +(def (candidate-bindings node) + (list 'bindings + (map (lambda (binding) + (list (if (eq? (syntax-node-kind binding) 'TupleBinding) 'tuple 'names) + (field-texts binding 'name) + (candidate-expression (field-node binding 'domain)))) + (field-nodes node 'binding)))) + +(def (candidate-function node) + (list "$NonRecursiveFcnSpec" + (candidate-bindings node) + (candidate-expression (field-node node 'body)))) + +(def (candidate-expression node) + (case (syntax-node-kind node) + ((Expression) + (list (normalized-operator (field-text node 'operator)) + (candidate-expression (field-node node 'left)) + (candidate-expression (field-node node 'right)))) + ((GroupedExpression) + (candidate-expression (field-node node 'expression))) + ((JunctionExpression) + (cons (if (equal? (field-text node 'operator) "/\\") "$ConjList" "$DisjList") + (map candidate-expression (field-nodes node 'body)))) + ((IfExpression) + (list "$IfThenElse" + (candidate-expression (field-node node 'condition)) + (candidate-expression (field-node node 'consequent)) + (candidate-expression (field-node node 'alternative)))) + ((CaseExpression) + (cons "$Case" + (append + (map (lambda (arm) + (list "$Pair" (candidate-expression (field-node arm 'condition)) + (candidate-expression (field-node arm 'result)))) + (field-nodes node 'arm)) + (if (pair? (field-nodes node 'other)) + (list (list "$Pair" (list 'string "$Other") + (candidate-expression (field-node node 'other)))) '())))) + ((FunctionApplication) + (let (arguments (map candidate-expression (field-nodes node 'argument))) + (list "$FcnApply" (candidate-expression (field-node node 'function)) + (if (null? (cdr arguments)) (car arguments) (cons "$Tuple" arguments))))) + ((RecordFieldExpression) + (list "$RcdSelect" (candidate-expression (field-node node 'record)) + (list 'string (field-text node 'field)))) + ((FunctionSetExpression) + (list "$SetOfFcns" + (candidate-expression (field-node node 'domain)) + (candidate-expression (field-node node 'codomain)))) + ((NameExpression) + (field-text node 'name)) + ((QuantifiedExpression) + (let* ((quantifier (field-text node 'quantifier)) + (bounded? (pair? (field-nodes node 'binding))) + (operator + (cond ((equal? quantifier "\\AA") "$TemporalForall") + ((equal? quantifier "\\EE") "$TemporalExists") + ((equal? quantifier "\\A") + (if bounded? "$BoundedForall" "$UnboundedForall")) + (else (if bounded? "$BoundedExists" "$UnboundedExists"))))) + (list operator + (if bounded? (candidate-bindings node) + (list 'names (field-texts node 'name))) + (candidate-expression (field-node node 'predicate))))) + ((FunctionConstructor SetMapExpression SetFilterExpression) + (list (case (syntax-node-kind node) + ((FunctionConstructor) "$FcnConstructor") + ((SetMapExpression) "$SetOfAll") + (else "$SubsetOf")) + (candidate-bindings node) + (candidate-expression + (field-node node (if (eq? (syntax-node-kind node) 'SetFilterExpression) + 'predicate 'body))))) + ((LetExpression) + (list "LET" + (map (lambda (definition) + (list (field-text definition 'name) + (if (eq? (syntax-node-kind definition) 'LocalFunctionDefinition) + (candidate-function definition) + (candidate-expression (field-node definition 'body))))) + (field-nodes node 'definition)) + (candidate-expression (field-node node 'body)))) + ((PrefixExpression) + (list (field-text node 'operator) + (candidate-expression (field-node node 'operand)))) + ((FairnessExpression) + (list (if (equal? (field-text node 'operator) "WF_") "$WF" "$SF") + (candidate-expression (field-node node 'subscript)) + (candidate-expression (field-node node 'action)))) + ((OperatorApplication) + (let* ((name (field-text node 'operator)) + (arguments (map candidate-expression (field-nodes node 'argument)))) + (if (and (> (string-length name) 3) + (member (substring name 0 3) '("WF_" "SF_"))) + (begin + (unless (= (length arguments) 1) + (error "fairness requires one action" name)) + (list (if (equal? (substring name 0 3) "WF_") "$WF" "$SF") + (substring name 3 (string-length name)) (car arguments))) + (cons name arguments)))) + ((TemporalSubscriptExpression AngleActionExpression) + (list (if (eq? (syntax-node-kind node) 'TemporalSubscriptExpression) + "$SquareAct" "$AngleAct") + (candidate-expression (field-node node 'action)) + (candidate-expression (field-node node 'subscript)))) + ((TupleExpression) + (cons "$Tuple" + (map (lambda (field) + (candidate-expression + (or (first-descendant field syntax-node?) + (error "tuple field has no expression")))) + (filter (lambda (child) + (and (syntax-field? child) + (eq? (syntax-field-name child) 'item))) + (syntax-node-children node))))) + (else (error "unsupported differential expression" (syntax-node-kind node))))) + +(def (operator-definitions value) + (cond ((syntax-node? value) + (if (memq (syntax-node-kind value) '(OperatorDefinition FunctionDefinition)) + (list value) + (apply append (map operator-definitions (syntax-node-children value))))) + ((syntax-field? value) + (apply append (map operator-definitions (syntax-field-children value)))) + (else []))) + +(def (candidate-shapes source path) + (let (artifact (parse-tla-plus-sany-candidate source)) + (unless (and (parse-artifact-success? artifact) + (parse-artifact-valid? artifact) + (equal? source (parse-artifact-roundtrip artifact))) + (error "candidate did not parse and round-trip fixture" path)) + (map (lambda (definition) + (cons (field-text definition 'name) + (if (eq? (syntax-node-kind definition) 'FunctionDefinition) + (candidate-function definition) + (candidate-expression (field-node definition 'body))))) + (operator-definitions (parse-artifact->cst artifact))))) + +(def (check-case jar fixture-directory case) + (let* ((module-name (car case)) + (expected (cdr case)) + (path (path-expand (string-append module-name ".tla") fixture-directory)) + (source (call-with-input-file path read-all-as-string)) + (xml (run-process + ["java" "-cp" jar "tla2sany.xml.XMLExporter" "-o" path] + directory: fixture-directory)) + (official (sany-shapes xml module-name)) + (candidate (candidate-shapes source path))) + (unless (and (= (length official) (length expected)) + (= (length candidate) (length expected))) + (error "SANY differential definition count differs" module-name + (length official) (length candidate))) + (for-each + (lambda (name) + (let ((official-entry (assoc name official)) + (candidate-entry (assoc name candidate))) + (unless (and official-entry candidate-entry) + (error "SANY differential missing definition" module-name name)) + (unless (equal? (cdr official-entry) (cdr candidate-entry)) + (error "SANY differential tree differs" module-name name + (cdr official-entry) (cdr candidate-entry))))) + expected) + (displayln "SANY-DIFF-OK " module-name ": " (length expected) + " expression trees") + (force-output))) + +(def (line-starts-with? line prefix) + (and (>= (string-length line) (string-length prefix)) + (equal? prefix (substring line 0 (string-length prefix))))) + +(def (sany-root-syntax-rejected? path module-name status output) + (let (lines (string-split output #\newline)) + (and status (not (= status 0)) + (member (string-append "Fatal errors while parsing TLA+ spec in file " path) lines) + (member (string-append "In module " module-name) lines) + (null? (filter (lambda (line) + (line-starts-with? line "Cannot find source file for module ")) lines))))) + +(def (check-negative-case jar fixture-directory module-name) + (let* ((path (path-expand (string-append module-name ".tla") fixture-directory)) + (source (call-with-input-file path read-all-as-string)) + (exit-status #f) + (output + (run-process + ["java" "-cp" jar "tla2sany.xml.XMLExporter" "-o" path] + directory: fixture-directory + stderr-redirection: #t + check-status: (lambda (status settings) (set! exit-status status))))) + (unless (sany-root-syntax-rejected? path module-name exit-status output) + (error "SANY negative fixture did not fail during syntax parsing" + module-name exit-status output)) + (when (parse-artifact-success? (parse-tla-plus-sany-candidate source)) + (error "candidate accepted SANY-negative syntax" module-name)) + (displayln "SANY-DIFF-OK " module-name ": both reject syntax") + (force-output))) + +(def (check-oracle-boundary jar fixture-directory case) + (let* ((module-name (car case)) + (path (path-expand (string-append module-name ".tla") fixture-directory)) + (source (call-with-input-file path read-all-as-string)) + (status #f) + (output (run-process ["java" "-cp" jar "tla2sany.xml.XMLExporter" "-o" path] + directory: fixture-directory stderr-redirection: #t + check-status: (lambda (value settings) (set! status value)))) + (artifact (parse-tla-plus-sany-candidate source))) + (unless (and status (not (= status 0)) + (pair? (filter (lambda (line) (line-starts-with? line (cdr case))) + (string-split output #\newline))) + (not (sany-root-syntax-rejected? path module-name status output)) + (parse-artifact-success? artifact) + (parse-artifact-valid? artifact) + (equal? source (parse-artifact-roundtrip artifact))) + (error "SANY oracle failure was mistaken for syntax rejection" module-name status output)) + (displayln "SANY-DIFF-OK " module-name ": oracle failure excluded") + (force-output))) + +(def (run-differential!) + (let* ((jar (last (command-line))) + (fixture-directory + (path-expand "t/fixtures/tla-sany-differential" (current-directory))) + (digest (call-with-input-file jar + (lambda (port) + (sha256-bytes (read-all-as-u8vector port)))))) + (unless (equal? digest +jar-digest+) + (error "wrong SANY jar digest" digest)) + (for-each (lambda (case) (check-case jar fixture-directory case)) +cases+) + (for-each (lambda (name) (check-negative-case jar fixture-directory name)) +negative-cases+) + (for-each (lambda (case) (check-oracle-boundary jar fixture-directory case)) +oracle-boundaries+) + (displayln "SANY-DIFF-OK pinned jar, acceptance, and structure") + (force-output))) + +(run-differential!) diff --git a/t/fixtures/tla-sany-differential/corpus.ss b/t/fixtures/tla-sany-differential/corpus.ss new file mode 100644 index 00000000..997d7a91 --- /dev/null +++ b/t/fixtures/tla-sany-differential/corpus.ss @@ -0,0 +1,30 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Scheme owns the immutable corpus, parser, validity, and lossless receipts. +(import (only-in :std/string/misc string-trim) + (only-in :std/misc/process run-process) + (only-in :std/misc/ports read-all-as-string) + :gerbil-parser/languages/tla-plus/parser + :gerbil-parser/src/runtime/artifact) + +(def +examples-pin+ "ceeaa904140e3e03781cb2a79cd6c6d8b8b08e10") + +(def (main root) + (let* ((head (string-trim (run-process ["git" "-C" root "rev-parse" "HEAD"]))) + (status (string-trim (run-process ["git" "-C" root "status" "--porcelain"]))) + (paths (filter (lambda (path) (not (string-empty? path))) + (string-split (run-process ["git" "-C" root "ls-files" "*.tla"]) #\newline)))) + (unless (and (equal? head +examples-pin+) (string-empty? status) (= (length paths) 424)) + (error "corpus must be the clean immutable 424-file Examples checkout" head status (length paths))) + (displayln "CORPUS-PIN-OK " head " files=424") (force-output) + (for-each + (lambda (relative) + (displayln "CORPUS-START " relative) (force-output) + (let* ((source (call-with-input-file (path-expand relative root) read-all-as-string)) + (artifact (parse-tla-plus-sany-candidate source))) + (unless (and (parse-artifact-success? artifact) (parse-artifact-valid? artifact) + (equal? source (parse-artifact-roundtrip artifact))) + (error "corpus syntax or lossless artifact gate failed" relative + (parse-artifact-ref artifact 'diagnostics))) + (displayln "CORPUS-OK " relative) (force-output))) paths) + (displayln "CORPUS-OK accepted=424 rejected=0 invalid=0 roundtrip-failed=0"))) diff --git a/t/fixtures/tla-sany-differential/exit-child-process.ss b/t/fixtures/tla-sany-differential/exit-child-process.ss new file mode 100644 index 00000000..09799e55 --- /dev/null +++ b/t/fixtures/tla-sany-differential/exit-child-process.ss @@ -0,0 +1,14 @@ +;;; -*- Gerbil -*- +;;; Exit only a completed, disposable test child. Harness status and buffered +;;; receipts are published first; native library teardown is not a test phase. +(import :std/ffi) +(export test-child-process-exit!) +(C-ffi-macrology) +(C-include "") +(def-C-lambda _Exit (int) void) +(def (test-child-process-exit! status) + (unless (and (exact-integer? status) (<= 0 status 255)) + (error "invalid native test exit status" status)) + (force-output (current-output-port)) + (force-output (current-error-port)) + (_Exit status)) diff --git a/t/fixtures/tla-sany-differential/preload.ss b/t/fixtures/tla-sany-differential/preload.ss new file mode 100644 index 00000000..310336cb --- /dev/null +++ b/t/fixtures/tla-sany-differential/preload.ss @@ -0,0 +1,159 @@ +;;; -*- Gerbil -*- +;;; Report real compiled-module admission before the test entry imports it. +;;; Dependency boundaries keep cold imports subject to the same silence gate. +(def +compiled-files+ (make-table test: equal?)) + +(def (prefer-compiled-interfaces!) + ;; Called only by the native suite; source benchmark entries retain their + ;; source overlay. FFI C forms require the compiled module interface. + (let (compiled-root (getenv "GERBIL_PATH" #f)) + (when compiled-root + (let (library (path-expand "lib" compiled-root)) + (when (file-exists? library) + (add-load-path! library)))))) + +(def (compiled-file module suffix) + ;; Native products are admitted before the child starts. Cache successful + ;; lookups only, so a later load-path addition can still resolve a miss. + (let (key (cons module suffix)) + (or (table-ref +compiled-files+ key #f) + (let (path (find file-exists? + (map (lambda (root) + (path-expand (string-append module suffix) root)) + (cons (path-expand "lib" (gerbil-home)) (load-path))))) + (when path (table-set! +compiled-files+ key path)) + path)))) + +(def (compiled-dependencies value) + (cond ((and (pair? value) (eq? (car value) 'load-module) + (pair? (cdr value)) (string? (cadr value))) + (list (cadr value))) + ((pair? value) + (append (compiled-dependencies (car value)) + (compiled-dependencies (cdr value)))) + (else '()))) + +;; Runtime wrappers do not list compile-time imports from .ssi interfaces. +;; Use the expander's import parameter to report the real nested admission +;; work. Delegate unchanged and restore the parameter on return or exception. +(def (call-with-compiled-interface-trace thunk) + (let (importer (gx#current-expander-module-import)) + (parameterize + ((gx#current-expander-module-import + (lambda (path reload?) + (displayln "INTERFACE-IMPORT " path) (force-output) + (let (context (importer path reload?)) + (displayln "INTERFACE-IMPORTED " path) (force-output) + context)))) + (thunk)))) + +(def (import-compiled-interface! module) + (call-with-compiled-interface-trace + (lambda () + (gx#import-module (string->symbol (string-append ":" module)) #f #t)))) + +(def (preload-compiled-module! module interfaces? (visiting #f)) + ;; SDK states are authoritative, including loading and failed states. + ;; Inspect wrappers only for a cold module so its dependencies report real + ;; progress before the SDK initializes the entry. Warm entries never scan. + (let (visiting (or visiting (make-table test: equal?))) + (unless (table-ref visiting module #f) + (table-set! visiting module #t) + (unless (or (assoc module (list-modules)) (string-contains module "~")) + (let (source (compiled-file module ".scm")) + (when source + (for-each + (lambda (dependency) (preload-compiled-module! dependency interfaces? visiting)) + (call-with-input-file source + (lambda (port) + (let loop ((dependencies '())) + (let (value (read port)) + (if (eof-object? value) dependencies + (loop (append dependencies (compiled-dependencies value)))))))))))) + (displayln "MODULE-LOAD " module) (force-output) + (load-module module) + (displayln "MODULE-LOADED " module) (force-output) + (when (and interfaces? (not (string-contains module "~")) + (compiled-file module ".ssi")) + (displayln "MODULE-IMPORT " module) (force-output) + (import-compiled-interface! module) + (displayln "MODULE-IMPORTED " module) (force-output))))) + +(def (preload-module module) (preload-compiled-module! module #t)) + +;;; Compiled benchmark entries execute phase-zero procedures only. Loading +;;; source expander interfaces would retain unrelated compiler state in their +;;; runtime heap; dependency initialization still reports actual progress. +(def (preload-runtime-module module) (preload-compiled-module! module #f)) + +;; Admit compiled dependencies declared by tests and their source helpers. +;; Reading import forms does not evaluate a helper or replace gxtest's owner. +(def +preloaded-sources+ (make-table test: equal?)) + +;;; Relative symbol and string imports have the same native ownership as colon +;;; imports. Derive the module identity from the nearest actual package manifest. +(def (source-module-name file) + (let ((file (path-normalize file))) + (let loop ((directory (path-directory file))) + (let (manifest (path-expand "gerbil.pkg" directory)) + (cond + ((file-exists? manifest) + (let* ((declaration (call-with-input-file manifest read)) + (package (member 'package: declaration))) + (and package (pair? (cdr package)) (symbol? (cadr package)) + (string-append (symbol->string (cadr package)) "/" + (path-strip-extension + (substring file (string-length directory) (string-length file))))))) + ((equal? directory "/") #f) + (else (loop (path-directory + (substring directory 0 (- (string-length directory) 1)))))))))) + +(def (preload-relative-import value source) + (let (file (path-normalize + (path-expand + (if (string-suffix? ".ss" value) value (string-append value ".ss")) + (path-directory source)))) + (when (file-exists? file) + (let (module (source-module-name file)) + (if (and module (compiled-file module ".ssi")) + (preload-module module) + (preload-test-imports file)))))) + +(def (preload-import-set value source) + (cond + ((symbol? value) + (let (name (symbol->string value)) + (cond + ((string-prefix? ":" name) + (let (module (substring name 1 (string-length name))) + ;; A runtime wrapper without its interface is an incomplete build + ;; product. Let Gerbil import source instead of admitting that wrapper. + (when (compiled-file module ".ssi") + (preload-module module)))) + ((or (string-prefix? "./" name) (string-prefix? "../" name)) + (preload-relative-import name source))))) + ((string? value) + (preload-relative-import value source)) + ((pair? value) + (for-each (lambda (entry) (preload-import-set entry source)) value)))) + +(def (preload-test-imports file) + (let (source (path-expand file)) + (unless (table-ref +preloaded-sources+ source #f) + ;; The in-progress marker bounds cyclic source imports. Failed admission + ;; removes it so a fixed file can be inspected on the next attempt. + (table-set! +preloaded-sources+ source #t) + (with-catch + (lambda (exception) + (table-set! +preloaded-sources+ source #f) + (raise exception)) + (lambda () + (call-with-input-file source + (lambda (port) + (let loop () + (let (form (read port)) + (unless (eof-object? form) + (when (and (pair? form) (eq? (car form) 'import)) + (for-each (lambda (entry) (preload-import-set entry source)) + (cdr form))) + (loop))))))))))) diff --git a/t/fixtures/tla-sany-differential/test-suite.ss b/t/fixtures/tla-sany-differential/test-suite.ss new file mode 100644 index 00000000..34375b63 --- /dev/null +++ b/t/fixtures/tla-sany-differential/test-suite.ss @@ -0,0 +1,320 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; One runtime shares imported dependencies; worker-local harnesses own test state. +(import :gerbil/expander + (only-in :std/sync/barrier Barrier? make-barrier barrier-post! barrier-wait!) + (only-in :std/test/base TestSuite? TestModule TestHarness TestConfig test-run! test-result-ok?)) +(load "t/fixtures/tla-sany-differential/preload.ss") +(load "scripts/test-plan.ss") +(prefer-compiled-interfaces!) +;; Source-first callers still need the prepared native FFI module before the +;; public import. The interpreter cannot evaluate its C declarations. +(preload-module "gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process") +(def native-test-exit! + (eval '(begin + (import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process + test-child-process-exit!)) + test-child-process-exit!))) +(import (only-in :clan/poo/object .o) + (only-in :core/observability/testing-case + poo-flow-default-testing-case-profile + poo-flow-current-testing-case-profile)) + +;; The Case sampler measures the shared process heap, including every loaded +;; Suite. Align its absolute cap with the full runner budget; retain its growth, +;; sampling, failure, and duration policies through POO inheritance. +(def testing-case-profile + (.o (:: @ poo-flow-default-testing-case-profile) + (heap-limit-bytes (* 3 1024 1024 1024)))) + +(def (import-test-owner file) + (let (importer (current-expander-module-import)) + (parameterize + ((current-expander-module-import + (lambda (path reload?) + (displayln "INTERFACE-IMPORT " path) (force-output) + (let (context (importer path reload?)) + (displayln "INTERFACE-IMPORTED " path) (force-output) + context)))) + (import-module file #f #t)))) + +(def (test-files directory) + (if (not (eq? (file-type directory) 'directory)) + (list directory) + (apply append + (map (lambda (name) + (if (string-prefix? "." name) '() + (let (path (path-expand name directory)) + (cond ((eq? (file-type path) 'directory) (test-files path)) + ((string-suffix? "-test.ss" name) (list path)) + (else '()))))) + (list-sort string= (- now started) budget) + (emit "MODULE-BATCH-TIMEOUT " file " reason=total-budget") + (native-test-exit! 70)) + ((>= (- now last-progress) 5) + (emit "MODULE-BATCH-TIMEOUT " file " reason=idle-timeout") + (native-test-exit! 70)) + (else (watch)))) + result)))))))) + (try + (let (result + (parameterize ((current-output-port output) (current-error-port output)) + (try + (with-catch + (lambda (e) (display-exception e) (cons 'error e)) + (lambda () (cons 'value (thunk)))) + (finally (close-output-port output))))) + (let (cases (thread-join! monitor)) + (if (eq? (car result) 'error) (raise (cdr result)) + (values (cdr result) cases)))) + (finally (close-input-port input)))))) + +(def (run-test-owner owner emit) + (let (file (test-owner-file owner)) + (emit "MODULE-BATCH-START " file) + (let-values (((ok? cases) + (call-with-owner-budget file emit + (lambda () + (parameterize ((poo-flow-current-testing-case-profile + testing-case-profile)) + (test-result-ok? (test-run! (test-owner-harness owner))))) + (- 90 (test-owner-preparation-seconds owner))))) + (unless ok? (error "Scheme test owner failed" file)) + (when (zero? cases) (error "Scheme test module ran no cases" file)) + (emit "MODULE-BATCH-OK " file " cases=" cases)))) + +(def (main . arguments) + ;; CI reaches this mode only after the compiled conformance gate succeeds. + ;; Direct source qualification remains complete by default. + (let* ((workers? (and (pair? arguments) (equal? (car arguments) "--workers"))) + (capacity + (let (value (if workers? + (if (pair? (cdr arguments)) (string->number (cadr arguments)) #f) + (##core-count))) + (unless (and (integer? value) (exact? value) (> value 0)) + (error "--workers requires a positive integer")) + value)) + (arguments (if (and workers? (pair? (cdr arguments))) (cddr arguments) arguments)) + (remaining? (and (pair? arguments) (equal? (car arguments) "--remaining-conformance"))) + (roots (if remaining? (cdr arguments) arguments)) + (all-files (apply append (map test-files roots))) + (covered-files (if remaining? (conformance-covered-test-files) '())) + (covered (map (lambda (file) (path-normalize (path-expand file))) covered-files)) + (source-rows (if remaining? (conformance-source-suites) '())) + (partial (map (lambda (row) (path-normalize (path-expand (string-append (car row) ".ss")))) source-rows)) + (files (filter (lambda (file) + (let (path (path-normalize (path-expand file))) + (or (not (member path covered)) (member path partial)))) all-files)) + (lock (make-mutex)) (pending '()) (failures '()) + (seen-suites (make-table test: eq?)) (logger #f)) + (when remaining? + (unless (andmap (lambda (file) (member file (map (lambda (path) (path-normalize (path-expand path))) all-files))) covered) + (error "remaining test roots omit conformance owners")) + (displayln "TEST-PLAN compiled-owners=" (length covered) " source-owners=" (length files) " source-only-suites=" (length (apply append (map cdr source-rows)))) + (force-output)) + (unless (= (length all-files) (length (unique-test-files all-files))) + (error "Scheme test suite has duplicate module owners")) + (when (null? files) (error "Scheme test suite has no modules" roots)) + (def (covered-suite-names file) + (let (path (path-normalize (path-expand file))) + (if (and remaining? (member path partial)) + (let (owner (find (lambda (owner) (equal? path (path-normalize (path-expand owner)))) covered-files)) + (conformance-imported-test-names (path-strip-extension owner))) + '()))) + (def (locked thunk) + (mutex-lock! lock) + (try (thunk) (finally (mutex-unlock! lock)))) + (def (emit . values) + ;; Readers must ingest actual owner progress independently of global + ;; console throughput. Fatal watchdog receipts flush synchronously before + ;; native exit; ordinary messages preserve mailbox order. + (if (and logger (not (member (car values) + '("MODULE-BATCH-TIMEOUT " "MODULE-BATCH-READER-FAILED ")))) + (thread-send logger values) + (locked (lambda () (apply displayln values) (force-output))))) + (def (flush-logger!) + (when logger + (let (barrier (make-barrier 1)) + (thread-send logger barrier) + (barrier-wait! barrier)))) + (def (stop-logger!) + (when logger + (let (active logger) + (set! logger #f) + (thread-send active (void)) + (thread-join! active)))) + (def (next-owner index) + (locked + (lambda () + (let loop ((remaining pending) (skipped '())) + (cond + ((null? remaining) #f) + ((or (zero? index) (not (test-owner-affinity (car remaining)))) + (set! pending (append (reverse skipped) (cdr remaining))) + (car remaining)) + (else (loop (cdr remaining) (cons (car remaining) skipped)))))))) + ;; Declared runtime-affine owners share Worker zero. Other modules remain + ;; work-stealing jobs; no filename or language-specific scheduler rules. + (def (worker index) + (let loop () + (let (owner (next-owner index)) + (when owner + ;; Stop admitting new jobs after failure and join every started + ;; Worker. Never lose an exception in thread-join!. + (with-catch + (lambda (e) + (locked (lambda () (set! failures (cons (cons (test-owner-file owner) e) failures)) (set! pending '()))) + (emit "MODULE-BATCH-FAILED " (test-owner-file owner))) + (lambda () (run-test-owner owner emit))) + (loop))))) + ;; One console owner flushes bounded batches of real messages. No timer + ;; writes progress and no per-line acknowledgement blocks the pipe reader. + (set! logger + (spawn + (lambda () + (let loop ((message (thread-receive))) + (let (stop? + (locked + (lambda () + (let (stop? + (let drain ((message message) (remaining 128)) + (cond ((void? message) #t) + ((not message) #f) + ((Barrier? message) + (force-output) + (barrier-post! message) + #f) + (else + (apply displayln message) + (if (= remaining 1) #f + (drain (thread-receive 0 #f) (- remaining 1))))))) + (force-output) stop?)))) + (unless stop? (loop (thread-receive)))))))) + (try + ;; Dynamic module admission completes before any test Worker runs. The + ;; runtime loader/expander is a shared preparation owner, not a worker job. + (set! pending + (map (lambda (file) + (emit "MODULE-PREPARE-START " file) + (let (started (##current-time-point)) + (let-values (((owner _) + (call-with-owner-budget file emit + (lambda () (prepare-test-owner file (covered-suite-names file) seen-suites))))) + (test-owner-preparation-seconds-set! owner (- (##current-time-point) started)) + (emit "MODULE-PREPARE-OK " file) + owner))) files)) + (emit "TEST-SUITE-WORKERS " (min capacity (length files))) + ;; CPU/wall admission cannot share an execution phase with other owners. + ;; This is declared metadata, not a filename-specific scheduling rule. + (let (exclusive (filter (lambda (owner) (test-owner-exclusive? owner)) pending)) + (set! pending (filter (lambda (owner) (not (test-owner-exclusive? owner))) pending)) + ;; The caller is persistent Worker zero across both phases. Affine ABI + ;; owners must not migrate from a spawned Worker to the caller later. + (let (workers (map (lambda (index) (spawn (lambda () (worker index)))) + (iota (max 0 (- (min capacity (length pending)) 1)) 1))) + (worker 0) + (for-each thread-join! workers)) + (when (and (null? failures) (pair? exclusive)) + ;; Official Barrier acknowledges a real console drain, once per + ;; phase. It emits no heartbeat and adds no per-line round trip. + (flush-logger!) + (emit "TEST-SUITE-EXCLUSIVE modules=" (length exclusive)) + (set! pending exclusive) + (worker 0))) + (stop-logger!) + (unless (null? failures) (raise (cdar failures))) + (emit "TEST-SUITE-OK modules=" (length files)) + (emit "OK") + (finally (stop-logger!))) + ;; All workers joined and every owner passed. The test child must release + ;; its process now; SDK finalizers are outside the harness contract. + (native-test-exit! 0))) diff --git a/t/fixtures/tla-sany-differential/worker-control.ss b/t/fixtures/tla-sany-differential/worker-control.ss new file mode 100644 index 00000000..29ad3519 --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-control.ss @@ -0,0 +1,22 @@ +;;; A shared barrier proves two Workers execute in the same Scheme runtime. +(import (only-in :std/sync/barrier make-barrier barrier-post! barrier-wait!)) +(export worker-control-loaded! worker-control-loaded-count worker-control-rendezvous!) +(def ready (make-barrier 2)) +(def loaded 0) +(def (worker-control-loaded!) (set! loaded (+ loaded 1))) +(def (worker-control-loaded-count) loaded) +(def (worker-control-rendezvous!) (barrier-post! ready) (barrier-wait! ready)) + +(def runtime-thread #f) +(def (worker-control-runtime-owner!) + (unless runtime-thread (set! runtime-thread (current-thread))) + (eq? runtime-thread (current-thread))) +(export worker-control-runtime-owner!) + +(def finished-left? #f) +(def finished-right? #f) +(def (worker-control-finished! side) + (case side ((left) (set! finished-left? #t)) ((right) (set! finished-right? #t)) + (else (error "unknown Worker control side" side)))) +(def (worker-control-finished?) (and finished-left? finished-right?)) +(export worker-control-finished! worker-control-finished?) diff --git a/t/fixtures/tla-sany-differential/worker-exclusive-invalid.ss b/t/fixtures/tla-sany-differential/worker-exclusive-invalid.ss new file mode 100644 index 00000000..d03d0044 --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-exclusive-invalid.ss @@ -0,0 +1,6 @@ +(import :std/test) +(def test-worker-exclusive? 'invalid) +(export test-worker-exclusive? worker-exclusive-invalid-test) +(def worker-exclusive-invalid-test + (test-suite "invalid exclusivity declaration" + (test-case "invalid metadata cannot reach execution" (check #t => #t)))) diff --git a/t/fixtures/tla-sany-differential/worker-exclusive.ss b/t/fixtures/tla-sany-differential/worker-exclusive.ss new file mode 100644 index 00000000..a5a3901d --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-exclusive.ss @@ -0,0 +1,8 @@ +(import :std/test ./worker-control) +(def test-worker-exclusive? #t) +(export test-worker-exclusive? worker-exclusive-test) +(def worker-exclusive-test + (test-suite "exclusive execution admission" + (test-case "exclusive owner waits for every parallel owner to finish" + (check (worker-control-loaded-count) => 2) + (check (worker-control-finished?) => #t)))) diff --git a/t/fixtures/tla-sany-differential/worker-failure.ss b/t/fixtures/tla-sany-differential/worker-failure.ss new file mode 100644 index 00000000..07e4a242 --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-failure.ss @@ -0,0 +1,5 @@ +(import :std/test) +(export worker-failure-test) +(def worker-failure-test + (test-suite "failure propagation" + (test-case "intentional failing assertion" (check #f => #t)))) diff --git a/t/fixtures/tla-sany-differential/worker-left.ss b/t/fixtures/tla-sany-differential/worker-left.ss new file mode 100644 index 00000000..25ef3978 --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-left.ss @@ -0,0 +1,10 @@ +(import :std/test ./worker-control) +(export worker-left-test) +(worker-control-loaded!) +(def worker-left-test + (test-suite "worker left" + (test-case "all modules load before shared-runtime Workers rendezvous" + (check (worker-control-loaded-count) => 2) + (worker-control-rendezvous!) + (check (worker-control-loaded-count) => 2) + (worker-control-finished! 'left)))) diff --git a/t/fixtures/tla-sany-differential/worker-reexport.ss b/t/fixtures/tla-sany-differential/worker-reexport.ss new file mode 100644 index 00000000..b48396b5 --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-reexport.ss @@ -0,0 +1,2 @@ +(import ./worker-left) +(export worker-left-test) diff --git a/t/fixtures/tla-sany-differential/worker-right.ss b/t/fixtures/tla-sany-differential/worker-right.ss new file mode 100644 index 00000000..71acc865 --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-right.ss @@ -0,0 +1,10 @@ +(import :std/test ./worker-control) +(export worker-right-test) +(worker-control-loaded!) +(def worker-right-test + (test-suite "worker right" + (test-case "all modules load before shared-runtime Workers rendezvous" + (check (worker-control-loaded-count) => 2) + (worker-control-rendezvous!) + (check (worker-control-loaded-count) => 2) + (worker-control-finished! 'right)))) diff --git a/t/fixtures/tla-sany-differential/worker-suite-alias.ss b/t/fixtures/tla-sany-differential/worker-suite-alias.ss new file mode 100644 index 00000000..f53a1a07 --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-suite-alias.ss @@ -0,0 +1,5 @@ +(import :std/test) +(def alias-left-test + (test-suite "alias control" (test-case "should never execute" (error "duplicate Suite executed")))) +(def alias-right-test alias-left-test) +(export alias-left-test alias-right-test) diff --git a/t/fixtures/tla-sany-differential/worker-timeout.ss b/t/fixtures/tla-sany-differential/worker-timeout.ss new file mode 100644 index 00000000..b64c23d9 --- /dev/null +++ b/t/fixtures/tla-sany-differential/worker-timeout.ss @@ -0,0 +1,5 @@ +(import :std/test) +(export worker-timeout-test) +(def worker-timeout-test + (test-suite "silence watchdog" + (test-case "intentional real silence" (thread-sleep! 6)))) diff --git a/t/fold-buffer-consumer-test.ss b/t/fold-buffer-consumer-test.ss new file mode 100644 index 00000000..8ecf062c --- /dev/null +++ b/t/fold-buffer-consumer-test.ss @@ -0,0 +1,67 @@ +(import :std/test (only-in :std/vector/u8vector little u8vector-uint-set!) + ../src/runtime/source-lines ../src/runtime/fold-tape) +(export fold-buffer-consumer-test) +(def fold-buffer-consumer-test + (test-suite "Continuous source scan and fixed event records" + (test-case "all record alignments preserve the full unsigned wire domain" + (for-each + (lambda (alignment) + (let* ((header (make-u8vector alignment 77)) + (owner (make-fold-tape header)) + (reference (make-u8vector (+ alignment 23) 0))) + (u8vector-copy! reference 0 header) + (u8vector-set! reference alignment 1) + (u8vector-uint-set! reference (+ alignment 1) 65535 little 2) + (u8vector-set! reference (+ alignment 3) 2) + (u8vector-uint-set! reference (+ alignment 4) 65535 little 2) + (u8vector-uint-set! reference (+ alignment 6) 18446744073709551615 little 8) + (u8vector-uint-set! reference (+ alignment 14) 9223372036854775808 little 8) + (fold-tape-start! owner 65535) + (fold-tape-token! owner 65535 18446744073709551615 9223372036854775808) + (fold-tape-finish! owner) + (check (fold-tape-publish! owner) => reference))) + (iota 16))) + (test-case "line offsets equal the character reference for mixed endings" + (for-each + (lambda (source) + (let ((reference (fold-source-lines source '() + (lambda (_ from until rest) (cons (cons from until) rest)))) + (actual (fold-source-byte-spans (string->utf8 source) '() + (lambda (from until rest) (cons (cons from until) rest))))) + (check actual => reference))) + '("" "plain" "\r" "\n" "\r\n" "a\r\nb\rc\n" "é\r\nλ\rtail" "😀\n\r\nend"))) + (test-case "the scanner cannot precompute past a visitor mutation" + (let* ((bytes (string->utf8 "a\nbc")) + (spans (fold-source-byte-spans bytes '() + (lambda (from until rest) + (when (zero? from) (u8vector-set! bytes 2 10)) + (cons (cons from until) rest))))) + (check (reverse spans) => '((0 . 2) (2 . 3) (3 . 4))))) + (test-case "visitor failure still stops traversal immediately" + (let (calls 0) + (check-exception + (fold-source-byte-spans (string->utf8 "a\nb\n") #f + (lambda (_from _until _state) (set! calls (+ calls 1)) (error "stop"))) true) + (check calls => 1))) + (test-case "publication transfers an exact isolated buffer including zero size" + (let* ((header (u8vector 11 22 33)) (owner (make-fold-tape header)) + (published (fold-tape-publish! owner))) + (u8vector-set! header 0 99) + (check published => (u8vector 11 22 33)) + (check-exception (fold-tape-finish! owner) true) + (check-exception (fold-tape-publish! owner) true)) + (let (owner (make-fold-tape (u8vector))) + (check (fold-tape-publish! owner) => (u8vector)) + (check-exception (fold-tape-start! owner 0) true)) + ;; Transfer after repeated growth, then collect while the sealed owner + ;; remains live. Publication must preserve exact bytes and stop writes. + (let (owner (make-fold-tape (u8vector 77))) + (for-each (lambda (index) (fold-tape-token! owner 9 index (+ index 1))) + (iota 100)) + (let* ((published (fold-tape-publish! owner)) + (snapshot (u8vector-copy published))) + (##gc) + (check (u8vector-length published) => 1901) + (check published => snapshot) + (check-exception (fold-tape-token! owner 9 0 1) true) + (check-exception (fold-tape-publish! owner) true)))))) diff --git a/t/fold-byte-dispatch-test.ss b/t/fold-byte-dispatch-test.ss new file mode 100644 index 00000000..b0396623 --- /dev/null +++ b/t/fold-byte-dispatch-test.ss @@ -0,0 +1,136 @@ +(import :std/test ../src/compiler/event-fold-byte-dispatch) +(export fold-byte-dispatch-test) +(def fold-byte-dispatch-test + (test-suite "Prepared byte dispatch effects" + (test-case "distinct bytes select only relevant ordered state branches" + (let (plan (fold-line-negative-region + '((if (and (state active) (line-starts-with "#+")) + ((set-bool active (bool #f))) ((token Line start end))) + (if (line-has-word-after-prefix? "Clock:") + ((finish-node)) ())))) + (check (vector-ref plan 0) => '((start 67 108 111 99 107 58) (start 35 43))) + (check (vector-ref plan 1) + => '((if (and (%line-prefix-hit) (and (state active) (line-starts-with "#+"))) + ((set-bool active (bool #f))) ((token Line start end))) + (if (and (%line-prefix-hit) (line-has-word-after-prefix? "Clock:")) + ((finish-node)) ())))) + (for-each + (lambda (predicate) + (check (fold-line-negative-region + (list `(if ,predicate ((token Line start end)) ()))) => #f)) + '((and (future-heading-title? "*" " " (uint 1) "END") (line-starts-with "#")) + (line-starts-with ""))) + (check (fold-line-negative-region + '((with-source-bounds start end + ((if (line-starts-with "#") ((token Line start end)) ()))))) => #f) + (let (plan (fold-line-negative-region + '((if (and (state active) + (line-byte-equal? (line-skip-horizontal start) 35)) + ((token Line start end)) ())))) + (check (vector-ref plan 0) => '((horizontal 35)))) + (check (fold-line-negative-region + '((if (and (future-heading-title? "*" " " (uint 1) "END") + (line-byte-equal? (line-skip-horizontal start) 35)) + ((token Line start end)) ()))) => #f) + (let ((plan (fold-byte-dispatch + '(if (and (state ready) (line-byte-equal? (line-index cursor) 97)) + ((set-bool ready (bool #f))) + ((if (line-byte-equal? (line-index cursor) 98) + ((set-bool ready (bool #t))) + ((if (line-byte-equal? (line-index cursor) 97) + ((token Line start end)) ()))))) 'cursor))) + (check (map car (vector-ref plan 0)) => '(97 98)) + (check (cdr (assv 98 (vector-ref plan 0))) => '((set-bool ready (bool #t)))) + (check (cdr (assv 97 (vector-ref plan 0))) + => '((if (and (state ready)) ((set-bool ready (bool #f))) ((token Line start end))))) + (check (vector-ref plan 1) => '()))) + (test-case "effects and non-loop positions cannot select byte regions" + (let* ((forms '((if (and (state ready) (line-byte-equal? (line-index cursor) 97)) + ((set-bool ready (bool #f))) ((token Line start end))) + (if (line-byte-equal? (line-index cursor) 98) + ((set-bool ready (bool #t))) ()))) + (regions (fold-byte-regions forms 'cursor))) + (check (map car (vector-ref regions 0)) => '((97) (98))) + (check (vector-ref regions 1) => '((token Line start end))) + (check (cdr (cadr (vector-ref regions 0))) + => '((token Line start end) + (if (line-byte-equal? (line-index cursor) 98) + ((set-bool ready (bool #t))) ()))) + (for-each + (lambda (blocked) + (check (fold-byte-regions + (list `(if ,blocked ((token Line start end)) ())) 'cursor) => #f)) + '((and (future-heading-title? "*" " " (uint 1) "END") + (line-byte-equal? (line-index cursor) 97)) + (line-byte-equal? start 97))) + ;; Repeated byte classes share one ordered body, not duplicate machines. + (let (plan (fold-byte-regions + '((if (line-bytes-any-in? (line-index cursor) + (line-step (line-index cursor)) (97 98)) + ((token Line start end)) ())) 'cursor)) + (check (map car (vector-ref plan 0)) => '((97 98))))) + (let ((nested + '((join-once handled + ((if (state ready) + ((if (line-byte-equal? (line-index cursor) 97) + ((set-bool handled (bool #t))) ())) ())) + ((with-source-bounds start end + ((if (line-byte-equal? (line-index cursor) 98) + ((token Line start end)) ())))))))) + (let (plan (fold-byte-regions nested 'cursor)) + (check (map car (vector-ref plan 0)) => '((97))) + (check (vector-ref plan 1) + => '((join-once handled ((if (state ready) () ())) + ((with-source-bounds start end + ((if (line-byte-equal? (line-index cursor) 98) + ((token Line start end)) ()))))))))) + (let (choice '(or (and (line-byte-equal? (line-index cursor) 97) + (line-byte-equal? (line-step (line-index cursor)) 98)) + (and (line-byte-equal? (line-index cursor) 99) + (line-byte-equal? (line-step (line-index cursor)) 100)))) + (check (fold-necessary-bytes choice 'cursor) => '(97 99)) + (check (fold-byte-prefilter `(and (state active) ,choice) 'cursor) + => `(and (line-bytes-any-in? (line-index cursor) + (line-step (line-index cursor)) (97 99)) + (state active) ,choice))) + (check (fold-necessary-bytes + '(or (line-byte-equal? (line-index cursor) 97) (bool #t)) 'cursor) => #f) + (check (fold-necessary-bytes + '(or (and (future-heading-title? "*" " " (uint 1) "END") + (line-byte-equal? (line-index cursor) 97)) + (line-byte-equal? (line-index cursor) 98)) 'cursor) => #f) + (check (fold-byte-prefilter + '(and (state first) (and (state second) (and (state third) + (line-byte-equal? (line-index cursor) 97))) (state last)) 'cursor) + => '(and (line-byte-equal? (line-index cursor) 97) + (state first) (state second) (state third) (state last))) + (check (fold-byte-prefilter + '(and (state active) + (and (and (line-byte-equal? (line-index cursor) 104) + (line-byte-equal? (line-step (line-index cursor)) 116)) + (line-byte-equal? (line-step (line-step (line-index cursor))) 116))) + 'cursor) + => '(and (line-byte-equal? (line-index cursor) 104) (state active) + (line-byte-equal? (line-step (line-index cursor)) 116) + (line-byte-equal? (line-step (line-step (line-index cursor))) 116))) + (check (fold-byte-prefilter + '(and (state active) + (and (future-heading-title? "*" " " (uint 1) "END") + (line-byte-equal? (line-index cursor) 104))) 'cursor) => #f) + (check (fold-byte-prefilter + '(and (state active) (line-bytes-any-in? (line-index cursor) + (line-step (line-index cursor)) (36 92)) (state ready)) 'cursor) + => '(and (line-bytes-any-in? (line-index cursor) + (line-step (line-index cursor)) (36 92)) (state active) (state ready))) + (check (fold-byte-prefilter + '(and (future-heading-title? "*" " " (uint 1) "END") + (line-byte-equal? (line-index cursor) 97)) 'cursor) => #f) + (check (fold-byte-prefilter + '(and (state active) (line-bytes-any-in? start end (36))) 'cursor) => #f) + (check (fold-byte-dispatch + '(if (and (future-heading-title? "*" " " (uint 1) "END") + (line-byte-equal? (line-index cursor) 97)) () + ((if (line-byte-equal? (line-index cursor) 98) () ()))) 'cursor) => #f) + (check (fold-byte-dispatch + '(if (line-byte-equal? start 97) () ((if (line-byte-equal? start 98) () ()))) + 'cursor) => #f)))) diff --git a/t/fold-call-environment-test.ss b/t/fold-call-environment-test.ss new file mode 100644 index 00000000..d43f89c0 --- /dev/null +++ b/t/fold-call-environment-test.ss @@ -0,0 +1,37 @@ +(import :std/test ../src/compiler/event-fold-scheme-context) +(export fold-call-environment-test) + +(def (shared-call code bindings indices statements?) + (parameterize ((current-fold-scheme-expressions (make-table test: equal?)) + (current-fold-scheme-definitions '()) + (current-fold-scheme-counter 0) + (current-fold-scheme-tape-kinds #f)) + (let (call (if statements? + (car (fold-scheme-share-statements (list code) bindings indices)) + (fold-scheme-share-expression code bindings indices))) + (list call (car (current-fold-scheme-definitions)))))) + +(def (large-code expression) `(begin ,@(make-list 32 expression))) +(def fold-call-environment-test + (test-suite "Dependency-derived private Fold calling convention" + (test-case "state-only helpers omit source bounds and owner" + (let* ((result (shared-call (large-code '(vector-ref slots 0)) '() '() #f)) + (call (car result)) (definition (cadr result))) + (check (cdr call) => '(slots)) + (check (cdadr definition) => '(slots)))) + (test-case "quoted names are not environment dependencies" + (let (result (shared-call (large-code ''(source-bytes start end slots events)) '() '() #f)) + (check (cdr (car result)) => '()) + (check (cdadr (cadr result)) => '()))) + (test-case "source and lexical cursor retain their positional binding" + (let* ((result (shared-call (large-code '(u8vector-ref source-bytes cursor)) + '() '((index . cursor)) #f)) + (call (car result)) (definition (cadr result))) + (check (cdr call) => '(source-bytes cursor)) + (check (cdadr definition) => '(source-bytes %fold-free-0)))) + (test-case "list owner publication remains in the lexical caller" + (let* ((result (shared-call (large-code '(vector-set! slots 0 1)) '() '() #t)) + (call (car result)) (definition (cadr result))) + (check (take call 2) => '(set! events)) + (check (cdr (caddr call)) => '(slots events)) + (check (cdadr definition) => '(slots events)))))) diff --git a/t/fold-call-plan-test.ss b/t/fold-call-plan-test.ss new file mode 100644 index 00000000..4e85902f --- /dev/null +++ b/t/fold-call-plan-test.ss @@ -0,0 +1,52 @@ +(import :std/test ../src/compiler/event-fold-call-plan + ../src/compiler/event-fold-scheme-context) +(export fold-call-plan-test) +(def (fuse forms (budget 256)) + (parameterize ((current-fold-scheme-counter 0) (current-fold-scheme-prefix 'test)) + (fold-fuse-single-use-calls forms budget))) +(def (execute forms) + (eval `(let () ,@(map (lambda (form) (cons 'define (cdr form))) forms) (entry)))) +(def fold-call-plan-test + (test-suite "Whole generated call-edge fusion" + (test-case "single edge preserves nested and caller shadowing" + (let* ((forms '((def (%event-fold-test-expression-0 value) + (let (value (+ value 1)) (let* ((next value) (value (+ next 2))) value))) + (def (entry) (let ((value 100)) (%event-fold-test-expression-0 7))))) + (result (fuse forms))) + (check (length result) => 1) + (check (execute result) => (execute forms)))) + (test-case "shared consumers are not duplicated" + (let (forms '((def (%event-fold-test-expression-0 x) (+ x 1)) + (def (entry) (+ (%event-fold-test-expression-0 1) + (%event-fold-test-expression-0 2))))) + (check (fuse forms) => forms))) + (test-case "effectful arguments retain their boundary and order" + (let (forms '((def (%event-fold-test-expression-0 x) x) + (def (entry) (%event-fold-test-expression-0 (vector 1))))) + (check (fuse forms) => forms))) + (test-case "unselected recognition branches do not execute errors" + (let* ((forms '((def (%event-fold-test-expression-0 x) (error "rejected" x)) + (def (entry) (if #f (%event-fold-test-expression-0 0) 7)))) + (result (fuse forms))) + (check (execute result) => 7) + (check (execute result) => (execute forms)))) + (test-case "expanded budget rejection retains reachable child definitions" + (let* ((forms '((def (%event-fold-test-expression-0 x) (+ x x x x x x x x)) + (def (%event-fold-test-expression-1 x) + (+ (%event-fold-test-expression-0 x) x x x x x x x)) + (def (entry) (%event-fold-test-expression-1 2)))) + (result (fuse forms 30))) + (check (execute result) => (execute forms)))) + (test-case "named-loop lexical owner and tape mutation survive fusion" + (let* ((forms '((def (%event-fold-test-expression-0 owner count) + (let next ((left count)) + (if (= left 0) owner + (begin (vector-set! owner 0 (+ (vector-ref owner 0) 1)) + (next (- left 1)))))) + (def (entry) + (let ((next 999) (owner (vector 0))) + (%event-fold-test-expression-0 owner 3) + (vector-ref owner 0))))) + (result (fuse forms))) + (check (execute result) => 3) + (check (execute result) => (execute forms)))))) diff --git a/t/fold-condition-test.ss b/t/fold-condition-test.ss new file mode 100644 index 00000000..f2cfed69 --- /dev/null +++ b/t/fold-condition-test.ss @@ -0,0 +1,18 @@ +(import :std/test ../src/compiler/event-fold-scheme-expressions) +(export fold-condition-test) +(def bindings '((left . left) (right . right))) +(def fold-condition-test + (test-suite "Conditional truth versus Boolean publication" + (test-case "conditional nesting does not normalize values" + (check (fold-scheme-condition '(and (state left) (or (state right) (bool #f))) bindings '()) + => '(and left (or right #f)))) + (test-case "published Boolean state still normalizes" + (check (fold-scheme-predicate '(and (state left) (state right)) bindings '()) + => '(if (and left right) #t #f))) + (test-case "a condition cannot leak its lowering context" + (fold-scheme-condition '(or (state left) (state right)) bindings '()) + (check (fold-scheme-predicate '(or (state left) (state right)) bindings '()) + => '(if (or left right) #t #f))) + (test-case "short circuit source order remains explicit" + (check (fold-scheme-condition '(and (bool #f) (line-starts-with "x")) bindings '()) + => '(and #f (fold-byte-prefix? source-bytes start end '#u8(120) #f)))))) diff --git a/t/fold-consumer-state-test.ss b/t/fold-consumer-state-test.ss new file mode 100644 index 00000000..a71df457 --- /dev/null +++ b/t/fold-consumer-state-test.ss @@ -0,0 +1,29 @@ +(import :std/test ../src/compiler/event-fold-consumer-state) +(export fold-consumer-state-test) +(def a '(uint-positive? (state a))) +(def b '(uint-positive? (state b))) +(def hit '(token Hit start end)) +(def fold-consumer-state-test + (test-suite "Consumer-local selector dependency support" + (test-case "positive and Boolean selectors refine their own consumer" + (check (fold-consumer-selector-body `((if ,a ((if ,a (,hit) ())) ()))) + => `((if ,a (,hit) ()))) + (check (fold-consumer-selector-body '((if (state enabled) () ((if (state enabled) ((token Wrong start end)) ()))))) + => '((if (state enabled) () ())))) + (test-case "unrelated consumer writes preserve eligibility" + (check (fold-consumer-selector-body `((if ,a ((set-uint b (uint 1)) (if ,a (,hit) ())) ()))) + => `((if ,a ((set-uint b (uint 1)) ,hit) ())))) + (test-case "dependent writes and barriers reject stale reuse" + (for-each + (lambda (mutation) + (let (forms `((if ,a (,mutation (if ,a (,hit) ())) ()))) + (check (fold-consumer-selector-body forms) => forms))) + '((set-uint a (uint 0)) (opaque-call) (call-source-helper helper start end)))) + (test-case "composite selectors retain event order and independent support" + (check (fold-consumer-selector-body `((if (and ,a ,b) ((token First start end) (if ,b (,hit) ())) ()))) + => `((if (and ,a ,b) ((token First start end) ,hit) ())))) + (test-case "branch joins and iterations do not inherit unproven facts" + (let (forms `((if ,a ((set-uint b (uint 0))) ()) (if ,a (,hit) ()))) + (check (fold-consumer-selector-body forms) => forms)) + (let (forms `((if ,a ((for-line-bytes cursor start end ((if ,a (,hit) ())))) ()))) + (check (fold-consumer-selector-body forms) => forms))))) diff --git a/t/fold-cost-test.ss b/t/fold-cost-test.ss new file mode 100644 index 00000000..2e7f96d2 --- /dev/null +++ b/t/fold-cost-test.ss @@ -0,0 +1,26 @@ +(import :std/test ../src/runtime/fold-cost) +(export fold-cost-test) +(def fold-cost-test + (test-suite "Request-scoped Fold diagnostics" + (test-case "disabled result and exception semantics" + (fold-cost-count! 'disabled) + (check (call-with-fold-cost 'disabled (lambda () 42)) => 42) + (check-exception (call-with-fold-cost 'disabled (lambda () (error "expected"))) true)) + (test-case "nested counts timing and exception restoration" + (let (session (make-fold-cost-session)) + (parameterize ((current-fold-cost-session session)) + (fold-cost-count! 'count-only) + (fold-cost-count! 'count-only) + (fold-cost-count! 'count-only 3) + (check-exception (fold-cost-count! 'count-only -1) true) + (check (call-with-fold-cost 'parent + (lambda () (call-with-fold-cost 'child (lambda () 7)))) => 7) + (check-exception (call-with-fold-cost 'failed (lambda () (error "expected"))) true) + (check (call-with-fold-cost 'after (lambda () 9)) => 9)) + (let (rows (fold-cost-rows session)) + (check (length rows) => 5) + (check (cdr (assq 'count-only rows)) => '(5 0 0 0)) + (check (list-ref (assq 'failed rows) 4) => 1) + (for-each (lambda (row) + (check (cadr row) => (if (eq? (car row) 'count-only) 5 1)) + (check (>= (caddr row) (cadddr row) 0) => #t)) rows)))))) diff --git a/t/fold-frame-facts-test.ss b/t/fold-frame-facts-test.ss new file mode 100644 index 00000000..41c29cb6 --- /dev/null +++ b/t/fold-frame-facts-test.ss @@ -0,0 +1,53 @@ +;;; Concrete lowering controls; actual Org AOT remains a separate gate. +(import :std/test ../src/compiler/event-fold-scheme-statements + ../src/compiler/event-fold-scheme-context) +(export fold-frame-facts-test) +(def scan '(line-skip-horizontal start)) +(def bindings '((at . at) (ready . ready))) +(def (scan-loops value) + (cond ((not (pair? value)) 0) + ((and (eq? (car value) 'let) (pair? (cdr value)) + (eq? (cadr value) 'loop)) 1) + ((eq? (car value) 'quote) 0) + (else (+ (scan-loops (car value)) (scan-loops (cdr value)))))) +(def (contains? value target) + (or (equal? value target) + (and (pair? value) + (or (contains? (car value) target) (contains? (cdr value) target))))) +(def (lower forms (indices '())) + (parameterize ((current-fold-scheme-counter 0)) + (fold-scheme-frame-statements forms bindings indices '()))) +(def fold-frame-facts-test + (test-suite "Immutable frame facts and ordered transitions" + (test-case "multiple offsets share one actual scan" + (check (scan-loops (lower `((set-uint at (offset ,scan)) + (token Text ,scan (line-step ,scan))))) => 1) + (parameterize ((current-fold-scheme-tape-kinds '((Text 0)))) + (let (code (lower '((set-uint at (offset (line-scan-until start "]"))) + (set-bool ready (bool #t)) + (token Text start (line-scan-until start "]"))))) + (check (scan-loops code) => 1) + (check (contains? code '(set! ready #t)) => #t)))) + (test-case "single and state-dependent scans are not hoisted" + (check (scan-loops (lower `((token Text ,scan end)))) => 1) + (check (scan-loops (lower '((set-uint at (offset (line-skip-horizontal (state-offset at)))) + (token Text (line-skip-horizontal (state-offset at)) end)))) => 2)) + (test-case "nested frames cannot inherit an outer fact" + (check (scan-loops + (lower `((with-source-bounds start end + ((set-uint at (offset ,scan)) (token Text ,scan end)))) + '((%frame-horizontal . forbidden-outer-value)))) => 1)) + (test-case "state writes and event order remain in the frame" + (let (code (lower `((set-uint at (offset ,scan)) + (set-bool ready (bool #t)) (start-node Text) + (token Text ,scan end) (finish-node)))) + (check (contains? code '(set! ready #t)) => #t) + (check (contains? code '(set! events (cons '(start Text) events))) => #t) + (check (contains? code '(set! events (cons '(finish) events))) => #t))) + (test-case "conditional queries retain short circuit evaluation" + (let (code (lower `((if (and (state ready) + (future-heading-title? "*" " " 1 "END")) + ((token Text ,scan end)) ()) + (token Text ,scan end)))) + (check (scan-loops code) => 1) + (check (contains? code 'ready) => #t))))) diff --git a/t/fold-inert-test.ss b/t/fold-inert-test.ss new file mode 100644 index 00000000..701645ea --- /dev/null +++ b/t/fold-inert-test.ss @@ -0,0 +1,35 @@ +(import :std/test (only-in ../src/compiler/event-fold-analysis fold-eliminate-inert-branches)) +(export fold-inert-test) + +(def fold-inert-test + (test-suite "Effect-admitted empty Fold branches" + (test-case "nested pure state observations disappear" + (check (fold-eliminate-inert-branches + '((if (uint-positive? (state mode)) + ((if (offset-less? start (state-offset cursor)) () ())) ()))) + => '())) + (test-case "range errors queries and unknown effects remain" + (for-each + (lambda (predicate) + (let (forms (list (list 'if predicate '() '()))) + (check (fold-eliminate-inert-branches forms) => forms))) + '((line-bytes-any-in? start end (32)) + (future-heading-title? "x") (unknown-predicate)))) + (test-case "writes events and frame validation retain order" + (check (fold-eliminate-inert-branches + '((if (bool #t) ((set-uint mode (uint 1))) ()) + (with-source-bounds start end ((if (bool #f) () ()))) + (token Text start end))) + => '((if (bool #t) ((set-uint mode (uint 1))) ()) + (with-source-bounds start end ()) + (token Text start end)))) + (test-case "only validated ordered cursor ranges admit discarded scans" + (let* ((forward '(if (line-bytes-any-in? (line-index byte) + (line-step (line-index byte)) (32)) () ())) + (backward '(if (line-bytes-any-in? (line-step (line-index byte)) + (line-index byte) (32)) () ())) + (frame `(with-source-bounds start end (,forward)))) + (check (fold-eliminate-inert-branches (list forward) 'byte) => '()) + (check (fold-eliminate-inert-branches (list forward)) => (list forward)) + (check (fold-eliminate-inert-branches (list backward) 'byte) => (list backward)) + (check (fold-eliminate-inert-branches (list frame) 'byte) => (list frame)))))) diff --git a/t/fold-run-test.ss b/t/fold-run-test.ss new file mode 100644 index 00000000..75e31717 --- /dev/null +++ b/t/fold-run-test.ss @@ -0,0 +1,137 @@ +(import :std/test ../src/compiler/event-fold-ordinary-run + (only-in ../src/compiler/event-fold-scheme-runs fold-scheme-run-loop) + (only-in ../src/compiler/event-fold-analysis fold-zero-support-preserved?) + (only-in ../src/compiler/event-fold-zero-state fold-zero-state-plan)) +(export fold-run-test) + +(def special '(if (line-byte-equal? (line-index cursor) 58) ((token Text start end)) ())) +(def overwrite '(set-bool seen-space + (line-bytes-any-in? (line-index cursor) (line-step (line-index cursor)) (32)))) +(def counter '(if (line-bytes-any-in? (line-index cursor) + (line-step (line-index cursor)) (65 66)) + ((set-uint count (uint-add (state count) (uint 1)))) + ((set-uint count (uint 0))))) +(def fold-run-test + (test-suite "Overwrite-only ordinary runs" + (test-case "ordered consumer invariant admits only zero preserving writes" + (check (fold-zero-support-preserved? + '((if (uint-positive? (state mode)) ((set-uint pending (uint 0))) ())) + '(pending)) => #t) + (check (fold-zero-support-preserved? '((call-source-helper unknown)) '(pending)) => #f) + (check (fold-zero-support-preserved? '((token Text start end)) '(pending)) => #f)) + (test-case "nonzero transient support writes cannot admit a local run" + (check (fold-zero-support-preserved? + '((set-uint pending (uint 1)) (set-uint pending (uint 0))) '(pending)) => #f) + (check (fold-consumer-run-plan + (list special '(set-uint pending (uint 1)) + '(if (uint-positive? (state pending)) ((token Text start end)) ()) + '(set-uint pending (uint 0))) 'cursor '(pending)) => #f)) + (test-case "interval selection retains the narrow inactive alphabet" + (let (plan (fold-ordinary-run-plan + '((if (uint-positive? (state active-mode)) + ((if (line-byte-equal? (line-index cursor) 65) ((token Text start end)) ())) ()) + (if (line-byte-equal? (line-index cursor) 58) ((set-uint active-mode (uint 1))) ())) + 'cursor)) + (check (vector-ref plan 0) => '()) + (check (vector-ref (vector-ref plan 5) 0) => '(active-mode)) + (check (vector-ref (vector-ref plan 5) 1) => '(58)) + (check (and (memv 65 (vector-ref plan 1)) #t) => #t))) + (test-case "only a dependent consumer keeps its interval premise" + (let* ((body '((if (uint-positive? (state blocked-mode)) ((token Text start end)) ()) + (if (uint-positive? (state active-mode)) ((set-uint active-mark (uint 7))) ()) + (if (line-byte-equal? (line-index cursor) 58) + ((set-uint blocked-mode (uint 1)) (set-uint active-mode (uint 1))) ()))) + (plan (fold-ordinary-run-plan body 'cursor))) + (check (vector-ref plan 0) => '(blocked-mode)) + (check (vector-ref plan 3) + => '((if (uint-positive? (state active-mode)) ((set-uint active-mark (uint 7))) ()))))) + (test-case "independent active consumers do not need an all-idle guard" + (let* ((body (list special + `(if (uint-positive? (state first-mode)) (,overwrite) ()) + '(if (uint-positive? (state second-mode)) + ((set-uint second-mark (uint 7))) ()))) + (plan (fold-consumer-run-plan body 'cursor))) + (check (vector-ref plan 0) => '()) + (check (vector-ref plan 3) => (cdr body)))) + (test-case "consumer dependencies and changing selectors reject interval execution" + (for-each (lambda (body) (check (fold-consumer-run-plan body 'cursor) => #f)) + (list (list special '(set-uint mode (uint 1)) + '(if (uint-positive? (state mode)) ((set-bool flag (bool #t))) ())) + (list special '(if (line-byte-equal? (line-index cursor) 32) + ((set-bool flag (bool #t))) + ((set-bool flag (bool #f)))) + '(if (state flag) ((set-uint mark (uint 1))) ())) + (list special '(if (future-heading-title? "x") + ((set-bool flag (bool #t))) ()))))) + (test-case "counter class no longer forces exceptional state execution" + (let (plan (fold-ordinary-run-plan (list special counter overwrite) 'cursor)) + (check (vector-ref plan 1) => '(58)) + (check (vector-ref plan 4) => '((count 65 66))) + (check (vector-ref plan 3) => (list overwrite))) + ;; The span summary must retain reset/carry behavior, including counters + ;; beyond the C integer domain; no counter value crosses the leaf ABI. + (let* ((plan (vector '() '(58) '(bool #t) '() + '((count 65 66) (second-count 66 67)))) + (code (fold-scheme-run-loop '(for-each-byte cursor start end ()) + plan 'start 'end 'cursor 'limit 'byte + '((count . count) (second-count . second-count)) '() + (lambda (_forms _bindings _indices) '((void)))))) + ;; Independent scalar reference: include NUL (truthy), empty and + ;; exact-end spans so a speculative out-of-bounds load must fail. + (for-each + (lambda (input) + (let (reference + (foldl (lambda (byte counts) + (list (if (memv byte '(65 66)) (+ 1 (car counts)) 0) + (if (memv byte '(66 67)) (+ 1 (cadr counts)) 0))) + '(18446744073709551616 7) (u8vector->list input))) + (check (eval `(let ((source-bytes ',input) (start 0) + (end ,(u8vector-length input)) + (count 18446744073709551616) (second-count 7)) + ,code (list count second-count))) => reference))) + (list #u8() #u8(0) #u8(65 66) #u8(65 0 66) #u8(0 65 66 66))))) + (test-case "observed counters reject summaries; conditional updates retain their selector" + (for-each (lambda (body) (check (fold-ordinary-run-plan body 'cursor) => #f)) + (list (list special counter '(set-bool seen (uint-positive? (state count)))) + (list special counter '(set-uint count (uint 9))) + (list special counter counter))) + (let (plan (fold-consumer-run-plan (list special `(if (state enabled) (,counter) ())) 'cursor)) + (check (vector-ref plan 4) => '()) + (check (vector-ref plan 3) + => '((if (state enabled) ((set-uint count (uint 0))) ()))))) + (test-case "byte-dependent overwrites require no state observation" + (let (plan (fold-ordinary-run-plan (list special overwrite) 'cursor)) + (check (vector-ref plan 1) => '(58)) + (check (vector-ref plan 3) => (list overwrite)))) + (test-case "events state reads queries and frames reject coalescing" + (for-each + (lambda (form) + (check (fold-ordinary-run-plan (list special form) 'cursor) => #f)) + '((token Text start end) + (set-uint count (uint-add (state count) (uint 1))) + (set-bool flag (future-heading-title? "x")) + (with-source-bounds start end ())))) + (test-case "prefix bound must remain stable throughout the run" + (let (plan (fold-ordinary-run-plan + `((if (offset-less? (line-index cursor) (state-offset bound)) + () (,special ,overwrite))) 'cursor)) + (check (vector-ref plan 2) + => '(not (offset-less? (line-index cursor) (state-offset bound))))) + (check (fold-ordinary-run-plan + `((if (offset-less? (line-index cursor) (state-offset bound)) + () (,special (set-uint bound (offset (line-index cursor)))))) 'cursor) + => #f)) + (test-case "zero cofactoring covers literals but not subsequent writes" + (let* ((branch '(if (uint-positive? (state mode)) ((set-uint mode (uint 2))) ())) + (literal '(if (uint-equal? (state mode) (uint 4)) ((token Text start end)) ())) + (plan (fold-zero-state-plan (list branch literal) #t))) + (check (vector-ref plan 1) => '()) + (check (fold-zero-state-plan + (list branch '(set-uint mode (uint 4)) literal) #t) + => (vector '(mode) (list '(set-uint mode (uint 4)) literal))))) + (test-case "region facts come from the explicit entry guard" + (let (literal '(if (uint-equal? (state mode) (uint 4)) ((token Text start end)) ())) + (check (vector-ref (fold-zero-state-plan (list literal) #t '(mode)) 1) => '()) + (check (fold-zero-state-plan (list literal) #t '()) => #f) + (check (fold-zero-state-plan + (list '(opaque-call) literal) #t '(mode)) => #f))))) diff --git a/t/fold-shared-probes-test.ss b/t/fold-shared-probes-test.ss new file mode 100644 index 00000000..39228a7e --- /dev/null +++ b/t/fold-shared-probes-test.ss @@ -0,0 +1,36 @@ +(import :std/test ../src/compiler/event-fold-shared-probes) +(export fold-shared-probes-test) +(def (condition members (from 'start)) + `(line-bytes-any-in? ,from (line-content-end) ,members)) +(def (decision members body (from 'start)) + `(if ,(condition members from) ,body ())) +(def fold-shared-probes-test + (test-suite "Shared source probes" + (test-case "one byte class preserves independent ordered decisions" + (let* ((forms (list (decision '(36 92) '((set-uint saved (uint 1)))) + (decision '(36 97) '((token Line start end))))) + (plan (car (fold-fuse-byte-probes forms)))) + (check (car plan) => '%shared-byte-probes) + (check (cadr plan) => (car forms)) + (check (caddr plan) => (cadr forms)) + (check (map (cut u8vector-ref (list-ref plan 3) <>) '(36 92 97 98)) + => '(3 1 2 0)) + (check (list-ref plan 4) => 3) + (check (length (fold-fuse-byte-probes + (list (decision '(97) + '((for-line-bytes cursor start end + ((if (line-bytes-any-in? (line-index cursor) + (line-step (line-index cursor)) (97)) + ((set-uint saved (uint 1))) ())))) + (decision '(98) '()))))) => 1) + (check (list-ref (car (fold-fuse-byte-probes + (list (decision '() '()) (decision '(97) '())))) 4) => 2))) + (test-case "range writes helpers and frame changes prevent fusion" + (for-each + (lambda (body) + (let (forms (list (decision '(97) body '(state-offset saved)) + (decision '(98) '() '(state-offset saved)))) + (check (fold-fuse-byte-probes forms) => forms))) + '(((set-uint saved (uint 1))) + ((call-source-helper slice start end)) + ((with-source-bounds start end ()))))))) diff --git a/t/fold-statement-scans-test.ss b/t/fold-statement-scans-test.ss new file mode 100644 index 00000000..17cc4f1b --- /dev/null +++ b/t/fold-statement-scans-test.ss @@ -0,0 +1,34 @@ +(import :std/test ../src/compiler/event-fold-analysis) +(export fold-statement-scans-test) +(def scan '(line-skip-horizontal start)) +(def first `(set-uint at (offset ,scan))) +(def last `(token Text ,scan end)) +(def fold-statement-scans-test + (test-suite "Cross-statement dominating source scans" + (test-case "source scan spans independent state writes and ordered events" + (let (body (list first '(set-bool ready (bool #t)) '(start-node Text) last)) + (check (fold-share-statement-scans body) + => (list (list '%shared-scans (list scan) body))))) + (test-case "conditionally reached first scan cannot be hoisted" + (let (body `((if (state ready) (,first) ()) ,last)) + (check (fold-share-statement-scans body) => body))) + (test-case "source frame helper opaque and loop boundaries block sharing" + (for-each + (lambda (barrier) + (let (body (list first barrier last)) + (check (fold-share-statement-scans body) => body))) + '((with-source-bounds start end ()) (call-source-helper nested start end) + (unknown-transition) (for-line-bytes cursor start end ())))) + (test-case "ordinary short circuit keeps queries and errors ordered" + (let (body `((if (and (future-heading-title? "*" " " 1 "END") + (offset-less? ,scan end)) () ()) ,last)) + (check (fold-share-statement-scans body) => body))) + (test-case "state-dependent ranges and unique scans are not reused" + (let (body '((set-uint at (offset (line-skip-horizontal (state-offset before)))) + (token Text (line-skip-horizontal (state-offset before)) end))) + (check (fold-share-statement-scans body) => body)) + (check (fold-share-statement-scans (list first)) => (list first)) + (let* ((delimiter '(line-scan-until start "]")) + (body `((set-uint at (offset ,delimiter)) (token Text start ,delimiter)))) + (check (fold-share-statement-scans body #t) + => (list (list '%shared-scans (list delimiter) body))))))) diff --git a/t/fold-tape-oracle.ss b/t/fold-tape-oracle.ss new file mode 100644 index 00000000..6ccf9dcf --- /dev/null +++ b/t/fold-tape-oracle.ss @@ -0,0 +1,67 @@ +;;; Independent canonical-list wire oracle; never a production consumer. +(import (only-in :std/vector/u8vector little u8vector-uint-set!) + (only-in ../src/language/descriptor language-grammar-ir language-grammar-machine) + (only-in ../src/compiler/machine parser-machine-grammar-digest)) +(export canonical-fold-tape make-legacy-fold-tape-consumer) + +;; Match the existing Org two-pass consumer, including admission, with its +;; grammar/header/catalog prepared before request timing. The independent +;; reference encoder below remains separate from both production writers. +(def (make-legacy-fold-tape-consumer grammar) + (let ((header (string->utf8 (string-append "OEV1" + (parser-machine-grammar-digest (language-grammar-machine grammar))))) + (indices (make-hash-table-eq))) + (let loop ((rest (cdr (assq 'syntax-kinds (language-grammar-ir grammar)))) (index 0)) + (unless (null? rest) + (hash-put! indices (caar rest) (cons index (cadar rest))) + (loop (cdr rest) (+ index 1)))) + (def (put! bytes offset width value) + (unless (and (exact-integer? value) (<= 0 value) (< value (expt 256 width))) + (error "legacy Fold tape integer out of range" value)) + (u8vector-uint-set! bytes offset value little width)) + (lambda (events) + (let* ((header-size (u8vector-length header)) + (size (foldl (lambda (event total) + (+ total (case (car event) + ((start) 3) ((token) 19) ((finish) 1) + (else (error "invalid legacy Fold event" event))))) + header-size events)) + (bytes (make-u8vector size 0))) + (u8vector-copy! bytes 0 header) + (let loop ((rest events) (offset header-size)) + (unless (null? rest) + (let* ((event (car rest)) (tag (car event)) + (width (case tag ((start) 3) ((token) 19) (else 1)))) + (unless (eq? tag 'finish) + (let ((kind (hash-get indices (cadr event)))) + (unless (and kind (eq? (cdr kind) (if (eq? tag 'start) 'node 'token))) + (error "invalid legacy Fold kind" event)) + (u8vector-set! bytes offset (if (eq? tag 'start) 1 2)) + (put! bytes (+ offset 1) 2 (car kind)))) + (when (eq? tag 'token) + (put! bytes (+ offset 3) 8 (caddr event)) + (put! bytes (+ offset 11) 8 (cadddr event))) + (loop (cdr rest) (+ offset width))))) + bytes)))) +(def (canonical-fold-tape grammar events) + (let ((kinds (map car (cdr (assq 'syntax-kinds (language-grammar-ir grammar))))) + (header (string->utf8 (string-append "OEV1" + (parser-machine-grammar-digest (language-grammar-machine grammar)))))) + (def (unsigned value width) + (if (= width 0) '() + (cons (modulo value 256) (unsigned (quotient value 256) (- width 1))))) + (def (index kind) + (let loop ((rest kinds) (ordinal 0)) + (cond ((null? rest) (error "missing oracle tape kind" kind)) + ((eq? (car rest) kind) ordinal) + (else (loop (cdr rest) (+ ordinal 1)))))) + (list->u8vector + (append (u8vector->list header) + (foldr append '() + (map (lambda (event) + (case (car event) + ((finish) '(0)) + ((start) (cons 1 (unsigned (index (cadr event)) 2))) + ((token) (append (cons 2 (unsigned (index (cadr event)) 2)) + (unsigned (caddr event) 8) (unsigned (cadddr event) 8))) + (else (error "invalid oracle tape event" event)))) events)))))) diff --git a/t/fold-tape-tail-test.ss b/t/fold-tape-tail-test.ss new file mode 100644 index 00000000..1979128a --- /dev/null +++ b/t/fold-tape-tail-test.ss @@ -0,0 +1,28 @@ +(import :std/test ../src/compiler/event-fold-scheme-context) +(export fold-tape-tail-test) + +(def (shared-statement tape? effect-only?) + (parameterize ((current-fold-scheme-expressions (make-table)) + (current-fold-scheme-definitions '()) + (current-fold-scheme-counter 0) + (current-fold-scheme-tape-kinds tape?) + (current-fold-scheme-effect-only effect-only?)) + (let (statements (fold-scheme-share-statements + (map (lambda (_) '(fold-tape-finish! events)) (iota 12)) '() '())) + (vector statements (current-fold-scheme-definitions))))) + +(def fold-tape-tail-test + (test-suite "Direct tape effect-only transitions" + (test-case "private tape calls do not reassign or return the owner" + (let* ((result (shared-statement '((Text 1)) #t)) + (call (caar (vector-ref result 0))) + (body (caddar (vector-ref result 1)))) + (check (symbol? call) => #t) + (check (eq? call 'set!) => #f) + (check (last body) => '(fold-tape-finish! events)))) + (test-case "list producers and disabled candidates retain threading" + (for-each + (lambda (result) + (check (caar (vector-ref result 0)) => 'set!) + (check (last (caddar (vector-ref result 1))) => 'events)) + (list (shared-statement #f #t) (shared-statement '((Text 1)) #f)))))) diff --git a/t/fold-tape-test.ss b/t/fold-tape-test.ss new file mode 100644 index 00000000..6391c986 --- /dev/null +++ b/t/fold-tape-test.ss @@ -0,0 +1,33 @@ +(import :std/test ../src/runtime/fold-tape + ./fold-tape-oracle ./event-strategy-fixture + ../src/compiler/event-fold-scheme) +(export fold-tape-test) +(def fold-tape-test + (test-suite "Direct Fold tape ownership" + (test-case "wire spans growth and publication are exact" + (let* ((header (u8vector 79 69 86 49)) (owner (make-fold-tape header))) + (fold-tape-start! owner 513) + (for-each (lambda (index) (fold-tape-token! owner 258 (+ 256 index) index)) (iota 1000)) + (fold-tape-finish! owner) + (let ((published (fold-tape-publish! owner))) + (check (u8vector-length published) => 19008) + (check (u8vector->list (subu8vector published 4 10)) => '(1 1 2 2 2 1)) + (check (u8vector-ref published 19007) => 0) + (check-exception (fold-tape-token! owner 1 0 1) true) + (check-exception (fold-tape-publish! owner) true) + (let ((other (make-fold-tape header))) + (fold-tape-finish! other) + (check (fold-tape-publish! other) => (u8vector 79 69 86 49 0))) + (check (u8vector-length published) => 19008)))) + (test-case "numeric bounds reject truncation" + (let ((owner (make-fold-tape (u8vector)))) + (check-exception (fold-tape-start! owner 65536) true) + (check-exception (fold-tape-token! owner 0 -1 0) true) + (check-exception (fold-tape-token! owner 0 0 (expt 2 64)) true))) + (test-case "direct product cannot materialize event lists" + (let ((source (event-fold-scheme-source 'direct-product event-lines-language-grammar + 'Document '() '((start-node Text) (token Line start end) (finish-node)) + '() '() '() 'tape))) + (check (not (not (string-contains source "fold-tape-token!"))) => #t) + (check (not (string-contains source "(cons '(finish)")) => #t) + (check (not (string-contains source "(list 'token")) => #t))))) diff --git a/t/fold-zero-state-test.ss b/t/fold-zero-state-test.ss new file mode 100644 index 00000000..de913885 --- /dev/null +++ b/t/fold-zero-state-test.ss @@ -0,0 +1,58 @@ +(import :std/test ../src/compiler/event-fold-zero-state) +(export fold-zero-state-test) +(def guard '(uint-positive? (state mode))) +(def branch `(if ,guard ((set-uint mode (uint 2))) ())) +(def fold-zero-state-test + (test-suite "Guarded zero-state plan" + (test-case "guard support excludes payload in an eliminated consumer" + (let* ((forms '((if (uint-positive? (state mode)) + ((if (uint-equal? (state payload) (uint 0)) + ((set-uint payload (uint 3))) ()) + (set-uint mode (uint 2))) ()))) + (plan (fold-zero-state-support forms #t))) + (check (vector-ref plan 0) => '(mode)) + (check (vector-ref plan 1) => '()) + (check (fold-zero-state-plan forms #t '()) => #f))) + (test-case "consumer selectors refine only their local unsigned support" + (let* ((other '(if (uint-positive? (state other)) ((token Other start end)) ())) + (forms `(if ,guard ((token Active start end)) + ((if ,guard ((token Impossible start end)) ()) ,other)))) + (check (fold-consumer-state-body (list forms)) + => `((if ,guard ((token Active start end)) (,other))))) + (check (fold-consumer-state-body `((set-uint mode (uint 0)) ,branch)) + => '((set-uint mode (uint 0))))) + (test-case "consumer writes barriers and joins preserve ordered observations" + (let (forms `((if ,guard () + ((set-uint mode (uint 1)) ,branch)))) + (check (fold-consumer-state-body forms) => forms)) + (let (forms `((if ,guard () ((opaque-call) ,branch)))) + (check (fold-consumer-state-body forms) => forms)) + (let (forms `((if (bool-state enabled) ((set-uint mode (uint 0))) ()) ,branch)) + (check (fold-consumer-state-body forms) => forms))) + (test-case "idle guards simplify only behind external zero guard" + (let (plan (fold-zero-state-plan (list branch))) + (check (vector-ref plan 0) => '(mode)) + (check (vector-ref plan 1) => '())) + (let* ((configuration '(if (uint-positive? (state policy)) () ())) + (plan (fold-zero-state-plan (list configuration branch)))) + (check (vector-ref plan 0) => '(mode)) + (check (vector-ref plan 1) => (list configuration)))) + (test-case "writes and opaque effects invalidate facts" + (let (plan (fold-ordinary-state-plan + '((if (line-byte-equal? (line-index cursor) 99) ((token Text start end)) ())) 'cursor)) + (check (vector-ref plan 0) => '()) + (check (vector-ref plan 1) => '(99)) + (check (vector-ref plan 2) => '())) + (let (plan (fold-zero-state-plan `((token Text start end) ,branch))) + (check (vector-ref plan 1) => '((token Text start end)))) + (check (fold-zero-state-plan `((set-uint mode (uint 1)) ,branch)) => #f) + (check (fold-zero-state-plan `((opaque-call) ,branch)) => #f)) + (test-case "unknown branch joins preserve only common facts" + (let* ((forms `((if (bool-state enabled) ((set-uint mode (uint 1))) ()) ,branch)) + (plan (fold-zero-state-plan forms))) + (check plan => #f))) + (test-case "unknown predicate effects retain ordered guards" + (check (fold-zero-state-plan `((if (and (opaque-predicate) ,guard) () ()))) => #f) + (check (fold-zero-state-plan `((if (opaque-predicate) (,branch) ()))) => #f)) + (test-case "local join binding shadows external facts" + (check (fold-zero-state-plan `((join-once mode (,branch) ()))) => #f)))) diff --git a/t/follow-partition-language-test.ss b/t/follow-partition-language-test.ss new file mode 100644 index 00000000..10546c4b --- /dev/null +++ b/t/follow-partition-language-test.ss @@ -0,0 +1,175 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Grammar IR qualification of the optional Scheme LR construction. + +(import (only-in :gerbil-parser/languages/hcl/parser-test hcl-test-language) + (only-in :gerbil-parser/language-support/development + language-loader-fixtures language-loader-fixture-count) + (only-in :gerbil-parser/language-support/entry language-parser-entry-ref) + :std/test + (only-in :core/observability/testing-case poo-flow-test-case) + (only-in :gerbil-parser/languages/arithmetic/grammar + arithmetic-grammar arithmetic-parser) + (only-in :gerbil-parser/languages/hcl/grammar + hcl-grammar hcl-parser + hcl-parser-ir) + + (only-in :gerbil-parser/language-support/fixture + syntax-fixture-id syntax-fixture-source) + (only-in :gerbil-parser/src/compiler/parser-ir + compile-parser parser-ir-ref) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr lr-spec-ref) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse) + (only-in :gerbil-parser/src/runtime/significant + parser-significant-tokens) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/t/fixtures/lr1-construction + genuine-reduce-conflict-rules lr1-not-lalr-rules + precedence-expression-rules)) + +(def (grammar-ir rules) + (list (cons 'schema "gerbil-parser.grammar-ir.v1") + (cons 'grammar 'follow-partition-witness) + (cons 'syntax-kinds '((SourceFile node ()))) + (cons 'terminals '()) + (cons 'lexical-rules '()) + (cons 'rules rules) + (cons 'extras '()) + (cons 'keywords '()) + (cons 'parser-entrypoints '((source-file parse pure))) + (cons 'recoveries '()) + (cons 'flow '((source lexical) (lexical cst))) + (cons 'conflict-policy 'reject) + (cons 'case-insensitive? #f))) + +(def (character-tokens source) + (let loop ((index 0) (tokens '())) + (if (= index (string-length source)) + (reverse tokens) + (loop (+ index 1) + (cons (make-token 'punctuation + (string (string-ref source index)) + index (+ index 1)) + tokens))))) + +(def (parsed-root spec source) + (let-values (((root rest) (lr-parse spec (character-tokens source)))) + (unless (null? rest) + (error "Grammar IR parser left a token suffix" source rest)) + root)) + +(def (parsed-language-root spec tokens) + (let-values (((root rest) (lr-parse spec tokens))) + (unless (null? rest) + (error "language parser left a token suffix" rest)) + root)) + +(def (check-grammar-ir-equivalence rules sources) + (let* ((grammar (grammar-ir rules)) + (canonical + (parser-ir-ref (compile-parser grammar 'canonical-lr1) 'lr-spec)) + (direct + (parser-ir-ref (compile-parser grammar 'follow-partition-lr1) + 'lr-spec))) + (check (lr-spec-ref direct 'algorithm) => 'follow-partition-lr1-v1) + (check (> (lr-spec-ref direct 'state-count) 0) => #t) + (for-each + (lambda (source) + (check (equal? (parsed-root canonical source) + (parsed-root direct source)) + => #t)) + sources))) + +(def follow-partition-language-test + (test-suite "follow partition through Grammar IR" + (poo-flow-test-case "default Grammar IR construction remains LALR" + (let (grammar (grammar-ir precedence-expression-rules)) + (check (parser-ir-ref (compile-parser grammar) 'lr-spec) + => (parser-ir-ref (compile-parser grammar 'lalr) 'lr-spec)))) + (test-case "explicit adaptive construction selects by conflict" + (let* ((ordinary (compile-parser + (grammar-ir precedence-expression-rules) + 'lalr-then-follow)) + (ordinary-spec (parser-ir-ref ordinary 'lr-spec)) + (split (compile-parser + (grammar-ir lr1-not-lalr-rules) + 'lalr-then-follow)) + (split-spec (parser-ir-ref split 'lr-spec)) + (reference (parser-ir-ref + (compile-parser (grammar-ir lr1-not-lalr-rules) + 'canonical-lr1) + 'lr-spec))) + (check (lr-spec-ref ordinary-spec 'algorithm) + => 'lalr1-lr0-fixed-point-v1) + (check ordinary-spec + => (parser-ir-ref + (compile-parser (grammar-ir precedence-expression-rules) + 'lalr) + 'lr-spec)) + (check (lr-spec-ref split-spec 'algorithm) + => 'follow-partition-lr1-v1) + (for-each + (lambda (source) + (check (parsed-root split-spec source) + => (parsed-root reference source))) + '("acd" "bce" "ace" "bcd")))) + (test-case "adaptive construction preserves genuine conflict rejection" + (check-exception + (compile-lr-spec genuine-reduce-conflict-rules 'source-file + 'reject #f 'lalr-then-follow) + true)) + (test-case "selective GLR policy keeps its LALR fork route" + (let (spec + (compile-lr-spec genuine-reduce-conflict-rules 'source-file + 'selective-glr #f 'lalr-then-follow)) + (check (lr-spec-ref spec 'algorithm) + => 'lalr1-lr0-fixed-point-v1))) + (poo-flow-test-case "LR(1)-only grammar keeps its parse products" + (check-grammar-ir-equivalence + lr1-not-lalr-rules '("acd" "bce" "ace" "bcd"))) + (poo-flow-test-case "precedence grammar keeps its parse products" + (check-grammar-ir-equivalence + precedence-expression-rules '("a" "a+a" "a*a" "a+a*a" "a*a+a"))) + (poo-flow-test-case "generated arithmetic language keeps parse products" + (let* ((canonical + (parser-ir-ref + (compile-parser arithmetic-grammar 'canonical-lr1) + 'lr-spec)) + (direct + (parser-ir-ref + (compile-parser arithmetic-grammar 'follow-partition-lr1) + 'lr-spec))) + (for-each + (lambda (source) + (let (tokens + (parser-significant-tokens + arithmetic-parser + (lex-source arithmetic-parser source))) + (check (equal? (parsed-language-root canonical tokens) + (parsed-language-root direct tokens)) + => #t))) + '("1 + 2 * (3 - 4)" "foo-7/2" "-x+3")))) + (test-case "official HCL corpus keeps LALR and direct parse products" + (let ((lalr (parser-ir-ref hcl-parser-ir 'lr-spec)) + (direct + (parser-ir-ref + (compile-parser hcl-grammar 'follow-partition-lr1) + 'lr-spec))) + (check (lr-spec-ref direct 'algorithm) => 'follow-partition-lr1-v1) + (check (lr-spec-ref direct 'state-count) + => (lr-spec-ref lalr 'state-count)) + (for-each + (lambda (fixture) + (let* ((source (syntax-fixture-source fixture)) + (tokens + (parser-significant-tokens + hcl-parser + (lex-source hcl-parser source)))) + (check (equal? (parsed-language-root lalr tokens) + (parsed-language-root direct tokens)) + => #t))) + (language-loader-fixtures hcl-test-language 'accepted)))))) + +(export follow-partition-language-test) diff --git a/t/fused-reduction-test.ss b/t/fused-reduction-test.ss new file mode 100644 index 00000000..5171330b --- /dev/null +++ b/t/fused-reduction-test.ss @@ -0,0 +1,204 @@ +;;; Generic POO generation and both semantic backends use two independent packs. +(import (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-test-language) + (only-in :gerbil-parser/languages/hcl/parser-test hcl-test-language) + (only-in :gerbil-parser/languages/hcl/parser-test hcl-official-fixtures) + :std/test + :gerbil-parser/src/grammar/algebra + (only-in :gerbil-parser/src/language/grammar deflanguage) + (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release) + (only-in :std/misc/ports read-all-as-string) + (only-in :clan/poo/object .o .cc .ref) + (only-in :clan/poo/mop validate) + (only-in :gerbil-parser/language-build-support + FusedReductionStrategy. FusedReductionStrategyContract + make-fused-reduction-strategy emit-build-strategy + emit-language-build-strategy) + (only-in :gerbil-parser/src/compiler/machine parser-machine-for-current-semantic-backend) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-event-program-enabled? lr-runtime-event-program?) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime) + (only-in :gerbil-parser/src/testing/language-strategy parse-indexed-reference) + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine) + (only-in :gerbil-parser/src/language/descriptor language-grammar-ir) + (only-in :gerbil-parser/src/language/descriptor + +language-grammar-schema+ make-language-grammar language-grammar-language language-grammar-version + language-grammar-contract language-grammar-grammar language-grammar-observability) + (only-in :gerbil-parser/src/compiler/lr lr-spec-ref production-table production-rhs) + (only-in :gerbil-parser/src/runtime/artifact make-success-parse-artifact + parse-artifact-valid? parse-artifact-roundtrip) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/recognition make-recognition-node make-recognition-child) + (only-in :gerbil-parser/src/compiler/machine parser-machine-grammar-digest parser-machine-trivia) + (only-in :gerbil-parser/language-support/fixture syntax-fixture-source syntax-fixture-expected-status) + (only-in :gerbil-parser/languages/hcl/parser hcl-language-grammar) + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-language-grammar) + (only-in :gerbil-parser/language-support/development LanguageDevelopmentLoaderContract) + (prefix-in :gerbil-parser/src/compiler/hcl-reductions hcl-) + (prefix-in "fixtures/arithmetic-reductions.ss" arithmetic-)) +(export fused-reduction-test) + +(deflanguage nullable-boundary-probe + (syntax + (lexical + (root source-file) + (lex (word Word (identifier))) + (extras) (keywords) (recoveries) (conflicts selective-glr) + (case-insensitive #f))) + (rules + (source-file (node SourceFile (seq (node Before (empty)) word (node After (empty))))))) +(bind-fixture-grammar-release nullable-boundary-probe + "nullable-boundary-probe" "test" "nullable-boundary-probe.test") + +(def (emitted strategy) + (call-with-output-string (lambda (port) (emit-build-strategy strategy port)))) + +(def (cdr-expression-count form) + (if (pair? form) + (+ (if (eq? (car form) 'cdr) 1 0) + (cdr-expression-count (car form)) (cdr-expression-count (cdr form))) + 0)) + +(def (check-linear-stack-reads descriptor) + (let* ((form (call-with-input-string (emitted (make-fused-reduction-strategy descriptor)) read)) + (table (production-table (lr-spec-ref (cdr (assq 'lr-spec (language-grammar-ir descriptor))) 'productions))) + (steps (filter (lambda (definition) + (and (pair? definition) (eq? (car definition) 'def) + (pair? (cadr definition)))) (cdr form)))) + (check (length steps) => 2) + (for-each + (lambda (step) + (for-each + (lambda (clause) + (when (pair? (car clause)) + (let* ((id (caar clause)) + (width (length (production-rhs (vector-ref table id)))) + (bindings (cadr (cadr clause)))) + (let loop ((remaining bindings) (reads 0)) + (if (eq? (caar remaining) 'offset) + (check reads => (* 2 width)) + (loop (cdr remaining) (+ reads (cdr-expression-count (cadar remaining))))))))) + (cddr (caddr step)))) steps))) +(def (check-reductions descriptor sources step event-step) + (let (machine (language-grammar-machine descriptor)) + (for-each + (lambda (source) + (let ((baseline (parse-indexed-reference machine source)) + (fused (parse-indexed-reference machine source step))) + (check fused => baseline) + (check (parse-artifact-valid? fused) => #t) + (check (parse-artifact-roundtrip fused) => source) + (parameterize ((current-lr-event-program-enabled? #t)) + (let* ((events-machine (parser-machine-for-current-semantic-backend machine)) + (event-artifact (parse-indexed-reference events-machine source event-step))) + (check (lr-runtime-event-program? (parser-machine-runtime events-machine)) => #t) + (check event-artifact => baseline))))) + sources))) + +(def fused-reduction-test + (test-suite "generic POO fused reduction strategy" + (test-case "generated recognition and event reducers preserve nullable sibling origins" + (let* ((descriptor nullable-boundary-probe-language-grammar) + (machine (language-grammar-machine descriptor)) + (token (make-token 'word "a" 0 1)) + (root (make-recognition-node 'SourceFile 0 1 + (list (make-recognition-child #f (make-recognition-node 'Before 0 0 '())) + (make-recognition-child #f token) + (make-recognition-child #f (make-recognition-node 'After 1 1 '()))))) + (expected (make-success-parse-artifact (parser-machine-grammar-digest machine) + "a" (list token) root (parser-machine-trivia machine))) + (strategy (.cc (make-fused-reduction-strategy descriptor) + 'step-name 'nullable-step 'event-name 'nullable-event-step + 'digest-name 'nullable-digest))) + (eval (call-with-input-string (emitted strategy) read)) + (check (parse-artifact-valid? expected) => #t) + (check (parse-indexed-reference machine "a") => expected) + (check (parse-indexed-reference machine "a" (eval 'nullable-step)) => expected) + (parameterize ((current-lr-event-program-enabled? #t)) + (check (parse-indexed-reference (parser-machine-for-current-semantic-backend machine) + "a" (eval 'nullable-event-step)) => expected)))) + (test-case "both emitted backends traverse each stack suffix once" + (for-each check-linear-stack-reads (list arithmetic-language-grammar hcl-language-grammar))) + (test-case "two descriptors generate independently and deterministically" + (let* ((hcl (make-fused-reduction-strategy hcl-language-grammar)) + (arithmetic (make-fused-reduction-strategy arithmetic-language-grammar)) + (first (emitted hcl))) + (check first => (call-with-input-file "src/compiler/hcl-reductions.ss" read-all-as-string)) + (check (emitted arithmetic) => (call-with-input-file "t/fixtures/arithmetic-reductions.ss" read-all-as-string)) + (check (equal? first (emitted arithmetic)) => #f) + (check (emitted hcl) => first))) + (test-case "POO inherited slots rename output and carry extension metadata" + (let* ((prototype (.o (:: self FusedReductionStrategy.) + step-name: 'reduce-step event-name: 'reduce-events digest-name: 'reduce-digest + metadata: (.o consumer: 'downstream))) + (strategy (make-fused-reduction-strategy arithmetic-language-grammar prototype)) + (form (call-with-input-string (emitted strategy) read))) + (check (.ref (.ref strategy 'metadata) 'consumer) => 'downstream) + (check (and (member '(export reduce-step reduce-events reduce-digest) (cdr form)) #t) => #t) + (check (equal? (emitted strategy) (emitted (make-fused-reduction-strategy arithmetic-language-grammar))) => #f))) + (test-case "effective overrides reject stale identity and malformed names before emission" + (let ((hcl (make-fused-reduction-strategy hcl-language-grammar)) + (port (open-output-string))) + (for-each + (lambda (invalid) + (check-exception (validate FusedReductionStrategyContract invalid) true) + (check-exception (emit-build-strategy invalid port) true)) + (list (.cc hcl 'digest "sha256:stale") + (.cc hcl 'descriptor arithmetic-language-grammar 'digest hcl-direct-grammar-digest) + (.cc hcl 'event-name 'direct-step) + (.cc hcl 'step-name 'if) + (.cc hcl 'step-name 'values) + (.cc hcl 'digest-name 'token-start) + (.cc hcl 'step-name '|bad name|) + (.cc hcl 'metadata #f))) + (check (get-output-string port) => ""))) + (test-case "unresolved reference overrides reject before either fused backend emits" + (def original arithmetic-language-grammar) + (def ir + (map (lambda (entry) + (if (eq? (car entry) 'lr-spec) + (cons 'lr-spec + (map (lambda (row) + (if (eq? (car row) 'productions) + (cons 'productions + (append (cdr row) + (list (list (length (cdr row)) 'unused + '((nonterminal missing)) 'pass #f)))) + row)) (cdr entry))) + entry)) (language-grammar-ir original))) + (def descriptor + (make-language-grammar +language-grammar-schema+ + (language-grammar-language original) (language-grammar-version original) + (language-grammar-contract original) (language-grammar-grammar original) + ir (language-grammar-machine original) (language-grammar-observability original))) + (def candidate (.cc (make-fused-reduction-strategy original) 'descriptor descriptor)) + (def port (open-output-string)) + (check-exception (validate FusedReductionStrategyContract candidate) true) + (check-exception (emit-build-strategy candidate port) true) + (check (get-output-string port) => "")) + (test-case "Loader slots admit declared strategies and reject foreign or duplicated bindings" + (let (output (call-with-output-string (lambda (port) (emit-language-build-strategy hcl-test-language 'fused-reductions port)))) + (check output => (emitted (make-fused-reduction-strategy hcl-language-grammar))) + (check (length (.ref arithmetic-test-language 'build-strategies)) => 2) + (check-exception (validate LanguageDevelopmentLoaderContract + (.cc hcl-test-language 'build-strategies (.ref arithmetic-test-language 'build-strategies))) true) + (check-exception (validate LanguageDevelopmentLoaderContract + (.cc hcl-test-language 'build-strategies + (append (.ref hcl-test-language 'build-strategies) (.ref hcl-test-language 'build-strategies)))) true) + (check-exception (emit-language-build-strategy hcl-test-language 'missing (open-output-string)) true))) + (test-case "HCL field and alias reductions preserve both native semantic backends" + (check-reductions hcl-language-grammar + (map syntax-fixture-source (filter (lambda (fixture) (eq? (syntax-fixture-expected-status fixture) 'accepted)) hcl-official-fixtures)) + hcl-direct-step hcl-direct-event-step)) + (test-case "Arithmetic precedence and zero-width reductions preserve both backends" + (check-reductions arithmetic-language-grammar + '("1" "a" "1 + 2 * 3" "(1 + 2) * 3" "-1 + +2" "a / (b - c)" "1 - 2 - 3" "α + 2" "1\r\n + 2") + arithmetic-direct-step arithmetic-direct-event-step)) + (test-case "malformed sources reject with generic and generated reductions" + (for-each + (lambda (row) + (let ((machine (language-grammar-machine (car row))) (step (cadr row))) + (for-each (lambda (source) + (check-exception (parse-indexed-reference machine source) true) + (check-exception (parse-indexed-reference machine source step) true)) + (cddr row)))) + (list (list hcl-language-grammar hcl-direct-step "x =\n" "b { x = 1\n") + (list arithmetic-language-grammar arithmetic-direct-step "1 +" "(1 + 2")))))) diff --git a/t/generate-binding-names.ss b/t/generate-binding-names.ss new file mode 100644 index 00000000..a7b6e748 --- /dev/null +++ b/t/generate-binding-names.ss @@ -0,0 +1,35 @@ +#!/usr/bin/env gxi +;;; Real production BindingProfile names lower through the shared TextProfile IR. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :gerbil-parser/t/fixtures/bash-products bash-parts) + (only-in :gerbil-parser/src/language/part-profile compile-part-profile part-plan-recipe part-plan-binding binding-plan-name-end) + (only-in :gerbil-parser/src/compiler/rust-runtime rust-text-profile-expression-source)) +(def controls '("" "α_123" "123x" "@rest" "_rest" "α-β" "λ中😀" "1α" "!α" "?α" "aβγ")) +(def (byte-at source at) (u8vector-length (string->utf8 (substring source 0 at)))) +(def (main output) + (let* ((plan (compile-part-profile bash-parts)) (recipes (list-ref (part-plan-recipe plan) 3)) + (count 0)) + (call-with-output-file output (lambda (port) + (display "// @generated from production Bash BindingProfile declarations.\n" port) + (display "pub static PROFILES: &[(&str, gerbil_parser_runtime::TextProfile)] = &[\n" port) + (for-each (lambda (role) + (display "(" port) (write (symbol->string role) port) (display ", " port) + (display (rust-text-profile-expression-source (car (cadr (assq role recipes)))) port) + (display "),\n" port)) '(simple parameter assignment)) + (display "];\npub static TRACES: &[(usize, &str, usize, usize, Option)] = &[\n" port) + (for-each (lambda (index) + (let* ((role (list-ref '(simple parameter assignment) index)) (binding (part-plan-binding plan role))) + (for-each (lambda (text) + (let (source (string-append "前:" text ":tail")) + (for-each (lambda (width) + (let* ((start 2) (limit (+ start width)) + (end (binding-plan-name-end binding source start limit))) + (display "(" port) (display index port) (display ", " port) (write source port) + (display ", " port) (display (byte-at source start) port) + (display ", " port) (display (byte-at source limit) port) (display ", " port) + (if (> end start) (begin (display "Some(" port) (display (byte-at source end) port) (display ")" port)) + (display "None" port)) + (display "),\n" port) (set! count (+ count 1)))) + (iota (+ (string-length text) 1))))) controls))) '(0 1 2)) + (display "];\n" port))) + (displayln "BINDING-NAMES-RUST-GENERATED traces=" count) (force-output) (test-child-process-exit! 0))) diff --git a/t/generate-command-source.ss b/t/generate-command-source.ss new file mode 100644 index 00000000..51430240 --- /dev/null +++ b/t/generate-command-source.ss @@ -0,0 +1,97 @@ +#!/usr/bin/env gxi +;;; Recognition controls come from the actual Scheme Source engine, including fields and deferred links. +(import (only-in :gerbil-parser/src/language/source declare-source-language) + (only-in :gerbil-parser/src/ffi/language-artifact-codec bind-language-abi abi-descriptor-payload abi-parse-binary-payload) + (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :clan/poo/object .o .ref) + (only-in :gerbil-parser/t/fixtures/bash-products bash-results bash-parts bash-word-regions bash-commands bash-command-scanner) + (only-in :gerbil-parser/src/compiler/rust-scanner command-source-rust-module-source) + (only-in :gerbil-parser/src/runtime/source-engines ShellSourceStrategy.) + (only-in :gerbil-parser/src/compiler/rust-runtime result-profile-rust-source) + (only-in :gerbil-parser/src/language/result-profile compile-result-profile result-plan-recipe) + (only-in :gerbil-parser/src/language/part-profile compile-part-profile) + (only-in :gerbil-parser/src/language/command-profile compile-command-profile) + (only-in :gerbil-parser/src/language/scanner-profile compile-scanner-profile) + (only-in :gerbil-parser/src/runtime/contextual-scanner prepare-contextual-scanner contextual-scanner-initial-state contextual-scanner-step) + (only-in :gerbil-parser/src/runtime/shell-parser make-shell-parser shell-here-document-link-marker-start shell-here-document-link-body-start) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success? parse-artifact-events event-kind token-event-token-kind event-start event-end)) +(def controls + '("" " # hi\n" "echo α😀" "a=中 x+=λ echo \"${x:-中}\" 2>out tail" "arr=(α \"中\"\nλ) echo ok" + "echo α|&cat && echo 中 || echo λ" "time -p ! echo α | cat" "! echo α" "time echo α" + "if echo α; then echo 中; elif echo λ; then echo β; else echo 😀; fi" + "while echo α; do echo 中; done" "until echo α; do echo 中; done" + "for x in α 中; do echo $x; done" "for ((i=0;i<2;i++)); do echo $i; done" + "select x in α 中; do echo $x; done" + "case α in α|β) echo 中;; (λ) ;; esac" + "function f() { echo α; } >out" "f ( ) { echo 中; }" "[[ α == 中 && -n $x ]]" + "((1+(2*3)))" "{ echo α; } 2>out" "(echo α; echo 中)" + "cat <")) +;;; A second POO declaration changes the pipeline role and form keyword, using identical engine code. +(def extended + (.o (:: self bash-commands) + roles: (map (lambda (row) (if (eq? (car row) 'pipeline) '(pipeline (operator "&&")) row)) (.ref bash-commands 'roles)) + projections: (map (lambda (row) (if (eq? (car row) 'BashFile) '(BashFile CommandList) row)) (.ref bash-commands 'projections)) + forms: (map (lambda (row) (case (car row) + ((brace-group) '(brace-group (word "开始") 0 BraceGroup (take open (word "开始")) (list body (word "结束") #f) (take close (word "结束")))) + ((pipeline-prefix) '(pipeline-prefix (manual) 0 Pipeline)) (else row))) (.ref bash-commands 'forms)))) +(def (scan ir source) + (let* ((scanner (prepare-contextual-scanner ir source)) (state (contextual-scanner-initial-state scanner))) + (let loop ((state state) (tokens '())) + (let-values (((token next) (contextual-scanner-step scanner state 'source))) + (if token (loop next (cons token tokens)) (reverse tokens)))))) +(def (emit-bytes port bytes) + (let (length (u8vector-length bytes)) + (for-each (lambda (shift) (write-u8 (bitwise-and 255 (arithmetic-shift length (- shift))) port)) '(0 8 16 24))) + (write-u8vector bytes port)) +(def (main output native-output) + (let* ((results (compile-result-profile bash-results)) (recipe (result-plan-recipe results)) + (nodes (map car (car recipe))) (terminals (cadr recipe)) (ir (compile-scanner-profile bash-command-scanner)) (count 0)) + (def (index kind values base) + (let loop ((values values) (at base)) (cond ((null? values) (error "unknown result kind" kind)) + ((eq? kind (car values)) at) (else (loop (cdr values) (+ at 1)))))) + (call-with-output-file native-output (lambda (native-port) + (call-with-output-file output (lambda (port) + (display "// @generated by actual Scheme Source recognition; do not edit.\n" port) + (display "pub struct Control {pub source:&'static str,pub accepted:bool,pub events:&'static [gerbil_parser_runtime::TreeEvent],pub fields:&'static [&'static str],pub links:&'static [(usize,usize)]}\n" port) + (for-each (lambda (entry) + (let* ((name (car entry)) (profile (cadr entry)) (sources (caddr entry)) + (strategy (.o (:: self ShellSourceStrategy.) regions: bash-word-regions results: bash-results parts: bash-parts commands: profile scanner: bash-command-scanner)) + (native (bind-language-abi (declare-source-language name "test" "source-native-control" strategy)))) + (display "pub mod " port) (display name port) (display " {\nuse gerbil_parser_runtime::TreeEvent;\n" port) + (display (command-source-rust-module-source (.o (:: self ShellSourceStrategy.) regions: bash-word-regions results: bash-results parts: bash-parts commands: profile scanner: bash-command-scanner)) port) + (display "pub static CONTROLS:&[super::Control]=&[\n" port) + (let-values (((parse receipt) (make-shell-parser bash-word-regions results (compile-part-profile bash-parts) (compile-command-profile profile)))) + (for-each (lambda (source) + (displayln "COMMAND-SOURCE-CONTROL " name " " count) (force-output) + (let-values (((artifact links) (receipt source (lambda (source) (scan ir source)) "control"))) + ;; These bytes come from the public language-independent ABI codec. + (emit-bytes native-port (string->utf8 name)) + (emit-bytes native-port (string->utf8 (abi-descriptor-payload native))) + (emit-bytes native-port (string->utf8 source)) + (emit-bytes native-port (abi-parse-binary-payload native source)) + (write-u8 (if (parse-artifact-success? artifact) 1 0) native-port) + (display "super::Control {source:" port) (write source port) + (display (if (parse-artifact-success? artifact) ",accepted:true,events:&[" ",accepted:false,events:&[") port) + (when (parse-artifact-success? artifact) + (for-each (lambda (event) + (case (event-kind event) + ((start-node) (display "TreeEvent::StartNode(" port) (display (index (vector-ref event 2) nodes 0) port) (display ")," port)) + ((finish-node) (display "TreeEvent::FinishNode," port)) + ((token) (display "TreeEvent::Token {kind:" port) (display (index (token-event-token-kind event) terminals (length nodes)) port) + (display ",start:" port) (display (event-start event) port) (display ",end:" port) (display (event-end event) port) (display "}," port)))) (parse-artifact-events artifact))) + (display "],fields:&[" port) + (when (parse-artifact-success? artifact) + (for-each (lambda (event) (when (eq? (event-kind event) 'start-field) (write (symbol->string (vector-ref event 1)) port) (display "," port))) (parse-artifact-events artifact))) + (display "],links:&[" port) + (for-each (lambda (link) (display "(" port) (display (shell-here-document-link-marker-start link) port) + (display "," port) (display (shell-here-document-link-body-start link) port) (display ")," port)) links) + (display "]},\n" port) (set! count (+ count 1)))) sources)) + (display "];\n}\n" port))) + (list (list "bash" bash-commands controls) + (list "extended" extended '("开始 echo α; 结束" "echo α && echo 中" "{ echo α; }" "开始 结束" "time echo α")))))))) + (displayln "COMMAND-SOURCE-RUST-GENERATED controls=" count) (force-output) (test-child-process-exit! 0))) diff --git a/t/generate-event-fold-executable.ss b/t/generate-event-fold-executable.ss new file mode 100644 index 00000000..dba24194 --- /dev/null +++ b/t/generate-event-fold-executable.ss @@ -0,0 +1,344 @@ +;;; -*- Gerbil -*- +;;; Build-time oracle produces literal controls for independently compiled code. +(import (only-in :gerbil-parser/t/event-strategy-fixture event-lines-language-grammar) + (only-in :gerbil-parser/src/compiler/event-fold-scheme event-fold-scheme-source) + (only-in :gerbil-parser/src/compiler/event-fold-runtime run-event-fold) + (only-in :gerbil-parser/src/compiler/event-fold-ir event-fold-ir)) + (import (only-in :gerbil-parser/t/fold-tape-oracle canonical-fold-tape)) +(export main) + +(def predicates + (append '((and (offset-less? (line-skip-horizontal start) end) + (offset-less? (line-skip-horizontal start) (line-trim-end))) + (or (state ready) + (offset-less? (line-skip-horizontal start) (line-skip-horizontal start))) + (and (line-starts-with-ascii-ci "A") (state ready)) + (or (line-starts-with "ab") (line-starts-with "a") + (line-starts-with "é") (line-starts-with "λ")) + (or (line-starts-with-ascii-ci "ABC") (line-starts-with-ascii-ci "A") + (line-starts-with-ascii-ci "B") (line-starts-with-ascii-ci "AB")) + (or (state ready) (line-starts-with "a") + (line-starts-with "ab") (line-starts-with "é")) + (bool #t) (state ready) + (uint-positive? (uint 1)) (uint-equal? (uint 1) (uint 1)) + (uint-not-equal? (uint 1) (uint 2)) (uint-greater? (uint 2) (uint 1)) + (offset-less? start end) (stack-nonempty? frames) + (not (state ready)) (and (bool #t) (state ready)) + (or (state ready) (bool #t)) + (line-starts-with "a") (line-starts-with-ascii-ci "A") + (line-prefix-boundary-ascii-ci "a") (line-marker-ascii-ci "a") + (line-blank?) (line-has-word-after-prefix? "a") + (line-has-key-after-prefix? "a") + (line-byte-equal? start 97) + (line-bytes-all-in? start end (9 10 13 32 97 98)) + (line-bytes-any-in? start end (97 98)) + (line-bytes-all-in? start (line-step start) (97 98)) + (line-bytes-any-in? start (line-step start) (97 98)) + (line-bytes-all-in? end (line-step end) (97 98)) + (line-bytes-any-in? end (line-step end) (97 98)) + (line-bytes-all-in? start (line-step start) ()) + (line-bytes-any-in? start (line-step start) ()) + (line-bytes-in-set? start (line-content-end) ("a" "ab")) + (source-slices-equal? start end start end) + (source-slices-equal-ascii-ci? start end start end) + (future-line-marker-before-boundary? "END" "" "*" " " #t #f "") + (future-heading-title? "*" " " (uint 1) "END") + (future-named-line-marker-before-boundary? + start (line-content-end) "END " "" "" "*" " " #t #f #t) + (future-named-line-marker-before-boundary? + start (line-content-end) "END " "" "" "*" " " #t #f #t + start (line-content-end) "STOP " "" #t)) + (list + (cons 'and (append (make-list 96 '(state ready)) '((not (state ready))))) + (cons 'or (append (make-list 96 '(not (state ready))) '((state ready)))) + (cons 'or (make-list 96 '(not (state ready)))) + `(line-bytes-in-set? start (line-content-end) + ,(map (lambda (index) (string (integer->char (+ 32 index)))) (iota 95)))))) + +(def offsets + '(start end (line-prefix-end "a") (line-marker-end "*" " ") + (line-skip-horizontal start) (line-scan-word start) (line-scan-key start) + (line-scan-nonspace-until start ":") (line-scan-until start ":") + (line-physical-end start) (line-step start) (line-trim-end) + (line-trim-end-from start) (line-content-end) (state-offset saved))) + +(def initial '((ready #f) (count 0) (saved 0) (frames (uint-stack)))) +(def helpers + '((slice ((parameter 0) (ready #f)) + ((start-node Text) + (if (uint-positive? (state parameter)) + ((token Line start end)) ()) + (if (line-starts-with-ascii-ci "A") + ((start-node Heading) (finish-node)) ()) + (with-source-bounds start end + ((if (line-bytes-any-in? start end (97)) + ((token Line start end)) ()))) + (finish-node)) + (parameter)))) + +(def specifications + (list + (list 'native-predicates initial + (append '((set-bool ready (bool #t)) (push-frame frames (uint 1))) + (map (lambda (predicate) + `(if ,predicate + ((start-node Heading) (token Line start end) (finish-node)) + ((start-node Text) (token Line start end) (finish-node)))) + predicates) + '((pop-frame frames))) + '() '()) + (list 'native-offsets initial + (append '((set-uint saved (offset start)) + (if (line-bytes-any-in? start (line-content-end) (97 98)) + ((set-bool ready (bool #t))) ()) + (if (line-bytes-any-in? start (line-content-end) (98 99)) + ((start-node Heading) (finish-node) + (for-line-bytes look start (line-content-end) + ((token Line (line-index look) (line-content-end)))) + (with-source-bounds end end + ((if (line-bytes-all-in? (line-content-end) end ()) + ((start-node Text) (finish-node)) ())))) ())) + (map (lambda (offset) `(token Line ,offset end)) offsets) + '((for-line-bytes cursor start end + ((if (and (state ready) (line-byte-equal? (line-index cursor) 97)) + ((start-node Text) (finish-node)) + ((if (line-byte-equal? (line-index cursor) 98) + ((start-node Heading) (finish-node)) + ((if (line-byte-equal? (line-index cursor) 97) + ((start-node Section) (finish-node)) ()))))) + (token Line (line-index cursor) (line-step (line-index cursor))) + (if (and (state ready) + (and (line-byte-equal? (line-index cursor) 97) + (line-byte-equal? (line-step (line-index cursor)) 98))) + ((start-node Section) (finish-node)) ()) + (if (and (state ready) + (line-bytes-any-in? (line-index cursor) + (line-step (line-index cursor)) (97 98))) + ((start-node Heading) (finish-node)) ()) + (if (and (state ready) + (or (and (line-byte-equal? (line-index cursor) 97) + (line-byte-equal? (line-step (line-index cursor)) 98)) + (and (line-byte-equal? (line-index cursor) 98) + (line-byte-equal? (line-step (line-index cursor)) 97)))) + ((start-node Section) (finish-node)) ()) + (if (line-bytes-all-in? (line-index cursor) + (line-step (line-index cursor)) ()) + ((start-node Text) (finish-node)) ()) + (with-source-bounds start end + ((if (line-bytes-any-in? (line-index cursor) + (line-step (line-index cursor)) (97 98)) + ((start-node Section) (finish-node)) ()))))))) + '() '()) + (list 'native-states initial + '((set-uint count (uint-add (state count) (uint 1))) + (set-uint saved (uint-divide (uint-multiply (uint 2) (uint 3)) (uint 0))) + (set-uint saved (line-indent-column 8)) + (set-uint saved (line-marker-level "*" " ")) + (open-level frames (state count) Section) + (if (uint-positive? (stack-top frames)) ((token Line start end)) ()) + (close-through frames (state count)) + (open-level frames (uint 1) Section) + (close-all frames) + (push-frame frames (uint 2)) + (close-frame frames 2) + (push-frame frames (uint 3)) + (close-frames-while frames (uint-positive? (stack-top frames)) 2) + (push-frame frames (uint 4)) + (close-all-frames frames 2)) + '((if (uint-positive? (state count)) ((start-node Text) (finish-node)) ())) '()) + (list 'native-helpers initial + '((join-once handled + ((if (line-starts-with "a") + ((set-bool handled (bool #t)) + (call-source-helper slice start end ((uint 1)))) ())) + ((with-source-bounds start end + ((call-source-helper slice start end ((uint 1)))))))) + '() helpers) + (list 'native-markers + '((present #f) (column 0) (ordered #f) (bullet-start 0) (bullet-end 0) (content 0)) + '((scan-list-marker "-+" #t 8 present column ordered bullet-start bullet-end content) + (if (state present) + ((token Line (state-offset bullet-start) (state-offset content)) + (token Line (state-offset content) end)) + ((token Line start end)))) + '() '()) + (list 'native-saved '((parameter 0) (saved 0) (limit 0)) + '((set-uint saved (offset start)) (set-uint limit (offset end))) + '((with-source-bounds (state-offset saved) (state-offset limit) + ((call-source-helper slice start end ((state parameter)))))) + helpers))) + +(def corpus + '("" "a" "ab\n" "a b:\r\nλ" " \t\r\n" "** END\nbody\n" + "END a\nEND ab\n" "- item\n 12) ordered\n" "é\r\nλ\rtail")) + +(def (generation-controls) + (def (rejected initial line finish helpers) + (unless + (with-catch (lambda (_) #t) + (lambda () + (event-fold-scheme-source 'invalid event-lines-language-grammar 'Document + initial line finish helpers) + #f)) + (error "native compiler admitted invalid declaration"))) + (rejected '() '((unknown-transition)) '() '()) + (rejected '((flag #f)) '((set-uint flag (uint 1))) '() '()) + (rejected '() '((call-source-helper missing start end)) '() '()) + (rejected '() '() '() + '((recursive () ((call-source-helper recursive start end))))) + (let (cycle (list '(token Line start end))) + (set-cdr! cycle cycle) + (rejected '() cycle '() '())) + (displayln "GENERATION-OK rejection-controls") + (force-output)) + +(def (native-boundary-controls port) + ;; The interpreted decoder is an independent oracle for borrowed UTF-8 + ;; bounds, including valid empty spans inside a multibyte character. + (map + (lambda (spec) + (let* ((name (car spec)) (line (cadr spec)) (helpers (caddr spec)) + (input "é\n") (initial '((cut 1))) + (expected + (with-catch (lambda (_) 'rejected) + (lambda () (run-event-fold input 'Document initial line '() helpers))))) + (unless (eq? (eq? expected 'rejected) (cadddr spec)) + (error "UTF-8 boundary oracle disagrees with control" name expected)) + (display (event-fold-scheme-source name event-lines-language-grammar 'Document + initial line '() helpers) port) + `(begin + (unless (equal? (with-catch (lambda (_) 'rejected) + (lambda () (,name ,input))) ',expected) + (error "native borrowed UTF-8 boundary mismatch" ',name)) + (displayln "CASE-OK " ',name) (force-output)))) + '((native-split-start + ((with-source-bounds (state-offset cut) end ((token Line start end)))) () #t) + (native-split-end + ((with-source-bounds start (state-offset cut) ((token Line start end)))) () #t) + (native-split-helper + ((call-source-helper slice (state-offset cut) end)) + ((slice () ((token Line start end)))) #t) + (native-empty-midpoint + ((with-source-bounds (state-offset cut) (state-offset cut) ((token Line start end)))) () #f)))) + +(def (main output) + (generation-controls) + (call-with-output-file output + (lambda (port) + (let (controls '()) + (for-each + (lambda (spec) + (let* ((name (car spec)) (initial (cadr spec)) (line (caddr spec)) + (finish (cadddr spec)) (helpers (list-ref spec 4)) + (digest (hash-get + (event-fold-ir name event-lines-language-grammar 'Document + initial line finish helpers) + "parser_digest")) + (binder (string->symbol (string-append "bind-" (symbol->string name))))) + (let (source + (event-fold-scheme-source + name event-lines-language-grammar 'Document initial line finish helpers)) + (unless (equal? source + (event-fold-scheme-source + name event-lines-language-grammar 'Document initial line finish helpers)) + (error "native fold source is not deterministic" name)) + (display source port)) + (set! controls (cons `(unless (eq? (,binder ,digest) ,name) + (error "native binding did not return executor")) + controls)) + (set! controls + (cons `(unless + (with-catch (lambda (_) #t) + (lambda () (,binder "wrong-digest") #f)) + (error "native binding admitted wrong identity")) + controls)) + (for-each + (lambda (source) + (let (expected (run-event-fold source 'Document initial line finish helpers)) + (set! controls + (cons `(begin + (unless (equal? (,name ,source) ',expected) + (error "native EventFold mismatch" ',name ,source)) + (let* ((bytes (string->utf8 ,source)) + (before (u8vector->list bytes))) + (unless (and (equal? (,name bytes) ',expected) + (equal? before (u8vector->list bytes))) + (error "byte-input EventFold parity or ownership mismatch" ',name))) + (displayln "CASE-OK " ',name) + (force-output)) + controls)))) + corpus) + (let (parameter (car (filter + (lambda (entry) (exact-integer? (cadr entry))) initial))) + (for-each + (lambda (source) + (for-each + (lambda (value) + (let* ((overrides (list (cons (car parameter) value))) + (expected (run-event-fold source 'Document initial + line finish helpers overrides))) + (let ((tape-name (string->symbol (string-append (symbol->string name) "-tape"))) + (wire (canonical-fold-tape event-lines-language-grammar expected))) + (set! controls + (cons `(begin + (unless (equal? (,tape-name ,source ',overrides) ',wire) + (error "direct Fold tape override mismatch" ',name)) + (displayln "CASE-TAPE-OK parameters " ',name) (force-output)) controls))) + (set! controls + (cons `(begin + (unless (equal? (,name ,source ',overrides) ',expected) + (error "native fold override or isolation mismatch" ',name)) + (displayln "CASE-OK parameters " ',name) + (force-output)) + controls)))) + '(0 7 0))) + '("a\n" ""))) + ;; Hold published output while reusing the executor, then mutate + ;; only its caller-owned spine. Neither later calls nor the + ;; empty root may share those cells with a prior publication. + (let ((first-source "é\r\nλ\rtail") + (expected (run-event-fold "é\r\nλ\rtail" 'Document initial line finish helpers)) + (empty (run-event-fold "" 'Document initial line finish helpers))) + (set! controls + (cons `(let* ((held (,name ,first-source)) (other (,name ""))) + (unless (and (equal? held ',expected) (equal? other ',empty) + (equal? (,name ,first-source) ',expected) + (equal? held ',expected) (not (eq? held other))) + (error "native publication retained a request spine" ',name)) + (set-cdr! held '()) + (set-cdr! other '()) + (unless (and (equal? (,name ,first-source) ',expected) + (equal? (,name "") ',empty)) + (error "native publication shares caller-owned cells" ',name)) + (displayln "CASE-OK publication " ',name) + (force-output)) + controls))) + (displayln "GENERATION-OK " name) + (let (tape-name (string->symbol (string-append (symbol->string name) "-tape"))) + (display (event-fold-scheme-source tape-name event-lines-language-grammar 'Document + initial line finish helpers '() 'tape) port) + (for-each + (lambda (text) + (let (expected (canonical-fold-tape event-lines-language-grammar + (run-event-fold text 'Document initial line finish helpers))) + (set! controls + (cons `(let* ((first (,tape-name ,text)) (held (u8vector->list first)) + (input (string->utf8 ,text)) (before (u8vector->list input))) + (unless (and (equal? first ',expected) + (equal? (,tape-name input) ',expected) + (equal? before (u8vector->list input))) + (error "direct Fold tape wire/input mismatch" ',tape-name)) + (,tape-name "") + (unless (equal? held (u8vector->list first)) + (error "direct Fold tape publication changed" ',tape-name)) + (when (> (u8vector-length first) 0) (u8vector-set! first 0 0)) + (unless (equal? (,tape-name ,text) ',expected) + (error "direct Fold tape aliases published bytes" ',tape-name)) + (displayln "CASE-TAPE-OK " ',tape-name) (force-output)) controls)))) + corpus)) + (force-output))) + specifications) + (set! controls (append (reverse (native-boundary-controls port)) controls)) + (write '(export main) port) (newline port) + (write `(def (main . args) ,@(reverse controls) (displayln "OK") (force-output)) port) + (newline port))))) diff --git a/t/generate-event-strategy-grammar.ss b/t/generate-event-strategy-grammar.ss index f40e5004..30bc67eb 100644 --- a/t/generate-event-strategy-grammar.ss +++ b/t/generate-event-strategy-grammar.ss @@ -1,14 +1,14 @@ #!/usr/bin/env gxi ;;; -*- Gerbil -*- -;;; Produce the Rowan descriptor from the same grammar as the event algorithm. +;;; Produce the Rust descriptor from the same grammar as the event algorithm. (import (only-in "event-strategy-fixture.ss" event-lines-language-grammar) - (only-in :gerbil-parser/src/compiler/rust-rowan - generate-language-rust-rowan-module)) + (only-in :gerbil-parser/src/compiler/rust-runtime + generate-language-rust-runtime-module)) (def arguments (command-line)) (unless (> (length arguments) 2) (error "usage: generate-event-strategy-grammar.ss OUTPUT.rs")) -(generate-language-rust-rowan-module +(generate-language-rust-runtime-module (car (reverse arguments)) event-lines-language-grammar) diff --git a/t/generate-header-delimiters.ss b/t/generate-header-delimiters.ss new file mode 100644 index 00000000..410e6339 --- /dev/null +++ b/t/generate-header-delimiters.ss @@ -0,0 +1,17 @@ +;;; Canonical Rust outputs from two independent Scheme language declarations. +(import (only-in :gerbil-parser/language-build-support + make-rust-runtime-strategy emit-build-strategy) + (only-in :clan/poo/object .ref) + (only-in :gerbil-parser/languages/hl7/parser hl7-language) + (only-in "header-delimiter-test.ss" header-record-language)) +(export main) +(def (emit loader output) + (let (strategy (make-rust-runtime-strategy (.ref loader 'descriptor))) + (call-with-output-file output + (lambda (port) (emit-build-strategy strategy port))))) +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(def (main hl7-output record-output) + (emit hl7-language hl7-output) + (displayln "HEADER-DELIMITER-HL7-GENERATED") (force-output) + (emit header-record-language record-output) + (displayln "HEADER-DELIMITER-RECORD-GENERATED") (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-language-abi-alignment.ss b/t/generate-language-abi-alignment.ss new file mode 100644 index 00000000..a4e6a56c --- /dev/null +++ b/t/generate-language-abi-alignment.ss @@ -0,0 +1,42 @@ +;;; -*- Gerbil -*- +;;; Native ABI fixture publication has a build owner separate from test assertions. +(import (prefix-in (only-in :gerbil-parser/src/ffi/language-artifact-codec bind-language-abi abi-descriptor-payload abi-parse-binary-payload) codec-) + (only-in :gerbil-parser/language-support/entry language-metadata-ref) + (only-in :clan/poo/object .ref) + (only-in :gerbil-parser/t/fixtures/language-entries language-entry-pairs) + (only-in :gerbil-parser/language-support/development language-loader-fixtures) + (only-in :gerbil-parser/language-support/fixture syntax-fixture-source) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success?)) +(export write-language-abi-alignment!) +;;; Each native backend is checked from its configured public parser descriptor. +(def (write-language-abi-alignment! output) + (def (emit-bytes port bytes) + (let (length (u8vector-length bytes)) + (for-each (lambda (shift) (write-u8 (bitwise-and 255 (arithmetic-shift length (- shift))) port)) + '(0 8 16 24))) + (write-u8vector bytes port)) + (call-with-output-file output + (lambda (port) + (for-each + (lambda (row) + (let* ((entry (car row)) (development (cadr row)) + (metadata (.ref entry 'metadata)) + (name (language-metadata-ref metadata 'contract)) + (native (codec-bind-language-abi (.ref entry 'descriptor)))) + (for-each + (lambda (status) + (let (fixtures (language-loader-fixtures development status)) + (when (null? fixtures) (error "missing native alignment control" name status)) + (let* ((source (syntax-fixture-source (car fixtures))) + (artifact ((.ref entry '.parse) source))) + (unless (eq? (parse-artifact-success? artifact) (eq? status 'accepted)) + (error "native alignment fixture status mismatch" name status)) + (emit-bytes port (string->utf8 name)) + (emit-bytes port (string->utf8 (codec-abi-descriptor-payload native))) + (emit-bytes port (string->utf8 source)) + (emit-bytes port (codec-abi-parse-binary-payload native source)) + (write-u8 (if (eq? status 'accepted) 1 0) port) + (displayln "NATIVE-ALIGNMENT-CASE " name " " status) (force-output)))) + '(accepted rejected)))) + language-entry-pairs))) + (displayln "LANGUAGE-ABI-ALIGNMENT-GENERATED") (force-output)) diff --git a/t/generate-module-source.ss b/t/generate-module-source.ss new file mode 100644 index 00000000..225d4ef6 --- /dev/null +++ b/t/generate-module-source.ss @@ -0,0 +1,15 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :gerbil-parser/src/compiler/rust-runtime language-rust-runtime-module-source) + (only-in :gerbil-parser/t/module-source-test module-framing-language-grammar) + (only-in "module-source-cases.ss" module-source-cases)) +(def (main output) + (call-with-output-file output (lambda (port) + (display (language-rust-runtime-module-source module-framing-language-grammar) port) + (display "\npub const MODULE_CASES: &[(&str, usize, Option)] = &[\n" port) + (for-each (lambda (row) + (display "(" port) (write (car row) port) (display ", " port) (display (cadr row) port) (display ", " port) + (if (caddr row) (begin (display "Some(" port) (display (caddr row) port) (display ")" port)) (display "None" port)) + (display "),\n" port)) module-source-cases) + (display "];\n" port))) + (displayln "MODULE-SOURCE-RUST-GENERATED") (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-ordered-results.ss b/t/generate-ordered-results.ss new file mode 100644 index 00000000..9adf62f6 --- /dev/null +++ b/t/generate-ordered-results.ss @@ -0,0 +1,69 @@ +#!/usr/bin/env gxi +;;; Scheme-owned lexical field order for catalog constructors used by Commands. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :gerbil-parser/t/fixtures/bash-products bash-results) + (only-in :gerbil-parser/src/compiler/rust-runtime result-profile-rust-source) + (only-in :gerbil-parser/src/language/result-profile compile-result-profile result-plan-node result-plan-recipe) + (only-in :gerbil-parser/src/runtime/recognition make-recognition-child recognition-node-kind recognition-node-start recognition-node-end recognition-node-children recognition-child-field recognition-child-value) + (only-in :gerbil-parser/src/runtime/token make-token token? token-kind token-start token-end)) +;;; Repeated fields retain lexical order, including interleaved fields. +(def controls + '(("abc" (SimpleCommand 0 3 + (name (Word 0 1 (part (token LiteralPart 0 1)))) + (redirect (Redirection 1 2 (operator (token operator 1 2)))) + (argument (Word 2 3 (part (token LiteralPart 2 3)))))) + ("a|b" (Pipeline 0 3 + (command (SimpleCommand 0 1 (name (Word 0 1 (part (token LiteralPart 0 1)))))) + (operator (token operator 1 2)) + (command (SimpleCommand 2 3 (name (Word 2 3 (part (token LiteralPart 2 3)))))))) + ("αβ😀" (BashFile 0 8 + (command (SimpleCommand 0 8 + (argument (Word 0 2 (part (token LiteralPart 0 2)))) + (name (Word 2 4 (part (token LiteralPart 2 4)))) + (argument (Word 4 8 (part (token LiteralPart 4 8)))))))) + ("" (BashFile 0 0)))) +(def (main output) + (let* ((plan (compile-result-profile bash-results)) (recipe (result-plan-recipe plan)) + (kinds (append (map car (car recipe)) (cadr recipe)))) + (def (index value (token? #f)) + (let loop ((values (if token? (cadr recipe) (map car (car recipe)))) (at (if token? (length (car recipe)) 0))) (cond ((null? values) (error "unknown result kind" value)) + ((eq? value (car values)) at) (else (loop (cdr values) (+ at 1)))))) + (def (recognize row source) + (if (eq? (car row) 'token) + (make-token (cadr row) "" (caddr row) (cadddr row)) + (result-plan-node plan (car row) (cadr row) (caddr row) + (map (lambda (child) (make-recognition-child (car child) (recognize (cadr child) source))) (cdddr row))))) + (def (emit port node) + (display "profile.bind_node(" port) (write (symbol->string (recognition-node-kind node)) port) + (display ")?.build(source," port) (display (recognition-node-start node) port) (display ".." port) (display (recognition-node-end node) port) + (display ",vec![" port) + (for-each (lambda (child) + (let (value (recognition-child-value child)) + (display "ResultChildCapture {field:" port) (write (symbol->string (recognition-child-field child)) port) + (display ",value:ProjectedValue::" port) + (if (token? value) + (begin (display "Token {kind:" port) (display (index (token-kind value) #t) port) (display ",span:" port) + (display (token-start value) port) (display ".." port) (display (token-end value) port) (display "}" port)) + (begin (display "Node(" port) (emit port value) (display "?)" port))) + (display "}," port))) (recognition-node-children node)) + (display "])" port)) + (def (events port value) + (if (token? value) + (begin (display "TreeEvent::Token {kind:" port) (display (index (token-kind value) #t) port) + (display ",start:" port) (display (token-start value) port) (display ",end:" port) (display (token-end value) port) (display "}," port)) + (begin (display "TreeEvent::StartNode(" port) (display (index (recognition-node-kind value)) port) (display ")," port) + (for-each (lambda (child) (events port (recognition-child-value child))) (recognition-node-children value)) + (display "TreeEvent::FinishNode," port)))) + (call-with-output-file output (lambda (port) + (display (result-profile-rust-source plan) port) + (display "use gerbil_parser_runtime::{PreparedResultProfile, ProjectedNode, ResultChildCapture, ProjectedValue, Diagnostic, TreeEvent};\n" port) + (for-each (lambda (row at) + (let (node (recognize (cadr row) (car row))) + (display "pub fn replay_" port) (display at port) + (display "<'s>(profile:&PreparedResultProfile,source:&'s str)->Result,Diagnostic>{" port) + (emit port node) (display "}\n" port))) controls (iota (length controls))) + (display "pub static SOURCES:&[&str]=&[" port) (for-each (lambda (row) (write (car row) port) (display "," port)) controls) + (display "];\npub static EVENTS:&[&[TreeEvent]]=&[" port) + (for-each (lambda (row) (display "&[" port) (events port (recognize (cadr row) (car row))) (display "]," port)) controls) + (display "];\n" port))) + (displayln "ORDERED-RESULTS-RUST-GENERATED controls=" (length controls)) (force-output) (test-child-process-exit! 0))) diff --git a/t/generate-quoted-string-profile.ss b/t/generate-quoted-string-profile.ss new file mode 100644 index 00000000..a47386fd --- /dev/null +++ b/t/generate-quoted-string-profile.ss @@ -0,0 +1,8 @@ +#!/usr/bin/env gxi +;;; Generate the Rust witness exclusively from the Scheme grammar declaration. +(import (only-in :gerbil-parser/src/compiler/rust-runtime generate-language-rust-runtime-module) + (only-in "quoted-string-profile-test.ss" quoted-profile-language-grammar)) +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(def (main output) + (generate-language-rust-runtime-module output quoted-profile-language-grammar) + (displayln "QUOTED-PROFILE-RUST-GENERATED") (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-region-boundaries.ss b/t/generate-region-boundaries.ss new file mode 100644 index 00000000..56009c7f --- /dev/null +++ b/t/generate-region-boundaries.ss @@ -0,0 +1,55 @@ +#!/usr/bin/env gxi +;;; Scheme executes production and generic declarations; Rust recognizes sources independently. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :gerbil-parser/t/fixtures/bash-products bash-word-regions) + (only-in :gerbil-parser/src/compiler/rust-scanner region-plan-rust-source) + (only-in :gerbil-parser/src/runtime/region-scanner prepare-region-plan region-plan-specification + region-plan-end prepare-scoped-region-source region-source-pair-end region-source-quote-end)) +(def generic (prepare-region-plan '((";") ((#\« #t ("λ{"))) + (("λ{" #\{ #\} 1) ("[" #\[ #\] 1)) #f))) +(def generic-scopes '(("λ{" "λ{") ("[" "λ{" "["))) +(def sources '("α😀" "'α😀'" "\"α${x:-中}\"" "${arr[${i:-1}]:-λ}" "$((1+$(echo α)))" + "<(echo \"中\")>stop" "`echo α`" "x\\α;tail" "x tail" "'unterminated" + "${x:-\"bad}" "${x:-${y:-z}}" "λ{[}tail" "[λ{[}λ{中}]" "«λ{中}»;tail" + "«\\»中»" "[λ{中}" ";tail" "${x}" "λ{λ{中}}")) +(def (byte-at text at) (u8vector-length (string->utf8 (substring text 0 at)))) +(def (attempt text thunk) + (with-catch (lambda (_) -1) + (lambda () (let (end (thunk)) (if end (byte-at text end) -2))))) +(def (main output) + (call-with-output-file output + (lambda (port) + (display "// @generated by Scheme region execution; do not edit.\n" port) + (for-each + (lambda (entry) + (let* ((name (car entry)) (plan (cadr entry)) (scopes (caddr entry)) + (data (region-plan-specification plan))) + (display "pub mod " port) (display name port) (display " {\n" port) + (display (region-plan-rust-source plan scopes) port) + (display "type Request = (u32, usize, i64);\npub static TRACES: &[(&str, &[Request])] = &[\n" port) + (for-each + (lambda (text) + (let (source (prepare-scoped-region-source plan text scopes)) + (display "(" port) (write text port) (display ", &[" port) + ;; Plain word matching has no scope; scoped boundary requests use source caches. + (display "(0,0," port) (display (attempt text (lambda () (region-plan-end plan text 0))) port) (display ")," port) + (let loop ((at 0)) + (when (< at (string-length text)) + (when (ormap (lambda (row) + (let (prefix (car row)) + (and (<= (+ at (string-length prefix)) (string-length text)) + (string=? prefix (substring text at (+ at (string-length prefix))))))) (caddr data)) + (display "(1," port) (display (byte-at text at) port) (display "," port) + (display (attempt text (lambda () (region-source-pair-end source at))) port) (display ")," port)) + (let (quote (assv (string-ref text at) (cadr data))) + (when quote + (display "(" port) (display (char->integer (car quote)) port) (display "," port) + (display (byte-at text at) port) (display "," port) + (display (attempt text (lambda () (region-source-quote-end source at (car quote)))) port) + (display ")," port))) + (loop (+ at 1)))) + (display "]),\n" port))) sources) + (display "];\n}\n" port))) + (list (list "bash" bash-word-regions '()) (list "generic" generic generic-scopes))))) + (displayln "REGION-BOUNDARIES-RUST-GENERATED sources=" (* 2 (length sources))) + (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-result-projections.ss b/t/generate-result-projections.ss new file mode 100644 index 00000000..11f397c2 --- /dev/null +++ b/t/generate-result-projections.ss @@ -0,0 +1,81 @@ +#!/usr/bin/env gxi +;;; Scheme owns tables and recognized captures; Rust independently executes the +;;; construction IR. These controls do not claim Rust Word recognition. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :gerbil-parser/t/fixtures/bash-products bash-results bash-parts bash-word-regions) + (only-in :gerbil-parser/src/compiler/rust-runtime result-profile-rust-source) + (only-in :gerbil-parser/src/language/result-profile compile-result-profile result-plan-recipe) + (only-in :gerbil-parser/src/language/part-profile compile-part-profile) + (only-in :gerbil-parser/src/runtime/shell-word make-shell-word-parser) + (only-in :gerbil-parser/src/runtime/token make-token token? token-kind token-start token-end) + (only-in :gerbil-parser/src/runtime/recognition recognition-node-kind recognition-node-start recognition-node-end + recognition-node-children recognition-child-field recognition-child-value)) +(def controls '((word "") (word "α😀") (word "'α😀'") (word "\"α${x:-中}\"") + (word "$'a\\n'") (word "${arr[${i:-1}]:-λ}") (word "$(echo α)") + (word "$((1+2))") (word "<(echo 中)") (word "`echo α`") (word "$x\\α") + (assignment "name+=\"α${x:-中}\"") (here "α${x:-中}\\\n"))) +(def (index-of values predicate) + (let loop ((values values) (at 0)) + (cond ((null? values) (error "unknown result kind")) ((predicate (car values)) at) + (else (loop (cdr values) (+ at 1)))))) +(def (span port value) + (display (if (token? value) (token-start value) (recognition-node-start value)) port) + (display ".." port) (display (if (token? value) (token-end value) (recognition-node-end value)) port)) +(def (main output) + (let* ((plan (compile-result-profile bash-results)) (recipe (result-plan-recipe plan)) + (nodes (car recipe)) (terminals (cadr recipe))) + (def (node-id kind) (index-of nodes (lambda (row) (eq? (car row) kind)))) + (def (token-id kind) (+ (length nodes) (index-of terminals (lambda (value) (eq? value kind))))) + (def (replay port node) + (let* ((id (recognition-node-kind node)) (projection (assq id (caddr recipe))) + (children (recognition-node-children node))) + (display "profile.project(" port) (write (symbol->string id) port) (display ", source, " port) + (span port node) (display ", vec![" port) + (for-each (lambda (operation) + (let (values (map recognition-child-value (filter (lambda (child) (eq? (recognition-child-field child) (cadr operation))) children))) + (case (car operation) + ((token) (if (null? values) (display "ResultCapture::Absent" port) + (begin (display "ResultCapture::Span(" port) (span port (car values)) (display ")" port)))) + ((one) (if (null? values) (display "ResultCapture::Absent" port) + (begin (display "ResultCapture::Node(" port) (replay port (car values)) (display "?)" port)))) + ((many) (display "ResultCapture::Parts(vec![" port) + (for-each (lambda (value) (replay port value) (display "?," port)) values) (display "])" port))) + (display "," port))) (cdddr projection)) + (display "])" port))) + (def (events port value) + (if (token? value) + (begin (display "TreeEvent::Token {kind: " port) (display (token-id (token-kind value)) port) + (display ", start: " port) (display (token-start value) port) (display ", end: " port) + (display (token-end value) port) (display "}," port)) + (begin (display "TreeEvent::StartNode(" port) (display (node-id (recognition-node-kind value)) port) (display ")," port) + (for-each (lambda (child) (events port (recognition-child-value child))) (recognition-node-children value)) + (display "TreeEvent::FinishNode," port)))) + (def (fields port node) + (for-each (lambda (child) + (display "(" port) (write (symbol->string (recognition-node-kind node)) port) (display "," port) + (write (symbol->string (recognition-child-field child)) port) (display ")," port) + (unless (token? (recognition-child-value child)) (fields port (recognition-child-value child)))) + (recognition-node-children node))) + (let-values (((word assignment here) (make-shell-word-parser bash-word-regions plan (compile-part-profile bash-parts)))) + (let (recognized (map (lambda (control) + (let* ((source (cadr control)) (token (make-token 'word source 0 (u8vector-length (string->utf8 source)))) + (parser (case (car control) ((word) word) ((assignment) assignment) ((here) here)))) + (let-values (((node tokens) (parser token))) (unless node (error "control failed recognition" control)) node))) controls)) + (call-with-output-file output (lambda (port) + (display (result-profile-rust-source plan) port) + (display "use gerbil_parser_runtime::{PreparedResultProfile, ProjectedNode, ResultCapture, Diagnostic, TreeEvent};\n" port) + (display "pub static SOURCES: &[&str] = &[" port) + (for-each (lambda (row) (write (cadr row) port) (display "," port)) controls) (display "];\n" port) + (for-each (lambda (node index) + (display "fn replay_" port) (display index port) + (display "<'s>(profile: &PreparedResultProfile, source: &'s str) -> Result, Diagnostic> {" port) + (replay port node) (display "}\n" port)) recognized (iota (length controls))) + (display "pub fn replay<'s>(profile: &PreparedResultProfile, source: &'s str, index: usize) -> Result, Diagnostic> {match index {\n" port) + (for-each (lambda (index) (display index port) (display " => replay_" port) (display index port) + (display "(profile, source),\n" port)) (iota (length controls))) + (display "_ => panic!(\"unknown generated control\")}}\n" port) + (display "pub static EXPECTED: &[&[TreeEvent]] = &[" port) + (for-each (lambda (node) (display "&[" port) (events port node) (display "]," port)) recognized) (display "];\n" port) + (display "pub static FIELDS: &[&[(&str,&str)]] = &[" port) + (for-each (lambda (node) (display "&[" port) (fields port node) (display "]," port)) recognized) (display "];\n" port))) + (displayln "RESULT-PROJECTIONS-RUST-GENERATED controls=" (length controls)) (force-output) (test-child-process-exit! 0))))) diff --git a/t/generate-scheme-alphabetic-class.ss b/t/generate-scheme-alphabetic-class.ss new file mode 100644 index 00000000..d9753f1d --- /dev/null +++ b/t/generate-scheme-alphabetic-class.ss @@ -0,0 +1,29 @@ +#!/usr/bin/env gxi +;;; Freeze the producer's Scheme alphabetic character class for Rust execution. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(def (main output) + (displayln "SCHEME-ALPHABETIC-CLASS-SCAN") (force-output) + (let (ranges + (let loop ((code 0) (start #f) (rows '())) + (if (= code #x110000) + (reverse (if start (cons (cons start (- code 1)) rows) rows)) + (let (alphabetic? (and (not (<= #xd800 code #xdfff)) + (char-alphabetic? (integer->char code)))) + (cond ((and alphabetic? (not start)) (loop (+ code 1) code rows)) + ((and (not alphabetic?) start) + (loop (+ code 1) #f (cons (cons start (- code 1)) rows))) + (else (loop (+ code 1) start rows))))))) + (call-with-output-file output + (lambda (port) + (display "// @generated by t/generate-scheme-alphabetic-class.ss; do not edit.\n" port) + (display "// Source authority: pinned native Scheme char-alphabetic? class.\n" port) + (display "const RANGES: &[(u32, u32)] = &[\n" port) + (for-each (lambda (range) + (display " (0x" port) (display (number->string (car range) 16) port) + (display ", 0x" port) (display (number->string (cdr range) 16) port) + (display "),\n" port)) ranges) + (display "];\n\npub(super) fn is_scheme_alphabetic(character: char) -> bool {\n" port) + (display " let code = character as u32;\n" port) + (display " let index = RANGES.partition_point(|&(start, _)| start <= code);\n" port) + (display " index > 0 && code <= RANGES[index - 1].1\n}\n" port))) + (displayln "SCHEME-ALPHABETIC-CLASS-GENERATED ranges=" (length ranges))) (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-scheme-numeric-class.ss b/t/generate-scheme-numeric-class.ss new file mode 100644 index 00000000..f364476f --- /dev/null +++ b/t/generate-scheme-numeric-class.ss @@ -0,0 +1,29 @@ +#!/usr/bin/env gxi +;;; Freeze the producer's Scheme numeric character class for Rust execution. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(def (main output) + (displayln "SCHEME-NUMERIC-CLASS-SCAN") (force-output) + (let (ranges + (let loop ((code 0) (start #f) (rows '())) + (if (= code #x110000) + (reverse (if start (cons (cons start (- code 1)) rows) rows)) + (let (numeric? (and (not (<= #xd800 code #xdfff)) + (char-numeric? (integer->char code)))) + (cond ((and numeric? (not start)) (loop (+ code 1) code rows)) + ((and (not numeric?) start) + (loop (+ code 1) #f (cons (cons start (- code 1)) rows))) + (else (loop (+ code 1) start rows))))))) + (call-with-output-file output + (lambda (port) + (display "// @generated by t/generate-scheme-numeric-class.ss; do not edit.\n" port) + (display "// Source authority: pinned native Scheme char-numeric? class.\n" port) + (display "pub(super) fn is_scheme_numeric(character: char) -> bool {\n" port) + (display " character.is_ascii_digit() || matches!(character as u32,\n" port) + (let loop ((rest (cdr ranges))) + (unless (null? rest) + (display " 0x" port) (display (number->string (caar rest) 16) port) + (display "..=0x" port) (display (number->string (cdar rest) 16) port) + (display (if (null? (cdr rest)) "\n" " |\n") port) + (loop (cdr rest)))) + (display " )\n}\n" port))) + (displayln "SCHEME-NUMERIC-CLASS-GENERATED ranges=" (length ranges))) (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-scheme-whitespace-class.ss b/t/generate-scheme-whitespace-class.ss new file mode 100644 index 00000000..ba8bd1fc --- /dev/null +++ b/t/generate-scheme-whitespace-class.ss @@ -0,0 +1,29 @@ +#!/usr/bin/env gxi +;;; Freeze the producer's Scheme whitespace character class for Rust execution. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(def (main output) + (displayln "SCHEME-WHITESPACE-CLASS-SCAN") (force-output) + (let (ranges + (let loop ((code 0) (start #f) (rows '())) + (if (= code #x110000) + (reverse (if start (cons (cons start (- code 1)) rows) rows)) + (let (whitespace? (and (not (<= #xd800 code #xdfff)) + (char-whitespace? (integer->char code)))) + (cond ((and whitespace? (not start)) (loop (+ code 1) code rows)) + ((and (not whitespace?) start) + (loop (+ code 1) #f (cons (cons start (- code 1)) rows))) + (else (loop (+ code 1) start rows))))))) + (call-with-output-file output + (lambda (port) + (display "// @generated by t/generate-scheme-whitespace-class.ss; do not edit.\n" port) + (display "// Source authority: pinned native Scheme char-whitespace? class.\n" port) + (display "const RANGES: &[(u32, u32)] = &[\n" port) + (for-each (lambda (range) + (display " (0x" port) (display (number->string (car range) 16) port) + (display ", 0x" port) (display (number->string (cdr range) 16) port) + (display "),\n" port)) ranges) + (display "];\n\npub(crate) fn is_scheme_whitespace(character: char) -> bool {\n" port) + (display " let code = character as u32;\n" port) + (display " let index = RANGES.partition_point(|&(start, _)| start <= code);\n" port) + (display " index > 0 && code <= RANGES[index - 1].1\n}\n" port))) + (displayln "SCHEME-WHITESPACE-CLASS-GENERATED ranges=" (length ranges))) (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-structured-lexical.ss b/t/generate-structured-lexical.ss new file mode 100644 index 00000000..7ebb0893 --- /dev/null +++ b/t/generate-structured-lexical.ss @@ -0,0 +1,16 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/src/compiler/rust-runtime language-rust-runtime-module-source) + (only-in :gerbil-parser/t/structured-lexical-test structured-lexical-language-grammar) + (only-in "structured-lexical-cases.ss" structured-lexical-cases)) +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(def (main output) + (call-with-output-file output (lambda (port) + (display (language-rust-runtime-module-source structured-lexical-language-grammar) port) + (display "\npub const PROFILE_CASES: &[(&str, &str, Option)] = &[\n" port) + (for-each (lambda (group) (for-each (lambda (row) + (display "(" port) (write (symbol->string (car group)) port) (display ", " port) + (write (car row) port) (display ", " port) + (if (cadr row) (begin (display "Some(" port) (display (cadr row) port) (display ")" port)) (display "None" port)) + (display "),\n" port)) (cdr group))) structured-lexical-cases) + (display "];\n" port))) + (displayln "STRUCTURED-LEXICAL-RUST-GENERATED") (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-text-profile.ss b/t/generate-text-profile.ss new file mode 100644 index 00000000..3bf442f3 --- /dev/null +++ b/t/generate-text-profile.ss @@ -0,0 +1,25 @@ +#!/usr/bin/env gxi +;;; Full language and endpoint witnesses are produced by the Scheme owner. +(import (only-in :gerbil-parser/src/compiler/rust-runtime language-rust-runtime-module-source) + (only-in :gerbil-parser/languages/fhirpath/parser fhirpath-language-grammar) + (only-in "text-profile-cases.ss" fhirpath-profile-cases)) +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!)) +(def (main output) + (call-with-output-file output + (lambda (port) + (display (language-rust-runtime-module-source fhirpath-language-grammar) port) + (display "\npub const PROFILE_CASES: &[(&str, &str, Option)] = &[\n" port) + (for-each + (lambda (group) + (for-each + (lambda (row) + (display "(" port) (write (symbol->string (car group)) port) + (display ", " port) (write (car row) port) (display ", " port) + (if (cadr row) + (begin (display "Some(" port) (display (cadr row) port) (display ")" port)) + (display "None" port)) + (display "),\n" port)) + (cdr group))) + fhirpath-profile-cases) + (display "];\n" port))) + (displayln "TEXT-PROFILE-RUST-GENERATED") (force-output) (test-child-process-exit! 0)) diff --git a/t/generate-word-parts.ss b/t/generate-word-parts.ss new file mode 100644 index 00000000..15776420 --- /dev/null +++ b/t/generate-word-parts.ss @@ -0,0 +1,91 @@ +#!/usr/bin/env gxi +;;; Independent Rust recognition controls executed by the actual Scheme Word engine. +(import (only-in :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process test-child-process-exit!) + (only-in :clan/poo/object .o .ref) + (only-in :gerbil-parser/t/fixtures/bash-products bash-results bash-parts bash-word-regions bash-parameter-binding) + (only-in :gerbil-parser/src/compiler/rust-scanner part-profile-rust-source) + (only-in :gerbil-parser/src/compiler/rust-runtime result-profile-rust-source) + (only-in :gerbil-parser/src/language/result-profile compile-result-profile result-plan-recipe) + (only-in :gerbil-parser/src/language/part-profile compile-part-profile) + (only-in :gerbil-parser/src/runtime/region-scanner prepare-region-plan region-plan-specification) + (only-in :gerbil-parser/src/runtime/shell-word make-shell-word-parser) + (only-in :gerbil-parser/src/runtime/token make-token token? token-kind token-start token-end) + (only-in :gerbil-parser/src/runtime/recognition recognition-node-kind recognition-node-start recognition-node-end + recognition-node-children recognition-child-field recognition-child-value)) +(def controls + '((word "") (word "α😀") (word "'α😀'") (word "\"α${x:-中}\"") (word "$'a\\n'") + (word "${arr[${i:-1}]:-λ}") (word "$(echo α)") (word "$((1+2))") (word "<(echo 中)") + (word "`echo α`") (word "$x\\α") (assignment "name+=\"α${x:-中}\"") (here "α${x:-中}\\\n") + (word "$😀") (word "$?") (word "${#}") (word "${!arr[@]}") (word "${x:-${y:-λ}}") + (word "${x[1[2]]:=中}") (word "${x[${y:-[}]:-λ}") (word "${x//α/β}") (word "${123:-α}") + (word "\\") (word "a\\😀b") (word "\"a\\$x\\q\"") (word "$'a\\😀b'") (word "a$(echo 中)b${x}c") + (here "'raw\"text") (here "\\x\\α\\$x") (here "$(echo α)\n${x:-λ}") (here "\\`x\\\n") + (assignment "α_2=😀") (assignment "name=α${x[0]:-中}") (assignment "name=") + (assignment "name") (assignment "1name=α") (assignment "α-β=中") + (word "'") (word "\"unterminated") (word "${") (word "$(") (word "$((1)") + (word "${x[0}") (assignment "name='bad") (here "${x") + (word "λ{！arr［λ{i→中}]→😀}") (word "«αλ{x→中}«") (word "※😀tail") + (word "λ{x→λ{y→中}}") (word "λ{x［0]→中}") (word "λ{x［0}") + (word "a$😀$x") (here "literal※😀«raw") (word "λ{！}") (word "λ{！x→α}"))) +(def generic-binding (.o (:: self bash-parameter-binding) prefixes: '("！") operators: '("→") subscript: '("［" #\]))) +(def generic-parts + (.o (:: self bash-parts) + bindings: (map (lambda (row) (if (eq? (car row) 'parameter) (cons 'parameter generic-binding) row)) (.ref bash-parts 'bindings)) + rules: (append '(((word) (prefix "$" next) (name SimpleParameter simple)) + ((word DoubleQuoted HereDocument) (prefix "λ{" any) (parameter)) + ((word) (prefix "«" any) (quote DoubleQuoted 1 1 #\«)) + ((word) (prefix "※" any) (escape EscapeSequence))) (map (lambda (row) (if (string=? (cadadr row) "$") + (cons (remq 'word (car row)) (cdr row)) row)) (.ref bash-parts 'rules))))) +(def generic-regions + (let (data (region-plan-specification bash-word-regions)) + (prepare-region-plan (list (car data) (append (cadr data) '((#\« #t ("λ{")))) + (append (caddr data) '(("λ{" #\{ #\} 1))) (cadddr data))))) +(def (index-of values predicate) + (let loop ((values values) (at 0)) + (cond ((null? values) (error "unknown result kind")) ((predicate (car values)) at) + (else (loop (cdr values) (+ at 1)))))) +(def (main output) + (let* ((results (compile-result-profile bash-results)) (recipe (result-plan-recipe results)) + (nodes (car recipe)) (terminals (cadr recipe)) (count 0)) + (def (node-id kind) (index-of nodes (lambda (row) (eq? (car row) kind)))) + (def (token-id kind) (+ (length nodes) (index-of terminals (lambda (value) (eq? value kind))))) + (def (events port value) + (if (token? value) + (begin (display "TreeEvent::Token {kind: " port) (display (token-id (token-kind value)) port) + (display ", start: " port) (display (token-start value) port) (display ", end: " port) + (display (token-end value) port) (display "}," port)) + (begin (display "TreeEvent::StartNode(" port) (display (node-id (recognition-node-kind value)) port) (display ")," port) + (for-each (lambda (child) (events port (recognition-child-value child))) (recognition-node-children value)) + (display "TreeEvent::FinishNode," port)))) + (def (fields port node) + (for-each (lambda (child) + (display "(" port) (write (symbol->string (recognition-node-kind node)) port) (display "," port) + (write (symbol->string (recognition-child-field child)) port) (display ")," port) + (unless (token? (recognition-child-value child)) (fields port (recognition-child-value child)))) + (recognition-node-children node))) + (call-with-output-file output (lambda (port) + (display "// @generated by Scheme Word recognition; do not edit.\n" port) + (display "pub struct Control { pub mode: &'static str, pub source: &'static str, pub start: usize, pub end: usize, pub status: u8, pub events: &'static [gerbil_parser_runtime::TreeEvent], pub fields: &'static [(&'static str,&'static str)] }\n" port) + (for-each (lambda (entry) + (let* ((name (car entry)) (parts (cadr entry)) (regions (caddr entry))) + (display "pub mod " port) (display name port) (display " {\nuse gerbil_parser_runtime::TreeEvent;\n" port) + (display (result-profile-rust-source results) port) + (display (part-profile-rust-source parts results regions) port) + (display "pub static CONTROLS: &[super::Control] = &[\n" port) + (let-values (((word assignment here) (make-shell-word-parser regions results (compile-part-profile parts)))) + (for-each (lambda (control) + (let* ((text (cadr control)) (start (u8vector-length (string->utf8 "前:"))) + (end (+ start (u8vector-length (string->utf8 text)))) + (parser (case (car control) ((word) word) ((assignment) assignment) ((here) here))) + (outcome (with-catch (lambda (_) 'rejected) + (lambda () (let-values (((node tokens) (parser (make-token 'word text start end)))) node))))) + (display "super::Control { mode: " port) (write (symbol->string (car control)) port) + (display ", source: " port) (write (string-append "前:" text ":尾") port) + (display ", start: " port) (display start port) (display ", end: " port) (display end port) + (display ", status: " port) (display (cond ((eq? outcome 'rejected) 0) (outcome 1) (else 2)) port) + (display ", events: &[" port) (when (and outcome (not (eq? outcome 'rejected))) (events port outcome)) + (display "], fields: &[" port) (when (and outcome (not (eq? outcome 'rejected))) (fields port outcome)) + (display "] },\n" port) (set! count (+ count 1)))) controls)) + (display "];\n}\n" port))) + (list (list "bash" bash-parts bash-word-regions) (list "generic" generic-parts generic-regions))))) + (displayln "WORD-PARTS-RUST-GENERATED controls=" count) (force-output) (test-child-process-exit! 0))) diff --git a/t/generic-dispatch-contract-test.ss b/t/generic-dispatch-contract-test.ss new file mode 100644 index 00000000..7d8741db --- /dev/null +++ b/t/generic-dispatch-contract-test.ss @@ -0,0 +1,58 @@ +;;; Core CLOS preparation and contextual conflict contracts. +;;; These executable design controls do not install a parser backend. +(import :std/test + (only-in :core/poo-clos/interface .defgeneric .defmethod) + (only-in :gerbil-parser/src/compiler/contextual-dispatch + compile-contextual-dispatch contextual-dispatch-ref) + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-role make-contextual-method)) +(export generic-dispatch-contract-test) + +(def selections 0) +(.defgeneric test-select-semantic-plan (backend capability)) +(.defmethod (test-select-semantic-plan backend capability) + (error "unsupported test semantic plan" backend capability)) +(.defmethod (test-select-semantic-plan (backend (eql 'recognition)) capability) + (set! selections (+ selections 1)) + (lambda (value) (list 'recognition value))) +(.defmethod (test-select-semantic-plan (backend (eql 'event)) + (capability (eql 'plain))) + (set! selections (+ selections 1)) + (lambda (value) (list 'event value))) + +(.defgeneric test-context-selection (mode position) + (argument-precedence-order mode position)) +(.defmethod (test-context-selection (mode (eql 'shell)) position) 'mode-choice) +(.defmethod (test-context-selection mode (position (eql 'command))) 'position-choice) + +(def (contextual-intersection first-result second-result) + (compile-contextual-dispatch + (list (make-contextual-role 'example + (list (make-contextual-method 'by-mode 'shell 'any 'any first-result) + (make-contextual-method 'by-position 'any 'command 'any second-result)))) + '(shell) '(command) '(word))) + +(def generic-dispatch-contract-test + (test-suite "generic preparation and contextual contracts" + (test-case "normal values select a fixed kernel once at preparation" + (set! selections 0) + (let (kernel (test-select-semantic-plan 'event 'plain)) + (for-each (lambda (value) (check (kernel value) => (list 'event value))) + '(a b c d)) + (check selections => 1)) + (check ((test-select-semantic-plan 'recognition 'layout) 'input) + => '(recognition input)) + (check-exception (test-select-semantic-plan 'event 'layout) + (lambda (condition) + (equal? (error-message condition) + "unsupported test semantic plan")))) + (test-case "argument precedence cannot silently replace contextual conflict admission" + (check (test-context-selection 'shell 'command) => 'mode-choice) + (check-exception (contextual-intersection 'mode-choice 'position-choice) + (lambda (condition) + (equal? (error-message condition) "ambiguous contextual dispatch")))) + (test-case "equal contextual results retain both declaration origins" + (let (ir (contextual-intersection 'same-result 'same-result)) + (check (contextual-dispatch-ref ir 'cells) + => '((shell command word + (same-result ((example by-mode) (example by-position)))))))))) diff --git a/t/gql/actor-test.ss b/t/gql/actor-test.ss new file mode 100644 index 00000000..d1df8a72 --- /dev/null +++ b/t/gql/actor-test.ss @@ -0,0 +1,31 @@ +;;; -*- Gerbil -*- +(import :std/test + (only-in :gerbil-parser/parser-actor-support + spawn-parser-actor parser-actor-parse parser-actor-stop!) + :gerbil-parser/languages/gql/parser + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success?)) +(export gql-actor-test) +(def gql-actor-test + (test-suite "GQL optional library actor" + (test-case "concurrent callers preserve exact independent artifacts" + (let* ((actor (spawn-parser-actor gql-parser)) + (sources '("RETURN 1" "RETURN '你好'" "RETURN @" "RETURN 2")) + (expected (map parse-gql sources))) + (try + (let (clients (map (lambda (source) + (spawn (lambda () (parser-actor-parse actor source)))) + sources)) + (check (map (lambda (client) (thread-join! client 3)) clients) + => expected)) + (finally (check (parser-actor-stop! actor) => 'shutdown))))) + (test-case "invalid requests do not poison the actor" + (let (actor (spawn-parser-actor gql-parser)) + (try + (check-exception (parser-actor-parse actor 42) true) + (check-exception (parser-actor-parse actor "RETURN 1" 0) true) + (check (parse-artifact-success? (parser-actor-parse actor "RETURN 1")) => #t) + (finally (parser-actor-stop! actor))))) + (test-case "shutdown terminates the owned actor" + (let (actor (spawn-parser-actor gql-parser)) + (check (parser-actor-stop! actor) => 'shutdown) + (check-exception (parser-actor-parse actor "RETURN 1" .05) true))))) diff --git a/t/gql/benchmark-profile-test.ss b/t/gql/benchmark-profile-test.ss new file mode 100644 index 00000000..a619c585 --- /dev/null +++ b/t/gql/benchmark-profile-test.ss @@ -0,0 +1,139 @@ +;;; -*- Gerbil -*- +;;; Profiling must preserve lossless Unicode artifacts and LR token admission. +(import :std/test + (only-in :gerbil-parser/languages/gql/parser +gql-representative-query+) + (only-in "../benchmarks/gql/runtime/reduction-counts" profile-gql-reductions) + (only-in "../benchmarks/gql/runtime/execution-counts" profile-gql-prepared-execution) + (only-in "../benchmarks/gql/runtime/matched-stages" profile-gql-stages) + (only-in "../benchmarks/parser-stage-cost/benchmark" + sample-at-percentile measure-parser-batch + gc-statistics-snapshot sample-gc-snapshot + component-allocation-summary) + (only-in :gerbil-parser/src/runtime/parse-cost admit-parser-allocation)) +(export gql-benchmark-profile-test) +(def gql-benchmark-profile-test + (test-suite "GQL language performance diagnostics" + (test-case "timed batches reject truncation and changed products" + (let* ((calls 0) + (thunk (lambda () (set! calls (+ calls 1)) 'complete)) + (row (measure-parser-batch 'shared-control 7 4 thunk 'complete 2))) + (check calls => 2) + (check (cdr (assq 'sample row)) => 7) + (for-each + (lambda (sizes) + (check-exception + (measure-parser-batch 'invalid 0 (car sizes) thunk 'complete (cadr sizes)) true)) + '((0 1) (3 2) (4 0) (4.0 1))) + (check calls => 2) + (check-exception + (measure-parser-batch 'changed 0 1 (lambda () 'wrong) 'complete) true))) + (test-case "allocation summaries exclude GC intervals and retain unavailable ownership" + (let* ((rows + (map (lambda (observation) + (list (cons 'allocated-bytes + (admit-parser-allocation (car observation) (cadr observation))))) + '((1000 0) (2000 0) (999999 1) (-100 0)))) + (summary (component-allocation-summary rows 10 'single-caller))) + (check (cdr (assq 'allocationSampleCount summary)) => 2) + (check (cdr (assq 'allocatedBytesPerParse summary)) => 100) + (check (cdr (assq 'allocationP95BytesPerParse summary)) => 200) + (for-each + (lambda (unavailable) + (check (cdr (assq 'allocationSampleCount unavailable)) => 0) + (check (cdr (assq 'allocatedBytesPerParse unavailable)) => #f) + (check (cdr (assq 'allocationP95BytesPerParse unavailable)) => #f)) + (list (component-allocation-summary rows 10 'process-with-thread-switches) + (component-allocation-summary '(((allocated-bytes . #f))) 10 'single-caller))) + (check (cdr (assq 'allocatedBytesPerParse + (component-allocation-summary '(((allocated-bytes . 0))) 10 'single-caller))) + => 0))) + (test-case "Unicode and trivia retain exact artifacts through all stages" + (let (reports (profile-gql-stages "RETURN '你好' /* retained */\n" 2 2)) + (check (map (lambda (row) (cdr (assq 'stage row))) reports) + => '(global-lexing prepared-lr artifact-publication full-source)) + (for-each + (lambda (row) + (check (cdr (assq 'sampleCount row)) => 2) + (check (cdr (assq 'parsesPerSample row)) => 2) + (check (length (cdr (assq 'samples row))) => 2)) reports))) + (test-case "selected derivation counts expose reduction and action costs" + (let (counts (profile-gql-reductions +gql-representative-query+)) + (check (cdr (assq 'scope counts)) => 'selected-derivation) + (check (cdr (assq 'reductions counts)) => 288) + (check (cdr (assq 'concatenatingReductions counts)) => 75) + (check (+ (cdr (assq 'emptyReductions counts)) + (cdr (assq 'unaryReductions counts)) + (cdr (assq 'multiOperandReductions counts))) => 288) + (check (> (cdr (assq 'unaryReductions counts)) 0) => #t) + (check (cdr (assq 'identityOperands counts)) => 192) + (check (cdr (assq 'decoratedOperands counts)) => 132)) + (check-exception (profile-gql-reductions "RETURN @") true)) + (test-case "stepped executor accounts for actions and preserves directed artifacts" + (let (counts (profile-gql-prepared-execution +gql-representative-query+)) + (check (cdr (assq 'scope counts)) => 'prepared-lr-single-action-checkpoints) + (check (cdr (assq 'shifts counts)) => 37) + (check (cdr (assq 'reductions counts)) => 288) + (check (cdr (assq 'actions counts)) => 325) + (check (cdr (assq 'accepts counts)) => 1) + (check (cdr (assq 'literalFreeNonEofObservations counts)) => 0) + (check (cdr (assq 'caseFoldObservations counts)) => 21) + (check (cdr (assq 'caseFoldTokens counts)) => 14) + (check (cdr (assq 'repeatedCaseFoldObservations counts)) => 7) + (check (+ (cdr (assq 'nonEofObservations counts)) + (cdr (assq 'eofObservations counts))) => 326)) + (check (cdr (assq 'accepts (profile-gql-prepared-execution "match (n) return n"))) => 1) + (let ((upper (profile-gql-prepared-execution "MATCH (n) RETURN n")) + (lower (profile-gql-prepared-execution "match (n) return n"))) + ;; Exact literal hits short-circuit folding; lowercase contextual + ;; keywords add conversion observations without changing shifts. + (check (cdr (assq 'shifts upper)) => (cdr (assq 'shifts lower))) + (check (> (cdr (assq 'caseFoldObservations lower)) + (cdr (assq 'caseFoldObservations upper))) => #t)) + (check-exception (profile-gql-prepared-execution "RETURN @") true)) + (test-case "wall percentile retains its own CPU and GC observation" + (let* ((rows (map (lambda (sample) + (list (cons 'sample sample) (cons 'wall-ms sample) + (cons 'cpu-ms (- 21 sample)) + (cons 'gc-wall-ms (if (= sample 19) 11 0)))) + (iota 20 1))) + (tail (sample-at-percentile (reverse rows) 'wall-ms 95))) + (check (cdr (assq 'sample tail)) => 19) + (check (cdr (assq 'cpu-ms tail)) => 2) + (check (cdr (assq 'gc-wall-ms tail)) => 11))) + (test-case "GC snapshots retain VM live heap and distinguish stale collections" + (let ((before (make-f64vector 20 0.0)) + (after (make-f64vector 20 0.0))) + (f64vector-set! before 6 4.0) + (f64vector-set! after 6 4.0) + (f64vector-set! after 12 0.002) + (f64vector-set! after 13 0.001) + (f64vector-set! after 14 0.009) + (f64vector-set! after 15 8192.0) + (f64vector-set! after 16 6144.0) + (f64vector-set! after 17 2048.0) + (f64vector-set! after 18 1536.0) + (f64vector-set! after 19 512.0) + (check (sample-gc-snapshot before after) => #f) + ;; Multiple collections still expose only the last one's snapshot. + (f64vector-set! after 6 6.0) + (let (snapshot (sample-gc-snapshot before after)) + (check snapshot => (gc-statistics-snapshot after)) + (check (cdr (assq 'scope snapshot)) => 'whole-vm-latest-collection) + (check (cdr (assq 'gcCount snapshot)) => 6.0) + (check (cdr (assq 'cpuMs snapshot)) => 3.0) + (check (cdr (assq 'wallMs snapshot)) => 9.0) + (check (cdr (assq 'heapBytes snapshot)) => 8192.0) + (check (cdr (assq 'allocatedBytes snapshot)) => 6144.0) + (check (cdr (assq 'liveBytes snapshot)) => 2048.0) + (check (cdr (assq 'movableBytes snapshot)) => 1536.0) + (check (cdr (assq 'stillBytes snapshot)) => 512.0)))) + (test-case "equal wall observations use sample identity for tie breaking" + (let (rows '(((sample . 2) (wall-ms . 1)) + ((sample . 0) (wall-ms . 1)) + ((sample . 1) (wall-ms . 1)))) + (check (cdr (assq 'sample (sample-at-percentile rows 'wall-ms 50))) => 1) + (check (cdr (assq 'sample (sample-at-percentile rows 'wall-ms 100))) => 2))) + (test-case "rejected inputs cannot publish misleading stage receipts" + (check-exception (profile-gql-stages "RETURN @" 2 2) true)) + (test-case "invalid measurement sizes fail before work" + (check-exception (profile-gql-stages "RETURN 1" 0 2) true)))) diff --git a/t/gql-runtime-benchmark-test.ss b/t/gql/runtime-benchmark-test.ss similarity index 58% rename from t/gql-runtime-benchmark-test.ss rename to t/gql/runtime-benchmark-test.ss index 840038b4..b120ad62 100644 --- a/t/gql-runtime-benchmark-test.ss +++ b/t/gql/runtime-benchmark-test.ss @@ -5,15 +5,16 @@ (import :std/test (only-in :std/source this-source-file) :asp-gerbil-scheme/src/benchmark/framework - (only-in :gerbil-parser/languages/gql/iso-39075-2024/grammar + (only-in :gerbil-parser/languages/gql/parser +gql-representative-query+) - (only-in :gerbil-parser/languages/gql/iso-39075-2024/parser - parse-gql-iso-39075-2024) + (only-in :gerbil-parser/languages/gql/parser + parse-gql) (only-in :gerbil-parser/src/runtime/artifact - parse-artifact-success?)) + parse-artifact-success?) + (only-in :gerbil-parser/t/fixtures/progress report-test-progress!)) (def benchmark-path - (path-expand "benchmarks/gql-runtime/benchmark.ss" + (path-expand "../benchmarks/gql/runtime/benchmark.ss" (path-directory (this-source-file)))) (def +parse-count+ 100) @@ -22,9 +23,10 @@ (unless (zero? remaining) (unless (parse-artifact-success? - (parse-gql-iso-39075-2024 +gql-representative-query+)) + (parse-gql +gql-representative-query+)) (error "GQL runtime benchmark parse failed")) - (loop (- remaining 1))))) + (loop (- remaining 1)))) + (report-test-progress! "GQL-BATCH-OK parses=" +parse-count+)) (def gql-runtime-benchmark-tests (test-suite "ISO GQL parser runtime benchmark" @@ -36,6 +38,12 @@ (benchmark-contract-run benchmark-path parse-gql-batch)) (write receipt) (newline) + (force-output) (check (benchmark-contract-receipt-pass? receipt) => #t))))) -(export gql-runtime-benchmark-tests) +(def gql-runtime-benchmark-test gql-runtime-benchmark-tests) +(export gql-runtime-benchmark-tests gql-runtime-benchmark-test) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/grammar-composition-contract-test.ss b/t/grammar-composition-contract-test.ss new file mode 100644 index 00000000..3337620e --- /dev/null +++ b/t/grammar-composition-contract-test.ss @@ -0,0 +1,154 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- + +(import :std/test + (only-in :clan/poo/object .o) + :gerbil-parser/src/grammar/algebra + :gerbil-parser/src/grammar/lexical-algebra + :gerbil-parser/src/compiler/normalize + :gerbil-parser/src/compiler/parser-ir + :gerbil-parser/src/compiler/lr + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + :gerbil-parser/src/language/grammar + :gerbil-parser/src/modules/parser/interface + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-roundtrip parse-artifact-success?) + (only-in :gerbil-parser/src/runtime/funcs vector-intern-map) + (only-in :gerbil-parser/src/runtime/incremental + apply-edit make-edit make-incremental-session + incremental-session-artifact parse-incremental-session + parse-source/incremental) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + :gerbil-parser/src/runtime/recognition + (only-in :gerbil-parser/src/runtime/lr-parser + lr-checkpoint? lr-checkpoint-deterministic-actions + lr-checkpoint-deterministic-shifts + lr-checkpoint-remaining-token-count + lr-checkpoint-advance lr-checkpoint-resume + lr-initial-checkpoint lr-parse lr-parse/receipt lr-prepare + lr-runtime-lexical-mode-catalog) + (only-in :gerbil-parser/src/runtime/parser parse-source) + :gerbil-parser/src/runtime/token + :gerbil-parser/src/compiler/machine + :gerbil-parser/languages/arithmetic/parser) +(import :gerbil-parser/t/fixtures/grammar-composition-models) + +(def grammar-composition-contract-test + (test-suite "grammar composition contracts" + (test-case "POO contracts own grammar values and section dispatch" + (check (grammar-role? base-lexical-role) => #t) + (check (grammar? base-object-grammar) => #t) + (check (grammar-role-name base-lexical-role) => 'base-lexical-role) + (check (grammar-role-ref base-lexical-role 'terminals) + => '((punctuation Punctuation))) + (let (refined-role + (.o (:: @ base-lexical-role) + (.section + (lambda (field) + (if (eq? field 'terminals) + '((identifier Identifier)) + (grammar-role-ref base-lexical-role field)))))) + (check (grammar-role-ref refined-role 'terminals) + => '((identifier Identifier))) + (check (grammar-role-ref base-lexical-role 'terminals) + => '((punctuation Punctuation)))) + (check (row-ref (grammar->alist base-object-grammar) 'schema) + => "gerbil-parser.grammar.v1") + (check-exception + (make-grammar 'projection-is-not-a-parent + (list arithmetic-grammar) + (list base-lexical-role)) + true)) + (test-case "normalization is deterministic" + (check (grammar-ir-canonical (compile-grammar arithmetic-grammar)) + => + (grammar-ir-canonical (compile-grammar arithmetic-grammar)))) + (test-case "standard vector traversal interns projected catalogs" + (let-values + (((catalog canonical) + (vector-intern-map + '#(left right left right) + (lambda (value) value) + (lambda (key id) (cons id key))))) + (check (vector-length canonical) => 2) + (check (eq? (vector-ref catalog 0) (vector-ref catalog 2)) => #t) + (check (eq? (vector-ref catalog 1) (vector-ref catalog 3)) => #t))) + (test-case "LR states direct identical lexical spellings" + (let ((global-tokens (lex-source directed-lexical-mode-parser "xx")) + (artifact (parse-source directed-lexical-mode-parser "xx"))) + (check (map token-kind global-tokens) => '(first first)) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => "xx"))) + (test-case "mode-local regular DFA keeps LR token identities" + (let ((global-tokens (lex-source directed-regular-mode-parser "x x")) + (artifact (parse-source directed-regular-mode-parser "x x"))) + (check (map token-kind global-tokens) => '(first space first)) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => "x x"))) + (test-case "character certificates veto competing token identities" + (for-each + (lambda (machine) + (let* ((catalog (lr-runtime-lexical-mode-catalog (parser-machine-runtime machine))) + (lexer (parser-machine-lex machine)) (competing 0)) + (for-each + (lambda (i) + (for-each + (lambda (j) + (let-values (((left left-end) (lexer "x" 0 0 (vector-ref catalog i))) + ((right right-end) (lexer "x" 0 0 (vector-ref catalog j)))) + (unless (eq? (token-kind left) (token-kind right)) + (set! competing (+ competing 1)) + (check (parser-machine-lexical-modes-compatible? machine i j #\x) => #f)))) + (iota (vector-length catalog)))) + (iota (vector-length catalog))) + (check (> competing 0) => #t))) + (list directed-lexical-mode-parser directed-regular-mode-parser))) + (test-case "incremental reuse honors LR lexical modes on identical spellings" + (let* ((source (make-string 96 #\x)) + (source-edit (make-edit 48 1 "y")) + (session + (make-incremental-session + directed-lexical-mode-parser source)) + (fresh + (parse-source directed-lexical-mode-parser + (apply-edit source source-edit)))) + (let-values (((next receipt) + (parse-incremental-session session source-edit)) + ((one-shot one-shot-receipt) + (parse-source/incremental + directed-lexical-mode-parser source + (parse-source directed-lexical-mode-parser source) + source-edit))) + (check (incremental-session-artifact next) => fresh) + (check one-shot => fresh) + (check (or (> (or (row-ref receipt 'checkpointReusedShiftCount) + 0) 0) + (> (or (row-ref receipt 'reusedRecognitionEventCount) + 0) 0)) + => #t) + (check (> (row-ref receipt 'reusedSuffixTokenCount) 0) => #t) + (check (row-ref one-shot-receipt 'freshFallback?) => #f)))) + (test-case "explicit POO composition emits an identity-bearing receipt" + (let-values (((ir receipt) + (compile-grammar/receipt explicit-composed-grammar))) + (check (grammar-ir-ref ir 'terminals) + => '((punctuation Punctuation) (identifier Identifier))) + (check (row-ref receipt 'schema) + => "gerbil-parser.grammar-composition-receipt.v1") + (check (grammar-ir-ref ir 'compositionDigest) + => (row-ref receipt 'compositionDigest)) + (check (map (lambda (step) (row-ref step 'operation)) + (row-ref receipt 'steps)) + => '(merge append)))) + (test-case "explicit append fails closed on an existing identity" + (let (conflicting + (make-grammar + 'conflicting-explicit-append + (list base-object-grammar) + '() + (list (cons 'append base-lexical-role)))) + (check (condition-message (lambda () (compile-grammar conflicting))) + => "grammar append target already exists"))) +)) +(export grammar-composition-contract-test) diff --git a/t/grammar-composition-execution-test.ss b/t/grammar-composition-execution-test.ss new file mode 100644 index 00000000..8b523fb5 --- /dev/null +++ b/t/grammar-composition-execution-test.ss @@ -0,0 +1,180 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- + +(import :std/test + (only-in :clan/poo/object .o) + :gerbil-parser/src/grammar/algebra + :gerbil-parser/src/grammar/lexical-algebra + :gerbil-parser/src/compiler/normalize + :gerbil-parser/src/compiler/parser-ir + :gerbil-parser/src/compiler/lr + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + :gerbil-parser/src/language/grammar + :gerbil-parser/src/modules/parser/interface + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-roundtrip parse-artifact-success?) + (only-in :gerbil-parser/src/runtime/funcs vector-intern-map) + (only-in :gerbil-parser/src/runtime/incremental + apply-edit make-edit make-incremental-session + incremental-session-artifact parse-incremental-session + parse-source/incremental) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + :gerbil-parser/src/runtime/recognition + (only-in :gerbil-parser/src/runtime/lr-parser + lr-checkpoint? lr-checkpoint-deterministic-actions + lr-checkpoint-deterministic-shifts + lr-checkpoint-remaining-token-count + lr-checkpoint-advance lr-checkpoint-resume + lr-initial-checkpoint lr-parse lr-parse/receipt lr-prepare + lr-runtime-lexical-mode-catalog) + (only-in :gerbil-parser/src/runtime/parser parse-source) + :gerbil-parser/src/runtime/token + :gerbil-parser/src/compiler/machine + :gerbil-parser/languages/arithmetic/parser) +(import :gerbil-parser/t/fixtures/grammar-composition-models) + +(def grammar-composition-execution-test + (test-suite "grammar composition execution" + (test-case "selective GLR commits the only successful session branch" + (let (spec + (compile-lr-spec selective-session-rules + 'source-file 'selective-glr)) + (let-values (((root rest) + (lr-parse + spec + (list (make-token 'punctuation "SESSION" 0 7) + (make-token 'punctuation "SET" 8 11) + (make-token 'punctuation "SESSION" 12 19) + (make-token 'punctuation "CLOSE" 20 25))))) + (check (recognition-node-kind root) => 'SourceFile) + (check rest => '())) + (let-values (((root rest) + (lr-parse + spec + (list (make-token 'punctuation "SESSION" 0 7) + (make-token 'punctuation "SET" 8 11) + (make-token 'punctuation "SESSION" 12 19) + (make-token 'punctuation "SET" 20 23))))) + (check (recognition-node-kind root) => 'SourceFile) + (check rest => '())))) + (test-case "static selective GLR merges equivalent completed branches" + (let (spec + (compile-lr-spec static-equivalent-ambiguity-rules + 'source-file 'selective-glr)) + (let-values (((root rest receipt) + (lr-parse/receipt + spec (list (make-token 'identifier "x" 0 1))))) + (check (recognition-node-kind root) => 'SourceFile) + (check rest => '()) + (check (row-ref receipt 'branchesExplored) => 2) + (check (row-ref receipt 'mergedBranches) => 1) + (check (row-ref receipt 'ambiguousBranches) => 0)))) + (test-case "static selective GLR rejects distinct completed branches" + (let (spec + (compile-lr-spec static-distinct-ambiguity-rules + 'source-file 'selective-glr)) + (check + (with-catch + (lambda (condition) (error-message condition)) + (lambda () + (call-with-values + (lambda () + (lr-parse/receipt + spec (list (make-token 'identifier "x" 0 1)))) + (lambda _ #f)))) + => "selective GLR ambiguity is unresolved"))) + (test-case "dynamic precedence requires selective GLR admission" + (check (condition-message + (lambda () + (compile-lr-spec dynamic-precedence-rules 'source-file))) + => "dynamic precedence requires selective GLR admission") + (let (spec + (compile-lr-spec dynamic-precedence-rules + 'source-file 'selective-glr)) + (let-values (((root rest receipt) + (lr-parse/receipt + spec (list (make-token 'identifier "x" 0 1))))) + (check (recognition-node-kind root) => 'SourceFile) + (check rest => '()) + (check (row-ref receipt 'dynamicScore) => 1) + (check (row-ref receipt 'schema) + => "gerbil-parser.selective-glr-receipt.v1")))) + (test-case "dynamic precedence ranks completed reduce branches" + (check + (condition-message + (lambda () + (compile-lr-spec competing-dynamic-precedence-rules + 'source-file))) + => "dynamic precedence requires selective GLR admission") + (let (spec + (compile-lr-spec competing-dynamic-precedence-rules + 'source-file 'selective-glr)) + (let-values (((root rest receipt) + (lr-parse/receipt + spec (list (make-token 'identifier "x" 0 1))))) + (check (recognition-node-kind root) => 'HighPath) + (check rest => '()) + (check (row-ref receipt 'branchesExplored) => 2) + (check (row-ref receipt 'successfulCompletions) => 2) + (check (row-ref receipt 'distinctCompletions) => 2) + (check (row-ref receipt 'winnerReason) => 'dynamic-precedence) + (check (row-ref receipt 'dynamicScore) => 2)))) + (test-case "immutable checkpoints resume the sole LR executor" + (let* ((spec (compile-lr-spec nonassociative-rules 'source-file)) + (runtime (lr-prepare spec)) + (tokens + (list (make-token 'number "1" 0 1) + (make-token 'punctuation "<" 1 2) + (make-token 'number "2" 2 3))) + (initial (lr-initial-checkpoint runtime tokens))) + (check (lr-checkpoint? initial) => #t) + (check (lr-checkpoint-deterministic-actions initial) => 0) + (check (lr-checkpoint-remaining-token-count initial) => 3) + (let-values (((status checkpoint) (lr-checkpoint-advance initial 1))) + (check status => 'checkpoint) + (check (lr-checkpoint-deterministic-actions checkpoint) => 1) + (check (lr-checkpoint-deterministic-shifts checkpoint) => 1) + (check (lr-checkpoint-remaining-token-count checkpoint) => 2) + (let-values (((resumed-root resumed-rest) + (lr-checkpoint-resume checkpoint)) + ((fresh-root fresh-rest) (lr-parse spec tokens))) + (check resumed-root => fresh-root) + (check resumed-rest => fresh-rest))))) + (test-case "non-associative precedence rejects only chained operators" + (let (spec (compile-lr-spec nonassociative-rules 'source-file)) + (let-values (((root rest) + (lr-parse + spec + (list (make-token 'number "1" 0 1) + (make-token 'punctuation "<" 1 2) + (make-token 'number "2" 2 3))))) + (check (recognition-node-kind root) => 'SourceFile) + (check rest => '())) + (check-exception + (lr-parse + spec + (list (make-token 'number "1" 0 1) + (make-token 'punctuation "<" 1 2) + (make-token 'number "2" 2 3) + (make-token 'punctuation "<" 3 4) + (make-token 'number "3" 4 5))) + true))) + (test-case "same lexical identity with a different definition is rejected" + (check-exception + (compile-grammar conflicting-arithmetic-grammar) + true)) + (test-case "the generic machine executes the declared entrypoint" + (let-values (((root rest) + ((parser-machine-parse arithmetic-parser) + (list (make-token 'number "1" 0 1))))) + (check (recognition-node-kind root) => 'SourceFile) + (check rest => '()))) + (test-case "extras generate the sole trivia predicate" + (check ((parser-machine-trivia arithmetic-parser) + (make-token 'whitespace " " 0 1)) + => '(whitespace)) + (check ((parser-machine-trivia arithmetic-parser) + (make-token 'punctuation "+" 0 1)) + => #f)))) +(export grammar-composition-execution-test) diff --git a/t/grammar-composition-lowering-test.ss b/t/grammar-composition-lowering-test.ss new file mode 100644 index 00000000..75dbcd81 --- /dev/null +++ b/t/grammar-composition-lowering-test.ss @@ -0,0 +1,111 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- + +(import :std/test + (only-in :clan/poo/object .o) + :gerbil-parser/src/grammar/algebra + :gerbil-parser/src/grammar/lexical-algebra + :gerbil-parser/src/compiler/normalize + :gerbil-parser/src/compiler/parser-ir + :gerbil-parser/src/compiler/lr + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + :gerbil-parser/src/language/grammar + :gerbil-parser/src/modules/parser/interface + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-roundtrip parse-artifact-success?) + (only-in :gerbil-parser/src/runtime/funcs vector-intern-map) + (only-in :gerbil-parser/src/runtime/incremental + apply-edit make-edit make-incremental-session + incremental-session-artifact parse-incremental-session + parse-source/incremental) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + :gerbil-parser/src/runtime/recognition + (only-in :gerbil-parser/src/runtime/lr-parser + lr-checkpoint? lr-checkpoint-deterministic-actions + lr-checkpoint-deterministic-shifts + lr-checkpoint-remaining-token-count + lr-checkpoint-advance lr-checkpoint-resume + lr-initial-checkpoint lr-parse lr-parse/receipt lr-prepare + lr-runtime-lexical-mode-catalog) + (only-in :gerbil-parser/src/runtime/parser parse-source) + :gerbil-parser/src/runtime/token + :gerbil-parser/src/compiler/machine + :gerbil-parser/languages/arithmetic/parser) +(import :gerbil-parser/t/fixtures/grammar-composition-models) + +(def grammar-composition-lowering-test + (test-suite "grammar composition lowering" + (test-case "parser IR preserves declared flow" + (check (parser-ir-ref arithmetic-parser-ir 'schema) + => "gerbil-parser.parser-ir.v1") + (check (parser-ir-ref arithmetic-parser-ir 'flow) + => '((source lexical) (lexical parser) (parser cst)))) + (test-case "deterministic LALR(1) compilation admits declared precedence" + (let (lr-spec (parser-ir-ref arithmetic-parser-ir 'lr-spec)) + (check (lr-spec-ref lr-spec 'schema) + => "gerbil-parser.lr-spec.v1") + (check (lr-spec-ref lr-spec 'algorithm) + => 'lalr1-lr0-fixed-point-v1) + (check (lr-spec-ref lr-spec 'lr0-state-visit-count) + => (lr-spec-ref lr-spec 'state-count)) + (check (> (lr-spec-ref lr-spec 'lookahead-item-visit-count) 0) + => #t) + (check (> (lr-spec-ref lr-spec 'state-count) 0) => #t))) + (test-case "language declarations materialize LR tables during AOT expansion" + (check (parser-ir-ref arithmetic-parser-ir 'materialization) + => 'aot-expansion) + (check (pair? (parser-ir-ref arithmetic-parser-ir 'lr-spec)) => #t)) + (test-case "canonical parser IR including LALR tables is deterministic" + (check (parser-ir-canonical (compile-parser arithmetic-grammar)) + => + (parser-ir-canonical (compile-parser arithmetic-grammar)))) + (test-case "grammar algebra preserves productions and entrypoint" + (check (parser-ir-ref arithmetic-parser-ir 'root-rule) => 'source-file) + (check (length (parser-ir-ref arithmetic-parser-ir 'terminals)) => 5) + (check (length (parser-ir-ref arithmetic-parser-ir 'lexical-rules)) => 5) + (check (length (parser-ir-ref arithmetic-parser-ir 'rules)) => 6) + (check (parser-ir-ref arithmetic-parser-ir 'extras) + => '((whitespace))) + (let* ((rules (parser-ir-ref arithmetic-parser-ir 'rules)) + (source-expression (cadr (assq 'source-file rules)))) + (check (grammar-expression-kind source-expression) => 'alias) + (check source-expression + => '(alias SourceFile + (field expression (reference expression)))))) + (test-case "nullable repetition is rejected at construction" + (check-exception + (grammar-expression + (repeat (optional (token identifier)))) + true)) + (test-case "lexical literals reject empty spellings" + (check-exception + (lexical-expression (literals "")) + true)) + (test-case "every terminal requires exactly one lexical rule" + (check-exception + (compile-composed missing-lexical-rule-grammar) + true)) + (test-case "unresolved production references are rejected" + (check-exception + (compile-composed unresolved-reference-grammar) + true)) + (test-case "terminals require token syntax kinds" + (check-exception + (compile-composed invalid-terminal-kind-grammar) + true)) + (test-case "precedence-bearing grammars compile to the LR owner" + (check (lr-spec-ref + (parser-ir-ref + (compile-composed unsupported-precedence-grammar) + 'lr-spec) + 'schema) + => "gerbil-parser.lr-spec.v1")) + (test-case "unresolved LR ambiguity fails closed" + (check (condition-message + (lambda () + (compile-lr-spec ambiguous-left-recursive-rules + 'source-file))) + => "unresolved shift/reduce conflict requires precedence")) +)) +(export grammar-composition-lowering-test) diff --git a/t/grammar-composition-test.ss b/t/grammar-composition-test.ss deleted file mode 100644 index 55dde4f5..00000000 --- a/t/grammar-composition-test.ss +++ /dev/null @@ -1,548 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- - -(import :std/test - (only-in :clan/poo/object .o) - :gerbil-parser/src/grammar/algebra - :gerbil-parser/src/grammar/lexical-algebra - :gerbil-parser/src/compiler/normalize - :gerbil-parser/src/compiler/parser-ir - :gerbil-parser/src/compiler/lr - (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) - :gerbil-parser/src/language/grammar - :gerbil-parser/src/modules/parser/interface - (only-in :gerbil-parser/src/runtime/artifact - parse-artifact-roundtrip parse-artifact-success?) - (only-in :gerbil-parser/src/runtime/funcs vector-intern-map) - (only-in :gerbil-parser/src/runtime/lexer lex-source) - :gerbil-parser/src/runtime/recognition - (only-in :gerbil-parser/src/runtime/lr-parser - lr-checkpoint? lr-checkpoint-deterministic-actions - lr-checkpoint-deterministic-shifts - lr-checkpoint-remaining-token-count - lr-checkpoint-advance lr-checkpoint-resume - lr-initial-checkpoint lr-parse lr-parse/receipt lr-prepare) - (only-in :gerbil-parser/src/runtime/parser parse-source) - :gerbil-parser/src/runtime/token - :gerbil-parser/src/compiler/machine - :gerbil-parser/languages/arithmetic/v1/parser) - -;;; Both rules recognize the same spelling. Only the LR state can select the -;;; correct token identity for each position; global longest-match necessarily -;;; selects the declaration-first identity twice. -(deflanguage directed-lexical-mode - (identity "directed-lexical-mode" "v1" "directed-lexical-mode.v1") - (root source-file) - (lex - (first FirstToken (literals "x")) - (second SecondToken (literals "x"))) - (rules - (source-file - (node SourceFile - (seq (field first first) (field second second))))) - (extras) - (keywords) - (recoveries) - (conflicts reject) - (case-insensitive #f)) - -(defgrammar-role base-lexical-role - (syntax-kinds - (Punctuation token (text))) - (terminals - (punctuation Punctuation)) - (lexical-rules - (punctuation (literals "+" "-" "*" "**" "/" "(" ")"))) - (rules) - (extras) - (keywords) - (parser-entrypoints) - (recoveries) - (flow)) - -(defgrammar base-object-grammar - (supers) - (roles base-lexical-role)) - -(defgrammar-role appended-identifier-role - (syntax-kinds (Identifier token (text))) - (terminals (identifier Identifier)) - (lexical-rules (identifier (identifier))) - (rules) - (extras) - (keywords) - (parser-entrypoints) - (recoveries) - (flow)) - -(defgrammar-compose explicit-composed-grammar - (supers base-object-grammar) - (compose (append appended-identifier-role))) - -(defgrammar-role conflicting-lexical-role - (syntax-kinds - (Punctuation token (text))) - (terminals - (punctuation Punctuation)) - (lexical-rules - (punctuation (literals "+" "-" "*" "/" "(" ")"))) - (rules) - (extras) - (keywords) - (parser-entrypoints) - (recoveries) - (flow)) - -(defgrammar conflicting-arithmetic-grammar - (supers base-object-grammar) - (roles conflicting-lexical-role)) - -(defgrammar-role unresolved-reference-role - (syntax-kinds - (SourceFile node ()) - (Unknown token (text))) - (terminals - (unknown Unknown)) - (lexical-rules - (unknown (fallback))) - (rules - (source-file (reference missing-rule))) - (extras) - (keywords) - (parser-entrypoints - (source-file parse pure)) - (recoveries) - (flow - (source lexical) - (lexical cst))) - -(defgrammar unresolved-reference-grammar - (supers) - (roles unresolved-reference-role)) - -(defgrammar-role invalid-terminal-kind-role - (syntax-kinds - (SourceFile node (value))) - (terminals - (value SourceFile)) - (lexical-rules - (value (identifier))) - (rules - (source-file - (alias SourceFile (field value (token value))))) - (extras) - (keywords) - (parser-entrypoints - (source-file parse pure)) - (recoveries) - (flow - (source lexical) - (lexical cst))) - -(defgrammar invalid-terminal-kind-grammar - (supers) - (roles invalid-terminal-kind-role)) - -(defgrammar-role missing-lexical-rule-role - (syntax-kinds - (SourceFile node (value)) - (Identifier token (text))) - (terminals - (identifier Identifier)) - (lexical-rules) - (rules - (source-file - (alias SourceFile (field value (token identifier))))) - (extras) - (keywords) - (parser-entrypoints - (source-file parse pure)) - (recoveries) - (flow - (source lexical) - (lexical cst))) - -(defgrammar missing-lexical-rule-grammar - (supers) - (roles missing-lexical-rule-role)) - -(defgrammar-role unsupported-precedence-role - (syntax-kinds - (SourceFile node (value)) - (Identifier token (text))) - (terminals - (identifier Identifier)) - (lexical-rules - (identifier (identifier))) - (rules - (source-file - (alias SourceFile - (field value - (prec left 10 (token identifier)))))) - (extras) - (keywords) - (parser-entrypoints - (source-file parse pure)) - (recoveries) - (flow - (source lexical) - (lexical cst))) - -(defgrammar unsupported-precedence-grammar - (supers) - (roles unsupported-precedence-role)) - -(def (condition-message thunk) - (with-catch (lambda (condition) (error-message condition)) thunk)) - -(def (row-ref row key) - (let (entry (assq key row)) (and entry (cdr entry)))) - -(def (compile-composed grammar) - (compile-parser (compile-grammar grammar))) - -(def dynamic-precedence-rules - '((source-file - (precedence dynamic 1 - (alias SourceFile (field value (token identifier))))))) - -(def competing-dynamic-precedence-rules - '((source-file - (choice (reference low-path) (reference high-path))) - (low-path - (precedence dynamic 1 - (alias LowPath (field value (token identifier))))) - (high-path - (precedence dynamic 2 - (alias HighPath (field value (token identifier))))))) - -;;; The two reductions accept the same token and publish the same tree. A -;;; correct selective-GLR runtime must execute both and merge their completed -;;; candidates even when neither production carries dynamic precedence. -(def static-equivalent-ambiguity-rules - '((source-file - (choice (reference first-path) (reference second-path))) - (first-path - (alias SourceFile (field value (token identifier)))) - (second-path - (alias SourceFile (field value (token identifier)))))) - -;;; The same reduce/reduce shape deliberately publishes distinct roots. The -;;; runtime must not turn declaration order into an implicit ambiguity policy. -(def static-distinct-ambiguity-rules - '((source-file - (choice (reference first-path) (reference second-path))) - (first-path - (alias FirstPath (field value (token identifier)))) - (second-path - (alias SecondPath (field value (token identifier)))))) - -(def ambiguous-left-recursive-rules - '((source-file - (choice - (sequence (reference source-file) (reference source-file)) - (alias SourceFile (field value (token identifier))))))) - -(def selective-session-rules - '((source-file - (alias SourceFile - (sequence (reference session-activity) - (optional (reference session-close))))) - (session-activity (repeat1 (reference session-set))) - (session-set (sequence (literal "SESSION") (literal "SET"))) - (session-close (sequence (literal "SESSION") (literal "CLOSE"))))) - -(def nonassociative-rules - '((source-file - (alias SourceFile (field expression (reference expression)))) - (expression - (choice - (precedence none 10 - (alias ComparisonExpression - (sequence - (field left (reference expression)) - (field operator (literal "<")) - (field right (reference expression))))) - (alias NumberExpression (field value (token number))))))) - -(def grammar-composition-tests - (test-suite "grammar composition" - (test-case "POO contracts own grammar values and section dispatch" - (check (grammar-role? base-lexical-role) => #t) - (check (grammar? base-object-grammar) => #t) - (check (grammar-role-name base-lexical-role) => 'base-lexical-role) - (check (grammar-role-ref base-lexical-role 'terminals) - => '((punctuation Punctuation))) - (let (refined-role - (.o (:: @ base-lexical-role) - (.section - (lambda (field) - (if (eq? field 'terminals) - '((identifier Identifier)) - (grammar-role-ref base-lexical-role field)))))) - (check (grammar-role-ref refined-role 'terminals) - => '((identifier Identifier))) - (check (grammar-role-ref base-lexical-role 'terminals) - => '((punctuation Punctuation)))) - (check (row-ref (grammar->alist base-object-grammar) 'schema) - => "gerbil-parser.grammar.v1") - (check-exception - (make-grammar 'projection-is-not-a-parent - (list arithmetic-grammar) - (list base-lexical-role)) - true)) - (test-case "normalization is deterministic" - (check (grammar-ir-canonical (compile-grammar arithmetic-grammar)) - => - (grammar-ir-canonical (compile-grammar arithmetic-grammar)))) - (test-case "standard vector traversal interns projected catalogs" - (let-values - (((catalog canonical) - (vector-intern-map - '#(left right left right) - (lambda (value) value) - (lambda (key id) (cons id key))))) - (check (vector-length canonical) => 2) - (check (eq? (vector-ref catalog 0) (vector-ref catalog 2)) => #t) - (check (eq? (vector-ref catalog 1) (vector-ref catalog 3)) => #t))) - (test-case "LR states direct identical lexical spellings" - (let ((global-tokens (lex-source directed-lexical-mode-parser "xx")) - (artifact (parse-source directed-lexical-mode-parser "xx"))) - (check (map token-kind global-tokens) => '(first first)) - (check (parse-artifact-success? artifact) => #t) - (check (parse-artifact-roundtrip artifact) => "xx"))) - (test-case "explicit POO composition emits an identity-bearing receipt" - (let-values (((ir receipt) - (compile-grammar/receipt explicit-composed-grammar))) - (check (grammar-ir-ref ir 'terminals) - => '((punctuation Punctuation) (identifier Identifier))) - (check (row-ref receipt 'schema) - => "gerbil-parser.grammar-composition-receipt.v1") - (check (grammar-ir-ref ir 'compositionDigest) - => (row-ref receipt 'compositionDigest)) - (check (map (lambda (step) (row-ref step 'operation)) - (row-ref receipt 'steps)) - => '(merge append)))) - (test-case "explicit append fails closed on an existing identity" - (let (conflicting - (make-grammar - 'conflicting-explicit-append - (list base-object-grammar) - '() - (list (cons 'append base-lexical-role)))) - (check (condition-message (lambda () (compile-grammar conflicting))) - => "grammar append target already exists"))) - (test-case "parser IR preserves declared flow" - (check (parser-ir-ref arithmetic-parser-ir 'schema) - => "gerbil-parser.parser-ir.v1") - (check (parser-ir-ref arithmetic-parser-ir 'flow) - => '((source lexical) (lexical parser) (parser cst)))) - (test-case "deterministic LALR(1) compilation admits declared precedence" - (let (lr-spec (parser-ir-ref arithmetic-parser-ir 'lr-spec)) - (check (lr-spec-ref lr-spec 'schema) - => "gerbil-parser.lr-spec.v1") - (check (lr-spec-ref lr-spec 'algorithm) - => 'lalr1-lr0-fixed-point-v1) - (check (lr-spec-ref lr-spec 'lr0-state-visit-count) - => (lr-spec-ref lr-spec 'state-count)) - (check (> (lr-spec-ref lr-spec 'lookahead-item-visit-count) 0) - => #t) - (check (> (lr-spec-ref lr-spec 'state-count) 0) => #t))) - (test-case "language declarations materialize LR tables during AOT expansion" - (check (parser-ir-ref arithmetic-parser-ir 'materialization) - => 'aot-expansion) - (check (pair? (parser-ir-ref arithmetic-parser-ir 'lr-spec)) => #t)) - (test-case "canonical parser IR including LALR tables is deterministic" - (check (parser-ir-canonical (compile-parser arithmetic-grammar)) - => - (parser-ir-canonical (compile-parser arithmetic-grammar)))) - (test-case "grammar algebra preserves productions and entrypoint" - (check (parser-ir-ref arithmetic-parser-ir 'root-rule) => 'source-file) - (check (length (parser-ir-ref arithmetic-parser-ir 'terminals)) => 5) - (check (length (parser-ir-ref arithmetic-parser-ir 'lexical-rules)) => 5) - (check (length (parser-ir-ref arithmetic-parser-ir 'rules)) => 6) - (check (parser-ir-ref arithmetic-parser-ir 'extras) - => '((whitespace))) - (let* ((rules (parser-ir-ref arithmetic-parser-ir 'rules)) - (source-expression (cadr (assq 'source-file rules)))) - (check (grammar-expression-kind source-expression) => 'alias) - (check source-expression - => '(alias SourceFile - (field expression (reference expression)))))) - (test-case "nullable repetition is rejected at construction" - (check-exception - (grammar-expression - (repeat (optional (token identifier)))) - true)) - (test-case "lexical literals reject empty spellings" - (check-exception - (lexical-expression (literals "")) - true)) - (test-case "every terminal requires exactly one lexical rule" - (check-exception - (compile-composed missing-lexical-rule-grammar) - true)) - (test-case "unresolved production references are rejected" - (check-exception - (compile-composed unresolved-reference-grammar) - true)) - (test-case "terminals require token syntax kinds" - (check-exception - (compile-composed invalid-terminal-kind-grammar) - true)) - (test-case "precedence-bearing grammars compile to the LR owner" - (check (lr-spec-ref - (parser-ir-ref - (compile-composed unsupported-precedence-grammar) - 'lr-spec) - 'schema) - => "gerbil-parser.lr-spec.v1")) - (test-case "unresolved LR ambiguity fails closed" - (check (condition-message - (lambda () - (compile-lr-spec ambiguous-left-recursive-rules - 'source-file))) - => "unresolved shift/reduce conflict requires precedence")) - (test-case "selective GLR commits the only successful session branch" - (let (spec - (compile-lr-spec selective-session-rules - 'source-file 'selective-glr)) - (let-values (((root rest) - (lr-parse - spec - (list (make-token 'punctuation "SESSION" 0 7) - (make-token 'punctuation "SET" 8 11) - (make-token 'punctuation "SESSION" 12 19) - (make-token 'punctuation "CLOSE" 20 25))))) - (check (recognition-node-kind root) => 'SourceFile) - (check rest => '())) - (let-values (((root rest) - (lr-parse - spec - (list (make-token 'punctuation "SESSION" 0 7) - (make-token 'punctuation "SET" 8 11) - (make-token 'punctuation "SESSION" 12 19) - (make-token 'punctuation "SET" 20 23))))) - (check (recognition-node-kind root) => 'SourceFile) - (check rest => '())))) - (test-case "static selective GLR merges equivalent completed branches" - (let (spec - (compile-lr-spec static-equivalent-ambiguity-rules - 'source-file 'selective-glr)) - (let-values (((root rest receipt) - (lr-parse/receipt - spec (list (make-token 'identifier "x" 0 1))))) - (check (recognition-node-kind root) => 'SourceFile) - (check rest => '()) - (check (row-ref receipt 'branchesExplored) => 2) - (check (row-ref receipt 'mergedBranches) => 1) - (check (row-ref receipt 'ambiguousBranches) => 0)))) - (test-case "static selective GLR rejects distinct completed branches" - (let (spec - (compile-lr-spec static-distinct-ambiguity-rules - 'source-file 'selective-glr)) - (check - (with-catch - (lambda (condition) (error-message condition)) - (lambda () - (call-with-values - (lambda () - (lr-parse/receipt - spec (list (make-token 'identifier "x" 0 1)))) - (lambda _ #f)))) - => "selective GLR ambiguity is unresolved"))) - (test-case "dynamic precedence requires selective GLR admission" - (check (condition-message - (lambda () - (compile-lr-spec dynamic-precedence-rules 'source-file))) - => "dynamic precedence requires selective GLR admission") - (let (spec - (compile-lr-spec dynamic-precedence-rules - 'source-file 'selective-glr)) - (let-values (((root rest receipt) - (lr-parse/receipt - spec (list (make-token 'identifier "x" 0 1))))) - (check (recognition-node-kind root) => 'SourceFile) - (check rest => '()) - (check (row-ref receipt 'dynamicScore) => 1) - (check (row-ref receipt 'schema) - => "gerbil-parser.selective-glr-receipt.v1")))) - (test-case "dynamic precedence ranks completed reduce branches" - (check - (condition-message - (lambda () - (compile-lr-spec competing-dynamic-precedence-rules - 'source-file))) - => "dynamic precedence requires selective GLR admission") - (let (spec - (compile-lr-spec competing-dynamic-precedence-rules - 'source-file 'selective-glr)) - (let-values (((root rest receipt) - (lr-parse/receipt - spec (list (make-token 'identifier "x" 0 1))))) - (check (recognition-node-kind root) => 'HighPath) - (check rest => '()) - (check (row-ref receipt 'branchesExplored) => 2) - (check (row-ref receipt 'successfulCompletions) => 2) - (check (row-ref receipt 'distinctCompletions) => 2) - (check (row-ref receipt 'winnerReason) => 'dynamic-precedence) - (check (row-ref receipt 'dynamicScore) => 2)))) - (test-case "immutable checkpoints resume the sole LR executor" - (let* ((spec (compile-lr-spec nonassociative-rules 'source-file)) - (runtime (lr-prepare spec)) - (tokens - (list (make-token 'number "1" 0 1) - (make-token 'punctuation "<" 1 2) - (make-token 'number "2" 2 3))) - (initial (lr-initial-checkpoint runtime tokens))) - (check (lr-checkpoint? initial) => #t) - (check (lr-checkpoint-deterministic-actions initial) => 0) - (check (lr-checkpoint-remaining-token-count initial) => 3) - (let-values (((status checkpoint) (lr-checkpoint-advance initial 1))) - (check status => 'checkpoint) - (check (lr-checkpoint-deterministic-actions checkpoint) => 1) - (check (lr-checkpoint-deterministic-shifts checkpoint) => 1) - (check (lr-checkpoint-remaining-token-count checkpoint) => 2) - (let-values (((resumed-root resumed-rest) - (lr-checkpoint-resume checkpoint)) - ((fresh-root fresh-rest) (lr-parse spec tokens))) - (check resumed-root => fresh-root) - (check resumed-rest => fresh-rest))))) - (test-case "non-associative precedence rejects only chained operators" - (let (spec (compile-lr-spec nonassociative-rules 'source-file)) - (let-values (((root rest) - (lr-parse - spec - (list (make-token 'number "1" 0 1) - (make-token 'punctuation "<" 1 2) - (make-token 'number "2" 2 3))))) - (check (recognition-node-kind root) => 'SourceFile) - (check rest => '())) - (check-exception - (lr-parse - spec - (list (make-token 'number "1" 0 1) - (make-token 'punctuation "<" 1 2) - (make-token 'number "2" 2 3) - (make-token 'punctuation "<" 3 4) - (make-token 'number "3" 4 5))) - true))) - (test-case "same lexical identity with a different definition is rejected" - (check-exception - (compile-grammar conflicting-arithmetic-grammar) - true)) - (test-case "the generic machine executes the declared entrypoint" - (let-values (((root rest) - ((parser-machine-parse arithmetic-parser) - (list (make-token 'number "1" 0 1))))) - (check (recognition-node-kind root) => 'SourceFile) - (check rest => '()))) - (test-case "extras generate the sole trivia predicate" - (check ((parser-machine-trivia arithmetic-parser) - (make-token 'whitespace " " 0 1)) - => '(whitespace)) - (check ((parser-machine-trivia arithmetic-parser) - (make-token 'punctuation "+" 0 1)) - => #f)))) - -(export grammar-composition-tests) diff --git a/t/grammar-contract-admission-test.ss b/t/grammar-contract-admission-test.ss new file mode 100644 index 00000000..18be67f6 --- /dev/null +++ b/t/grammar-contract-admission-test.ss @@ -0,0 +1,91 @@ +;;; Boolean admission must match Core's complete evidence protocol. +(import :std/test + (only-in :clan/poo/object .o .cc .ref .all-slots) + (only-in :clan/poo/mop element? validate) + (only-in :core/types poo-flow-contract-admit + poo-flow-validation-evidence-accepted? + poo-flow-classification-evidence) + :gerbil-parser/src/modules/parser/interface) + +(def (evidence-member? contract candidate) + (poo-flow-validation-evidence-accepted? + (poo-flow-contract-admit contract candidate #f))) +(def (empty-role (rules '())) + (make-grammar-role 'role '() '() '() rules '() '() '() '() '())) +(def (rejects? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) + +(def grammar-contract-admission-test + (test-suite "grammar boolean admission and complete evidence" + (test-case "every responsibility and missing ancestry are checked" + (let* ((role (empty-role)) (grammar (make-grammar 'grammar '() (list role)))) + (for-each + (lambda (pair) + (let ((contract (car pair)) (candidate (cdr pair))) + (check (element? contract candidate) => (evidence-member? contract candidate)) + (check (eq? (validate contract candidate) candidate) => #t) + (for-each + (lambda (slot) + (let (invalid (.cc candidate slot 0)) + (check (evidence-member? contract invalid) => #f) + (check (element? contract invalid) => #f) + (check (rejects? (lambda () (validate contract invalid))) => #t))) + (.all-slots (.ref contract 'responsibilities))))) + (list (cons GrammarRoleContract role) (cons GrammarContract grammar))) + (check (element? GrammarRoleContract (.o name: 'role)) => #f) + (check (element? GrammarContract (.o name: 'grammar)) => #f))) + (test-case "descriptor refinements keep additional responsibility admission" + (let* ((role (empty-role)) + (responsibilities (.cc (.ref GrammarRoleContract 'responsibilities) + 'extra ParserSymbol)) + (refined (.cc GrammarRoleContract 'responsibilities responsibilities)) + (valid (.cc role 'extra 'symbol)) + (invalid (.cc role 'extra 0))) + (check (element? refined valid) => (evidence-member? refined valid)) + (check (element? refined invalid) => #f) + (check (rejects? (lambda () (validate refined invalid))) => #t))) + (test-case "descriptor refinements retain custom classifier and obligations" + (let* ((role (empty-role)) + (reclassified (.cc GrammarRoleContract '.classify + (lambda (candidate context) + (poo-flow-classification-evidence + (.ref GrammarRoleContract 'identity) candidate #f + '(refined-rejection) context)))) + (constrained (.cc GrammarRoleContract '.obligations + (lambda (_candidate _context) '(refined-obligation))))) + (for-each + (lambda (contract) + (check (evidence-member? contract role) => #f) + (check (element? contract role) => #f) + (check (rejects? (lambda () (validate contract role))) => #t)) + (list reclassified constrained)))) + (test-case "refined validation invokes its classifier exactly once" + (let (calls 0) + (let* ((role (empty-role)) + (refined (.cc GrammarRoleContract '.classify + (lambda (candidate context) + (set! calls (+ calls 1)) + (poo-flow-classification-evidence + (.ref GrammarRoleContract 'identity) candidate #f + '(refined-rejection) context))))) + (check (rejects? (lambda () (validate refined role))) => #t) + (check calls => 1)))) + (test-case "mutable declaration lists are rechecked on every call" + (let* ((rules (list 'rule)) (role (empty-role rules))) + (check (element? GrammarRoleContract role) => #t) + (set-cdr! rules 'improper-tail) + (check (element? GrammarRoleContract role) => #f) + (check (element? GrammarRoleContract role) => (evidence-member? GrammarRoleContract role)) + (check (rejects? (lambda () (validate GrammarRoleContract role))) => #t))) + (test-case "composition obligations reject foreign roles and parents" + (let* ((role (empty-role)) (grammar (make-grammar 'grammar '() (list role)))) + (for-each + (lambda (invalid) + (check (evidence-member? GrammarContract invalid) => #f) + (check (element? GrammarContract invalid) => #f) + (check (rejects? (lambda () (validate GrammarContract invalid))) => #t)) + (list (.cc grammar 'parents (list (.o))) + (.cc grammar 'roles (list (.o))) + (.cc grammar 'composition (list (cons 'unknown role))) + (.cc grammar 'composition (list (cons 'append (.o)))))))))) +(export grammar-contract-admission-test) diff --git a/t/graph-projection-aot-test.ss b/t/graph-projection-aot-test.ss index a8c04825..034af315 100644 --- a/t/graph-projection-aot-test.ss +++ b/t/graph-projection-aot-test.ss @@ -2,13 +2,13 @@ ;;; POO graph rules are validated and resolve against one grammar identity. (import (only-in :std/test check test-case test-suite) - (only-in :gerbil-parser/languages/arithmetic/v1/grammar + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-language-grammar) (only-in :gerbil-parser/graph-projection-support make-graph-projection make-graph-node make-graph-field graph-projection? graph-projection-digest)) -(import (only-in :gerbil-parser/src/compiler/graph-projection-rowan - graph-projection-rowan-source)) +(import (only-in :gerbil-parser/src/compiler/graph-projection-runtime + graph-projection-runtime-source)) (export graph-projection-aot-test) (def graph-projection-aot-test @@ -19,7 +19,7 @@ (list (make-graph-node 'Expression "document" "root" (list (make-graph-field 'Number "value")))))) - (source (graph-projection-rowan-source + (source (graph-projection-runtime-source arithmetic-language-grammar projection)) (digest (graph-projection-digest arithmetic-language-grammar projection))) @@ -46,7 +46,7 @@ (list (make-graph-node 'Expression "document" "root" (list (make-graph-field 'Number "value" 'each)))))) - (source (graph-projection-rowan-source + (source (graph-projection-runtime-source arithmetic-language-grammar each-rule))) (check (if (string-contains source "mode: GraphFieldMode::Each") #t #f) @@ -68,13 +68,27 @@ 'Expression "document" "root" (list (make-graph-field 'Number "value" 'append-or-empty)))))) - (source (graph-projection-rowan-source + (source (graph-projection-runtime-source arithmetic-language-grammar projection))) (check (graph-projection? projection) => #t) (check (if (string-contains source "mode: GraphFieldMode::AppendOrEmpty") #t #f) => #t))) + (test-case "node text fields resolve against node kinds" + (let* ((projection + (make-graph-projection + (list (make-graph-node + 'Expression "document" "root" + (list (make-graph-field + 'Expression "source" 'node-text)))))) + (source (graph-projection-runtime-source + arithmetic-language-grammar projection))) + (check (graph-projection? projection) => #t) + (check (if (string-contains + source "mode: GraphFieldMode::NodeText") + #t #f) + => #t))) (test-case "duplicate graph node owners are rejected" (check (with-catch @@ -90,7 +104,7 @@ (with-catch (lambda (error) #t) (lambda () - (graph-projection-rowan-source + (graph-projection-runtime-source arithmetic-language-grammar (make-graph-projection (list (make-graph-node 'Number "document" "root" '())))) diff --git a/t/graph-query-test.ss b/t/graph-query-test.ss new file mode 100644 index 00000000..2d2e3cfd --- /dev/null +++ b/t/graph-query-test.ss @@ -0,0 +1,54 @@ +;;; -*- Gerbil -*- +;;; Generic POO graph query semantics are independent of Org labels. + +(import (only-in :std/test check test-case test-suite) + (only-in :clan/poo/object .o .ref) + (only-in :gerbil-parser/graph-query-support + graph-query-context? make-graph-query-context + graph-query-map graph-query-lineage? graph-query-select)) +(export graph-query-test) + +(def (view source-records) + (.o records: source-records + id-of: (lambda (record) (.ref record 'id)) + parent-of: (lambda (record) (.ref record 'parent)) + kind-of: (lambda (record) (.ref record 'kind)))) + +(def (ids records) + (map (lambda (record) (.ref record 'id)) records)) + +(def graph-query-test + (test-suite "POO graph query context" + (test-case "scope and relations accept non-preorder language views" + (let (context + (make-graph-query-context + (view (list (.o id: 0 parent: #f kind: "root") + (.o id: 1 parent: 0 kind: "section") + (.o id: 2 parent: 1 kind: "item") + (.o id: 3 parent: 0 kind: "section") + (.o id: 4 parent: 1 kind: "item"))))) + (check (graph-query-context? context) => #t) + (check (ids (graph-query-map context "item" (lambda (_) #t))) + => '(2 4)) + (check (graph-query-lineage? context 1 4) => #t) + (check (graph-query-lineage? context 3 4) => #f) + (check (ids (graph-query-select context "item" 1 'child-of '(1) + (lambda (_) #t))) + => '(2 4)) + (check (ids (graph-query-select context "item" 0 'descendant-of + '(1 3) (lambda (_) #t))) + => '(2 4)) + (check (ids (graph-query-select context "item" 0 'any '() + (lambda (record) + (= (.ref record 'id) 4)))) + => '(4)))) + (test-case "invalid graph ancestry is rejected at context admission" + (check + (with-catch + (lambda (_) #t) + (lambda () + (make-graph-query-context + (view (list (.o id: 0 parent: #f kind: "root") + (.o id: 1 parent: 8 kind: "item")))) + #f)) + => #t)))) diff --git a/t/graph-syntax-test.ss b/t/graph-syntax-test.ss new file mode 100644 index 00000000..c5910b35 --- /dev/null +++ b/t/graph-syntax-test.ss @@ -0,0 +1,44 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- + +(import (only-in :clan/poo/object .o) + (only-in :std/test check test-case test-suite) + (only-in :gerbil-parser/languages/gql/parser parse-gql) + (only-in :gerbil-parser/languages/cypher/parser parse-opencypher) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success? parse-artifact-roundtrip) + :gerbil-parser/graph-syntax-support) + +(export graph-syntax-syntax-test) + +(def Program + (.o (:: @ GraphSyntaxProgram.) + match: + (.o (:: @ GraphSyntaxPath.) + start: (.o (:: @ GraphSyntaxNode.) binding: 's label: 'Scenario) + next: + (.o (:: @ GraphSyntaxStep.) relation: 'HAS_CASE + target: (.o (:: @ GraphSyntaxNode.) binding: 'c label: 'Case))) + where: + (.o (:: @ GraphSyntaxEquals.) + left: (.o (:: @ GraphSyntaxProperty.) binding: 's property: 'identity) + right: + (.o (:: @ GraphSyntaxLiteral.) + literal-kind: 'string value: "patient's-case")) + project: + (.o (:: @ GraphSyntaxProjection.) + expression: + (.o (:: @ GraphSyntaxProperty.) binding: 'c property: 'id)))) + +(def graph-syntax-syntax-test + (test-suite "shared POO graph syntax" + (test-case "projects canonical source from the engine graph" + (check (graph-syntax-program? Program) => #t) + (check (graph-syntax-program->source Program) + => "MATCH (s:Scenario)-[:HAS_CASE]->(c:Case)\nWHERE s.identity = 'patient''s-case'\nRETURN c.id\n")) + (test-case "GQL and Cypher parse the same inherited engine syntax" + (let (source (graph-syntax-program->source Program)) + (for-each (lambda (parse) + (let (artifact (parse source)) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + (list parse-gql parse-opencypher)))))) diff --git a/t/header-delimiter-test.ss b/t/header-delimiter-test.ss new file mode 100644 index 00000000..415d6bbb --- /dev/null +++ b/t/header-delimiter-test.ss @@ -0,0 +1,81 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; Closed header-derived scanning is shared by independent POO language Loaders. +(import :std/test + (only-in :clan/poo/object .o .cc .ref) + :gerbil-parser/language-support/projection + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-ref) + (only-in :clan/poo/mop validate) + (only-in :gerbil-parser/language-support deflanguage deflanguage-parser-loader LanguageLoader. LanguageLoaderContract) + (only-in :gerbil-parser/src/grammar/lexical-algebra lexical-expression?) + (only-in :gerbil-parser/src/runtime/lexical-source scan-header-delimiter scan-header-data) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success? parse-artifact-valid? parse-artifact-roundtrip) + (only-in :gerbil-parser/languages/hl7/parser hl7-language)) +(export header-delimiter-test header-record-language-grammar header-record-language) + +(begin + (deflanguage header-record + (syntax + (lexical + (root document) + (lex (field Separator (header-delimiter "記" 2 0)) + (component ComponentSeparator (header-delimiter "記" 2 1)) + (data Data (header-data "記" 2 ";"))))) + (rules (document (node SourceFile + (literal "記") field component (field value data) field)))) + (bind-fixture-grammar-release header-record "header-record" "v1" "header-record.v1") ) +(deflanguage-parser-loader (header-record-language :: self LanguageLoader.) + (grammar header-record-language-grammar) + (parse parse-header-record) + (slots metadata: (.o dialect: 'header-record))) + +(deflanguage-projection header-record-projection + (grammar header-record-language-grammar) + (strategy (.o (:: self RecordProjection.) prefix: "記" delimiter-count: 2 + constants: '((schema . "header-record.output.v1")) + columns: '((value split (record "記" 1) 1 1))))) + +(def header-delimiter-test + (test-suite "closed header-derived delimiter rules" + (test-case "independent record projection preserves its own source identity" + (let* ((artifact (parse-header-record "記|§α|")) + (projected (header-record-projection artifact))) + (check (cdr (assq 'value projected)) => "α") + (check (cdr (assq 'sourceDigest projected)) => (parse-artifact-ref artifact 'sourceDigest)))) + (test-case "engine rules derive separators and data without language callbacks" + (check (scan-header-delimiter "MSH|^~\\&|" 3 "MSH" 5 0) => 4) + (check (scan-header-data "MSH|^~\\&α x|" 8 "MSH" 5 "\r\n") => 11) + (check (scan-header-data "MSH|^~\\&\r" 8 "MSH" 5 "\r\n") => #f) + (check (scan-header-delimiter "記|§α§" 2 "記" 2 1) => 3) + (check (scan-header-data "記|§α§" 3 "記" 2 ";") => 4) + (for-each (lambda (source) + (check (scan-header-delimiter source 3 "MSH" 5 0) => #f)) + '("MSH|^~||" "MSH1^~\\&" "MSHα^~\\&" "MSH|^" "REC|^~\\&")) + (for-each (lambda (offset) + (check (scan-header-data "記|§α§" offset "記" 2 ";") => #f)) + '(-1 1.5 5 100))) + (test-case "closed recipe admission rejects malformed data" + (for-each (lambda (expression) (check (lexical-expression? expression) => #f)) + '((header-delimiter "" 2 0) (header-delimiter "記" 0 0) + (header-delimiter "記" 33 0) (header-delimiter "記" 2 2) + (header-delimiter "記" 2 -1) (header-delimiter "記" 2.5 0) + (header-data "記" 2 callback) (header-data "記" 2) + (header-data "記" 0 ";") (header-data "記" 33 ";"))) + (check (lexical-expression? '(header-data "記" 2 "")) => #t)) + (test-case "independent POO Loaders use the same rules and remain lossless" + (validate LanguageLoaderContract header-record-language) + (let (extended (.cc header-record-language 'metadata + (.cc (.ref header-record-language 'metadata) 'consumer 'downstream))) + (validate LanguageLoaderContract extended) + (for-each + (lambda (row) + (let* ((source (car row)) (artifact ((.ref extended '.parse) source))) + (check (parse-artifact-success? artifact) => (cdr row)) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + '(("記|§α|" . #t) ("記*$α*" . #t) ("記|§a b|" . #t) + ("記||α|" . #f) ("記a§αa" . #f) ("記|§|" . #f) ("記|§α;|" . #f)))) + (let* ((source "MSH*$%!?*LEGACY*AU*FHIR*AU*202609170900**ADT$A08*1*P*2.5.1\r") + (artifact ((.ref hl7-language '.parse) source))) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))))) diff --git a/t/incremental-suffix-benchmark-test.ss b/t/incremental-suffix-benchmark-test.ss index afde01c5..b94051a0 100644 --- a/t/incremental-suffix-benchmark-test.ss +++ b/t/incremental-suffix-benchmark-test.ss @@ -20,6 +20,7 @@ (let (algorithm-receipt (incremental-suffix-scenario)) (write algorithm-receipt) (newline) + (force-output) (check (incremental-suffix-scenario-pass? algorithm-receipt) => #t)) (run-scenario) (##gc) @@ -27,6 +28,12 @@ (benchmark-contract-run benchmark-path run-scenario)) (write benchmark-receipt) (newline) + (force-output) (check (benchmark-contract-receipt-pass? benchmark-receipt) => #t))))) -(export incremental-suffix-benchmark-tests) +(def incremental-suffix-benchmark-test incremental-suffix-benchmark-tests) +(export incremental-suffix-benchmark-tests incremental-suffix-benchmark-test) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/indentation-invariance-test.ss b/t/indentation-invariance-test.ss index feb78b5a..3dcebe88 100644 --- a/t/indentation-invariance-test.ss +++ b/t/indentation-invariance-test.ss @@ -3,10 +3,11 @@ ;;; Non-semantic whitespace may change source/trivia spans, never syntax. (import (only-in :std/test check test-case test-suite) - :gerbil-parser/languages/cypher/opencypher-2024-1/parser - :gerbil-parser/languages/gql/iso-39075-2024/parser - :gerbil-parser/languages/hcl/v2-24/parser - :gerbil-parser/languages/tla-plus/v1/parser + :gerbil-parser/languages/cypher/parser + :gerbil-parser/languages/gql/parser + :gerbil-parser/languages/hcl/parser + :gerbil-parser/languages/tla-plus/parser + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine) (only-in :gerbil-parser/src/compiler/machine parser-machine-trivia) (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success?) @@ -55,20 +56,24 @@ (test-suite "native AST indentation invariance" (test-case "non-semantic indentation preserves native syntax shape" (check-indentation-invariance - opencypher-2024-1-parser parse-opencypher-2024-1 + opencypher-parser parse-opencypher "MATCH (n)\nRETURN n\n" " MATCH (n)\n RETURN n\n") (check-indentation-invariance - gql-iso-parser parse-gql-iso-39075-2024 + gql-parser parse-gql "MATCH (n)\nRETURN n\n" " MATCH (n)\n RETURN n\n") (check-indentation-invariance - hcl-v2-24-parser parse-hcl-v2-24 + hcl-parser parse-hcl "resource \"x\" \"y\" {\nvalue = 1\n}\n" "resource \"x\" \"y\" {\n value = 1\n}\n") (check-indentation-invariance - tla-plus-v1-parser parse-tla-plus-v1 + (language-grammar-machine tla-plus-core-language-grammar) parse-tla-plus-core "---- MODULE Indent ----\nVARIABLE x\nInit == x = 1\n====\n" "---- MODULE Indent ----\n VARIABLE x\n Init == x = 1\n====\n")))) (export indentation-invariance-tests) + +;; gxtest discovers only exported names ending in -test. +(def indentation-invariance-test indentation-invariance-tests) +(export indentation-invariance-test) diff --git a/t/inline-link-event-test.ss b/t/inline-link-event-test.ss new file mode 100644 index 00000000..3cdb24a9 --- /dev/null +++ b/t/inline-link-event-test.ss @@ -0,0 +1,88 @@ +;;; Non-Org link declarations exercise the engine's native commit boundary. +(import :std/test + (only-in :gerbil-parser/src/modules/parser/interface + make-source-event-scope make-text-line make-inline-link source-inline-link-initial + source-inline-link-scan-forms) + (only-in :gerbil-parser/src/compiler/event-fold-runtime run-event-fold)) +(def scope (make-source-event-scope 'native-test)) +(export inline-link-event-test) +(def rule (make-text-line 'Text 'Raw + (make-inline-link "<<" "::" ">>" 'Reference 'Target 'Label 'Boundary))) +(def (events source (literal? #f) (invalid-target 'recover-span)) + (run-event-fold source 'Document + (append (source-inline-link-initial rule scope) '((scan-cursor 0) (policy 7))) + `((for-line-bytes scan-index start (line-content-end) + ,(source-inline-link-scan-forms rule scope index: 'scan-index cursor: 'scan-cursor + description-helper: 'label parameters: '(policy) + description-node: 'Description literal-description?: literal? + invalid-target: invalid-target)) + (token Raw (state-offset scan-cursor) end)) + '() + '((label ((policy 0)) + ((if (uint-equal? (state policy) (uint 7)) + ((token Label start end)) ((token Wrong start end)))) (policy))))) +(def (starts result node) + (length (filter (lambda (event) (equal? event (list 'start node))) result))) +(def (covers? result source) + (let loop ((rest result) (offset 0) (depth 0)) + (if (null? rest) + (and (= offset (u8vector-length (string->utf8 source))) (= depth 0)) + (let (event (car rest)) + (case (car event) + ((token) (and (= offset (caddr event)) + (<= offset (cadddr event)) + (loop (cdr rest) (cadddr event) depth))) + ((start) (loop (cdr rest) offset (+ depth 1))) + ((finish) (and (> depth 0) (loop (cdr rest) offset (- depth 1))))))))) +(def inline-link-event-test + (test-suite "native inline link declarations" + (test-case "custom markers and Unicode commit one source-backed node" + (let* ((source "α <<β::γ>> z") (result (events source))) + (check (starts result 'Reference) => 1) + (check (starts result 'Description) => 1) + (check (covers? result source) => #t))) + (test-case "adjacent links do not reconsume committed delimiters" + (let* ((source "<><>") (result (events source))) + (check (starts result 'Reference) => 2) + (check (covers? result source) => #t))) + (test-case "target-only form has no artificial description" + (check (starts (events "<>") 'Description) => 0) + (check (covers? (events "<>\r\n") "<>\r\n") => #t)) + (test-case "literal description bypasses the registered nested helper" + (let* ((source "<>") (result (events source #t))) + (check (starts result 'Reference) => 1) + (check (starts result 'Description) => 0) + (check (and (member '(token Label 5 6) result) #t) => #t) + (check (covers? result source) => #t))) + (test-case "invalid and truncated candidates never leak partial nodes" + (for-each + (lambda (source) + (let (result (events source)) + (check (starts result 'Reference) => 0) + (check (covers? result source) => #t))) + '("" "<<" "<>" "<<::b>>"))) + (test-case "malformed policy and missing link declarations are rejected" + (check-exception (source-inline-link-scan-forms rule scope) true) + (check-exception (source-inline-link-initial (make-text-line 'Text 'Raw) scope) true) + (check-exception (source-inline-link-scan-forms rule scope index: 'scan-index cursor: 'scan-cursor parameters: '(policy)) true) + (check-exception (source-inline-link-scan-forms rule scope index: 'scan-index cursor: 'scan-cursor description-helper: 'label parameters: '(p p)) true) + (check-exception (source-inline-link-scan-forms rule scope index: 'scan-index cursor: 'scan-cursor description-node: "bad") true) + (check-exception (source-inline-link-scan-forms rule scope index: 'scan-index cursor: 'scan-cursor invalid-target: 'unknown) true)) + (test-case "recovery scope is declared rather than inherited from Org" + (let (source "<<::x>> then <>") + (check (starts (events source) 'Reference) => 1) + (check (starts (events source #f 'recover-region) 'Reference) => 0) + (check (covers? (events source) source) => #t) + (check (covers? (events source #f 'recover-region) source) => #t))) + (test-case "opening marker bytes cannot become a premature separator" + (let* ((alternate (make-text-line 'Text 'Raw + (make-inline-link "<:" ":" ">" 'Reference 'Target 'Label 'Boundary))) + (source "<:a:b>") + (result (run-event-fold source 'Document + (append (source-inline-link-initial alternate scope) '((scan-cursor 0))) + `((for-line-bytes scan-index start (line-content-end) + ,(source-inline-link-scan-forms alternate scope index: 'scan-index cursor: 'scan-cursor)) + (token Raw (state-offset scan-cursor) end)) '()))) + (check (starts result 'Reference) => 1) + (check (and (member '(token Target 2 3) result) #t) => #t) + (check (covers? result source) => #t))))) diff --git a/t/installed-language-consumer-test.ss b/t/installed-language-consumer-test.ss index 27915190..f7324019 100644 --- a/t/installed-language-consumer-test.ss +++ b/t/installed-language-consumer-test.ss @@ -4,10 +4,10 @@ ;;; proves that generated language modules can resolve their packaged IR files. (import (only-in :std/test check test-case test-suite) - (only-in :gerbil-parser/languages/cypher/opencypher-2024-1/parser - parse-opencypher-2024-1) - (only-in :gerbil-parser/languages/gql/iso-39075-2024/parser - parse-gql-iso-39075-2024) + (only-in :gerbil-parser/languages/cypher/parser + parse-opencypher) + (only-in :gerbil-parser/languages/gql/parser + parse-gql) (only-in :gerbil-parser/src/runtime/artifact parse-artifact-roundtrip parse-artifact-success? @@ -25,13 +25,17 @@ (test-suite "installed GQL and openCypher consumers" (test-case "ISO GQL resolves its installed sidecars" (check (consumer-receipt - parse-gql-iso-39075-2024 + parse-gql "MATCH (n) RETURN n\n") => '(#t #t #t))) (test-case "openCypher resolves its installed sidecars" (check (consumer-receipt - parse-opencypher-2024-1 + parse-opencypher "MATCH (n) RETURN n\n") => '(#t #t #t))))) (export installed-language-consumer-tests) + +;; gxtest discovers only exported names ending in -test. +(def installed-language-consumer-test installed-language-consumer-tests) +(export installed-language-consumer-test) diff --git a/t/iso-bnf-source-test.ss b/t/iso-bnf-source-test.ss index 106d457e..9ab971a5 100644 --- a/t/iso-bnf-source-test.ss +++ b/t/iso-bnf-source-test.ss @@ -3,8 +3,9 @@ (import :std/test :gerbil-parser/language-support - :gerbil-parser/languages/cypher/opencypher-2024-1/grammar - :gerbil-parser/languages/cypher/opencypher-2024-1/source) + (only-in :gerbil-parser/language-support/iso-bnf + iso-bnf-source->datum iso-bnf-source-from-datum) + :gerbil-parser/languages/cypher/parser) ;; Test projections use one fail-closed lookup boundary instead of repeating ;; the source-map representation throughout behavioral assertions. @@ -18,27 +19,37 @@ (def iso-bnf-source-tests (test-suite "ISO WG3 BNF grammar source" + (test-case "frozen source catalogs retain all production fields" + (let* ((data (iso-bnf-source->datum opencypher-bnf)) + (copy (iso-bnf-source-from-datum data))) + (check (iso-bnf-source->datum copy) => data) + (check (iso-bnf-source-production copy "program") + => (iso-bnf-source-production opencypher-bnf "program"))) + (check-exception (iso-bnf-source-from-datum '(invalid)) true) + (check-exception + (iso-bnf-source-from-datum + (list +iso-bnf-source-schema+ "test" "v1" "commit" "digest" '(bad-row))) true)) (test-case "the complete openCypher production catalog is immutable" - (check (iso-bnf-source-language opencypher-2024-1-bnf) + (check (iso-bnf-source-language opencypher-bnf) => "opencypher") - (check (iso-bnf-source-version opencypher-2024-1-bnf) + (check (iso-bnf-source-version opencypher-bnf) => +opencypher-version+) - (check (iso-bnf-source-commit opencypher-2024-1-bnf) + (check (iso-bnf-source-commit opencypher-bnf) => +opencypher-commit+) - (check (iso-bnf-source-digest opencypher-2024-1-bnf) + (check (iso-bnf-source-digest opencypher-bnf) => +opencypher-bnf-digest+) - (check (length (iso-bnf-source-productions opencypher-2024-1-bnf)) + (check (length (iso-bnf-source-productions opencypher-bnf)) => 352) - (check (iso-bnf-source-production opencypher-2024-1-bnf "program") + (check (iso-bnf-source-production opencypher-bnf "program") ? iso-bnf-production?) - (check (iso-bnf-source-production opencypher-2024-1-bnf + (check (iso-bnf-source-production opencypher-bnf "value expression") ? iso-bnf-production?) - (check (iso-bnf-source-production opencypher-2024-1-bnf + (check (iso-bnf-source-production opencypher-bnf "binary digit") ? iso-bnf-production?) (let* ((program - (iso-bnf-source-production opencypher-2024-1-bnf "program")) + (iso-bnf-source-production opencypher-bnf "program")) (ast (iso-bnf-production-ast program))) (check (car ast) => 'choice) (check (iso-bnf-production-references program) @@ -46,31 +57,31 @@ ;; ISO WG3 ellipsis is one-or-more. Compatibility corrections belong in ;; a separately receipted rule overlay, never in the notation parser. (check (iso-bnf-production-ast - (iso-bnf-source-production opencypher-2024-1-bnf + (iso-bnf-source-production opencypher-bnf "linear statement")) => '(sequence (repeat1 (reference "primitive statement")) (optional (reference "primitive result statement")))) (check (grammar-rule-expression - (iso-bnf-source-grammar-rules opencypher-2024-1-bnf) + (iso-bnf-source-grammar-rules opencypher-bnf) 'identifier) => '(alias identifier (choice (reference |regular identifier|) (reference |delimited identifier|) (reference |non-reserved word|)))) (check (grammar-rule-expression - (iso-bnf-source-grammar-rules opencypher-2024-1-bnf) + (iso-bnf-source-grammar-rules opencypher-bnf) '|delimited identifier|) => '(alias |delimited identifier| (reference |accent quoted character sequence|))) (check (grammar-rule-expression - (iso-bnf-source-grammar-rules opencypher-2024-1-bnf) + (iso-bnf-source-grammar-rules opencypher-bnf) '|accent quoted character sequence|) => '(alias |accent quoted character sequence| (token delimited-identifier))) (check (grammar-rule-expression - (iso-bnf-source-grammar-rules opencypher-2024-1-bnf) + (iso-bnf-source-grammar-rules opencypher-bnf) '|create node pattern filler|) => '(alias |create node pattern filler| (choice @@ -81,7 +92,7 @@ (reference |create node label and property set specification|)))) (check (grammar-rule-expression - (iso-bnf-source-grammar-rules opencypher-2024-1-bnf) + (iso-bnf-source-grammar-rules opencypher-bnf) '|create node label and property set specification|) => '(alias |create node label and property set specification| (choice @@ -93,7 +104,7 @@ (for-each (lambda (production) (check (pair? (iso-bnf-production-ast production)) => #t)) - (iso-bnf-source-productions opencypher-2024-1-bnf))) + (iso-bnf-source-productions opencypher-bnf))) (test-case "unresolved production references fail closed" (check-exception (parse-iso-bnf-source @@ -157,3 +168,7 @@ true))))) (export iso-bnf-source-tests) + +;; gxtest discovers only exported names ending in -test. +(def iso-bnf-source-test iso-bnf-source-tests) +(export iso-bnf-source-test) diff --git a/t/javacc-source-test.ss b/t/javacc-source-test.ss index 2b464de9..007f4cb1 100644 --- a/t/javacc-source-test.ss +++ b/t/javacc-source-test.ss @@ -3,17 +3,20 @@ (import :std/test :gerbil-parser/language-support - :gerbil-parser/languages/tla-plus/v1/source) + :gerbil-parser/languages/tla-plus/parser) + +(def metadata (language-parser-entry-ref tla-plus-layout-language 'metadata)) +(def sany-source (language-metadata-ref metadata 'source-catalog)) (def javacc-source-tests (test-suite "JavaCC native grammar source" (test-case "the complete pinned SANY production inventory is immutable" - (check (javacc-source-digest tla-plus-sany-source) - => +tla-plus-sany-grammar-digest+) - (check (length (javacc-source-productions tla-plus-sany-source)) => 99) + (check (javacc-source-digest sany-source) + => (language-metadata-ref metadata 'sany-grammar-digest)) + (check (length (javacc-source-productions sany-source)) => 99) (for-each (lambda (name) - (check (javacc-source-production tla-plus-sany-source name) + (check (javacc-source-production sany-source name) ? javacc-production?)) '("CompilationUnit" "Module" "Body" "Expression" "LetIn" "Proof" "Step" "PrimitiveExp" "ExtendableExpr" @@ -43,3 +46,7 @@ true)))) (export javacc-source-tests) + +;; gxtest discovers only exported names ending in -test. +(def javacc-source-test javacc-source-tests) +(export javacc-source-test) diff --git a/t/lalr-fixed-point-benchmark-test.ss b/t/lalr-fixed-point-benchmark-test.ss index 417470f3..83ea1272 100644 --- a/t/lalr-fixed-point-benchmark-test.ss +++ b/t/lalr-fixed-point-benchmark-test.ss @@ -23,6 +23,7 @@ (let (algorithm-receipt (lalr-fixed-point-scenario)) (write algorithm-receipt) (newline) + (force-output) (check (lalr-fixed-point-scenario-pass? algorithm-receipt) => #t)) (run-scenario-batch) (##gc) @@ -30,6 +31,12 @@ (benchmark-contract-run benchmark-path run-scenario-batch)) (write benchmark-receipt) (newline) + (force-output) (check (benchmark-contract-receipt-pass? benchmark-receipt) => #t))))) -(export lalr-fixed-point-benchmark-tests) +(def lalr-fixed-point-benchmark-test lalr-fixed-point-benchmark-tests) +(export lalr-fixed-point-benchmark-tests lalr-fixed-point-benchmark-test) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/language-abi-alignment-test.ss b/t/language-abi-alignment-test.ss new file mode 100644 index 00000000..c884a2a4 --- /dev/null +++ b/t/language-abi-alignment-test.ss @@ -0,0 +1,16 @@ +;;; Exact language ABI fixture parity is an ordinary Scheme semantic test. +(import :std/test :std/io/tempfile :std/misc/ports + (only-in ./generate-language-abi-alignment write-language-abi-alignment!)) +(export language-abi-alignment-test) +(def language-abi-alignment-test + (test-suite "all-language ABI byte alignment" + (test-case "accepted and rejected language products match the Rust fixture exactly" + (let (directory (make-temporary-file-name "parser-language-alignment-")) + (create-directory directory) + (try + (let (product (path-expand "language-artifacts.bin" directory)) + (write-language-abi-alignment! product) + (check (equal? (call-with-input-file product read-all-as-u8vector) + (call-with-input-file "rust/gerbil-parser-artifact/tests/fixtures/languages.bin" + read-all-as-u8vector)) => #t)) + (finally (delete-file-or-directory directory #t))))))) diff --git a/t/language-abi-benchmark-test.ss b/t/language-abi-benchmark-test.ss new file mode 100644 index 00000000..27c315d1 --- /dev/null +++ b/t/language-abi-benchmark-test.ss @@ -0,0 +1,94 @@ +;;; Performance gate lives in the Scheme/ASP test owner, outside the C product. +(import :std/test + :asp-gerbil-scheme/src/benchmark/framework + (only-in :asp-gerbil-scheme/src/benchmark/gate benchmark-run/result) + (only-in :gerbil-parser/t/fixtures/progress report-test-progress!) + (only-in :gerbil-parser/t/fixtures/ffi/language-probe parse-native-batch parse-native-sized-batch create-native-handle) + (only-in :gerbil-parser/src/ffi/language-handles release-language!)) +(def language-abi-benchmark-tests + (test-suite "native language C performance" + (test-case "scaled UTF-8 C calls preserve whole-call allocation observations" + (let (handle (create-native-handle)) + (dynamic-wind + void + (lambda () + (for-each + (lambda (shape) + (let ((rows (car shape)) (unicode? (cadr shape))) + (check (parse-native-sized-batch handle rows unicode? 1) => 0) + (##gc) + (let-values (((receipt admission-row) + (benchmark-run/result + (benchmark-contract-read "t/benchmarks/language-abi/scaled-benchmark.ss") + (lambda () + (let* ((before (##process-statistics)) + (started (##current-time-point)) + (status (parse-native-sized-batch handle rows unicode? 1)) + (wall-ms (* 1000 (- (##current-time-point) started))) + (after (##process-statistics)) + (delta (lambda (index) + (- (f64vector-ref after index) + (f64vector-ref before index)))) + (row (list (cons 'rows rows) (cons 'unicode? unicode?) + (cons 'source-bytes (* rows (if unicode? 6 4))) + (cons 'calls 1) (cons 'wall-ms wall-ms) + (cons 'cpu-ms (* 1000 (+ (delta 0) (delta 1)))) + (cons 'gc-count (delta 6)) + (cons 'gc-wall-ms (* 1000 (delta 5))) + (cons 'allocated-bytes (delta 7))))) + (unless (zero? status) (error "scaled native batch rejected")) + (report-test-progress! "LANGUAGE-ABI-SCALE " row) + row))))) + (report-test-progress! "LANGUAGE-ABI-SCALE-ADMISSION " admission-row) + (check (benchmark-contract-receipt-pass? receipt) => #t)))) + '((256 #f) (4096 #f) (4096 #t)))) + (lambda () (release-language! handle))))) + (test-case "one hundred warm C calls satisfy the ASP contract" + (let (handle (create-native-handle)) + (check (positive? handle) => #t) + (check (parse-native-batch handle) => 0) + (##gc) + (let (sample 0) + (let-values (((receipt admission-row) + (benchmark-run/result + (benchmark-contract-read "t/benchmarks/language-abi/benchmark.ss") + (lambda () + (let* ((before (##process-statistics)) + (started (##current-time-point)) + (status (parse-native-batch handle)) + (wall-ms (* 1000 (- (##current-time-point) started))) + (after (##process-statistics)) + (delta (lambda (index) + (- (f64vector-ref after index) + (f64vector-ref before index))))) + (unless (zero? status) (error "native batch rejected")) + ;; Preserve each observation. Independently ranked CPU + ;; percentiles cannot explain the wall P95 sample. + (let (row (list (cons 'sample sample) + (cons 'wall-ms wall-ms) + (cons 'cpu-ms (* 1000 (+ (delta 0) (delta 1)))) + (cons 'gc-count (delta 6)) + (cons 'gc-wall-ms (* 1000 (delta 5))) + (cons 'allocated-bytes (delta 7)))) + (set! sample (+ sample 1)) + (report-test-progress! "LANGUAGE-ABI-BENCHMARK-BATCH calls=" 100 + " observation=" row) + row)))))) + ;; This is the actual ASP-admitted wall-ranked attempt, preserving + ;; its inner C batch counters even when progress output affects rank. + (report-test-progress! "LANGUAGE-ABI-ADMISSION-SAMPLE " admission-row) + (write receipt) (newline) (force-output) + (check (benchmark-contract-receipt-pass? receipt) => #t))) + (release-language! handle))))) +(export language-abi-benchmark-tests) + +(def language-abi-benchmark-test language-abi-benchmark-tests) +(export language-abi-benchmark-test) + +;;; Both ABI Suites share the persistent runtime owner thread. +(def test-worker-affinity 'language-abi-runtime) +(export test-worker-affinity) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/language-abi-test.ss b/t/language-abi-test.ss new file mode 100644 index 00000000..b5e2adfc --- /dev/null +++ b/t/language-abi-test.ss @@ -0,0 +1,254 @@ +;;; Native language admission uses independent descriptors, never builtin names. +(import :std/test + (only-in :gerbil-parser/src/ffi/language-abi-context language-abi-context-source-admission write-abi-row! ascii-source-bytes? copy-ascii-source-range! ascii-source-range?) + (only-in :clan/poo/object .o) + (only-in :gerbil-parser/src/language/source declare-source-language source-language-result-catalog source-language-digest) + (only-in :gerbil-parser/src/runtime/source-engines LineSourceStrategy.) + (only-in :gerbil-parser/languages/bash/parser bash-source-language) + (only-in :gerbil-parser/t/fixtures/ffi/language-probe probe-source-language) + (only-in :gerbil-parser/t/abi-datum-support abi-datum-read) + (only-in :std/vector/u8vector little u8vector-u32-ref u8vector-u64-ref) + (only-in :gerbil-parser/t/fixtures/shared-scanner/records records-language-grammar records-contextual-product) + (only-in :gerbil-parser/src/ffi/language-handles register-language! release-language! + language-handle-descriptor language-handle-parse + language-handle-parse/publish language-handle-scan-counts + language-handle-count) + (only-in :gerbil-parser/src/runtime/parser prepare-contextual-parser parse-source/contextual/prepared) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-event-program-enabled?) + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine + +language-grammar-schema+ language-grammar-language language-grammar-version + language-grammar-contract language-grammar-grammar language-grammar-ir + language-grammar-observability make-language-grammar) + (only-in :gerbil-parser/src/ffi/language-artifact-codec + prepare-language-abi-context abi-descriptor-payload + bind-language-abi abi-parse-binary-payload abi-parse-binary-payload/bytes) + (only-in :gerbil-parser/t/fixtures/progress report-test-progress!) + (only-in :gerbil-parser/src/language/entry parse-language-source) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-events parse-artifact-ref) + (only-in :gerbil-parser/src/compiler/language-pack generate-language-pack)) +(def language-abi-tests + (test-suite "independent language native admission" + (test-case "bounded GPA1 rows preserve little endian widths and adjacent bytes" + (let (bytes (make-u8vector 26 170)) + (write-abi-row! bytes 1 4294967295 2147483648 18446744073709551615 0 4294967295) + (check (u8vector-ref bytes 0) => 170) + (check (u8vector-ref bytes 25) => 170) + (check (u8vector-u32-ref bytes 1 little) => 4294967295) + (check (u8vector-u32-ref bytes 5 little) => 2147483648) + (check (u8vector-u64-ref bytes 9 little) => 18446744073709551615) + (check (u8vector-u32-ref bytes 17 little) => 0) + (check (u8vector-u32-ref bytes 21 little) => 4294967295) + (let (before (u8vector-copy bytes)) + (for-each (lambda (offset) + (check-exception (write-abi-row! bytes offset 1 2 3 4 5) true) + (check bytes => before)) '(-1 3 26 18446744073709551615)) + (check-exception (write-abi-row! bytes 1 4294967296 0 0 0 0) true) + (check bytes => before)))) + (test-case "inline fixnum row admission preserves full widths and rejects before writes" + (let (bytes (make-u8vector 26 170)) + (write-abi-row! bytes 1 4294967295 2147483648 4294967296 4294967295 0) + (check (u8vector-ref bytes 0) => 170) + (check (u8vector-ref bytes 25) => 170) + (check (u8vector-u32-ref bytes 1 little) => 4294967295) + (check (u8vector-u32-ref bytes 5 little) => 2147483648) + (check (u8vector-u64-ref bytes 9 little) => 4294967296) + (check (u8vector-u32-ref bytes 17 little) => 4294967295) + (check (u8vector-u32-ref bytes 21 little) => 0) + (let (before (u8vector-copy bytes)) + (check-exception (write-abi-row! bytes 1 1 2 3 4 4294967296) true) + (check bytes => before) + (check-exception (write-abi-row! bytes 1 1 2 -1 4 5) true) + (check bytes => before) + (check-exception (write-abi-row! bytes 1 1 2 3 4 5.5) true) + (check bytes => before)))) + (test-case "native Source handles own edit histories and publish fresh payload parity" + (let* ((baseline (language-handle-count)) + (a (register-language! bash-source-language)) + (b (register-language! bash-source-language)) + (language (bind-language-abi bash-source-language)) + (first "cat <utf8 first)) => (abi-parse-binary-payload language first)) + (check (language-handle-parse a (string->utf8 edit)) => (abi-parse-binary-payload language edit)) + (let-values (((scanned reused) (language-handle-scan-counts a))) + (check scanned => 1) (check (> reused 0) => #t)) + ;; A new handle cannot consume another handle's token/context history. + (check (language-handle-parse b (string->utf8 edit)) => (abi-parse-binary-payload language edit)) + (let-values (((scanned reused) (language-handle-scan-counts b))) + (check (> scanned 1) => #t) (check reused => 0)) + (for-each + (lambda (source) + (check (language-handle-parse a (string->utf8 source)) + => (abi-parse-binary-payload language source))) + '("echo猫 <<'A'\nα\nA\necho end\n" "cat < baseline) + (check-exception (language-handle-scan-counts a) true))) + (test-case "failed UTF-8 or result publication preserves the previous Source history" + (let* ((handle (register-language! bash-source-language)) + (first "cat <utf8 first)) + (let-values (((before-scanned before-reused) (language-handle-scan-counts handle))) + (check-exception (language-handle-parse handle #u8(255)) true) + ;; Simulate the C allocation/copy owner rejecting a complete payload. + (language-handle-parse/publish handle (string->utf8 "unrelated\n") (lambda (_) #f)) + (check-exception + (language-handle-parse/publish handle (string->utf8 "unrelated\n") + (lambda (_) (error "publication failed"))) true) + (let-values (((scanned reused) (language-handle-scan-counts handle))) + (check scanned => before-scanned) (check reused => before-reused))) + (language-handle-descriptor handle) + (language-handle-parse handle (string->utf8 edit)) + (let-values (((scanned reused) (language-handle-scan-counts handle))) + (check scanned => 1) (check (> reused 0) => #t)) + ;; Lexical rejection publishes a diagnostic but cannot authorize reuse. + (language-handle-parse handle (string->utf8 "cat < #f) (check reused => 0)) + (language-handle-parse handle (string->utf8 edit)) + (let-values (((scanned reused) (language-handle-scan-counts handle))) + (check (> scanned 1) => #t) (check reused => 0)) + (release-language! handle))) + (test-case "descriptor queries never execute a parser and retain bound identity" + (for-each + (lambda (descriptor) + (let* ((calls 0) + (language (prepare-language-abi-context + "metadata-only" descriptor + (lambda (_) (set! calls (+ calls 1)) + (error "metadata must not execute recognition")))) + (metadata (abi-datum-read (abi-descriptor-payload language)))) + (check calls => 0) + (check (string? (hash-get metadata "grammarDigest")) => #t))) + (list records-language-grammar bash-source-language)) + (let* ((language (bind-language-abi records-language-grammar records-contextual-product)) + (metadata (abi-datum-read (abi-descriptor-payload language)))) + (check (hash-get metadata "grammarDigest") => (cdr (assq 'digest records-contextual-product))) + (for-each + (lambda (source) + (let* ((plan (prepare-contextual-parser (language-grammar-machine records-language-grammar) + records-contextual-product)) + (artifact (parse-source/contextual/prepared plan source))) + (check (hash-get metadata "grammarDigest") => (parse-artifact-ref artifact 'grammarDigest)))) + '("a=1\n" "!")))) + (test-case "contextual and canonical Scheme trees preserve exact events" + (let (plan (prepare-contextual-parser (language-grammar-machine records-language-grammar) records-contextual-product)) + (for-each + (lambda (source) + (let ((canonical (parse-language-source records-language-grammar source)) + (contextual (parse-source/contextual/prepared plan source))) + (check (parse-artifact-ref contextual 'status) => (parse-artifact-ref canonical 'status)) + (check (parse-artifact-events contextual) => (parse-artifact-events canonical)))) + '("" "α=1\n" "a = 1\r\nb=\"β\"\n" "name=value")))) + (test-case "direct publication matches every canonical payload byte at scale" + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (let (language (bind-language-abi records-language-grammar records-contextual-product)) + (for-each + (lambda (source) + (check (abi-parse-binary-payload/bytes language (string->utf8 source)) + => (abi-parse-binary-payload language source)) + (report-test-progress! "NATIVE-PAYLOAD-PARITY-OK bytes=" + (u8vector-length (string->utf8 source)))) + (append '("" "α=1\n" "a = 1\r\nb=\"β\"\n" "name=value" "a=1\x0;!" "a=1\n!") + (map (lambda (row) (apply string-append (make-list 4096 row))) + '("a=1\n" "α=1\r\n"))))))) + '(#f #t))) + (test-case "ASCII canonical proof covers every byte and excludes all high bytes" + (let* ((language (bind-language-abi records-language-grammar records-contextual-product)) + (admit (language-abi-context-source-admission language)) + (bytes (list->u8vector (iota 128))) + (source (list->string (map integer->char (iota 128))))) + (check (ascii-source-bytes? bytes) => #t) + (check (admit bytes) => source) + (check (admit #u8()) => "") + (for-each (lambda (width) + (let (text (make-string width #\a)) + (check (admit (string->utf8 text)) => text) + (check (admit (string->utf8 (string-append text "猫"))) => (string-append text "猫")))) + '(4095 4096 4097 8191 8192 8193)) + (for-each (lambda (byte) + (check (ascii-source-bytes? (u8vector byte)) => (< byte 128))) (iota 256)))) + (test-case "ASCII leaves reject oversized ranges and short destinations before writing" + (let ((bytes (make-u8vector 4097 97)) (source (make-string 4097 #\z))) + (check (ascii-source-range? bytes 0 4097) => #f) + (check (copy-ascii-source-range! bytes source 0 4097) => #f) + (check source => (make-string 4097 #\z)) + (check (copy-ascii-source-range! bytes (make-string 4096 #\z) 0 4096) => #f) + (check (copy-ascii-source-range! bytes source 0 4096) => #t) + (check (string-ref source 4096) => #\z) + (check (copy-ascii-source-range! bytes source 4096 4097) => #t) + (check source => (make-string 4097 #\a)))) + (test-case "upstream bulk admission preserves Unicode scalars and chunk boundaries" + (let* ((language (bind-language-abi records-language-grammar records-contextual-product)) + (admit (language-abi-context-source-admission language)) + (scalars (map integer->char '(0 127 128 2047 2048 55295 57344 65535 65536 1114111)))) + (for-each + (lambda (width) + (let* ((source (list->string (append (make-list width #\a) scalars))) + (bytes (string->utf8 source))) + (check (admit bytes) => source))) + '(0 245 246 255 256 257 511 512)))) + (test-case "strict byte admission rejects noncanonical UTF-8 before publication" + (let (language (bind-language-abi records-language-grammar records-contextual-product)) + (for-each + (lambda (bytes) (check-exception (abi-parse-binary-payload/bytes language bytes) true)) + '(#u8(255) #u8(192 175) #u8(237 160 128) #u8(244 144 128 128) #u8(226 130))))) + (test-case "binary catalog errors escape rather than become rejected source" + (let* ((descriptor records-language-grammar) + (grammar (map (lambda (row) (if (eq? (car row) 'syntax-kinds) + (cons 'syntax-kinds '()) row)) + (language-grammar-grammar descriptor))) + (invalid (make-language-grammar + +language-grammar-schema+ (language-grammar-language descriptor) + (language-grammar-version descriptor) (language-grammar-contract descriptor) + grammar (language-grammar-ir descriptor) (language-grammar-machine descriptor) + (language-grammar-observability descriptor))) + (language (bind-language-abi invalid records-contextual-product))) + (check-exception (abi-parse-binary-payload/bytes language (string->utf8 "a=1\n")) + (lambda (e) (equal? (error-message e) + "ParseArtifact ABI symbol is outside descriptor"))))) + (test-case "two handles retain distinct lifetimes and descriptor identity" + (let ((a (register-language! records-language-grammar records-contextual-product)) + (b (register-language! records-language-grammar))) + (check (not (= a b)) => #t) + (let (descriptor (abi-datum-read (language-handle-descriptor a))) + (check (hash-get descriptor "language") => "record-assignments")) + (check (u8vector-u32-ref (language-handle-parse a (string->utf8 "α=1\n")) 8 little) => 0) + (release-language! a) (release-language! a) + (check-exception (language-handle-parse a #u8()) (lambda (e) (equal? (error-message e) "unknown or released language ABI handle"))) + (check (u8vector-u32-ref (language-handle-parse b #u8()) 8 little) => 0) + (release-language! b))) + (test-case "invalid UTF-8 and embedded NUL have distinct admission outcomes" + (let (handle (register-language! records-language-grammar records-contextual-product)) + (check-exception (language-handle-parse handle #u8(255)) true) + (check (u8vector-u32-ref (language-handle-parse handle #u8(97 61 49 0 33)) 8 little) => 1) + (release-language! handle))) + (test-case "source descriptors bind to the same length-delimited public C ABI" + (for-each (lambda (descriptor) + (check (probe-source-language (register-language! descriptor)) => 0)) + (list bash-source-language + (declare-source-language "independent-lines" "1" "lines.v1" + (.o (:: self LineSourceStrategy.) root-kind: 'IndependentFile token-kind: 'IndependentLine))))) + (test-case "source result catalog drives binary symbol indexes and rejected publications" + (let* ((native (bind-language-abi bash-source-language)) + (source "echo \"α${x:-中}\"\n")) + (check (abi-parse-binary-payload native source) + => (abi-parse-binary-payload/bytes native (string->utf8 source))) + (check (u8vector-u32-ref (abi-parse-binary-payload native "if true; then\n") 8 little) => 1) + (check-exception (bind-language-abi bash-source-language records-contextual-product) true))) + (test-case "C symbol injection fails before producing any output" + (check-exception + (generate-language-pack "/private/tmp/should-not-exist.ss" "/private/tmp/should-not-exist.h" + "pack/entry" "pack/grammar" 'grammar "bad;symbol") true)))) +(export language-abi-tests) + +(def language-abi-test language-abi-tests) +(export language-abi-test) + +;;; Both ABI Suites share the persistent runtime owner thread. +(def test-worker-affinity 'language-abi-runtime) +(export test-worker-affinity) diff --git a/t/language-artifact-test.ss b/t/language-artifact-test.ss index 280421fd..5fca39b6 100644 --- a/t/language-artifact-test.ss +++ b/t/language-artifact-test.ss @@ -6,13 +6,18 @@ (only-in :std/sync/barrier barrier-post! barrier-wait! make-barrier) (only-in :std/test check check-exception test-case test-suite) - (only-in :std/encoding/base64 base64-encode) + (only-in :gerbil-parser/src/runtime/embedded-image + pack-language-artifact-image unpack-language-artifact-image) (only-in :std/encoding/zlib compress) + (only-in :std/misc/ports read-all-as-u8vector) (only-in :gerbil-parser/src/compiler/language-artifact compile-language-declaration-artifacts/output-dirs + make-admitted-language-declaration compile-admitted-language-declaration compile-language-parser-artifact/output-dirs materialize-compiled-language-artifact/output-dirs) - (only-in :gerbil-parser/src/runtime/identity sha256-text) + (only-in :gerbil-parser/src/compiler/parser-ir compile-parser) + (only-in :gerbil-parser/src/compiler/lr current-grammar-source-map) + (only-in :gerbil-parser/src/runtime/identity sha256-bytes sha256-text) (only-in :gerbil-parser/src/runtime/language-artifact compiled-language-artifact-relative-path load-compiled-language-artifact/embedded @@ -59,6 +64,91 @@ (def language-artifact-tests (test-suite "compiled language artifact admission" + (test-case "Unicode publication preserves text identity and compressed bytes across both loaders" + (call-with-temporary-directory + (lambda (root) + (let* ((value `((schema . ,test-schema) + (payload . (中文 λ "line\r\nvalue" "𝄞")))) + (expected-bytes (string->utf8 (serialize value))) + (expected-digest (sha256-text (serialize value))) + (expected-compressed (compress expected-bytes compression: 9)) + (locator (materialize-compiled-language-artifact/output-dirs value (list root))) + (path (path-expand (car locator) root))) + (check locator => (list (compiled-language-artifact-relative-path expected-digest) + expected-digest)) + (check (call-with-input-file path read-all-as-u8vector) => expected-compressed) + (check (load-compiled-language-artifact/roots test-schema locator (list root)) => value) + (check (load-compiled-language-artifact/embedded + test-schema locator (pack-language-artifact-image expected-compressed)) => value) + (check (materialize-compiled-language-artifact/output-dirs value (list root)) => locator) + (check (call-with-input-file path read-all-as-u8vector) => expected-compressed))))) + (test-case "authenticated artifacts reject BOM framing and malformed UTF-8 in trailing comments" + (let (datum (string->utf8 (serialize test-value))) + (for-each + (lambda (bytes) + (let* ((digest (sha256-bytes bytes)) + (locator (list (compiled-language-artifact-relative-path digest) digest)) + (image (pack-language-artifact-image (compress bytes compression: 9)))) + (check-exception + (load-compiled-language-artifact/embedded test-schema locator image) exception?))) + (list (u8vector-append #u8(239 187 191) datum) + (u8vector-append datum (string->utf8 " ;") #u8(255)) + (u8vector-append datum (string->utf8 " ;") #u8(192 128)) + (u8vector-append datum (string->utf8 " ;") #u8(226 130)))))) + (test-case "declaration compiler passes selected origins into LR rejection and restores context" + (call-with-temporary-directory + (lambda (root) + (let* ((grammar '((schema . "gerbil-parser.grammar-ir.v1") + (grammar . nullable-origin) + (syntax-kinds (Start node ())) (terminals) (lexical-rules) + (rules (start (alias Start (repeat (reference item)))) (item (empty))) + (extras) (parser-entrypoints (start Start)) + (flow (source lexical) (lexical cst)))) + (source-map '((rule (start (componentOwner . selected-base)) + (item (componentOwner . selected-override))))) + (admitted (make-admitted-language-declaration grammar 'test-origin '() source-map)) + (condition + (with-catch (lambda (condition) condition) + (lambda () + (compile-admitted-language-declaration + admitted compile-parser + (lambda (_grammar _origin _lineage _sources) + '((schema . "gerbil-parser.bound-grammar-ir.v1"))) + output-dirs: (list root)) #f)))) + (check (error-message condition) => "resolved repetition operand accepts empty input") + (let* ((details (list-ref (error-irritants condition) 3)) + (references (cdr (assq 'nullableReferencePath details)))) + (check (cdr (assq 'componentOwner (cdr (assq 'source (car references))))) + => 'selected-override)) + (check (current-grammar-source-map) => '()))))) + (test-case "complete cached declarations cannot bypass resolved repetition admission" + (call-with-temporary-directory + (lambda (root) + (let* ((grammar '((schema . "gerbil-parser.grammar-ir.v1") (grammar . stale-admission) + (rules (start (repeat (reference item))) (item (empty))))) + (source-map '((rule (item (componentOwner . cached-override))))) + (origin 'test-origin) (lineage '()) + (admitted (make-admitted-language-declaration grammar origin lineage source-map))) + ;; Seed a complete old-generation receipt with its unchanged nullable facts. + (compile-language-declaration-artifacts/output-dirs + (list origin lineage source-map) grammar + (lambda () '((schema . "gerbil-parser.bound-grammar-ir.v1"))) + (lambda () '((schema . "gerbil-parser.parser-ir.v1") + (lr-spec (schema . "gerbil-parser.lr-spec.v1") (nullable item start)))) + (list root)) + (let ((calls 0) + (condition #f)) + (set! condition + (with-catch (lambda (condition) condition) + (lambda () + (compile-admitted-language-declaration admitted + (lambda (_) (set! calls (+ calls 1)) (error "unexpected generation")) + (lambda (_grammar _origin _lineage _sources) + (set! calls (+ calls 1)) (error "unexpected binding")) + output-dirs: (list root)) #f))) + (check calls => 0) + (check (error-message condition) => "resolved repetition operand accepts empty input") + (check (current-grammar-source-map) => '())))))) (test-case "content identities use a portable physical filename" (check (sha256-identity-filename @@ -80,21 +170,76 @@ (check (load-compiled-language-artifact/roots test-schema locator (list root)) => test-value))))) + (test-case "native image framing preserves every tail length and unsigned word" + (for-each + (lambda (size) + (let (bytes (make-u8vector size)) + (let loop ((index 0)) + (when (< index size) + (u8vector-set! bytes index (modulo index 256)) + (loop (fx+ index 1)))) + (check (unpack-language-artifact-image (pack-language-artifact-image bytes)) + => bytes))) + '(0 1 2 3 4 5 6 7 8 9 15 16 17 255 256 257 + 4095 4096 4097 8191 8192 8193 65535 65536 65537)) + (check (pack-language-artifact-image #u8(1 2 3 4 5 6 7 8)) + => '#u64(8 #x0102030405060708)) + (check (unpack-language-artifact-image '#u64(8 #xffffffffffffffff)) + => #u8(255 255 255 255 255 255 255 255)) + (for-each + (lambda (invalid) + (check-exception (unpack-language-artifact-image invalid) true)) + (list "encoded" #u8(1) '#u64() '#u64(1) '#u64(0 0) + '#u64(#xffffffffffffffff) '#u64(1 #x0100000000000001)))) + (test-case "native producer preserves high-bit full words and partial tails" + (check (pack-language-artifact-image #u8(255)) => '#u64(1 #xff00000000000000)) + (check (pack-language-artifact-image #u8(255 254 253 252 251 250 249)) + => '#u64(7 #xfffefdfcfbfaf900)) + (check (pack-language-artifact-image #u8(255 255 255 255 255 255 255 255 128)) + => '#u64(9 #xffffffffffffffff #x8000000000000000)) + (let* ((bytes (make-u8vector 17 255)) + (image (pack-language-artifact-image bytes))) + (u8vector-set! bytes 0 0) + (check (unpack-language-artifact-image image) => (make-u8vector 17 255)))) (test-case "an AOT image consumes the same compressed artifact without roots" (let* ((serialized (serialize test-value)) (digest (sha256-text serialized)) (locator (list (compiled-language-artifact-relative-path digest) digest)) (encoded - (base64-encode + (pack-language-artifact-image (compress (string->utf8 serialized) compression: 9)))) (check (load-compiled-language-artifact/embedded test-schema locator encoded) => test-value) (check-exception (load-compiled-language-artifact/embedded - test-schema locator "not-base64") + test-schema locator "not-a-native-image") true))) + (test-case "byte hashing preserves Unicode artifact identity in sidecars and AOT images" + (let* ((value `((schema . ,test-schema) (payload . "λ中😀é\x0;"))) + (serialized (serialize value)) (bytes (string->utf8 serialized)) + (digest (sha256-text serialized)) + (locator (list (compiled-language-artifact-relative-path digest) digest))) + (check (sha256-bytes bytes) => digest) + (check (load-compiled-language-artifact/embedded + test-schema locator (pack-language-artifact-image (compress bytes compression: 9))) => value) + (call-with-temporary-directory + (lambda (root) + (write-serialized-sidecar root serialized) + (check (load-compiled-language-artifact/roots test-schema locator (list root)) => value))))) + (test-case "a matching raw digest cannot admit malformed UTF-8 inside an artifact string" + (for-each + (lambda (invalid) + (let* ((prefix (string->utf8 + (string-append "((schema . " (object->string test-schema) ") (payload . \""))) + (bytes (u8vector-append prefix invalid #u8(34 41 41))) + (digest (sha256-bytes bytes)) + (locator (list (compiled-language-artifact-relative-path digest) digest))) + (check-exception + (load-compiled-language-artifact/embedded + test-schema locator (pack-language-artifact-image (compress bytes compression: 9))) true))) + '(#u8(255) #u8(192 175) #u8(237 160 128) #u8(244 144 128 128) #u8(226 130)))) (test-case "locator identity prevents absolute and traversal reads" (call-with-temporary-directory (lambda (root) @@ -270,3 +415,7 @@ (check parser-count => 1)))))))))) (export language-artifact-tests) + +;; gxtest discovers only exported names ending in -test. +(def language-artifact-test language-artifact-tests) +(export language-artifact-test) diff --git a/t/language-catalog-test.ss b/t/language-catalog-test.ss new file mode 100644 index 00000000..2445364f --- /dev/null +++ b/t/language-catalog-test.ss @@ -0,0 +1,118 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +;;; Published ABI order is validated over the concise DSL's inferred syntax. +(import :std/test + (for-syntax (only-in :gerbil/expander core-expand)) + (only-in :gerbil-parser/src/language/grammar deflanguage) + (only-in :gerbil-parser/src/compiler/language-expander compile-language) + (only-in :gerbil-parser/src/compiler/normalize grammar-ir-ref) + (only-in :gerbil-parser/src/compiler/parser-ir parser-ir-ref) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-events parse-artifact-success? parse-artifact-roundtrip)) + +(begin + (deflanguage projected + (syntax + (lexical + (root source-file) + (lex (word WordToken (identifier)) + (whitespace Whitespace (whitespace+)) + (unknown Unknown (fallback))) + (extras whitespace) + (catalog + (syntax-kinds (Root node (reserved item)) (WordToken token (text)) + (EmptyLine node ()) (Name node (value)) + (Unknown token (text)) (Whitespace token (text)) (UnusedToken token (text))) + (terminals (unknown Unknown) (word WordToken) (whitespace Whitespace)) + (reserved-token-kinds UnusedToken)))) + (rules (source-file (node Root (field item name))) + (name (node Name (field value word))))) + (bind-fixture-grammar-release projected "catalog-witness" "v1" "catalog-witness.v1") ) + +(compile-language published + (identity "catalog-witness" "v1" "catalog-witness.v1") + (syntax-kinds (Root node (reserved item)) (WordToken token (text)) + (EmptyLine node ()) (Name node (value)) + (Unknown token (text)) (Whitespace token (text)) (UnusedToken token (text))) + (terminals (unknown Unknown) (word WordToken) (whitespace Whitespace)) + (lexical-rules (word (identifier)) (whitespace (whitespace+)) (unknown (fallback))) + (rules (source-file (alias Root (field item (reference name)))) + (name (alias Name (field value (token word))))) + (extras whitespace) (keywords) (parser-entrypoints (source-file parse pure)) + (recoveries) (conflicts reject) (case-insensitive #f) + (flow (source lexical) (lexical parser) (parser cst))) + +(defsyntax (catalog-rejection-messages stx) + (def (message option) + (with-syntax ((catalog-option (datum->syntax #'deflanguage option))) + (with-catch (lambda (condition) (error-message condition)) + (lambda () + (core-expand + #'(deflanguage invalid-catalog + (syntax + (lexical + (root source-file) + (lex (word WordToken (identifier))) + catalog-option)) + (rules (source-file (node Root (field value word)))))) + "UNEXPECTED-ADMISSION")))) + (datum->syntax + #'catalog-rejection-messages + (list 'quote + (map message + '((catalog (syntax-kinds) (terminals (word WordToken))) + (catalog (syntax-kinds (WordToken token (text)) (Root node (value))) + (terminals (word WordToken))) + (catalog (syntax-kinds (Root node (value)) (Root node (value)) (WordToken token (text))) + (terminals (word WordToken))) + (catalog (syntax-kinds (Root node ()) (WordToken token (text))) + (terminals (word WordToken))) + (catalog (syntax-kinds (Root node (value)) (WordToken node (text))) + (terminals (word WordToken))) + (catalog (syntax-kinds (Root node (value value)) (WordToken token (text))) + (terminals (word WordToken))) + (catalog (syntax-kinds (Root node (value)) (WordToken token (text)) (Invented token (text))) + (terminals (word WordToken))) + (catalog (syntax-kinds (Root node (value)) (WordToken token (text))) + (terminals (word Root))) + (catalog (syntax-kinds (Root node (value)) (WordToken token (text))) + (terminals (word WordToken) (word WordToken))) + (catalog (syntax-kinds (Root node (value)) (WordToken token (text))) + (terminals (word WordToken)) (reserved-token-kinds WordToken)) + (catalog (syntax-kinds (Root node (value)) (WordToken token (text))) + (terminals (word WordToken)) (reserved-token-kinds Missing)) + (catalog (syntax-kinds (Root node (value)) (WordToken token (text))))))))) + +(def language-catalog-test + (test-suite "validated published language catalog" + (test-case "retains reserved nodes, fields and interleaved public kind IDs" + (check (grammar-ir-ref projected-grammar 'syntax-kinds) + => (grammar-ir-ref published-grammar 'syntax-kinds)) + (check (parser-ir-ref projected-parser-ir 'root-kind) => 'Root) + (check (grammar-ir-ref projected-grammar 'terminals) + => (grammar-ir-ref published-grammar 'terminals))) + (test-case "preserves canonical rules, lexical priority and LR tables" + (for-each + (lambda (key) + (check (grammar-ir-ref projected-grammar key) => (grammar-ir-ref published-grammar key))) + '(lexical-rules rules extras keywords parser-entrypoints recoveries conflicts case-insensitive flow)) + (check (parser-ir-ref projected-parser-ir 'lr-spec) => (parser-ir-ref published-parser-ir 'lr-spec))) + (test-case "preserves accepted and rejected lossless artifacts" + (for-each + (lambda (source) + (let ((actual (parse-source projected-parser source)) + (expected (parse-source published-parser source))) + (check (parse-artifact-success? actual) => (parse-artifact-success? expected)) + (check (parse-artifact-roundtrip actual) => source) + (check (parse-artifact-events actual) => (parse-artifact-events expected)))) + '("alpha" " α " "?" "alpha beta" ""))) + (test-case "rejects incomplete, remapped and malformed catalogs during expansion" + (let (messages (catalog-rejection-messages)) + (check (length messages) => 12) + (for-each + (lambda (message) + (check (not (not (string-contains message "catalog"))) => #t) + (check (string-contains message "UNEXPECTED-ADMISSION") => #f)) + messages))))) +(export language-catalog-test) diff --git a/t/language-diagnostics-test.ss b/t/language-diagnostics-test.ss new file mode 100644 index 00000000..9fc14469 --- /dev/null +++ b/t/language-diagnostics-test.ss @@ -0,0 +1,337 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +;;; Existing concise admission errors retain author occurrence and context. +(import :std/test + (only-in ../src/language/grammar deflanguage) + (only-in ../src/compiler/language-expander compile-language) + (only-in ../src/grammar/algebra grammar-expression? grammar-expression-header?) + (only-in ./fixtures/language-diagnostics-vocabulary required-items broken-wrapper) + (rename-in (only-in ./fixtures/language-diagnostics-vocabulary required-items) + (required-items renamed-items)) + (for-syntax (only-in :gerbil/expander core-expand))) +(export language-diagnostics-test) + +(defsyntax (declaration-error stx) + (def (capture form) + (with-catch + (lambda (condition) + (unless (syntax-error? condition) (raise condition)) + (let* ((details (slot-ref condition 'irritants)) + (blame (car details)) (loc (stx-source blame)) + (start (and loc (##position->filepos (##locat-start-position loc))))) + (list (error-message condition) (syntax->datum blame) + (and start (list (##container->path (##locat-container loc)) + (+ 1 (##filepos-line start)) + (##filepos-col start))) + (map syntax->datum (cdr details))))) + (lambda () (core-expand form) (error "invalid declaration was admitted")))) + (syntax-case stx (source-free) + ((_ source-free form) + (datum->syntax #'declaration-error + (list 'quote (capture (datum->syntax #'declaration-error (syntax->datum #'form) #f #f))))) + ((_ form) + (datum->syntax #'declaration-error (list 'quote (capture #'form)))))) + +(defrules invalid-language () + ((_ expression) + (begin + (deflanguage diagnostic-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules (source-file (node SourceFile expression)))) + (bind-fixture-grammar-release diagnostic-study "diagnostic-study" "v1" "diagnostic-study.v1") ))) + +;; Pass the reader-authored expression through the declaration template without +;; rebuilding its datum, so its original occurrence remains available. +(defsyntax (invalid-expression-error stx) + (syntax-case stx (source-free) + ((_ source-free expression) + #'(declaration-error source-free (invalid-language expression))) + ((_ expression) + #'(declaration-error (invalid-language expression))))) + +(def direct-reference + (invalid-expression-error (reference missing-rule))) +(def direct-token + (invalid-expression-error (token missing-token))) +(def direct-constructor + (invalid-expression-error (misspelled identifier))) +(def direct-identifier + (invalid-expression-error missing-name)) +(def macro-reference + (invalid-expression-error + (required-items argument (reference missing-rule) ","))) +(def renamed-reference + (invalid-expression-error + (renamed-items argument (reference missing-rule) ","))) +(def macro-constructor + (invalid-expression-error (broken-wrapper identifier))) +(def repeated-reference + (invalid-expression-error (seq (reference repeated-missing) (reference repeated-missing)))) +(def source-free-reference + (invalid-expression-error source-free (reference missing-rule))) + +(def forward-field + (declaration-error + (begin + (deflanguage diagnostic-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules (source-file (node SourceFile (reference helper))) + (helper (field argument)))) + (bind-fixture-grammar-release diagnostic-study "diagnostic-study" "v1" "diagnostic-study.v1") ))) + +(defrules invalid-helper-error () + ((_ helper-expression) + (declaration-error + (begin + (deflanguage diagnostic-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules (source-file (node SourceFile (reference helper))) + (helper helper-expression))) + (bind-fixture-grammar-release diagnostic-study "diagnostic-study" "v1" "diagnostic-study.v1") )))) +(def forward-unary (invalid-helper-error (optional))) +(def forward-reference (invalid-helper-error (reference))) + +(defrules invalid-root-error () + ((_ expression) + (declaration-error + (begin + (deflanguage diagnostic-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules (source-file expression))) + (bind-fixture-grammar-release diagnostic-study "diagnostic-study" "v1" "diagnostic-study.v1") )))) +(def root-without-node (invalid-root-error identifier)) +(def macro-root-without-node + (invalid-root-error (required-items argument identifier ","))) +(def node-token-collision + (invalid-expression-error (node Identifier identifier))) +(def multi-body-root + (declaration-error + (begin + (deflanguage diagnostic-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules (source-file identifier identifier))) + (bind-fixture-grammar-release diagnostic-study "diagnostic-study" "v1" "diagnostic-study.v1") ))) +(def source-free-collision + (invalid-expression-error source-free (node Identifier identifier))) +(defrules token-node () + ((_ item) (node Identifier item))) +(def macro-node-token-collision + (invalid-expression-error (token-node identifier))) + +;; One admission matrix covers the author projection and canonical compiler entry. +(defrules invalid-core-error () + ((_ expression) + (declaration-error + (compile-language diagnostic-study + (identity "diagnostic-study" "v1" "diagnostic-study.v1") + (syntax-kinds (SourceFile node ()) (Identifier token (text))) + (terminals (identifier Identifier)) + (lexical-rules (identifier (identifier))) + (rules (source-file (alias SourceFile expression))) + (extras) (keywords) (parser-entrypoints (source-file parse pure)) + (recoveries) (flow (source lexical) (lexical cst)))))) + +(def scalar-errors + (list (invalid-expression-error "") + (invalid-expression-error (literal "")) + (invalid-expression-error (layout-start "")) + (invalid-expression-error (layout-next "")) + (invalid-expression-error (prec sideways 10 identifier)) + (invalid-expression-error (prec left "high" identifier)))) +(def scalar-blames + '("" (literal "") (layout-start "") (layout-next "") + (prec sideways 10 identifier) (prec left "high" identifier))) +(def core-errors + (list (invalid-core-error (optional)) + (invalid-core-error (field argument)) + (invalid-core-error (sequence)) + (invalid-core-error (token identifier extra)) + (invalid-core-error (literal "")) + (invalid-core-error (layout-end "")) + (invalid-core-error (precedence sideways 10 (token identifier))) + (invalid-core-error (precedence left "high" (token identifier))) + (invalid-core-error (optional . identifier)))) +(def core-blames + '((optional) (field argument) (sequence) (token identifier extra) + (literal "") (layout-end "") (precedence sideways 10 (token identifier)) + (precedence left "high" (token identifier)) (optional . identifier))) +(def dotted-surface + (invalid-expression-error (optional . identifier))) + +;; Default normalization must preserve the declaration the author actually wrote. +(def defaulted-root + (declaration-error + (begin + (deflanguage diagnostic-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier))))) + (rules (source-file identifier))) + (bind-fixture-grammar-release diagnostic-study "diagnostic-study" "v1" "diagnostic-study.v1") ))) +(def source-free-defaulted-root + (declaration-error source-free + (begin + (deflanguage diagnostic-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier))))) + (rules (source-file identifier))) + (bind-fixture-grammar-release diagnostic-study "diagnostic-study" "v1" "diagnostic-study.v1") ))) + +(def (source-line location) + (call-with-input-file (car location) + (lambda (port) + (let loop ((line 1)) + (let (text (read-line port)) + (when (eof-object? text) (error "diagnostic line missing" location)) + (if (= line (cadr location)) text (loop (+ line 1)))))))) +(def (check-error result message blame text) + (check (car result) => message) + (check (cadr result) => blame) + (check (pair? (cadddr result)) => #t) + (check (cadr (car (cadddr result))) => 'diagnostic-study) + (let* ((location (caddr result)) (line (source-line location)) + (offset (caddr location))) + (check (substring line offset (+ offset (string-length text))) => text))) + +(def language-diagnostics-test + (test-suite "concise declaration author diagnostics" + (test-case "direct unknown targets and constructor retain reader parameters" + (check-error direct-reference "explicit reference names an unknown rule" + 'missing-rule "missing-rule") + (check-error direct-token "explicit token names an unknown lexical declaration" + 'missing-token "missing-token") + (check-error direct-constructor "unknown concise GrammarExpr constructor" + 'misspelled "misspelled") + (check-error direct-identifier "unresolved concise grammar identifier" + 'missing-name "missing-name")) + (test-case "macro and renamed import report the author invocation" + (check-error macro-reference "explicit reference names an unknown rule" + '(required-items argument (reference missing-rule) ",") "(required-items") + (check-error renamed-reference "explicit reference names an unknown rule" + '(renamed-items argument (reference missing-rule) ",") "(renamed-items")) + (test-case "invalid generated constructors blame the author call" + (check-error macro-constructor "unknown concise GrammarExpr constructor" + (quote (broken-wrapper identifier)) "(broken-wrapper")) + (test-case "equal invalid expressions select the first occurrence" + (check-error repeated-reference "explicit reference names an unknown rule" + 'repeated-missing "repeated-missing") + (let* ((location (caddr repeated-reference)) (line (source-line location)) + (offset (caddr location))) + (check (substring line (- offset 11) offset) => "(reference ") + (check (substring line 0 offset) + => " (invalid-expression-error (seq (reference "))) + (test-case "forward helper shape is checked before field inference" + (check-error forward-field "invalid concise field expression" + (quote (field argument)) "(field argument")) + (test-case "malformed forward helpers remain typed syntax errors" + (check-error forward-unary "invalid concise unary grammar expression" + (quote (optional)) "(optional") + (check-error forward-reference "explicit reference names an unknown rule" + (quote (reference)) "(reference")) + (test-case "root obligation identifies the authored body" + (check-error root-without-node "concise language root rule must construct one node" + 'identifier "identifier") + (check-error macro-root-without-node "concise language root rule must construct one node" + '(required-items argument identifier ",") "(required-items") + (check-error multi-body-root "concise language root rule must construct one node" + '(source-file identifier identifier) "(source-file identifier identifier)")) + (test-case "node and token kind collision retains author context" + (check-error node-token-collision "concise syntax kind is both a node and a token" + 'Identifier "Identifier") + (check-error macro-node-token-collision "concise syntax kind is both a node and a token" + '(token-node identifier) "(token-node")) + (test-case "canonical scalar constraints fail at concise admission" + (for-each + (lambda (result blame) + (check-error result "invalid canonical GrammarExpr metadata" blame + (if (string? blame) "\"\"" "("))) + scalar-errors scalar-blames) + (check-error dotted-surface "invalid concise grammar expression" + '(optional . identifier) "(optional")) + (test-case "canonical entry rejects malformed constructors before list access" + (for-each + (lambda (result blame) + (check-error result "invalid GrammarExpr declaration" blame "(")) + core-errors core-blames)) + (test-case "canonical header checks and recursive membership stay distinct" + (for-each + (lambda (row) (check (and (grammar-expression? (car row)) #t) => (cadr row))) + '(((empty) #t) ((literal "a") #t) ((literal "") #f) + ((layout-start " ") #t) ((layout-next "") #f) + ((layout-end) #t) ((layout-end ")") #t) ((layout-end "") #f) + ((token name) #t) ((reference rule) #t) ((sequence (empty)) #t) + ((choice) #f) ((optional (empty)) #t) ((repeat (empty)) #t) + ((repeat1 (literal "a")) #t) ((field value (token name)) #t) + ((alias Node (reference rule)) #t) + ((precedence left 10 (token name)) #t) + ((precedence sideways 10 (token name)) #f) + ((precedence right "high" (token name)) #f) + ((optional . name) #f) ((optional invalid-child) #f))) + (check (and (grammar-expression-header? '(optional invalid-child)) #t) => #t) + ;; Nullability/liveness policy is owned by construction/resolved LR analysis, + ;; not by canonical structural membership or the new header predicate. + (check (grammar-expression? '(repeat (empty))) => #t)) + (test-case "defaulted declarations retain the original author context" + (check-error defaulted-root "concise language root rule must construct one node" + 'identifier "identifier") + (check (car (cadddr defaulted-root)) + => '(deflanguage diagnostic-study + (syntax + (lexical + (root source-file) + (lex (identifier Identifier (identifier))))) + (rules (source-file identifier)))) + (check (car source-free-defaulted-root) + => "concise language root rule must construct one node") + (check (caddr source-free-defaulted-root) => #f) + (check (cadddr source-free-defaulted-root) => (cadddr defaulted-root))) + (test-case "source-free input does not invent author coordinates" + (check (car source-free-reference) => "explicit reference names an unknown rule") + (check (cadr source-free-reference) => 'missing-rule) +(check (caddr source-free-reference) => #f) + (check (car source-free-collision) => "concise syntax kind is both a node and a token") + (check (cadr source-free-collision) => 'Identifier) + (check (caddr source-free-collision) => #f)))) diff --git a/t/language-entry-boundary-test.ss b/t/language-entry-boundary-test.ss new file mode 100644 index 00000000..570a0b37 --- /dev/null +++ b/t/language-entry-boundary-test.ss @@ -0,0 +1,109 @@ +;;; -*- Gerbil -*- +;;; Production dispatch and development conformance have distinct dependencies. +(import (only-in :gerbil-parser/src/ffi/language-artifact-codec bind-language-abi abi-descriptor-payload abi-parse-binary-payload) + (only-in :std/vector/u8vector little u8vector-u32-ref) + (for-syntax (only-in :gerbil/expander core-expand)) + :std/test + (only-in :clan/poo/object .ref .slot?) + (only-in :clan/poo/mop validate) + (only-in :gerbil-parser/language-support/entry LanguageLoaderContract deflanguage-parser-loader) + (only-in :gerbil-parser/language-support/development + LanguageDevelopmentLoaderContract language-loader-fixtures language-loader-fixture-count run-language-test) + (only-in :gerbil-parser/language-support/fixture syntax-fixture-source) + (only-in :gerbil-parser/t/fixtures/language-entries language-entry-pairs) + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-language) + (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-test-language) + (only-in :gerbil-parser/languages/gql/parser gql-language) + (only-in :gerbil-parser/languages/cypher/parser opencypher-language) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-events parse-artifact-roundtrip)) +(export language-entry-boundary-test) +(defsyntax (production-declaration-rejections stx) + (def (message form) + (with-catch (lambda (condition) (error-message condition)) + (lambda () (core-expand form) (error "invalid production declaration admitted")))) + (datum->syntax #'production-declaration-rejections + (list 'quote + (list + (message #'(deflanguage-parser-loader invalid-keyword + (grammar #f) (parse invalid-parse) (slots fixtures: '()))) + (message #'(deflanguage-parser-loader invalid-method + (grammar #f) (parse invalid-parse) (slots (tests '())))) + (message #'(deflanguage-parser-loader invalid-metadata + (grammar #f) (parse invalid-parse) (metadata '()))) + (message #'(deflanguage-parser-loader invalid-alias + (grammar old #f) (parse invalid-parse) (slots metadata: '()))))))) +(def language-entry-boundary-test + (test-suite "production and development entry separation" + (test-case "production declarations reject development services and legacy sections before binding" + (check (production-declaration-rejections) + => '("development service requires deflanguage-development-loader" + "development service requires deflanguage-development-loader" + "unknown language loader section" + "invalid language loader declaration"))) + (test-case "production declarations reject inherited development services" + (check (with-catch (lambda (condition) (error-message condition)) + (lambda () + (let () + (deflanguage-parser-loader (invalid-inherited :: self arithmetic-test-language) + (grammar (.ref arithmetic-language 'descriptor)) (parse invalid-parse)) + invalid-inherited))) + => "production parser entry contains a development service") + (check (with-catch (lambda (_) 'rejected) + (lambda () (run-language-test arithmetic-language 'fixtures) 'admitted)) + => 'rejected)) + + (test-case "native codec uses parser-owned descriptors for accepted and rejected UTF-8 source" + (for-each (lambda (row) + (let* ((language (bind-language-abi (.ref (car row) 'descriptor))) (source (cadr row)) (expected (caddr row)) + (descriptor (abi-descriptor-payload language)) + (payload (abi-parse-binary-payload language source))) + (check (positive? (string-length descriptor)) => #t) + (check (subu8vector payload 0 4) => #u8(71 80 65 49)) + (check (u8vector-u32-ref payload 8 little) => expected) + (check (u8vector-length payload) + => (+ 80 (* 24 (u8vector-u32-ref payload 12 little)))))) + (list (list gql-language "RETURN '你好'" 0) + (list opencypher-language "MATCH (n:Person) RETURN n\n" 0) + (list gql-language "RETURN (" 1) (list opencypher-language "RETURN (" 1)))) + (test-case "all ten production entries need no corpus or development services" + (check (length language-entry-pairs) => 10) + (for-each + (lambda (row) + (validate LanguageLoaderContract (car row)) + (for-each (lambda (name) (check (.slot? (car row) name) => #f)) + '(fixtures fixture-catalog tests scan-workers build-strategies native-test-profile))) + language-entry-pairs)) + (test-case "all development entries preserve the exact production descriptor and corpus" + (for-each + (lambda (row) + (let ((production (car row)) (development (cadr row))) + (validate LanguageDevelopmentLoaderContract development) + (validate LanguageLoaderContract development) + (check (eq? (.ref development 'descriptor) (.ref production 'descriptor)) => #t) + (check (language-loader-fixture-count development) => (caddr row)))) + language-entry-pairs)) + (test-case "all accepted and rejected corpora reproduce complete production events" + (for-each + (lambda (row) + (for-each + (lambda (fixture) + (let* ((source (syntax-fixture-source fixture)) + (production ((.ref (car row) '.parse) source)) + (development ((.ref (cadr row) '.parse) source))) + (check (parse-artifact-success? development) => (parse-artifact-success? production)) + (check (parse-artifact-events development) => (parse-artifact-events production)) + (check (parse-artifact-roundtrip development) => (parse-artifact-roundtrip production)) + (check (parse-artifact-roundtrip production) => source))) + (language-loader-fixtures (cadr row)))) + language-entry-pairs)) + (test-case "engine dispatch does not import common adapters or fixture services" + (let (forms (call-with-input-file "src/language/entry.ss" + (lambda (port) (read port)))) + (check (car forms) => 'import) + (for-each + (lambda (item) + (check (and (pair? item) (eq? (car item) 'only-in) + (member (cadr item) '(../../language-support/fixture + ../../language-support/development))) => #f)) + (cdr forms)))))) diff --git a/t/language-grammar-observability-test.ss b/t/language-grammar-observability-test.ss new file mode 100644 index 00000000..237096ae --- /dev/null +++ b/t/language-grammar-observability-test.ss @@ -0,0 +1,34 @@ +;;; -*- Gerbil -*- +;;; Language observation owns its full compiled openCypher grammar fixture. +(import :std/test + (only-in :gerbil-parser/src/runtime/observability call-with-parser-observed-phase) + "./scenarios/observability/opencypher-grammar-phases/scenario") +(def language-grammar-observability-tests + (test-suite "LanguageGrammar Core phase observation" + (test-case "disabled phase bodies preserve values and evaluate inputs once" + (let ((policies 0) (phases 0) (bodies 0)) + (let-values (((a b) + (call-with-parser-observed-phase + (begin (set! policies (1+ policies)) #f) + (begin (set! phases (1+ phases)) 'contract) + (lambda () (set! bodies (1+ bodies)) (values 'a 'b))))) + (check (list a b policies phases bodies) => '(a b 1 1 1))))) + (test-case "phase calls preserve first-class and computed thunk execution" + (let* ((calls 0) (invoke call-with-parser-observed-phase) + (thunk (lambda () (set! calls (1+ calls)) 'done))) + (check (invoke #f 'contract thunk) => 'done) + (check (call-with-parser-observed-phase #f 'contract thunk) => 'done) + (check calls => 2))) + (test-case "disabled phase bodies propagate the original exception" + (check (with-catch error-irritants + (lambda () + (call-with-parser-observed-phase #f 'contract + (lambda () (error "phase failure" 'receipt))))) + => '(receipt))) + (test-case "LanguageGrammar atomically configures POO Flow phase observation" + (let (receipt (opencypher-grammar-observability-scenario)) + (write receipt) (newline) (force-output) + (check (opencypher-grammar-observability-scenario-pass? receipt) + => #t))))) +(def language-grammar-observability-test language-grammar-observability-tests) +(export language-grammar-observability-tests language-grammar-observability-test) diff --git a/t/language-loader-test.ss b/t/language-loader-test.ss new file mode 100644 index 00000000..4848862e --- /dev/null +++ b/t/language-loader-test.ss @@ -0,0 +1,343 @@ +;;; POO language loaders bind effective inherited slots before engine dispatch. +(import :std/test + (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-source-receipt) + (for-syntax (only-in :gerbil/expander core-expand)) + (only-in :clan/poo/object .o .ref .cc object?) + (only-in :clan/poo/mop define-type element? validate) + (only-in :core/types PooFlowContract. poo-flow-classification-evidence) + (only-in :gerbil-parser/language-support/development + deflanguage-development-loader LanguageDevelopmentLoader. LanguageDevelopmentLoaderContract + language-parser-entry-ref check-language-loader-fixtures! + declare-language-source-scan-worker make-language-scan-worker + declare-language-fixture-test run-language-test) + (only-in :gerbil-parser/languages/arithmetic/parser + arithmetic-language-grammar) + (only-in :gerbil-parser/language-support/fixture + defsyntax-fixture defsyntax-corpus syntax-fixture-source syntax-fixture-source-digest) + (only-in :gerbil-parser/src/runtime/identity sha256-text) + (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-basic-fixture) + (only-in :gerbil-parser/src/language/source source-language-scanner-factory) + (only-in "fixtures/source-strategies.ss" test-source-strategy) + (only-in :gerbil-parser/src/runtime/source-scanner source-scanner-tokens) + (only-in :gerbil-parser/src/runtime/token token-lexeme) + (only-in :gerbil-parser/src/language/source declare-source-language) + (only-in :gerbil-parser/src/runtime/source-engines LineSourceStrategy.) + (only-in :gerbil-parser/languages/bash/parser bash-source-language) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-roundtrip)) + +(def Dialect. + (.o (:: @ LanguageDevelopmentLoader.) + dialect: 'arithmetic + ;; Identity and engine dispatch from parents must not take control. + language: "incorrect-parent" + (.parse (lambda (_) (error "parent parse must never run"))))) + +(def dialect-admissions 0) + +(define-type (DialectContract @ PooFlowContract.) + identity: 'loader-test/dialect + .classify: (lambda (candidate context) + (set! dialect-admissions (+ dialect-admissions 1)) + (let (ok? (eq? (.ref candidate 'dialect) 'arithmetic)) + (poo-flow-classification-evidence + 'loader-test/dialect candidate ok? + (if ok? '() '((invalid dialect))) context)))) + +(deflanguage-development-loader (arithmetic-loader :: self Dialect.) + (descriptor arithmetic-language-grammar) + (parse parse-loaded-arithmetic) + (slots dialect-name: "Arithmetic" + (summary (string-append (.ref self 'dialect-name) " dialect"))) + (contracts DialectContract)) + +(def (rejects? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) + +(defsyntax (engine-slot-rejection-message stx) + (def (message form) + (with-catch (lambda (condition) (error-message condition)) + (lambda () (core-expand form)))) + (datum->syntax + #'engine-slot-rejection-message + (string-append + (message + #'(deflanguage-development-loader (invalid-loader @ LanguageDevelopmentLoader.) + (descriptor #f) (parse invalid-parse) + (slots (.parse (lambda (_) #f))) (contracts))) + " | " + (message + #'(deflanguage-development-loader (invalid-keyword-loader @ LanguageDevelopmentLoader.) + (descriptor #f) (parse invalid-keyword-parse) + (slots schema: "invalid") (contracts)))))) + +(defsyntax (invalid-declaration-messages stx) + (def (message form) + (with-catch (lambda (condition) (error-message condition)) + (lambda () (core-expand form)))) + (datum->syntax + #'invalid-declaration-messages + (cons 'quote (list (list + (message #'(deflanguage-development-loader bad (grammar #f) (parse parse-bad) + (slots) (slots))) + (message #'(deflanguage-development-loader bad (grammar #f) (parse parse-bad) + (contracts) (slots))) + (message #'(deflanguage-development-loader bad (grammar #f) (parse parse-bad) + (options))) + (message #'(deflanguage-development-loader (bad marker self LanguageDevelopmentLoader.) + (descriptor #f) (parse parse-bad) (slots) (contracts))) + (message #'(deflanguage-development-loader bad (grammar #f) (parse "bad")))))))) + +(defsyntax (inline-fixture-rejections stx) + (def (rejects form) + (with-catch (lambda (_) #t) (lambda () (core-expand form) #f))) + (datum->syntax + #'inline-fixture-rejections + (list 'quote + (list + (rejects #'(defsyntax-fixture invalid-fixture + (identity "bad" "lang" "v1" "contract") + (text 17) (expect accepted Root ()))) + (rejects #'(defsyntax-fixture invalid-fixture + (identity "bad" "lang" "v1" "contract") + (text "x") (expect unknown Root ()))) + (rejects #'(defsyntax-corpus invalid-corpus + (identity "lang" "v1" "contract") + (accepted ("bad" invalid-fixture (text 17) Root ())) + (rejected))))))) + +(defsyntax-corpus inline-arithmetic-fixtures + (identity "arithmetic" "v1" "arithmetic-expression.v1") + (accepted ("loader/inline/unicode" inline-unicode (text "α + 2") SourceFile (Expression))) + (rejected ("loader/inline/incomplete" inline-incomplete (text "1 +") ))) + +(defsyntax-fixture foreign-language + (identity "foreign/language" "foreign" "v1" "arithmetic-expression.v1") + (text "1") (expect accepted SourceFile ())) +(defsyntax-fixture foreign-version + (identity "foreign/version" "arithmetic" "v2" "arithmetic-expression.v1") + (text "1") (expect accepted SourceFile ())) +(defsyntax-fixture foreign-contract + (identity "foreign/contract" "arithmetic" "v1" "foreign.v1") + (text "1") (expect accepted SourceFile ())) + +(def language-loader-test + (test-suite "POO language loader admission" + (test-case "source schema slots admit ordinary values and native inherited POO" + (def template '((schema . "test.source.v1") (namespace . test) + (sourceVersion . "v1") (upstreamCommit . "first"))) + (deflanguage-parser-loader value-entry (descriptor arithmetic-language-grammar) + (parse value-parse) (slots source-schema: template)) + (def parent (.o schema: "test.source.v1" namespace: 'test + sourceVersion: "v1" upstreamCommit: "first")) + (deflanguage-parser-loader inherited-entry (descriptor arithmetic-language-grammar) + (parse inherited-parse) + (slots source-schema: (.o (:: @ parent) upstreamCommit: "second"))) + (check (language-parser-source-receipt value-entry '(normalization factoring)) + => '((schema . "test.source.v1") (namespace . test) (name . factoring) + (kind . normalization) (sourceVersion . "v1") (upstreamCommit . "first"))) + (check (language-parser-source-receipt inherited-entry 'replacement) + => '((schema . "test.source.v1") (namespace . test) (name . replacement) + (sourceVersion . "v1") (upstreamCommit . "second"))) + (check (language-parser-source-receipt value-entry 'replacement) + => '((schema . "test.source.v1") (namespace . test) (name . replacement) + (sourceVersion . "v1") (upstreamCommit . "first"))) + (check (parse-artifact-success? (value-parse "1 + 2")) => #t) + (check (parse-artifact-success? (inherited-parse "1 + 2")) => #t) + (check (rejects? (lambda () + (language-parser-source-receipt value-entry '(unknown factoring)))) => #t)) + (test-case "malformed source schemas fail at production entry admission" + (for-each + (lambda (configuration) + (check (rejects? (lambda () + (deflanguage-parser-loader invalid-entry (descriptor arithmetic-language-grammar) + (parse invalid-parse) (slots source-schema: configuration)) + invalid-entry)) => #t)) + '(((schema . "test.source.v1") (namespace . test) (sourceVersion . "v1")) + ((schema . "test.source.v1") (namespace . test) (sourceVersion . "v1") (sourceVersion . "v2")) + ((schema . "test.source.v1") (namespace . "wrong") (sourceVersion . "v1") (upstreamCommit . "first")) + ((schema . "test.source.v1") (namespace . test) (sourceVersion . "v1") (upstreamCommit . "")) + ((schema . "test.source.v1") (namespace . test) (sourceVersion . "v1") (upstreamCommit . "first") (typo . 1))))) + + (test-case "inherits extensions and binds engine identity and dispatch" + (check (object? arithmetic-loader) => #t) + (check (element? LanguageDevelopmentLoaderContract arithmetic-loader) => #t) + (check (.ref arithmetic-loader 'dialect) => 'arithmetic) + (check (.ref arithmetic-loader 'dialect-name) => "Arithmetic") + (check (.ref arithmetic-loader 'summary) => "Arithmetic dialect") + (check (language-parser-entry-ref arithmetic-loader 'language) => "arithmetic") + (check (.ref arithmetic-loader 'capabilities) => '(grammar-ir parser-ir scheme)) + (let* ((source "a + 2 * b") (artifact (parse-loaded-arithmetic source))) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + (test-case "typed declarations compose with POO extension forms and optional sections" + (deflanguage-development-loader (typed-loader :: self Dialect.) + (grammar arithmetic-language-grammar) (parse parse-typed) + (slots label: "typed" (summary (.ref self 'label)))) + (deflanguage-development-loader (contract-loader @ Dialect.) + (grammar arithmetic-language-grammar) (parse parse-contract) + (contracts DialectContract)) + (deflanguage-development-loader (self-loader :: self Dialect.) + (grammar arithmetic-language-grammar) (parse parse-self)) + (deflanguage-development-loader (self-contract-loader :: self Dialect.) + (grammar arithmetic-language-grammar) (parse parse-self-contract) + (contracts DialectContract)) + (check (.ref typed-loader 'summary) => "typed") + (for-each (lambda (parse) (check (parse-artifact-success? (parse "a+1")) => #t)) + (list parse-typed parse-contract parse-self parse-self-contract))) + (test-case "source declarations use the same extension normalization and lossless dispatch" + (deflanguage-development-loader (source-loader :: self LanguageDevelopmentLoader.) + (source bash-source-language) (parse parse-source-loaded) + (slots label: "Bash" (summary (.ref self 'label)))) + (let* ((source "echo α\n") (artifact (parse-source-loaded source))) + (check (.ref source-loader 'capabilities) => '(scheme-source)) + (check (.ref source-loader 'summary) => "Bash") + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + (test-case "POO fixture values inherit and override without parser execution" + (def Pack. + (.o (:: self Dialect.) + metadata: (.o edition: "v1" version: "wrong" digest: "wrong") + fixtures: (list arithmetic-basic-fixture) + scan-workers: '())) + (deflanguage-development-loader (pack-loader :: self Pack.) + (grammar arithmetic-language-grammar) (parse parse-pack)) + (check (.ref pack-loader 'fixtures) => (list arithmetic-basic-fixture)) + (check (rejects? (lambda () (run-language-test pack-loader 'missing))) => #t) + (check (.ref (.ref pack-loader 'metadata) 'edition) => "v1") + (check (.ref (.ref pack-loader 'metadata) 'version) => (.ref pack-loader 'version)) + (check (equal? (.ref (.ref pack-loader 'metadata) 'digest) "wrong") => #f) + (check (.ref pack-loader 'grammars) => (list arithmetic-language-grammar)) + (let (artifacts (run-language-test pack-loader 'fixtures)) + (check (length artifacts) => 1) + (check (parse-artifact-success? (car artifacts)) => #t)) + (let (overridden (.cc pack-loader 'fixtures inline-arithmetic-fixtures)) + (check (map parse-artifact-success? (run-language-test overridden 'fixtures)) => '(#t #f))) + (let (extended (.cc pack-loader 'fixtures + (append (.ref pack-loader 'fixtures) (list inline-incomplete)))) + (check (map parse-artifact-success? (run-language-test extended 'fixtures)) => '(#t #f))) + (deflanguage-development-loader (worker-loader :: self LanguageDevelopmentLoader.) + (source bash-source-language) (parse parse-worker-source) + (slots scan-workers: (list (cons 'bash-command + (declare-language-source-scan-worker + bash-source-language))))) + (let* ((source "echo α\n") + (worker (make-language-scan-worker worker-loader 'bash-command source))) + (check (apply string-append (map token-lexeme (source-scanner-tokens worker 'command))) => source))) + (test-case "typed fixture services extend POO slots while engine checks outcomes" + (check (rejects? (lambda () (declare-language-fixture-test #f))) => #t) + (deflanguage-development-loader (extended-tests :: self LanguageDevelopmentLoader.) + (grammar arithmetic-language-grammar) (parse parse-extended-tests) + (slots fixtures: (list arithmetic-basic-fixture) + tests: (list (cons 'conformance (declare-language-fixture-test arithmetic-language-grammar))))) + (check (length (run-language-test extended-tests 'conformance)) => 1) + (let (fake (.cc extended-tests '.parse (lambda (_) #t))) + (check (rejects? (lambda () (run-language-test fake 'conformance))) => #t))) + (test-case "declared source workers reject foreign products and undeclared names" + (check (rejects? (lambda () + (declare-language-source-scan-worker arithmetic-language-grammar))) => #t) + (for-each + (lambda (factory) + (def bad-source (declare-source-language "bad-worker" "v1" "bad-worker.test" + (test-source-strategy factory (lambda (_) '()) (lambda _ #f)))) + (deflanguage-development-loader (bad-worker-loader :: self LanguageDevelopmentLoader.) + (source bad-source) (parse parse-bad-worker) + (slots scan-workers: (list (cons 'command + (declare-language-source-scan-worker bad-source))))) + (check (rejects? (lambda () (make-language-scan-worker bad-worker-loader 'command "echo α\n"))) => #t) + (check (rejects? (lambda () (make-language-scan-worker bad-worker-loader 'missing "echo α\n"))) => #t)) + (list (lambda (_) #f) (lambda (_) ((source-language-scanner-factory bash-source-language) "different source"))))) + (test-case "metadata admission rejects invalid descriptors and service registries" + (for-each + (lambda (row) + (check (element? LanguageDevelopmentLoaderContract (.cc arithmetic-loader (car row) (cdr row))) => #f)) + (list (cons 'grammars '()) (cons 'grammars (list #f)) + (cons 'metadata '()) + (cons 'metadata (.cc (.ref arithmetic-loader 'metadata) 'version "wrong")) + (cons 'metadata (.cc (.ref arithmetic-loader 'metadata) 'digest "wrong")) (cons 'fixtures #f) + (cons 'fixtures (lambda () '())) + (cons 'fixtures '(invalid)) + (cons 'tests (list (cons 'bad #f))) + (cons 'tests (list (cons 'fixtures (lambda (_) #t)))) + (cons 'tests (list (cons 'arbitrary (lambda (_) '(accepted))))) + (cons 'tests (list (cons 'renamed check-language-loader-fixtures!))) + (cons 'tests (list (cons 'fixtures check-language-loader-fixtures!))) + (cons 'tests (list (cons 'foreign (declare-language-fixture-test bash-source-language)))) + (cons 'tests (list (cons 'same (declare-language-fixture-test arithmetic-language-grammar)) + (cons 'same (declare-language-fixture-test arithmetic-language-grammar)))) + (cons 'scan-workers (list (cons 'bad #f))) + (cons 'scan-workers (list (cons 'undeclared (source-language-scanner-factory bash-source-language)))) + (cons 'scan-workers (list (cons 'foreign (declare-language-source-scan-worker + bash-source-language)))))) + (check (rejects? (lambda () + (check-language-loader-fixtures! + (.cc arithmetic-loader 'fixtures '(invalid))))) => #t)) + (test-case "inline corpora preserve source bytes, digest and positive/negative conformance" + (check (inline-fixture-rejections) => '(#t #t #t)) + (let* ((loader (.cc arithmetic-loader 'fixtures inline-arithmetic-fixtures)) + (artifacts (check-language-loader-fixtures! loader))) + (check (map parse-artifact-success? artifacts) => '(#t #f)) + (check (syntax-fixture-source inline-unicode) => "α + 2") + (check (syntax-fixture-source-digest inline-unicode) => (sha256-text "α + 2")))) + (test-case "fixture admission rejects foreign identities and duplicate IDs before parsing" + (def calls 0) + (for-each + (lambda (fixtures) + (let (loader (.cc (.cc arithmetic-loader 'fixtures fixtures) + '.parse (lambda (_) (set! calls (+ calls 1)) #f))) + (check (rejects? (lambda () (check-language-loader-fixtures! loader))) => #t))) + (list (list foreign-language) (list foreign-version) (list foreign-contract) + (list arithmetic-basic-fixture arithmetic-basic-fixture) + ;; Admission covers the whole corpus before parsing its valid prefix. + (list arithmetic-basic-fixture foreign-contract))) + (check calls => 0)) + (test-case "parsing reuses the loaded entry without extension admission" + (let (before dialect-admissions) + (for-each + (lambda (source) + (check (parse-artifact-success? (parse-loaded-arithmetic source)) => #t)) + '("1" "a + b" "a * (b + 2)")) + (check dialect-admissions => before))) + (test-case "rejects a missing descriptor and inconsistent identity" + (check (element? LanguageDevelopmentLoaderContract LanguageDevelopmentLoader.) => #f) + (check (rejects? (lambda () (validate LanguageDevelopmentLoaderContract LanguageDevelopmentLoader.))) => #t) + (check (element? LanguageDevelopmentLoaderContract (.cc arithmetic-loader 'language "bash")) => #f) + (check (element? LanguageDevelopmentLoaderContract (.cc arithmetic-loader 'descriptor #f)) => #f)) + (test-case "rejects engine slot overrides during macro expansion" + (check (engine-slot-rejection-message) + => "language loader extension overrides an engine slot | language loader extension overrides an engine slot")) + (test-case "normalization rejects malformed declarations before publishing a loader" + (check (invalid-declaration-messages) + => '("duplicate or out-of-order language loader slots" + "duplicate or out-of-order language loader slots" + "unknown language loader section" + "invalid language loader binding" + "language loader parse binding must be an identifier"))) + (test-case "grammar and source declarations enforce their descriptor kind" + (let (source-descriptor + (declare-source-language "source-test" "v1" "source-test.v1" + LineSourceStrategy.)) + (check + (rejects? (lambda () + (deflanguage-development-loader wrong-grammar + (grammar source-descriptor) (parse parse-wrong-grammar)) + wrong-grammar)) => #t) + (check + (rejects? (lambda () + (deflanguage-development-loader wrong-source + (source arithmetic-language-grammar) (parse parse-wrong-source)) + wrong-source)) => #t))) + (test-case "extension contracts validate the effective slot object" + (check + (rejects? + (lambda () + (deflanguage-development-loader (invalid-dialect @ Dialect.) + (descriptor arithmetic-language-grammar) + (parse parse-invalid-dialect) + (slots (dialect 'other)) + (contracts DialectContract)) + invalid-dialect)) => #t) + (check (element? DialectContract arithmetic-loader) => #t) + (check (rejects? (lambda () (validate DialectContract (.cc arithmetic-loader 'dialect 'other)))) => #t)))) + +(export language-loader-test) diff --git a/t/language-loader-value-test.ss b/t/language-loader-value-test.ss new file mode 100644 index 00000000..5895e44d --- /dev/null +++ b/t/language-loader-value-test.ss @@ -0,0 +1,128 @@ +;;; Stable loader identity and ordinary POO value slots use the unreleased v2 contract. +(import :std/test + (only-in :clan/poo/object .o .ref .cc) + (only-in :clan/poo/mop element?) + (only-in :gerbil-parser/language-support/development + deflanguage-development-loader LanguageDevelopmentLoader. LanguageDevelopmentLoaderContract + +language-parser-entry-schema+ run-language-test + language-loader-fixtures language-loader-fixture-count language-loader-fixture) + (only-in :gerbil-parser/src/language/source declare-source-language) + (only-in :gerbil-parser/src/runtime/source-engines LineSourceStrategy.) + (only-in :gerbil-parser/language-support/fixture defsyntax-fixture syntax-fixture-copy syntax-fixture-id syntax-fixture-source + syntax-fixture-source-digest syntax-fixture-expected-status)) +(export language-loader-value-test) + +(def descriptor + (declare-source-language "loader-value" "0.12.2" "loader-value.test" + LineSourceStrategy.)) +(def Pack. + (.o (:: self LanguageDevelopmentLoader.) + metadata: (.o version: "spoofed" digest: "spoofed" upstream-commit: "pinned") + fixtures: '() + (edition-summary (.ref (.ref self 'metadata) 'version)))) +(deflanguage-development-loader (loader :: self Pack.) + (source descriptor) (parse parse-loader)) +(defsyntax-fixture foreign + (identity "foreign" "another-language" "0.12.2" "loader-value.test") + (text "x") (expect rejected #f ())) +(def (rejects? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) + +(def fixture-descriptor + (declare-source-language "fixture-index" "0.12.2" "fixture-index.test" + (.o (:: self LineSourceStrategy.) root-kind: 'RecordFile token-kind: 'Record required-prefix: "ok:"))) +(defsyntax-fixture fixture-a + (identity "fixture-index/0000" "fixture-index" "0.12.2" "fixture-index.test") + (text "ok:α") (expect accepted RecordFile ())) +(defsyntax-fixture fixture-b + (identity "fixture-index/rejected" "fixture-index" "0.12.2" "fixture-index.test") + (text "bad") (expect rejected #f ())) +(defsyntax-fixture fixture-c + (identity "fixture-index/extended" "fixture-index" "0.12.2" "fixture-index.test") + (text "ok:中") (expect accepted RecordFile ())) +(def FixturePack. + (.o (:: self LanguageDevelopmentLoader.) fixtures: (list fixture-a fixture-b) + fixture-catalog: 'forged-parent + (accepted-count (language-loader-fixture-count self 'accepted)))) +(deflanguage-development-loader (fixture-loader :: self FixturePack.) + (source fixture-descriptor) (parse parse-fixture-index)) +(def language-loader-value-test + (test-suite "language loader value slots" + (test-case "engine fixture metadata partitions once and runs complete semantic conformance" + (check (language-loader-fixture-count fixture-loader) => 2) + (check (language-loader-fixture-count fixture-loader 'accepted) => 1) + (check (language-loader-fixture-count fixture-loader 'rejected) => 1) + (check (map syntax-fixture-id (language-loader-fixtures fixture-loader)) + => '("fixture-index/0000" "fixture-index/rejected")) + (check (map syntax-fixture-id (language-loader-fixtures fixture-loader 'accepted)) => '("fixture-index/0000")) + (check (map syntax-fixture-id (language-loader-fixtures fixture-loader 'rejected)) => '("fixture-index/rejected")) + (check (.ref fixture-loader 'accepted-count) => 1) + (check (eq? (.ref fixture-loader 'fixture-catalog) (.ref fixture-loader 'fixture-catalog)) => #t) + (check (length (run-language-test fixture-loader 'fixtures)) => 2) + (check (syntax-fixture-source (language-loader-fixture fixture-loader (string-copy "fixture-index/0000"))) => "ok:α") + (check (language-loader-fixture fixture-loader "missing") => #f)) + (test-case "POO replacement and extension recompute the effective fixture index" + (let* ((replacement (.cc fixture-loader 'fixtures (list fixture-c))) + (extension (.cc fixture-loader 'fixtures (append (.ref fixture-loader 'fixtures) (list fixture-c))))) + (check (element? LanguageDevelopmentLoaderContract replacement) => #t) + (check (element? LanguageDevelopmentLoaderContract extension) => #t) + (check (.ref replacement 'accepted-count) => 1) + (check (.ref extension 'accepted-count) => 2) + (check (language-loader-fixture-count replacement 'rejected) => 0) + (check (map syntax-fixture-id (language-loader-fixtures extension 'accepted)) + => '("fixture-index/0000" "fixture-index/extended")) + (check (eq? (.ref replacement 'fixture-catalog) (.ref fixture-loader 'fixture-catalog)) => #f) + (check (length (run-language-test extension 'fixtures)) => 3))) + (test-case "input and published fixture mutations cannot alter the admitted snapshot" + (let* ((input (syntax-fixture-copy fixture-a)) (owned-loader (.cc fixture-loader 'fixtures (list input))) + (snapshot (.ref owned-loader 'fixture-catalog)) + (published (language-loader-fixture owned-loader "fixture-index/0000"))) + (string-set! (syntax-fixture-source input) 0 #\x) + (string-set! (syntax-fixture-id published) 0 #\x) + (string-set! (syntax-fixture-source published) 0 #\x) + (set-car! (language-loader-fixtures owned-loader) fixture-b) + (check (syntax-fixture-source (language-loader-fixture owned-loader "fixture-index/0000")) => "ok:α") + (check (language-loader-fixture-count owned-loader 'accepted) => 1) + (check (length (run-language-test owned-loader 'fixtures)) => 1))) + (test-case "foreign identities, duplicate ids, invalid digests and forged derived slots reject at admission" + (let (corrupted (syntax-fixture-copy fixture-a)) + (string-set! (syntax-fixture-source corrupted) 0 #\x) + (for-each (lambda (candidate) (check (element? LanguageDevelopmentLoaderContract candidate) => #f)) + (list (.cc fixture-loader 'fixtures (list fixture-a (syntax-fixture-copy fixture-a))) + (.cc fixture-loader 'fixtures (list foreign)) + (.cc fixture-loader 'fixtures (list corrupted)) + (.cc fixture-loader 'fixtures (lambda () '())) + (.cc fixture-loader 'fixture-catalog 'forged) + (.cc fixture-loader 'fixture-catalog (.ref loader 'fixture-catalog))))) + (check (rejects? (lambda () (language-loader-fixtures fixture-loader 'unknown))) => #t) + (check (rejects? (lambda () (language-loader-fixture-count fixture-loader 'unknown))) => #t) + (check (rejects? (lambda () (language-loader-fixture fixture-loader 'not-a-string))) => #t)) + (test-case "metadata seals identity and preserves extension values without a schema bump" + (check +language-parser-entry-schema+ => "gerbil-parser.language-entry.v2") + (check (.ref (.ref loader 'metadata) 'language) => "loader-value") + (check (.ref (.ref loader 'metadata) 'version) => "0.12.2") + (check (.ref (.ref loader 'metadata) 'contract) => "loader-value.test") + (check (.ref (.ref loader 'metadata) 'digest-kind) => 'source-identity) + (check (equal? (.ref (.ref loader 'metadata) 'digest) "spoofed") => #f) + (check (.ref (.ref loader 'metadata) 'upstream-commit) => "pinned") + (check (.ref loader 'edition-summary) => "0.12.2") + (check (element? LanguageDevelopmentLoaderContract loader) => #t) + (check (element? LanguageDevelopmentLoaderContract + (.cc loader 'metadata (.cc (.ref loader 'metadata) 'version "spoofed"))) => #f)) + (test-case "fixture values inherit and replace without callback execution" + (check (.ref loader 'fixtures) => '()) + (check (run-language-test loader 'fixtures) => '()) + (check (element? LanguageDevelopmentLoaderContract (.cc loader 'fixtures (lambda () '()))) => #f) + (check (element? LanguageDevelopmentLoaderContract (.cc loader 'fixtures '(invalid))) => #f) + (check (rejects? (lambda () (run-language-test (.cc loader 'fixtures (list foreign)) 'fixtures))) => #t)) + (test-case "new descriptor revisions change metadata under the same public loader contract" + (def next-descriptor + (declare-source-language "loader-value" "0.12.3" "loader-value.test" + LineSourceStrategy.)) + (deflanguage-development-loader (next-loader :: self Pack.) + (source next-descriptor) (parse parse-next-loader)) + (check (.ref (.ref next-loader 'metadata) 'version) => "0.12.3") + (check (equal? (.ref (.ref next-loader 'metadata) 'digest) + (.ref (.ref loader 'metadata) 'digest)) => #f) + (check (.ref next-loader 'edition-summary) => "0.12.3") + (check (.ref next-loader 'schema) => (.ref loader 'schema))))) diff --git a/t/language-pack-metadata-test.ss b/t/language-pack-metadata-test.ss new file mode 100644 index 00000000..95fbf477 --- /dev/null +++ b/t/language-pack-metadata-test.ss @@ -0,0 +1,94 @@ +;;; Explicit POO pack entries expose grammar metadata and reusable test services. +(import (only-in :gerbil-parser/language-support/entry language-metadata-ref) (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-test-language) + (only-in :gerbil-parser/languages/hcl/parser-test hcl-test-language) + (only-in :gerbil-parser/languages/cypher/parser-test opencypher-test-language) + (only-in :gerbil-parser/languages/tla-plus/parser-test tla-plus-core-test-language) + (only-in :gerbil-parser/languages/bash/parser-test bash-test-language) + (only-in :gerbil-parser/languages/hl7/parser-test hl7-test-language) + (only-in :gerbil-parser/languages/fhirpath/parser-test fhirpath-test-language) + (only-in :gerbil-parser/languages/tla-plus/parser-test tla-plus-layout-test-language) + (only-in :gerbil-parser/languages/tla-plus/parser-test tla-plus-sany-candidate-test-language) + :std/test + (only-in :clan/poo/object .ref object?) + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-language) + (only-in :gerbil-parser/languages/hcl/parser hcl-language) + (only-in :gerbil-parser/languages/gql/parser gql-language) + (only-in :gerbil-parser/languages/gql/parser-test gql-test-language) + (only-in :gerbil-parser/languages/cypher/parser opencypher-language) + (only-in :gerbil-parser/languages/bash/parser bash-language) + (only-in :gerbil-parser/languages/hl7/parser hl7-language) + (only-in :gerbil-parser/languages/fhirpath/parser fhirpath-language) + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-core-language tla-plus-layout-language) + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-sany-candidate-language) + (only-in :gerbil-parser/src/runtime/source-scanner source-scanner-tokens) + (only-in :gerbil-parser/language-support/development make-language-scan-worker run-language-test +language-parser-entry-schema+) + (only-in :gerbil-parser/src/runtime/token token-lexeme) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid?) + (only-in :gerbil-parser/src/language/descriptor language-grammar-ir language-grammar-machine) + (only-in :gerbil-parser/src/language/source source-language? source-language-digest) + (only-in :gerbil-parser/src/compiler/machine parser-machine-grammar-digest)) + +;;; Count declarations in admitted IR, not filenames or copied result snapshots. +(def (external-site-count expression) + (cond ((not (pair? expression)) 0) + ((eq? (car expression) 'external) 1) + (else (apply + (map external-site-count expression))))) +(def (loader-external-count loader) + (let (descriptor (.ref loader 'descriptor)) + (if (source-language? descriptor) 'source-parser + (external-site-count (cdr (assq 'lexical-rules (language-grammar-ir descriptor))))))) + +(def language-pack-metadata-test + (test-suite "public POO language pack metadata" + (test-case "shipped entries expose their descriptor and value metadata" + (for-each + (lambda (row) + (let (loader (car row)) + (check (object? loader) => #t) + (for-each (lambda (key) + (check (language-metadata-ref (.ref loader 'metadata) key) => (.ref loader key))) + '(language version contract)) + (let (descriptor (.ref loader 'descriptor)) + (check (language-metadata-ref (.ref loader 'metadata) 'digest) + => (if (source-language? descriptor) (source-language-digest descriptor) + (parser-machine-grammar-digest (language-grammar-machine descriptor)))) + (check (language-metadata-ref (.ref loader 'metadata) 'digest-kind) + => (if (source-language? descriptor) 'source-identity 'parser-ir))) + (check (.ref loader 'schema) => +language-parser-entry-schema+) + (check (not (not (memq (.ref loader 'descriptor) (.ref loader 'grammars)))) => #t) + (check (language-metadata-ref (.ref loader 'metadata) 'grammar-format) => (cdr row)))) + (list (cons arithmetic-language 'concise-dsl) + (cons hcl-language 'concise-dsl) + (cons hl7-language 'concise-dsl) + (cons fhirpath-language 'concise-dsl) + (cons gql-language 'antlr4) + (cons opencypher-language 'iso-bnf) + (cons bash-language 'source-parser) + (cons tla-plus-core-language 'concise-dsl) + (cons tla-plus-layout-language 'concise-dsl) + (cons tla-plus-sany-candidate-language 'concise-dsl)))) + (test-case "the complete migration inventory follows all ten admitted descriptors" + (check (map loader-external-count + (list arithmetic-language hcl-language + gql-language opencypher-language + hl7-language fhirpath-language bash-language + tla-plus-core-language tla-plus-layout-language tla-plus-sany-candidate-language)) + => '(0 0 0 0 0 0 source-parser 0 0 0))) + (test-case "registered fixture services preserve native corpus conformance" + (for-each + (lambda (loader) + (let (artifacts (run-language-test loader 'fixtures)) + (check (pair? artifacts) => #t) + (check (every parse-artifact-valid? artifacts) => #t))) + (list arithmetic-test-language hcl-test-language + gql-test-language opencypher-test-language + hl7-test-language fhirpath-test-language bash-test-language + tla-plus-core-test-language tla-plus-layout-test-language tla-plus-sany-candidate-test-language))) + (test-case "Bash workers retain independent deferred scanner obligations" + (let* ((make-worker (lambda (source) (make-language-scan-worker bash-test-language 'command source))) + (first "cat < second) + (check (apply string-append (map token-lexeme (source-scanner-tokens left 'command))) => first) + (check (apply string-append (map token-lexeme (source-scanner-tokens left 'command))) => first))))) +(export language-pack-metadata-test) diff --git a/t/language-policy-test.ss b/t/language-policy-test.ss new file mode 100644 index 00000000..cd5a1c16 --- /dev/null +++ b/t/language-policy-test.ss @@ -0,0 +1,170 @@ +;;; Descriptor-bound policy parity across loader, contextual C ABI and AOT admission. +(import (only-in :gerbil-parser/t/abi-datum-support abi-datum-read) + :std/test + (only-in :std/vector/u8vector little u8vector-u32-ref) + (only-in :clan/poo/object .ref) + (only-in :gerbil-parser/language-support/development + deflanguage-development-loader parse-language-source call-with-language-parser-policy) + (only-in :gerbil-parser/src/language/descriptor + language-grammar-with-parser-policy language-grammar-parser-policy + language-grammar-with-observability require-portable-language-policy! + language-grammar-language language-grammar-version language-grammar-contract + language-grammar-machine language-grammar-ir) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-branch-budget) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-valid? parse-artifact-roundtrip + parse-artifact-ref) + (only-in :gerbil-parser/src/ffi/language-artifact-codec + bind-language-abi abi-descriptor-payload + abi-parse-binary-payload abi-parse-binary-payload/bytes) + (only-in :gerbil-parser/src/compiler/rust-runtime + generate-language-rust-runtime-module language-rust-runtime-module-source + rust-runtime-module-source) + (only-in :gerbil-parser/src/compiler/machine parser-machine-grammar-digest) + (only-in :gerbil-parser/src/compiler/rust-scanner generate-contextual-language-rust-runtime-module) + (only-in :gerbil-parser/t/fixtures/shared-scanner/records + records-language-grammar records-contextual-product)) + +(def calls 0) +(def seen-budget #f) +(def diagnostic + '((schema . "gerbil-parser.diagnostic.v1") (code . "POLICY-TEST") + (reasonKind . parse-rejected) (failureKind . policy-rejected) + (message . "rejected by test language policy") (byteOffset . 0))) +(def descriptor + (language-grammar-with-parser-policy + records-language-grammar "records.test-policy.v1" 17 + (lambda (_cst) + (set! calls (+ calls 1)) + (set! seen-budget (current-lr-branch-budget)) + diagnostic))) +(deflanguage-development-loader policy-loader (grammar descriptor) (parse parse-records-policy)) + +(def language-policy-test + (test-suite "descriptor-bound language policy" + (test-case "admission rejects malformed policy configuration" + (for-each + (lambda (row) + (check-exception (apply language-grammar-with-parser-policy row) true)) + (list (list #f "test.v1" 17 void) + (list records-language-grammar "" 17 void) + (list records-language-grammar "test.v1" 0 void) + (list records-language-grammar "test.v1" 1.5 void) + (list records-language-grammar "test.v1" 17 #f)))) + (test-case "policy is retained by observability configuration" + (check (language-grammar-parser-policy + (language-grammar-with-observability descriptor #f)) + => (language-grammar-parser-policy descriptor))) + (test-case "public loader and descriptor reject the same lossless artifact" + (let* ((source "α=1\r\n") + (a (parse-records-policy source)) + (b ((.ref policy-loader '.parse) source)) + (c (parse-language-source descriptor source))) + (check a => b) (check b => c) + (check (parse-artifact-success? a) => #f) + (check (parse-artifact-valid? a) => #t) + (check (parse-artifact-roundtrip a) => source) + (check (.ref policy-loader 'capabilities) => '(grammar-ir parser-ir scheme scheme-policy)))) + (test-case "budget is scoped and checker runs once only after recognition succeeds" + (parameterize ((current-lr-branch-budget 123)) + (let (before calls) + (parse-records-policy "a=1\n") + (check calls => (+ before 1)) + (check seen-budget => 17) + (check (current-lr-branch-budget) => 123) + (parse-records-policy "!") + (check calls => (+ before 1))) + (check-exception + (call-with-language-parser-policy descriptor "" (lambda () (error "recognition failed"))) true) + (check (current-lr-branch-budget) => 123))) + (test-case "validator exceptions restore the caller budget" + (let (throwing (language-grammar-with-parser-policy + records-language-grammar "records.throw.v1" 17 + (lambda (_) (error "validator failed")))) + (parameterize ((current-lr-branch-budget 123)) + (check-exception (parse-language-source throwing "a=1\n") true) + (check (current-lr-branch-budget) => 123)))) + (test-case "validator acceptance preserves the canonical artifact" + (let* ((source "β=1\n") + (accepting (language-grammar-with-parser-policy + records-language-grammar "records.accept.v1" 17 (lambda (_) #f)))) + (check (parse-language-source accepting source) + => (parse-language-source records-language-grammar source)))) + (test-case "invalid diagnostics escape rather than publish invalid artifacts" + (let (invalid (language-grammar-with-parser-policy + records-language-grammar "records.invalid.v1" 17 (lambda (_) 'invalid))) + (check-exception (parse-language-source invalid "a=1\n") true))) + (test-case "contextual native bytes cannot bypass the policy checker" + (for-each + (lambda (product) + (let* ((language (bind-language-abi descriptor product)) + (source "α=1\n") (before calls) + (payload (abi-parse-binary-payload/bytes language (string->utf8 source)))) + (check calls => (+ before 1)) + (check (u8vector-u32-ref payload 8 little) => 1) + (check payload => (abi-parse-binary-payload language source)) + (let* ((metadata (abi-datum-read (abi-descriptor-payload language) + )) + (policy (hash-get metadata "parserPolicy"))) + (check (hash-get policy "identity") => "records.test-policy.v1") + (check (hash-get policy "branchBudget") => 17) + (check (hash-get policy "portable") => #f)))) + (list #f records-contextual-product))) + (test-case "descriptor source admission retains portable recognition bytes" + (check-exception + (language-rust-runtime-module-source descriptor) + (lambda (e) + (equal? (error-message e) + "language parser policy is unsupported by standalone Rust AOT"))) + (check (language-rust-runtime-module-source records-language-grammar) + => (rust-runtime-module-source + (language-grammar-language records-language-grammar) + (language-grammar-version records-language-grammar) + (language-grammar-contract records-language-grammar) + (parser-machine-grammar-digest (language-grammar-machine records-language-grammar)) + (language-grammar-ir records-language-grammar)))) + (test-case "contextual Rust rejects policy before admission and output publication" + (let (path (path-expand "contextual-language-policy-rejected.rs" (getenv "TMPDIR" "/tmp"))) + (check (file-exists? path) => #f) + (dynamic-wind + void + (lambda () + (for-each + (lambda (product) + (check-exception + (generate-contextual-language-rust-runtime-module path descriptor product) + (lambda (e) + (equal? (error-message e) + "language parser policy is unsupported by standalone Rust AOT"))) + (check (file-exists? path) => #f)) + (list records-contextual-product #f))) + (lambda () (when (file-exists? path) (delete-file path)))))) + (test-case "policy rejection preserves an existing downstream product" + (let ((path (path-expand "language-policy-existing.rs" (getenv "TMPDIR" "/tmp"))) + (sentinel "existing downstream product α")) + (check (file-exists? path) => #f) + (dynamic-wind + (lambda () (call-with-output-file path (lambda (port) (display sentinel port)))) + (lambda () + (for-each + (lambda (emit) + (check-exception (emit) + (lambda (e) + (equal? (error-message e) + "language parser policy is unsupported by standalone Rust AOT"))) + (check (call-with-input-file path read-line) => sentinel)) + (list (lambda () (generate-language-rust-runtime-module path descriptor)) + (lambda () (generate-contextual-language-rust-runtime-module path descriptor records-contextual-product)) + (lambda () (generate-contextual-language-rust-runtime-module path descriptor #f))))) + (lambda () (delete-file path))))) + (test-case "standalone Rust rejects an unlowered policy before writing output" + (let (path (path-expand "language-policy-rejected.rs" (getenv "TMPDIR" "/tmp"))) + (check (file-exists? path) => #f) + (check-exception (generate-language-rust-runtime-module path descriptor) + (lambda (e) + (equal? (error-message e) + "language parser policy is unsupported by standalone Rust AOT"))) + (check (file-exists? path) => #f) + (check (require-portable-language-policy! records-language-grammar) + => records-language-grammar))))) +(export language-policy-test) diff --git a/t/language-product-test.ss b/t/language-product-test.ss new file mode 100644 index 00000000..07491341 --- /dev/null +++ b/t/language-product-test.ss @@ -0,0 +1,23 @@ +;;; Product harness controls exercise data quoting and temporary ownership. +(import :std/test + :gerbil-parser/language-test-support + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-core-language + qualify-tla-plus-core-model tla-plus-model-receipt?) + (only-in :gerbil-parser/src/testing/language-product language-model-test-receipt) + (only-in :gerbil-parser/src/runtime/identity sha256-text)) +(export language-product-test) +(def source "---- MODULE Quote ----\nVARIABLE enabled\nInit == enabled = FALSE\nNext == enabled' = TRUE\n====\n") +(def config "INIT Init\nNEXT Next\n") +(def output "TLC2 Version fixture'$(printf corrupted)\n") +(def language-product-test + (test-suite "closed model tool fixtures" + (test-case "stdout is inert data and all temporary files are removed" + (let* ((receipt (language-model-test-receipt tla-plus-core-language "Quote" source config (list 'stdout output 0) 1)) + (ref (lambda (key) (cdr (assq key receipt)))) + (directory (path-directory (ref 'tool-path)))) + (check (ref 'output-digest) => (sha256-text output)) + (check (ref 'tlc-version) => "fixture'$(printf corrupted)") + (check (ref 'source-digest) => (sha256-text source)) + (check (ref 'config-digest) => (sha256-text config)) + (check (ref 'admitted) => #f) + (check (file-exists? directory) => #f))))) diff --git a/t/language-source-contract-test.ss b/t/language-source-contract-test.ss new file mode 100644 index 00000000..3c7b700d --- /dev/null +++ b/t/language-source-contract-test.ss @@ -0,0 +1,130 @@ +;;; Engine ownership: source-contract admission and UTF-8 scanner checkpoints. +(import (only-in :std/test check test-case test-suite) + (only-in ./benchmarks/parser-stage-cost/benchmark measure-parser-stages) + (only-in :gerbil-parser/src/language/source declare-source-language parse-source-language) + (only-in "fixtures/source-strategies.ss" test-source-strategy) + (only-in :gerbil-parser/languages/bash/parser parse-bash parse-bash/receipt) + (only-in :gerbil-parser/languages/gql/parser parse-gql +gql-representative-query+) + (only-in :gerbil-parser/languages/arithmetic/parser parse-arithmetic) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-event-program-enabled?) + (only-in :gerbil-parser/src/runtime/source-engines LineSourceStrategy.) + (only-in :clan/poo/object .o) + (only-in :gerbil-parser/src/runtime/parse-cost current-parser-cost-observer) + (only-in :gerbil-parser/src/runtime/source-scanner + make-source-scanner source-scanner-initial-state + source-scanner-step source-scan-state-byte-offset) + (only-in :gerbil-parser/src/runtime/token token-end token-kind token-start)) + +(export language-source-contract-test) +(def language-source-contract-test + (test-suite "language source contracts" + (test-case "generated drive observation identifies the executed route under either preference" + (let* ((source "1 + 2") (reference (parse-arithmetic source))) + (for-each + (lambda (events?) + (let (rows '()) + (check (parameterize + ((current-lr-event-program-enabled? events?) + (current-parser-cost-observer + (lambda (stage receipt) (set! rows (cons (cons stage receipt) rows))))) + (parse-arithmetic source)) => reference) + (check (cdr (assq 'completed (cdr (assq 'generated-drive-execution rows)))) => #t) + (check (assq 'prepared-checkpoint-execution rows) => #f))) + '(#f #t)))) + (test-case "public GQL observes the selected streaming engine without changing products" + (for-each + (lambda (source) + (let ((reference (parse-gql source)) (rows '())) + (check + (parameterize ((current-parser-cost-observer + (lambda (stage receipt) + (set! rows (cons (cons stage receipt) rows))))) + (parse-gql source)) + => reference) + (check (current-parser-cost-observer) => #f) + (check (pair? (filter (lambda (row) (eq? 'source-recognition (car row))) rows)) => #t) + (check (pair? (assq 'prepared-checkpoint-execution rows)) => #t) + (check (pair? (filter (lambda (row) (eq? 'artifact-materialization (car row))) rows)) => #t))) + (list +gql-representative-query+ "MATCH (" ""))) + (test-case "stage diagnostics publish P50 and P95 for complete requests" + (let (rows (parameterize ((current-output-port (open-output-string))) + (measure-parser-stages parse-bash "echo hi\n" 20))) + (check (pair? rows) => #t) + (for-each + (lambda (row) + (let ((cpu50 (cdr (assq 'cpu-p50-ms row))) + (cpu95 (cdr (assq 'cpu-p95-ms row))) + (alloc50 (cdr (assq 'allocation-p50-bytes row))) + (alloc95 (cdr (assq 'allocation-p95-bytes row)))) + (check (cdr (assq 'samples row)) => 20) + (check (and (number? cpu50) (number? cpu95) (>= cpu95 cpu50)) => #t) + (check (if (zero? (cdr (assq 'allocation-samples row))) + (and (not alloc50) (not alloc95)) + (and (number? alloc50) (number? alloc95) (>= alloc95 alloc50))) + => #t))) rows))) + (test-case "source language rejects an artifact with another digest" + (let (other + (declare-source-language + "bash" "5.3" "different-contract" + (test-source-strategy (lambda (_) #f) (lambda (_) '()) + (lambda (source _scanner _digest) (parse-bash source))))) + (check + (with-catch + (lambda (_condition) #t) + (lambda () + (parse-source-language other "echo hi\n") + #f)) + => #t))) + (test-case "public source observations preserve admitted products and receipts" + (def (observe parse source (scan-completed? #t)) + (let ((rows '()) (expected (call-with-values (lambda () (parse source)) list))) + (let (actual + (parameterize ((current-parser-cost-observer + (lambda (name receipt) + (set! rows (cons (cons name receipt) rows))))) + (call-with-values (lambda () (parse source)) list))) + (check actual => expected)) + (check (current-parser-cost-observer) => #f) + (for-each + (lambda (stage) + (check (length (filter (lambda (row) (eq? stage (car row))) rows)) => 1)) + '(source-prepare source-scan artifact-validation)) + (for-each + (lambda (row) + (check (cdr (assq 'completed (cdr row))) + => (if (eq? (car row) 'source-scan) scan-completed? #t))) rows))) + (for-each + (lambda (source) + (let (completed? (not (string=? source "echo \""))) + (observe parse-bash source completed?) + (observe parse-bash/receipt source completed?))) + '("echo α\n" "echo \"" "")) + (let (descriptor + (declare-source-language "lines" "test" "stage-observation" + (.o (:: self LineSourceStrategy.) root-kind: 'LineFile token-kind: 'Line + required-prefix: "M"))) + (def (parse-lines source) (parse-source-language descriptor source)) + (observe parse-lines "Mα\n") + (observe parse-lines "rejected\n"))) + (test-case "scanner checkpoints retain byte offsets" + (let* ((scanner + (make-source-scanner + "αx" #f + (lambda (_source offset context _mode) + (if (= offset 2) + (values #f offset context) + (values 'character (fx+ offset 1) context))))) + (initial (source-scanner-initial-state scanner))) + (let-values (((first after-first) + (source-scanner-step scanner initial 'test))) + (check (token-start first) => 0) + (check (token-end first) => 2) + (check (source-scan-state-byte-offset initial) => 0) + (check (source-scan-state-byte-offset after-first) => 2) + (let-values (((second after-second) + (source-scanner-step scanner after-first 'test))) + (check (token-kind second) => 'character) + (check (token-start second) => 2) + (check (token-end second) => 3) + (check (source-scan-state-byte-offset after-second) => 3))))) +)) diff --git a/t/language-strategy-test.ss b/t/language-strategy-test.ss new file mode 100644 index 00000000..ad3279c6 --- /dev/null +++ b/t/language-strategy-test.ss @@ -0,0 +1,113 @@ +;;; Seed streams are executable controls copied from the prior HCL suite. +;;; Independent port-writing references retain the entire original sequence. +(import (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-test-language) + (only-in :gerbil-parser/languages/hcl/parser-test hcl-test-language) + :std/test + (only-in :gerbil-parser/src/testing/language-strategy generate-language-test-sources check-language-strategies) + (only-in :clan/poo/object .cc .o .ref) + (only-in :clan/poo/mop validate) + (only-in :gerbil-parser/language-support/development LanguageDevelopmentLoaderContract) +) +(export language-strategy-test) + +(def (reference-sources seed count build) + (def (next-random modulus) + (set! seed (modulo (+ (* seed 1103515245) 12345) 2147483648)) + (modulo seed modulus)) + (map (lambda (index) (build index next-random)) (iota count))) + +(def language-strategy-test + (test-suite "closed language strategy source admission" + (test-case "all 512 ASCII sources preserve the prior stream" + (check (generate-language-test-sources + '(seeded 1729 512 (repeat 0 32 (character "abcXYZ_eE0123= \t\r\n")))) + => (reference-sources 1729 512 + (lambda (_ draw) + (let (alphabet "abcXYZ_eE0123= \t\r\n") + (call-with-output-string + (lambda (port) + (let chars ((remaining (draw 32))) + (when (> remaining 0) + (display (string-ref alphabet (draw (string-length alphabet))) port) + (chars (- remaining 1))))))))))) + (test-case "all 128 quoted sources preserve the prior stream" + (check (generate-language-test-sources + '(seeded 3857 128 (concat "key = \"" (repeat 0 12 (choice "a" "0" " " "\\\\" "\\\"" "//")) "\"\n"))) + => (reference-sources 3857 128 + (lambda (_ draw) + (let (pieces '#("a" "0" " " "\\\\" "\\\"" "//")) + (call-with-output-string + (lambda (port) + (display "key = \"" port) + (let parts ((remaining (draw 12))) + (when (> remaining 0) + (display (vector-ref pieces (draw (vector-length pieces))) port) + (parts (- remaining 1)))) + (display "\"\n" port)))))))) + (test-case "all 128 comment sources preserve draw order and terminators" + (check (generate-language-test-sources + '(seeded 7103 128 (bind ((start (choice "#" "//" "/*"))) + (concat "key = 1 " start (repeat 0 16 (character "ab09 #/\t")) + (when-equal start "/*" "*/") "\r\n")))) + => (reference-sources 7103 128 + (lambda (_ draw) + (let ((start (vector-ref '#("#" "//" "/*") (draw 3))) (alphabet "ab09 #/\t")) + (call-with-output-string + (lambda (port) + (display "key = 1 " port) (display start port) + (let chars ((remaining (draw 16))) + (when (> remaining 0) + (display (string-ref alphabet (draw (string-length alphabet))) port) + (chars (- remaining 1)))) + (when (equal? start "/*") (display "*/" port)) + (display "\r\n" port)))))))) + (test-case "all 128 numeric sources preserve parity and draw bounds" + (check (generate-language-test-sources + '(seeded 7207 128 (concat "key = " (integer 1 999) "." (integer 0 1000) + (if-even "e+" "E-") (integer 0 30) "\n"))) + => (reference-sources 7207 128 + (lambda (index draw) + (string-append "key = " (number->string (+ 1 (draw 999))) "." + (number->string (draw 1000)) (if (even? index) "e+" "E-") + (number->string (draw 30)) "\n"))))) + (test-case "all 128 mixed corridors preserve row and value streams" + (check (generate-language-test-sources + '(seeded 4919 128 (repeat 1 8 + (concat (choice "a" "foo" "é" "name_2") " = " + (choice "1" "23" "foo" "true" "\"x\"" "\"雪\"") + (choice "\n" "\r\n" " # comment\n"))))) + => (reference-sources 4919 128 + (lambda (_ draw) + (let ((names '#("a" "foo" "é" "name_2")) + (values '#("1" "23" "foo" "true" "\"x\"" "\"雪\"")) + (ends '#("\n" "\r\n" " # comment\n"))) + (call-with-output-string + (lambda (port) + (let rows ((remaining (+ 1 (draw 8)))) + (when (> remaining 0) + (display (vector-ref names (draw (vector-length names))) port) + (display " = " port) + (display (vector-ref values (draw (vector-length values))) port) + (display (vector-ref ends (draw (vector-length ends))) port) + (rows (- remaining 1))))))))))) + (test-case "lexical-only coverage executes each candidate once" + (let* ((profile (.ref hcl-test-language 'native-test-profile)) + (lex (.ref profile 'lexer)) (calls 0) + (owner (.cc hcl-test-language 'native-test-profile + (.cc profile 'lexer (lambda (source) + (set! calls (+ calls 1)) (lex source)))))) + (validate LanguageDevelopmentLoaderContract owner) + (check-language-strategies owner '(sources "key = 1\n" "雪" "") + '((routes) (lexical optional) (coverage 0 0))) + (check calls => 3))) + (test-case "profiles reject stale digest, foreign source and missing lexer" + (let* ((profile (.ref hcl-test-language 'native-test-profile)) + (rejected? (lambda (owner) + (with-catch (lambda (_) #t) + (lambda () (validate LanguageDevelopmentLoaderContract owner) #f))))) + (check (rejected? (.cc hcl-test-language 'native-test-profile (.cc profile 'digest "sha256:stale"))) => #t) + (check (rejected? (.cc hcl-test-language 'native-test-profile (.cc profile 'source (lambda args #t)))) => #t) + (check (rejected? (.cc hcl-test-language 'native-test-profile (.cc profile 'lexer #f))) => #t) + (check (rejected? (.cc hcl-test-language 'native-test-profile (.cc profile 'profile 'unknown))) => #t) + (check (rejected? (.cc arithmetic-test-language 'native-test-profile profile)) => #t) + (check (rejected? (.cc hcl-test-language 'native-test-profile (.cc profile 'digest (.ref profile 'digest)))) => #f))))) diff --git a/t/language-surface-test.ss b/t/language-surface-test.ss index 775f87fb..2a813a3c 100644 --- a/t/language-surface-test.ss +++ b/t/language-surface-test.ss @@ -1,6 +1,8 @@ +(import (only-in :gerbil-parser/src/compiler/language-expander compile-language) + (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) ;;; -*- Gerbil -*- ;;; Boundary: executable witness for the concise v1 language authoring surface. -;;; Invariant: inferred declarations lower through deflanguage-grammar and +;;; Invariant: inferred and explicit declarations share canonical admission and ;;; publish the same Grammar IR, Parser IR, and ParseArtifact contracts. (import (only-in :std/test check test-case test-suite) @@ -11,15 +13,21 @@ (only-in :gerbil-parser/src/compiler/normalize grammar-ir-ref) (only-in :gerbil-parser/src/compiler/machine lexical-choice lexical-dispatch lexical-dispatch/ranked - lexical-end) + lexical-end install-parser-machine-backends! + parser-machine-grammar-digest parser-machine-direct-drive + parser-machine-direct-source parser-machine-backend-representation + parser-machine-for-current-semantic-backend install-parser-machine-direct-step!) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-event-program-enabled?) (only-in :gerbil-parser/src/language/grammar deflanguage - deflanguage-grammar defgrammar-syntax) (only-in :gerbil-parser/src/runtime/artifact parse-artifact-roundtrip parse-artifact-success?) + (only-in :gerbil-parser/src/runtime/lexer lex-source) (only-in :gerbil-parser/src/runtime/scan make-literal-end-scanner) + (only-in :gerbil-parser/src/runtime/token + token-kind token-start token-end) (only-in :gerbil-parser/src/runtime/parser parse-source)) ;;; These direct expansion witnesses keep the compiler's closed lexical algebra @@ -49,25 +57,27 @@ (defgrammar-syntax (named-node token-name) (node SourceFile (field name token-name))) -(deflanguage concise-v1-witness - (identity "concise-v1-witness" "v1" "concise-v1-witness.v1") - (root source-file) - (lex +(begin + (deflanguage concise-v1-witness + (syntax + (lexical + (root source-file) + (lex (whitespace Whitespace (whitespace+)) (identifier Identifier (identifier)) (unknown Unknown (fallback))) + (extras whitespace) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) (rules (source-file - (named-node identifier))) - (extras whitespace) - (keywords) - (recoveries) - (conflicts reject) - (case-insensitive #f)) + (named-node identifier)))) + (bind-fixture-grammar-release concise-v1-witness "concise-v1-witness" "v1" "concise-v1-witness.v1") ) -;;; Compatibility witness: the stable verbose v1 form remains executable while -;;; the concise surface is only an authoring projection over the same owner. -(deflanguage-grammar explicit-v1-witness +;;; Independent engine-IR control. This compiler entry is not a public author DSL. +(compile-language explicit-v1-witness (identity "explicit-v1-witness" "v1" "explicit-v1-witness.v1") (syntax-kinds (SourceFile node (name)) @@ -88,8 +98,71 @@ (recoveries) (flow (source lexical) (lexical cst))) +;;; A token/rule namespace collision is legal when references are explicit. +;;; Repeated node alternatives infer the union of their public fields. +(begin + (deflanguage alternative-fields-witness + (syntax + (lexical + (root source-file) + (lex (word WordToken (identifier)) + (punctuation Punctuation (literals "+")) + (unknown Unknown (fallback))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f) + (node-fields (Name reserved)))) + (rules + (source-file (node Root (field value (reference word)))) + (word (choice (node Name (field left (token word))) + (node Name (field right (literal "+"))))))) + (bind-fixture-grammar-release alternative-fields-witness "alternative-fields" "v1" "alternative-fields.v1") ) + +;;; Fields in recursive transparent helpers belong to their enclosing node. +(begin + (deflanguage transparent-fields-witness + (syntax + (lexical + (root source-file) + (lex (word WordToken (identifier)) + (punctuation Punctuation (literals "(" ")"))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file (node Root helper)) + (helper (choice (field item word) + (seq "(" helper ")" (field tail word)))))) + (bind-fixture-grammar-release transparent-fields-witness "transparent-fields" "v1" "transparent-fields.v1") ) + (def concise-v1-language-surface-tests (test-suite "concise v1 language surface" + (test-case "transparent recursive references infer fields without crossing nodes" + (check (assq 'Root (grammar-ir-ref transparent-fields-witness-grammar 'syntax-kinds)) + => '(Root node (item tail))) + (check (assq 'Root (grammar-ir-ref alternative-fields-witness-grammar 'syntax-kinds)) + => '(Root node (value))) + (for-each + (lambda (source) + (let (artifact (parse-source transparent-fields-witness-parser source)) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + '("alpha" "(beta)gamma" "((a)b)c"))) + (test-case "alternative node schemas and explicit namespaces share one DSL" + (check (assq 'Name (grammar-ir-ref alternative-fields-witness-grammar 'syntax-kinds)) + => '(Name node (reserved left right))) + (check (grammar-ir-ref alternative-fields-witness-grammar 'flow) + => '((source lexical) (lexical parser) (parser cst))) + (for-each + (lambda (source) + (let (artifact (parse-source alternative-fields-witness-parser source)) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + '("alpha" "+"))) (test-case "infers the stable grammar and parser contracts" (let* ((source "alpha") (artifact (parse-source concise-v1-witness-parser source))) @@ -152,6 +225,99 @@ (check external-scanner-witness => 2) (check (literal-trie-witness "<=" 0) => 2) (check (literal-trie-witness "MATCHED suffix" 0) => 7) - (check (literal-trie-witness "missing" 0) => #f)))) + (check (literal-trie-witness "missing" 0) => #f)) + (test-case "mode dispatch keeps Unicode and fallback byte boundaries" + (let (tokens (lex-source concise-v1-witness-parser "α β?")) + (check (map token-kind tokens) + => '(identifier whitespace identifier unknown)) + (check (map token-start tokens) => '(0 2 3 5)) + (check (map token-end tokens) => '(2 3 5 6)))))) (export concise-v1-language-surface-tests) + +;; gxtest discovers only exported names ending in -test. +(def language-surface-test concise-v1-language-surface-tests) +(export language-surface-test) + +;;; Generated source specializations may decline a shape and use ordinary LR. +(def backend-source-fallback (lambda (_machine _source) #f)) +(begin + (deflanguage backend-admission-witness + (syntax + (lexical + (root name) + (lex (identifier Identifier (identifier))))) + (rules (name (node Name (field value identifier))))) + (bind-fixture-grammar-release backend-admission-witness "backend-witness" "v1" "backend-witness.v1") + (install-parser-machine-backends! backend-admission-witness-parser + (list (list 'source (parser-machine-grammar-digest backend-admission-witness-parser) backend-source-fallback)))) + +(def (backend-rejects? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) + +(def language-backend-test + (test-suite "Engine-owned parser backend admission" + (test-case "captured representations distinguish generated entries from prepared preference" + (let* ((machine backend-admission-witness-parser) + (selected (parameterize ((current-lr-event-program-enabled? #t)) + (parser-machine-for-current-semantic-backend machine)))) + (check (eq? machine selected) => #f) + (check (parser-machine-backend-representation machine 'prepared) => 'recognition) + (check (parser-machine-backend-representation selected 'prepared) => 'event-program) + (check (parser-machine-backend-representation selected 'drive) => #f) + (check (parser-machine-backend-representation selected 'source) => 'parse-artifact) + (check (backend-rejects? + (lambda () + (install-parser-machine-backends! selected + (list (list 'drive (parser-machine-grammar-digest selected) (lambda _ #f)) + (list 'step (parser-machine-grammar-digest selected) (lambda _ #f)))))) => #t) + (check (parser-machine-backend-representation selected 'drive) => #f) + (check (backend-rejects? + (lambda () (install-parser-machine-direct-step! + selected (parser-machine-grammar-digest selected) (lambda _ #f)))) => #t) + (install-parser-machine-backends! selected + (list (list 'drive (parser-machine-grammar-digest selected) + (lambda _ (error "contract control must not execute generated drive"))))) + (check (parser-machine-backend-representation selected 'drive) => 'recognition) + (check (parser-machine-backend-representation machine 'drive) => #f) + (parameterize ((current-lr-event-program-enabled? #f)) + (check (parser-machine-backend-representation selected 'prepared) => 'event-program)) + (check (backend-rejects? (lambda () (parser-machine-backend-representation #f 'prepared))) => #t) + (check (backend-rejects? (lambda () (parser-machine-backend-representation machine 'unknown))) => #t))) + (test-case "declarative source backend retains the ordinary LR fallback" + (check (eq? (parser-machine-direct-source backend-admission-witness-parser) + backend-source-fallback) => #t) + (let (artifact (parse-source backend-admission-witness-parser "example")) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => "example"))) + (test-case "a stale later row cannot partially install an earlier backend" + (check + (backend-rejects? + (lambda () + (install-parser-machine-backends! + explicit-v1-witness-parser + (list (list 'drive (parser-machine-grammar-digest explicit-v1-witness-parser) + (lambda _ #f)) + (list 'step "sha256:stale" (lambda _ #f)))))) => #t) + (check (parser-machine-direct-drive explicit-v1-witness-parser) => #f)) + (test-case "unknown duplicate and occupied backend slots reject" + (let ((digest (parser-machine-grammar-digest explicit-v1-witness-parser)) + (procedure (lambda _ #f))) + (for-each + (lambda (rows) + (check (backend-rejects? + (lambda () (install-parser-machine-backends! + explicit-v1-witness-parser rows))) => #t) + (check (parser-machine-direct-drive explicit-v1-witness-parser) => #f)) + (list (list (list 'unknown digest procedure)) + (list (list 'drive digest procedure) (list 'drive digest procedure)) + (list (list 'drive digest #f)))) + (check + (backend-rejects? + (lambda () + (install-parser-machine-backends! + backend-admission-witness-parser + (list (list 'source + (parser-machine-grammar-digest backend-admission-witness-parser) + procedure))))) => #t))))) +(export language-backend-test) diff --git a/t/language-test-syntax-test.ss b/t/language-test-syntax-test.ss new file mode 100644 index 00000000..d309d2c3 --- /dev/null +++ b/t/language-test-syntax-test.ss @@ -0,0 +1,106 @@ +;;; Engine admission controls for the public language-test DSL. +(import (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-test-language) + (only-in :gerbil-parser/languages/hcl/parser-test hcl-test-language) + (only-in :gerbil-parser/languages/arithmetic/parser-test arithmetic-basic-fixture) + :std/test + (for-syntax (only-in :gerbil/expander core-expand)) + :gerbil-parser/language-test-support + (only-in :clan/poo/object .cc .ref) +) +(export language-test-syntax-test) +(defsyntax (invalid-test-declarations stx) + (def (rejects form) + (with-catch (lambda (_) #t) (lambda () (core-expand form) #f))) + (datum->syntax #'invalid-test-declarations + (list 'quote + (list + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (arbitrary "bad" #t))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "bad" "1" (nodes)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "bad" "1" (unknown x)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "bad" "1" (counts (Number -1))))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (identity "bad" (unknown #t)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (property "bad" (bindings)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (property "bad" (bindings) (begin #t)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "same" "1") (rejected "same" "2"))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted 17 "1"))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "bad" "1" (subtree (node X (lexemes)))))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "bad" "1" (subtree (node X (unknown 1)))))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "bad" "1" (field-counts X value (-1))))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (parser-ir "bad" #f (unknown 1)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (bound-rules "bad" #f (rule (unknown 1))))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (lr "bad" #f (initial-shifts)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (lr-receipt "bad" #f "x" 0 (rest '())))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (parallel "bad" 0 '("1")))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (incremental "bad" "1" (replace "" "2") (schema "x")))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (recovery-rejected "bad" "x"))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (installed "bad" arbitrary))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (sources "1") (routes unknown entry)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (sources "1") (routes entry entry)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (sources) (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (indexed 0 "x") (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (seeded -1 1 "x") (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (seeded 1 0 "x") (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (seeded 1 1 (character "")) (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (seeded 1 1 (integer 0 0)) (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (seeded 1 1 foreign) (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (seeded 1 1 (bind ((x "1") (x "2")) x)) (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (sources "1") (routes entry indexed) (accepted "true")))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (strategy-parity "bad" (sources "1") (routes entry indexed) (coverage 0 0)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (fixture-parity "bad" arbitrary (routes entry indexed)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (fixture-parity "bad" accepted (routes entry indexed) (diagnostics -1)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "bad" "1" (fields X)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted "bad" "1" (token-counts (number -1))))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (accepted-fixture "bad" -1))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (fixture-group "bad" unknown))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (fixture-catalog "bad" (unknown 1)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (syntax-kind "bad" X unknown ()))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (distinct-contract "bad"))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (native-entry "bad" "1" unknown))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (portable-rejected "bad" 1 "message"))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (model-receipt "bad" "../Escape" "source" "config" (unresolved) 1 (admitted #f)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (model-receipt "bad" "Model" "source" "config" (callback #t) 1 (admitted #f)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (model-receipt "bad" "Model" "source" "config" (stdout "data" 256) 1 (admitted #f)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (model-receipt "bad" "Model" "source" "config" (stdout "data" 0) 0 (admitted #f)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (model-receipt "bad" "Model" "source" "config" (unresolved) 1 (unknown #f)))) + (rejects #'(deflanguage-parser-tests bad "bad" (loader #f) (model-receipt "bad" "Model" "source" "config" (unresolved) 1))))))) +(def language-test-syntax-test + (test-suite "language parser test declarations" + (test-case "unknown, empty, malformed and duplicate declarations reject during expansion" + (check (invalid-test-declarations) => (make-list 50 #t))))) + +(def owner-calls 0) +(def source-calls 0) +(def parse-calls 0) +(def counting-loader + (.cc arithmetic-test-language '.parse + (lambda (source) + (set! parse-calls (+ parse-calls 1)) + ((.ref arithmetic-test-language '.parse) source)))) +(deflanguage-parser-tests language-test-evaluation-test "language-test single evaluation" + (loader (begin (set! owner-calls (+ owner-calls 1)) counting-loader)) + (accepted "source and parser execute once" + (begin (set! source-calls (+ source-calls 1)) "1") + (nodes NumberExpression) (tokens number) (counts (NumberExpression 1))) + (property "loader, source and parse are not repeated per expectation" + (bindings (owner 'caller-binding) (artifact 'caller-artifact)) + (equal owner-calls 1) (equal source-calls 1) (equal parse-calls 1) + (equal owner 'caller-binding) (equal artifact 'caller-artifact))) + +(deflanguage-parser-tests language-test-fixture-override-test "inherited fixture override" + (loader (.cc arithmetic-test-language 'fixtures (list arithmetic-basic-fixture))) + (fixtures "fixture service receives the effective loader")) + +(deflanguage-parser-tests language-test-structure-test "generic structural test semantics" + (loader arithmetic-test-language) + (accepted "queries observe real node text and field cardinality" "1" + (root SourceFile) (subtree (node NumberExpression (lexemes "1"))) + (without-subtree (node NumberExpression (lexemes "2"))) + (without-subtree (node MissingKind)) + (field-counts SourceFile expression (1)) (field-counts NumberExpression value (1))) + (parallel "parallel results retain their own sources" 5 '("1" "2" "1+2"))) + +(deflanguage-parser-tests language-test-byte-edit-test "incremental test byte offsets" + (loader hcl-test-language) + (incremental "Unicode before and inside the replaced text" "名称 = \"λ中😀\"\n" + (replace "λ中" "μ文") (schema "gerbil-parser.incremental-receipt.v1"))) diff --git a/t/language-topology-test.ss b/t/language-topology-test.ss new file mode 100644 index 00000000..263fc69b --- /dev/null +++ b/t/language-topology-test.ss @@ -0,0 +1,115 @@ +;;; Repository authoring contract; no implementation or generated products in packs. +(import :std/test) +(export language-topology-test) +(def (scheme-files root) + (apply append + (map (lambda (name) + (let (path (path-expand name root)) + (cond ((eq? (file-info-type (file-info path)) 'directory) (scheme-files path)) + ((string-suffix? ".ss" name) (list path)) + (else '())))) (directory-files root)))) +(def (forms path) + (call-with-input-file path (lambda (port) + (let loop ((result '())) + (let (form (read port)) + (if (eof-object? form) (reverse result) (loop (cons form result)))))))) +(def (declaration? path head) + (find (lambda (form) (and (pair? form) (eq? (car form) head))) (forms path))) +(def (development-import? form) + (cond ((pair? form) (or (development-import? (car form)) (development-import? (cdr form)))) + ((symbol? form) + (and (memq form (quote (:gerbil-parser/language-support/development + :gerbil-parser/language-support/fixture + :gerbil-parser/language-test-support + :gerbil-parser/language-build-support))) #t)) + (else #f))) +;;; Parser declarations use one public slot vocabulary, with no metadata sugar. +(def (canonical-parser-declaration? form) + (and (list? form) (>= (length form) 5) + (let ((descriptor (caddr form)) (parse (cadddr form)) + (sections (cddddr form))) + (and (list? descriptor) (= (length descriptor) 2) + (memq (car descriptor) '(grammar source descriptor)) + (list? parse) (= (length parse) 2) (eq? (car parse) 'parse) + (find (lambda (row) (and (list? row) (pair? row) + (eq? (car row) 'slots) (memq metadata: (cdr row)))) sections) + (every (lambda (row) (and (list? row) (pair? row) + (memq (car row) '(slots backends contracts)))) sections))))) +(def (authoring-form? form interface) + (and (list? form) (pair? form) + (or (eq? (car form) 'export) + (and (eq? (car form) 'import) + (or (eq? interface 'tests) (not (development-import? form)))) + (and (eq? (car form) 'def) (eq? interface 'parser) + (= (length form) 3) (symbol? (cadr form)) + (not (and (pair? (caddr form)) (eq? (caaddr form) 'lambda)))) + (and + (or (not (eq? (car form) 'deflanguage-parser-loader)) + (canonical-parser-declaration? form)) + (or (not (eq? interface 'grammar)) + (not (eq? (car form) 'deflanguage)) + (and (= (length form) 4) + (symbol? (cadr form)) + (list? (caddr form)) (= (length (caddr form)) 2) + (eq? (caaddr form) 'syntax) + (pair? (cadr (caddr form))) + (list? (cadddr form)) (pair? (cadddr form)) + (eq? (car (cadddr form)) 'rules))) + (memq (car form) + (case interface + ((grammar) '(deflanguage defgrammar-syntax deftext-profile + defsyntax-antlr4-source defsyntax-javacc-source defsyntax-iso-bnf-source)) + ((parser) '(deflanguage-parser-loader deflanguage-parser-receipt deflanguage-model-entry deflanguage-projection + defsyntax-javacc-source)) + ((tests) '(defsyntax-fixture defsyntax-corpus deflanguage-development-loader deflanguage-parser-tests)))))))) +(def language-topology-test + (test-suite "three-interface language authoring contract" + (test-case "reject procedure definitions and unrestricted parser tests" + (check (and (authoring-form? '(def (scan source) source) 'grammar) #t) => #f) + (check (and (authoring-form? '(def scan (lambda (source) source)) 'grammar) #t) => #f) + (check (and (authoring-form? '(def test (test-suite "manual")) 'tests) #t) => #f) + (check (authoring-form? (quote (defsyntax-corpus corpus)) (quote grammar)) => #f) + (check (authoring-form? (quote (deflanguage-development-loader entry)) (quote parser)) => #f) + (check (authoring-form? (quote (import :gerbil-parser/language-support/development)) (quote parser)) => #f)) + (test-case "parser declarations reject legacy metadata and descriptor aliases" + (check (and (authoring-form? '(deflanguage-parser-loader x (grammar old syntax) (parse parse-x) (metadata '())) 'parser) #t) => #f) + (check (and (authoring-form? '(deflanguage-parser-loader x (grammar syntax) (parse parse-x) (metadata '())) 'parser) #t) => #f) + (check (and (authoring-form? '(deflanguage-parser-loader x (grammar syntax) (parse parse-x) (slots metadata: '())) 'parser) #t) => #t)) + (test-case "release identity belongs to the parser interface" + (check (and (authoring-form? '(def +version+ "1") 'grammar) #t) => #f) + (check (and (authoring-form? '(def +version+ "1") 'parser) #t) => #t) + (check (and (authoring-form? '(deflanguage example (identity "x" "1" "x.v1") (root source)) 'grammar) #t) => #f) + (check (and (authoring-form? '(deflanguage example + (syntax + (lexical + (root source) + (lex))) + (rules)) 'grammar) #t) => #t)) + (test-case "author grammar excludes backend profile assembly" + (for-each (lambda (head) + (check (and (authoring-form? (list head 'backend) 'grammar) #t) => #f)) + '(deflanguage-projection deflanguage-source defregion-plan defscanner-profile defresult-profile defpart-profile defcommand-profile defbinding-profile)) + (check (and (authoring-form? '(defgrammar-syntax (loop kind) (node kind)) 'grammar) #t) => #t)) + (test-case "every pack exposes three interfaces with declarative private syntax modules" + (for-each + (lambda (language) + (let* ((root (path-expand language "languages")) + (files (scheme-files root)) + (interfaces (map (lambda (name) (path-expand name root)) + '("grammar.ss" "parser.ss" "parser-test.ss"))) + (private (filter (lambda (file) (not (member file interfaces))) files))) + (check (andmap (lambda (file) (string-prefix? (path-expand "syntax/" root) file)) private) => #t) + (for-each (lambda (file) + (check (andmap (lambda (form) + (and (memq (car form) '(import export deflanguage defgrammar-syntax deftext-profile)) + (authoring-form? form 'grammar) #t)) (forms file)) => #t)) private) + (for-each (lambda (row) + (check (andmap (lambda (form) (and (authoring-form? form (cdr row)) #t)) + (forms (path-expand (car row) root))) => #t)) + '(("grammar.ss" . grammar) ("parser.ss" . parser) ("parser-test.ss" . tests))) + (check (andmap (lambda (name) (and (member (path-expand name root) files) #t)) + '("grammar.ss" "parser.ss" "parser-test.ss")) => #t) + (check (and (declaration? (path-expand "parser.ss" root) 'deflanguage-parser-loader) #t) => #t) + (check (and (declaration? (path-expand "parser-test.ss" root) 'deflanguage-parser-tests) #t) => #t))) + (filter (lambda (name) (eq? (file-info-type (file-info (path-expand name "languages"))) 'directory)) + (directory-files "languages")))))) diff --git a/t/layout-boundary-test.ss b/t/layout-boundary-test.ss new file mode 100644 index 00000000..8c5176f1 --- /dev/null +++ b/t/layout-boundary-test.ss @@ -0,0 +1,203 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +;;; Explicit closing boundaries belong to the declaring language grammar. + +(import :std/test + :gerbil-parser/src/grammar/algebra + (only-in :gerbil-parser/src/language/grammar deflanguage) + (only-in :gerbil-parser/src/runtime/parser + parse-source parse-source/checkpoints + prepare-contextual-parser parse-source/contextual) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-runtime parser-machine-grammar-digest parser-machine-ir) + (only-in :gerbil-parser/src/modules/parser/contextual-objects + make-contextual-method make-contextual-role make-contextual-scan-rule) + (only-in :gerbil-parser/src/compiler/contextual-parser-ir compile-contextual-parser) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-parse/prepared lr-parse/prepared/receipt + current-lr-event-program-enabled? lr-runtime-for-current-semantic-backend + lr-initial-checkpoint lr-checkpoint-drive) + (only-in :gerbil-parser/src/runtime/lr-action-index index-action-row) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact + parse-artifact-success? parse-artifact-valid? parse-artifact-roundtrip) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/layout + make-layout-columns current-layout-columns current-layout-frames + layout-after-end layout-current-action-row)) +(export layout-boundary-test) + +(begin + (deflanguage closing-boundary-probe + (syntax + (lexical + (root source-file) + (lex (word Word (identifier)) + (punctuation Punctuation (literals "(" "|>")) + (space Space (whitespace+))) + (extras space) + (keywords) + (recoveries) + (conflicts selective-glr) + (case-insensitive #f))) + (rules + (source-file (node SourceFile (seq "(" (field items (reference items)) "END"))) + (items + (node Items + (seq (layout-start "|>") (field item word) + (repeat (seq (layout-next "|>") (field item word))) + (layout-end "END")))))) + (bind-fixture-grammar-release closing-boundary-probe "closing-boundary-probe" "v1" "closing-boundary-probe.v1") ) + +(def (layout-contextual-product machine) + (compile-contextual-parser + (parser-machine-ir machine) (parser-machine-grammar-digest machine) + (list (make-contextual-role + 'layout-probe + (list (make-contextual-method 'word 'any 'any 'word 'word)))) + '(normal) '(body) '(word) + (list (make-contextual-scan-rule 'word 'normal 'word '(identifier) 0 'keep)) + 'normal '((body () ())))) + +(def layout-boundary-test + (test-suite "Grammar-owned layout closing boundaries" + (test-case "layout continuations are rejected before contextual preparation" + (let* ((machine closing-boundary-probe-parser) + (runtime (parser-machine-runtime machine)) + (product (layout-contextual-product machine))) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (for-each + (lambda (create) + (check (with-catch (lambda (condition) (error-message condition)) create) + => "LR checkpoints require a runtime without layout")) + (list (lambda () (lr-initial-checkpoint runtime '()) #f) + (lambda () (prepare-contextual-parser machine product) #f) + (lambda () (parse-source/contextual machine product "( |> x END") #f))))) + '(#f #t)))) + (test-case "layout checkpoint source requests retain complete GLR artifacts" + (for-each + (lambda (source) + (let-values (((artifact tokens modes snapshots) + (parse-source/checkpoints closing-boundary-probe-parser source 1))) + (check artifact => (parse-source closing-boundary-probe-parser source)) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source) + (check tokens => #f) + (check modes => #f) + (check snapshots => '#()))) + '("( |> x END" "( |> x\n |> y END" "( |>"))) + (test-case "plain prepared runtimes ignore outer layout columns and frames" + (for-each + (lambda (rules) + (let ((runtime (lr-prepare (compile-lr-spec rules 'root 'selective-glr))) + (tokens (list (make-token 'word "x" 0 1))) + (columns (make-layout-columns "x")) + (frames '((100 . "outer")))) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (let (selected (lr-runtime-for-current-semantic-backend runtime)) + (let-values (((expected rest) (lr-parse/prepared selected tokens))) + (check rest => '()) + (def (artifact root) + (make-success-parse-artifact + (string-append "sha256:" (make-string 64 #\0)) "x" tokens root #f)) + (parameterize ((current-layout-columns columns) (current-layout-frames frames)) + (let-values (((root rest) (lr-parse/prepared selected tokens))) + (check rest => '()) + (check (artifact root) => (artifact expected))) + ;; The independent materialized GLR entry has its own + ;; fast and memoized action lookups, even for plain IR. + (let-values (((root rest receipt) (lr-parse/prepared/receipt selected tokens))) + (check rest => '()) + (check (artifact root) => (artifact expected))) + (check (eq? columns (current-layout-columns)) => #t) + (check (eq? frames (current-layout-frames)) => #t)))))) + '(#f #t)))) + '(((root (alias Root (token word)))) + ((root (alias Root (choice (reference first) (reference second)))) + (first (precedence dynamic 2 (token word))) + (second (precedence dynamic 1 (token word))))))) + (test-case "streaming LR callbacks can parse inside an active outer layout request" + (let* ((runtime (lr-prepare (compile-lr-spec '((root (alias Root (token word)))) 'root))) + (input (make-token 'word "x" 0 1)) + (pending (list input)) (shifted '()) + (columns (make-layout-columns "x")) (frames '((100 . "outer")))) + (parameterize ((current-layout-columns columns) (current-layout-frames frames)) + (let-values (((status payload) + (lr-checkpoint-drive + (lr-initial-checkpoint runtime '()) + (lambda (_mode) + (and (pair? pending) + (let (token (car pending)) + (set! pending (cdr pending)) token))) + (lambda (token _states _values _actions _shifts) + (set! shifted (cons token shifted)))))) + (check status => 'accepted) + (check (cadr payload) => '()) + (check shifted => (list input))) + (check (eq? columns (current-layout-columns)) => #t) + (check (eq? frames (current-layout-frames)) => #t)))) + (test-case "layout casefold lookup retains the declared row and guard decision" + (def entries + '(((terminal literal "END") layout-guard (shift 1 #f) (reduce 0)) + ((terminal literal "É") shift 2 #f) + ((terminal token word) shift 3 #f))) + (def row (index-action-row entries)) + (parameterize ((current-layout-columns (make-layout-columns "end é")) + (current-layout-frames '())) + (check (layout-current-action-row row (make-token 'word "end" 0 3) #t) + => '((terminal literal "END") shift 1 #f)) + (check (eq? (layout-current-action-row row (make-token 'word "é" 4 6) #t) + (cadr entries)) => #t) + (check (eq? (layout-current-action-row row (make-token 'word "end" 0 3) #f) + (caddr entries)) => #t) + (parameterize ((current-layout-frames '((100 . "outer")))) + (check (layout-current-action-row row (make-token 'word "end" 0 3) #t) + => '((terminal literal "END") reduce 0))))) + (test-case "nested source success and rejection restore the outer request context" + (let ((columns (make-layout-columns "outer")) (frames '((100 . "outer")))) + (parameterize ((current-layout-columns columns) (current-layout-frames frames)) + (for-each + (lambda (source) + (let ((expected (equal? source "( |> x END")) + (artifact (parse-source closing-boundary-probe-parser source))) + (check (parse-artifact-success? artifact) => expected) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source) + (check (eq? columns (current-layout-columns)) => #t) + (check (eq? frames (current-layout-frames)) => #t))) + '("( |> x END" "( |>" "( |> x END"))))) + (test-case "closing boundaries are validated nullable grammar values" + (check (grammar-expression? '(layout-end ")" "END")) => #t) + (check (grammar-expression-nullable? (grammar-expression (layout-end "END"))) => #t) + (check (grammar-expression? '(layout-end "")) => #f) + (check (grammar-expression? '(layout-end END)) => #f)) + (test-case "an arbitrary declared word closes a list to the right of its marker" + (for-each + (lambda (source) + (let (artifact (parse-source closing-boundary-probe-parser source)) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + '("( |> x END" "( |> x\n |> y END"))) + (test-case "undeclared right-hand boundaries retain the column restriction" + (parameterize ((current-layout-columns (make-layout-columns "( |> x END")) + (current-layout-frames '((3 . "|>")))) + (let (end (make-token 'word "END" 7 10)) + (check (layout-after-end end) => #f) + (check (layout-after-end end '("END")) => '())))) + (test-case "closing a nested list pops exactly one reference" + (parameterize ((current-layout-columns (make-layout-columns "( |> x END")) + (current-layout-frames '((3 . "|>") (1 . "outer")))) + (check (layout-after-end (make-token 'word "END" 7 10) '("END")) + => '((1 . "outer"))))) + (test-case "an aligned continuation remains in its list" + (parameterize ((current-layout-columns (make-layout-columns " |> x\n |> y")) + (current-layout-frames '((3 . "|>")))) + (check (layout-after-end (make-token 'punctuation "|>" 9 11) '("|>")) + => #f))))) diff --git a/t/layout-column-storage-test.ss b/t/layout-column-storage-test.ss new file mode 100644 index 00000000..6f3eef21 --- /dev/null +++ b/t/layout-column-storage-test.ss @@ -0,0 +1,45 @@ +;;; Source coordinates retain byte semantics across private storage choices. +(import :std/test + (only-in ./fixtures/layout-columns reference-layout-columns reference-packed-layout-columns layout-column-sources) + (only-in :gerbil-parser/src/runtime/layout + make-layout-columns current-layout-columns layout-token-column) + (only-in :gerbil-parser/src/runtime/token make-token)) +(export layout-column-storage-test) + +(def layout-column-storage-test + (test-suite "Layout source coordinate storage" + (test-case "packed source columns match the byte oracle including UTF-8 interiors and EOF" + (for-each + (lambda (source) + (let* ((expected (reference-layout-columns source)) + (columns (make-layout-columns source))) + (check (u32vector->list (reference-packed-layout-columns source)) => (vector->list expected)) + (check (u32vector? columns) => #t) + (check (u32vector-length columns) => (vector-length expected)) + (parameterize ((current-layout-columns columns)) + (let visit ((offset 0)) + (when (< offset (vector-length expected)) + (check (layout-token-column (make-token 'probe "" offset offset)) + => (vector-ref expected offset)) + (visit (+ offset 1))))))) + layout-column-sources)) + (test-case "ASCII fast path rejoins multibyte filling with exact byte columns" + (for-each (lambda (source) + (let (expected (vector->list (reference-layout-columns source))) + (check (u32vector->list (reference-packed-layout-columns source)) => expected) + (check (u32vector->list (make-layout-columns source)) => expected))) + (list (list->string (map integer->char (iota 128))) + (string-append (make-string 1024 #\space) "λ\t\r\n😀" (make-string 1024 #\space))))) + (test-case "long tab columns remain exact beyond u16 and retain CRLF resets" + (let* ((source (string-append (make-string 10000 #\tab) "中\r\n😀")) + (columns (make-layout-columns source))) + (parameterize ((current-layout-columns columns)) + (check (layout-token-column (make-token 'probe "中" 10000 10003)) => 80000) + (check (layout-token-column (make-token 'probe "😀" 10005 10009)) => 0) + (check (layout-token-column (make-token 'probe "" 10009 10009)) => 1)))) + (test-case "wide exact-integer storage and missing request context retain lookup semantics" + (parameterize ((current-layout-columns (vector 0 #x100000000 #x100000001))) + (check (layout-token-column (make-token 'probe "" 1 1)) => #x100000000) + (check (layout-token-column (make-token 'probe "" 2 2)) => #x100000001)) + (parameterize ((current-layout-columns #f)) + (check (layout-token-column (make-token 'probe "" 0 0)) => #f))))) diff --git a/t/lexical-character-run-test.ss b/t/lexical-character-run-test.ss new file mode 100644 index 00000000..d2d960c3 --- /dev/null +++ b/t/lexical-character-run-test.ss @@ -0,0 +1,28 @@ +;;; -*- Gerbil -*- +;;; Variable-width delimiters have one canonical Scheme lexical boundary. + +(import (only-in :std/test check test-case test-suite) + (only-in :gerbil-parser/src/compiler/machine lexical-end) + (only-in :gerbil-parser/src/grammar/lexical-algebra + lexical-expression?) + (only-in :gerbil-parser/src/runtime/scan scan-character-run)) +(export lexical-character-run-test) + +(def (scan-hyphens source offset) + (lexical-end source offset (character-run "-" 4))) + +(def lexical-character-run-test + (test-suite "character-run lexical primitive" + (test-case "canonical algebra requires one character and a positive minimum" + (check (lexical-expression? '(character-run "-" 4)) => #t) + (check (lexical-expression? '(character-run "é" 2)) => #t) + (check (lexical-expression? '(character-run "--" 4)) => #f) + (check (lexical-expression? '(character-run "-" 0)) => #f)) + (test-case "generated scanner consumes a maximal qualified run" + (check (scan-hyphens "---- MODULE" 0) => 4) + (check (scan-hyphens "------- MODULE" 0) => 7) + (check (scan-hyphens "--- MODULE" 0) => #f) + (check (scan-hyphens "x----" 1) => 5) + (check (scan-character-run "ééx" 0 "é" 2) => 2) + (check (scan-character-run "----" 0 "" 2) => #f) + (check (scan-character-run "----" 0 "-" 0) => #f)))) diff --git a/t/lexical-first-character-test.ss b/t/lexical-first-character-test.ss new file mode 100644 index 00000000..8eeab6a5 --- /dev/null +++ b/t/lexical-first-character-test.ss @@ -0,0 +1,119 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; Differential witness for admission filters on nonregular lexical rules. +(import :std/test + :gerbil-parser/src/language/grammar + (only-in :gerbil-parser/src/compiler/machine lexical-dispatch/ranked) + (only-in :gerbil-parser/src/compiler/lexical-lexer prepare-generated-lexer) + (only-in :gerbil-parser/src/runtime/lexer scan-source-token) + (only-in :gerbil-parser/src/runtime/scan scan-emit) + (only-in :gerbil-parser/src/runtime/source-scanner + make-source-scanner source-scanner-initial-state source-scanner-step + source-scan-state-character-offset source-scan-state-byte-offset) + (only-in :gerbil-parser/src/runtime/token token-kind token-lexeme token-start token-end)) + +(begin + (deflanguage first-character-witness + (syntax + (lexical + (root source-file) + (lex + (numeric Numeric (number-literal ("0x" "0b" "hex") "_" ("M") #t #t)) + (quoted Quoted (quoted-string "\"" "'")) + (escaped Escaped (escaped-quoted-string "@")) + (here Here (heredoc)) + (name Name (identifier)) + (other Other (fallback))) + (extras) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file (node SourceFile + (repeat (choice numeric quoted escaped here name other)))))) + (bind-fixture-grammar-release first-character-witness "first-character-witness" "v1" "first-character-witness.v1") ) + +(def (reference-match source offset) + (lexical-dispatch/ranked source offset + ((numeric (number-literal ("0x" "0b" "hex") "_" ("M") #t #t)) + (quoted (quoted-string "\"" "'")) + (escaped (escaped-quoted-string "@")) + (here (heredoc)) + (name (identifier)) + (other (fallback))))) + +(def lexical-first-character-test + (test-suite "closed lexical first-character admission" + (test-case "prepared competition preserves evaluation order and empty projections" + (let (observed '()) + (def (rule name rank first? extent) + (vector (lambda (terminals) #t) + (lambda (source offset) + (set! observed (cons name observed)) + (let (end (extent source offset)) + (and end (list name end rank)))) + #f name rank first? #f)) + (let-values (((lexer plans certificates) + (prepare-generated-lexer + (list + (rule 'first 0 char-alphabetic? (lambda (s o) (+ o 1))) + (rule 'long 0 (lambda (ch) (char=? ch #\a)) + (lambda (s o) (and (char=? (string-ref s o) #\a) (+ o 2)))) + (rule 'priority 9 char-alphabetic? (lambda (s o) (+ o 1)))) + (lambda () (error "unused certificate preparation")) '#()))) + (for-each + (lambda (row) + (set! observed '()) + (let-values (((token end) (lexer (car row) 0 7 #f))) + (check (list (token-kind token) (token-lexeme token) + (token-start token) (token-end token) end) + => (cadr row))) + (check (reverse observed) => (caddr row))) + '(("ab" (long "ab" 7 9 2) (priority long first)) + ("b" (priority "b" 7 8 1) (priority first)) + ("α" (priority "α" 7 9 1) (priority long first)))) + (set! observed '()) + (check-exception (lexer "?" 0 0 #f) true) + (check observed => '())))) + (test-case "engine token bytes preserve 1-4 byte scalars NUL and combining marks" + (let* ((source (string-append "Aé中😀" (string #\nul) "e" (string (integer->char #x301)))) + (scanner (make-source-scanner source #f + (lambda (text start context _mode) + (values (and (< start (string-length text)) 'scalar) + (+ start 1) context))))) + (let loop ((state (source-scanner-initial-state scanner)) + (index 0) (byte-start 0) (widths '(1 2 3 4 1 1 2))) + (if (null? widths) + (let-values (((token final) (source-scanner-step scanner state #f))) + (check token => #f) + (check (source-scan-state-character-offset final) => 7) + (check (source-scan-state-byte-offset final) => 14)) + (let* ((byte-end (+ byte-start (car widths))) + (direct (scan-emit source 'scalar index (+ index 1) (+ 7 byte-start)))) + (check (token-start direct) => (+ 7 byte-start)) + (check (token-end direct) => (+ 7 byte-end)) + (let-values (((token next) (source-scanner-step scanner state #f))) + (check (token-kind token) => 'scalar) + (check (token-lexeme token) => (substring source index (+ index 1))) + (check (token-start token) => byte-start) + (check (token-end token) => byte-end) + (loop next (+ index 1) byte-end (cdr widths)))))))) + (test-case "filtered scanner preserves maximal munch at every offset" + (for-each + (lambda (source) + (let loop ((offset 0)) + (when (< offset (string-length source)) + (let* ((match (reference-match source offset)) + (expected (scan-emit source (car match) offset (cadr match) 0))) + (let-values (((actual end) + (scan-source-token first-character-witness-parser + source offset 0 #f))) + (check (list (token-kind actual) (token-lexeme actual) + (token-end actual) end) + => (list (token-kind expected) (token-lexeme expected) + (token-end expected) (cadr match))))) + (loop (+ offset 1))))) + '("foo .25 1. 0xAB 0b10 hexAB 12_345M" + "\"a\"\"b\" 'c' @d\\@e@ @unterminated" + "αβ λ1 ٣.٢" "< ."))))) +(export lexical-first-character-test) diff --git a/t/lexical-plan-test.ss b/t/lexical-plan-test.ss new file mode 100644 index 00000000..09e23afa --- /dev/null +++ b/t/lexical-plan-test.ss @@ -0,0 +1,181 @@ +#!/usr/bin/env gxi +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +(import (only-in :gerbil-parser/languages/tla-plus/syntax/core tla-plus-core-syntax) + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine) + (only-in :gerbil-parser/src/language/entry parse-language-source) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid? parse-artifact-success?)) +(import :std/test + :gerbil-parser/src/language/grammar + (only-in :gerbil-parser/src/compiler/machine + current-lexical-plan-sharing-enabled? parser-machine-prepare-lexer + parser-machine-lexical-plans parser-machine-lexical-modes-compatible? parser-machine-runtime) + (only-in :gerbil-parser/src/runtime/lr-parser lr-runtime-lexical-mode-catalog) + (only-in :gerbil-parser/src/runtime/incremental + current-lr-lexical-plan-reuse-enabled? current-lr-source-index-enabled? make-incremental-session parse-incremental-session + incremental-session-artifact incremental-session-project-artifact apply-edit make-edit edit-start-byte) + (only-in :gerbil-parser/languages/hcl/parser hcl-parser parse-hcl) + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-parser)) +(begin + (deflanguage lexical-fallback-priority + (syntax + (lexical + (root source-file) + (lex (first FirstToken (literals "x")) + (second SecondToken (precedence 9 (fallback))) + (space Space (whitespace+))) + (extras space) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules (source-file (node SourceFile (seq (field first first) (field second second)))))) + (bind-fixture-grammar-release lexical-fallback-priority "lexical-fallback-priority" "v1" "lexical-fallback-priority.v1") ) +(begin + (deflanguage lexical-fallback-order + (syntax + (lexical + (root source-file) + (lex (second SecondToken (fallback)) + (first FirstToken (literals "x")) + (space Space (whitespace+))) + (extras space) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules (source-file (node SourceFile (seq (field first first) (field second second)))))) + (bind-fixture-grammar-release lexical-fallback-order "lexical-fallback-order" "v1" "lexical-fallback-order.v1") ) + +(def (unique-count plans) + (length (foldl (lambda (plan found) (if (memq plan found) found (cons plan found))) '() (vector->list plans)))) +(def (scan lexer text mode) + (with-catch (lambda (condition) (list 'rejected (error-message condition) (error-irritants condition))) + (lambda () (call-with-values (lambda () (lexer text 0 0 mode)) list)))) +(def lexical-plan-test + (test-suite "compiler-proved lexical scanner plans" + (test-case "shared engine prepares the regenerated TLA core with unchanged mode scans" + (let (machine (language-grammar-machine tla-plus-core-syntax)) + (let-values (((shared plans certificates) + (parameterize ((current-lexical-plan-sharing-enabled? #t)) + (parser-machine-prepare-lexer machine))) + ((separate controls separate-certificates) + (parameterize ((current-lexical-plan-sharing-enabled? #f)) + (parser-machine-prepare-lexer machine)))) + (check (< (unique-count plans) (vector-length plans)) => #t) + (check (unique-count controls) => (vector-length controls)) + (check (force certificates) => (force separate-certificates)) + (for-each + (lambda (mode) + (for-each (lambda (text) (check (scan shared text mode) => (scan separate text mode))) + '("MODULE" "λ中" "001" "<=>" "\\A" "(* nested (* x *) *)" "\"λ中😀\"" "@"))) + (vector->list (lr-runtime-lexical-mode-catalog (parser-machine-runtime machine)))))) + (for-each + (lambda (row) + (let (artifact (parse-language-source tla-plus-core-syntax (car row))) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-success? artifact) => (cadr row)))) + '(("---- MODULE M ----\nA == TRUE\n====\n" #t) + ("---- MODULE M ----\nA == IF TRUE THEN 1 ELSE 2\n====\n" #t) + ("---- MODULE M ----\nA == IF TRUE THEN\n====\n" #f)))) + (test-case "plan sharing changes construction count but preserves every mode scan" + (for-each + (lambda (machine) + (let-values (((shared plans shared-certificates) (parameterize ((current-lexical-plan-sharing-enabled? #t)) (parser-machine-prepare-lexer machine))) + ((separate controls control-certificates) (parameterize ((current-lexical-plan-sharing-enabled? #f)) (parser-machine-prepare-lexer machine)))) + (check (< (unique-count plans) (vector-length plans)) => #t) + (check (unique-count controls) => (vector-length controls)) + (for-each + (lambda (mode) + (for-each (lambda (text) (check (scan shared text mode) => (scan separate text mode))) + '("value" "λ中" "001" ".2" "**" "+" "/* λ */" "\n" "\"λ中😀\"" "@"))) + (vector->list (lr-runtime-lexical-mode-catalog (parser-machine-runtime machine)))))) + (list hcl-parser arithmetic-parser))) + (test-case "equivalent plans preserve token output across distinct LR modes" + (let* ((plans (parser-machine-lexical-plans hcl-parser)) + (catalog (lr-runtime-lexical-mode-catalog (parser-machine-runtime hcl-parser)))) + (let-values (((lexer ignored certificates) (parser-machine-prepare-lexer hcl-parser))) + (for-each + (lambda (i) + (for-each + (lambda (j) + (when (and (not (= i j)) (eq? (vector-ref plans i) (vector-ref plans j))) + (check (parser-machine-lexical-modes-compatible? hcl-parser i j) => #t) + (check (scan lexer "value" (vector-ref catalog i)) => (scan lexer "value" (vector-ref catalog j))))) + (iota (vector-length plans)))) + (iota (vector-length plans)))))) + (test-case "first-character certificates preserve competition across distinct modes" + (for-each + (lambda (machine) + (let* ((catalog (lr-runtime-lexical-mode-catalog (parser-machine-runtime machine))) + (n (vector-length catalog)) (refined 0) (vetoed 0)) + (let-values (((lexer plans certificates) (parser-machine-prepare-lexer machine))) + (for-each + (lambda (i) + (for-each + (lambda (j) + (for-each + (lambda (text) + (when (and (eq? machine hcl-parser) (equal? text "\n")) + (check (parser-machine-lexical-modes-compatible? machine i j #\newline) => #t)) + (let ((compatible (parser-machine-lexical-modes-compatible? machine i j (string-ref text 0))) + (left (scan lexer text (vector-ref catalog i))) + (right (scan lexer text (vector-ref catalog j)))) + (when compatible + ;; Rejection mode objects differ; compare outcome, while + ;; successful scans compare complete token and offset. + (check (if (eq? (car left) 'rejected) 'rejected left) + => (if (eq? (car right) 'rejected) 'rejected right)) + (unless (parser-machine-lexical-modes-compatible? machine i j) + (set! refined (+ refined 1)))) + (when (and (not compatible) (not (equal? left right))) + (set! vetoed (+ vetoed 1))))) + '("\n" " " "x" "value" "001" ".2" "**" "*" "+" "/* λ */" "\"λ中😀\"" "< refined 0) => #t) + (unless (eq? machine arithmetic-parser) (check (> vetoed 0) => #t)))) + (list hcl-parser arithmetic-parser + lexical-fallback-priority-parser lexical-fallback-order-parser))) + (test-case "Unicode byte shifts preserve certified probes and indexed mode provenance" + (for-each + (lambda (indexed?) + (parameterize ((current-lr-source-index-enabled? indexed?)) + (let* ((source (apply string-append (make-list 80 "value = 001\n"))) + (session (make-incremental-session hcl-parser source #t))) + (for-each + (lambda (edit) + (let-values (((next receipt) + (parameterize ((current-lr-lexical-plan-reuse-enabled? #t)) + (parse-incremental-session session edit)))) + (let-values (((control ignored) + (parameterize ((current-lr-lexical-plan-reuse-enabled? #f)) + (parse-incremental-session session edit)))) + (check (incremental-session-artifact next) => (incremental-session-artifact control))) + (set! source (apply-edit source edit)) + (check (incremental-session-artifact next) => (parse-hcl source)) + (check (incremental-session-project-artifact next) => (incremental-session-artifact next)) + (set! session next))) + (list (make-edit 0 0 "λ = \"中😀\"\n") (make-edit 0 15 "")))))) + '(#f #t))) + (test-case "six-edit token convergence preserves full artifacts and original folds" + (let* ((line "value = 001\n") (source (apply string-append (make-list 80 line))) + (session (make-incremental-session hcl-parser source #t))) + (for-each + (lambda (edit) + (let-values (((next receipt) (parameterize ((current-lr-lexical-plan-reuse-enabled? #t)) + (parse-incremental-session session edit)))) + (parameterize ((current-lr-lexical-plan-reuse-enabled? #f)) + (let-values (((control strict-receipt) (parse-incremental-session session edit))) + (when (and (zero? (edit-start-byte edit)) + (equal? source (apply string-append (make-list 80 line)))) + (check (< (cdr (assq 'fragmentCertificateProbeByteCount receipt)) + (cdr (assq 'fragmentCertificateProbeByteCount strict-receipt))) => #t) + (check (> (cdr (assq 'certifiedLexicalProbeTokenCount receipt)) 0) => #t)) + (check (incremental-session-artifact next) => (incremental-session-artifact control)))) + (set! source (apply-edit source edit)) + (check (incremental-session-artifact next) => (parse-hcl source)) + (check (incremental-session-project-artifact next) => (incremental-session-artifact next)) + (set! session next))) + (list (make-edit 0 0 "other = 002\n") (make-edit 492 0 "third = 003\n") + (make-edit 0 12 "") (make-edit 480 12 "") + (make-edit 960 0 "last = 004\n") (make-edit 960 11 ""))))))) +(export lexical-plan-test) diff --git a/t/lexical-source-test.ss b/t/lexical-source-test.ss new file mode 100644 index 00000000..fdd7436d --- /dev/null +++ b/t/lexical-source-test.ss @@ -0,0 +1,43 @@ +;;; Prepared captures belong to a request, including nested and parallel parses. +(import :std/test + (only-in :gerbil-parser/src/runtime/lexical-source + prepare-lexical-source-plan call-with-lexical-source + scan-header-delimiter scan-header-data)) +(export lexical-source-test) +(def capture-plan + (prepare-lexical-source-plan + '((field (header-delimiter "記" 2 0)) + (body (choice (header-data "記" 2 ";") (precedence 1 (header-delimiter "記" 2 1)))) + (other (header-data "REC" 2 "\n"))))) +(def (scan-record source) + (call-with-lexical-source capture-plan source + (lambda () + (list (scan-header-delimiter source 1 "記" 2 0) + (scan-header-delimiter source 2 "記" 2 1) + (scan-header-data source 3 "記" 2 ";"))))) +(def lexical-source-test + (test-suite "request-owned lexical captures" + (test-case "nested lexical choices deduplicate capture preparation" + (check capture-plan => '(("記" 2) ("REC" 2))) + (check (scan-record "記|§α|") => '(2 3 4)) + (check (scan-record "記*$α*") => '(2 3 4)) + (check (scan-record "記||α|") => '(#f #f #f))) + (test-case "nested foreign sources and failed scopes restore the parent capture" + (let (source "記|§α|") + (call-with-lexical-source capture-plan source + (lambda () + (check (scan-record "記*$α*") => '(2 3 4)) + (check (with-catch (lambda (_) 'caught) + (lambda () (call-with-lexical-source capture-plan "記||α|" + (lambda () (error "nested scope failed"))))) => 'caught) + (check (scan-header-delimiter source 1 "記" 2 0) => 2) + (check (scan-header-data source 3 "記" 2 ";") => 4) + ;; A raw scanner on a different source never reads a parent capture. + (check (scan-header-delimiter "記*$α*" 1 "記" 2 0) => 2))))) + (test-case "parallel source requests retain independent captures" + (let ((left (spawn (lambda () (let loop ((n 100) (rows '())) + (if (zero? n) rows (loop (- n 1) (cons (scan-record "記|§α|") rows))))))) + (right (spawn (lambda () (let loop ((n 100) (rows '())) + (if (zero? n) rows (loop (- n 1) (cons (scan-record "記*$α*") rows)))))))) + (check (andmap (cut equal? <> '(2 3 4)) (thread-join! left)) => #t) + (check (andmap (cut equal? <> '(2 3 4)) (thread-join! right)) => #t))))) diff --git a/t/line-event-test.ss b/t/line-event-test.ss new file mode 100644 index 00000000..255bf5ed --- /dev/null +++ b/t/line-event-test.ss @@ -0,0 +1,77 @@ +;;; Native execution of non-Org declarations through the public engine surface. +(import :std/test + (only-in :gerbil-parser/src/modules/parser/interface + make-table-line make-source-event-scope make-text-line source-table-row-initial + source-table-row-forms source-paragraph-initial + source-paragraph-close-form source-paragraph-line-form) + (only-in :gerbil-parser/src/compiler/event-fold-runtime run-event-fold)) +(def scope (make-source-event-scope 'native-test)) +(export line-event-test) +(def table (make-table-line ";" 'Table 'Row 'RuleRow 'Cell + 'Separator 'Value 'Trivia 'RuleText)) +(def text (make-text-line 'Blank 'Text #f 'Paragraph)) +(def (starts events node) + (length (filter (lambda (event) (equal? event (list 'start node))) events))) +(def (source-covered? events source) + (let loop ((rest events) (offset 0)) + (if (null? rest) (= offset (u8vector-length (string->utf8 source))) + (let (event (car rest)) + (if (eq? (car event) 'token) + (and (= (caddr event) offset) (<= offset (cadddr event)) + (loop (cdr rest) (cadddr event))) + (loop (cdr rest) offset)))))) +(def (table-events source (escape 33)) + (run-event-fold source 'Document (source-table-row-initial table scope) + (source-table-row-forms table scope escape-byte: escape + rule-bytes: '(61 9 32) rule-marker: 61) + '())) +(def (paragraph-events source) + (run-event-fold source 'Document (source-paragraph-initial text scope) + (list (source-paragraph-line-form text scope)) + (list (source-paragraph-close-form text scope #f '() #f)))) +(def line-event-test + (test-suite "native declarative line events" + (test-case "non-Org delimiter, escape, empty cell and Unicode source" + (let* ((source ";λ!;b;;z;\r\n") (events (table-events source))) + (check (starts events 'Cell) => 3) + (check (source-covered? events source) => #t))) + (test-case "escape parity and no-escape policy are declaration driven" + (check (starts (table-events ";a!!;b;") 'Cell) => 2) + (check (starts (table-events ";a!;b;" #f) 'Cell) => 2) + (check (starts (table-events ";a!;b;") 'Cell) => 1)) + (test-case "rule-row marker is language policy, not hardcoded Org bytes" + (check (starts (table-events ";===;\n") 'RuleRow) => 1) + (check (starts (table-events ";---;\n") 'Row) => 1) + (check (source-covered? (table-events ";===;\n") ";===;\n") => #t)) + (test-case "consecutive rows reset escape and separator state" + (let* ((source ";a!\n;b;\n") (events (table-events source))) + (check (starts events 'Row) => 2) + (check (starts events 'Cell) => 2) + (check (source-covered? events source) => #t))) + (test-case "paragraph lifetime handles CRLF, blank lines and EOF" + (let* ((source "α\r\nβ\n\nγ") (events (paragraph-events source))) + (check (starts events 'Paragraph) => 2) + (check (starts events 'Blank) => 1) + (check (source-covered? events source) => #t))) + (test-case "empty source and leading/trailing blank lines stay lossless" + (for-each + (lambda (source) + (check (source-covered? (paragraph-events source) source) => #t)) + '("" "\n\n" "\nα\n\n" "α"))) + (test-case "content helpers receive explicit named state parameters" + (let* ((forms (source-table-row-forms table scope 'content '(policy))) + (events (run-event-fold ";x;" 'Document + (append (source-table-row-initial table scope) '((policy 7))) + forms '() + '((content ((policy 0)) + ((if (uint-equal? (state policy) (uint 7)) + ((token Value start end)) + ((token Wrong start end)))) (policy)))))) + (check (and (member '(token Value 1 2) events) #t) => #t))) + (test-case "invalid declarations and bindings are rejected before execution" + (check-exception (source-table-row-forms #f scope) true) + (check-exception (source-paragraph-initial (make-text-line 'Blank 'Text) scope) true) + (check-exception (source-table-row-forms table scope 'content '(x x)) true) + (check-exception (source-table-row-forms table scope #f '(x)) true) + (check-exception (source-table-row-forms table scope escape-byte: 256) true) + (check-exception (source-table-row-forms table scope rule-marker: 61) true)))) diff --git a/t/line-structure-aot-test.ss b/t/line-structure-aot-test.ss index 54e93a12..2552a506 100644 --- a/t/line-structure-aot-test.ss +++ b/t/line-structure-aot-test.ss @@ -2,7 +2,7 @@ ;;; POO declaration owns contextual-line AOT admission. (import (only-in :std/test check check-exception test-case test-suite) - (only-in :gerbil-parser/languages/arithmetic/v1/grammar + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-language-grammar) (only-in :gerbil-parser/src/modules/parser/line-structure-objects line-structure? make-line-structure make-heading-line @@ -10,9 +10,9 @@ make-block-line make-block-header make-key-value-line make-inline-link make-text-line make-table-line make-list-line make-key-line) - (only-in :gerbil-parser/src/compiler/line-structure-rowan - line-structure-parser-digest line-structure-rowan-source - line-structure-rowan-syntax) + (only-in :gerbil-parser/src/compiler/line-structure-runtime + line-structure-parser-digest line-structure-runtime-source + line-structure-runtime-syntax) (only-in ./line-structure-assertions check-structural-aot)) (export line-structure-aot-test) @@ -29,7 +29,7 @@ (test-suite "POO structural-line AOT" (test-case "one checked POO declaration resolves canonical parser kinds" (let* ((structure (fixture-structure)) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check (line-structure? structure) => #t) (check-structural-aot syntax (identity) (block-opening "BEGIN")))) @@ -79,7 +79,7 @@ 'PrefixExpression 'Punctuation 'Number 'Punctuation 'close-at-eof #f #f #f 'elements)) (make-text-line 'NameExpression 'Number))) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar recursive))) (check-structural-aot syntax (block-contents "BlockContents::Elements")) @@ -110,7 +110,7 @@ '() (make-text-line 'NameExpression 'Number) #f list-rule)) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check-structural-aot syntax (list-markers "-+*" 8)) (check (equal? (line-structure-parser-digest @@ -139,7 +139,7 @@ 'PrefixExpression 'Punctuation 'Number 'Punctuation 'close-at-eof #f #f)) (make-text-line 'NameExpression 'Number))) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check-structural-aot syntax (heading-fields)) (check (equal? (line-structure-parser-digest @@ -159,7 +159,7 @@ 'PrefixExpression 'Punctuation 'Number 'Punctuation 'recover-as-text #t body)) (make-text-line 'NameExpression 'Number))) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check-structural-aot syntax (body-line)) (check (equal? (line-structure-parser-digest @@ -178,7 +178,7 @@ 'PrefixExpression 'Punctuation 'Number 'Punctuation 'close-at-eof #f #f header)) (make-text-line 'NameExpression 'Number))) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check-structural-aot syntax (block-header)) (check (equal? (line-structure-parser-digest @@ -198,7 +198,7 @@ 'PrefixExpression 'Punctuation 'Number 'Punctuation 'close-at-eof #f #f)) (make-text-line 'NameExpression 'Number link))) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check-structural-aot syntax (inline-link)) (check (equal? (line-structure-parser-digest @@ -216,7 +216,7 @@ 'PrefixExpression 'Punctuation 'Number 'Punctuation 'close-at-eof #f #f)) (make-text-line 'NameExpression 'Number #f 'NameExpression))) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check-structural-aot syntax (paragraph-node)) (check (equal? (line-structure-parser-digest @@ -234,7 +234,7 @@ '() (make-text-line 'NameExpression 'Number) #f #f (list rule))) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check-structural-aot syntax (key-lines '(("#+" () "KeyLineContext::Anywhere" @@ -264,7 +264,7 @@ '() (make-text-line 'NameExpression 'Number) table)) - (syntax (line-structure-rowan-syntax + (syntax (line-structure-runtime-syntax arithmetic-language-grammar structure))) (check-structural-aot syntax @@ -279,7 +279,7 @@ (check (with-catch (lambda (error) #t) (lambda () - (line-structure-rowan-source + (line-structure-runtime-source arithmetic-language-grammar (fixture-structure 'Number)) #f)) diff --git a/t/list-event-test.ss b/t/list-event-test.ss new file mode 100644 index 00000000..5e8989e5 --- /dev/null +++ b/t/list-event-test.ss @@ -0,0 +1,90 @@ +;;; Non-Org declaration controls for scoped native list structure and spans. +(import :std/test + (only-in :gerbil-parser/src/modules/parser/interface + make-list-line make-text-line make-source-event-scope + source-list-initial source-list-scan-form source-list-present-condition + source-list-ordered-condition source-list-active-condition + source-list-first-blank-condition source-list-continuation-condition + source-list-content-offset source-list-item-open-forms + source-list-paragraph-span-forms source-list-paragraph-close-form + source-list-close-forms source-list-first-blank-forms source-list-reset-blank-form) + (only-in :gerbil-parser/src/compiler/event-fold-runtime run-event-fold)) +(export list-event-test) +(def rule (make-list-line "+" #t 'Sequence 'Entry 'Marker 'Space 4)) +(def text (make-text-line 'Blank 'Raw #f 'Content)) +(def scope (make-source-event-scope 'non-org-list)) +(def (starts events node) + (length (filter (lambda (event) (equal? event (list 'start node))) events))) +(def (covers? events source) + (let loop ((rest events) (offset 0) (depth 0)) + (if (null? rest) + (and (= offset (u8vector-length (string->utf8 source))) (= depth 0)) + (let (event (car rest)) + (case (car event) + ((token) (and (= offset (caddr event)) (<= offset (cadddr event)) + (loop (cdr rest) (cadddr event) depth))) + ((start) (loop (cdr rest) offset (+ depth 1))) + ((finish) (and (> depth 0) (loop (cdr rest) offset (- depth 1))))))))) +(def (events source (span 'marker) (declaration rule)) + (def close (source-list-close-forms text scope)) + (def paragraph-close (source-list-paragraph-close-form text scope)) + (def (item ordered?) + `(,paragraph-close + ,@(source-list-item-open-forms declaration scope ordered? bullet-span: span) + ,@(source-list-paragraph-span-forms text scope (source-list-content-offset scope) 'end) + ,(source-list-reset-blank-form scope))) + (run-event-fold source 'Document (source-list-initial declaration scope text) + `(,(source-list-scan-form declaration scope) + (if ,(source-list-present-condition scope) + ((if ,(source-list-ordered-condition scope) ,(item #t) ,(item #f))) + ((if ,(source-list-active-condition scope) + ((if (line-blank?) + ((if ,(source-list-first-blank-condition scope) + ,(source-list-first-blank-forms declaration text scope) + (,@close (token Raw start end)))) + ((if ,(source-list-continuation-condition declaration scope) + (,@(source-list-paragraph-span-forms text scope 'start 'end) + ,(source-list-reset-blank-form scope)) + (,@close (token Raw start end)))))) + ((token Raw start end)))))) + (source-list-close-forms text scope #f '() #f))) +(def list-event-test + (test-suite "native declarative list transitions" + (test-case "non-Org siblings preserve Unicode CRLF and EOF" + (let* ((source "+ α\r\n+ β") (result (events source))) + (check (starts result 'Sequence) => 1) + (check (starts result 'Entry) => 2) + (check (covers? result source) => #t))) + (test-case "nested dedent and sibling transitions balance every node" + (let* ((source "+ a\n + b\n + c\n + d\n+ e\n") + (result (events source))) + (check (starts result 'Sequence) => 3) + (check (starts result 'Entry) => 5) + (check (covers? result source) => #t))) + (test-case "ordered kind changes replace a same-column container" + (let* ((source "+ a\n1. b\n+ c") (result (events source))) + (check (starts result 'Sequence) => 3) + (check (covers? result source) => #t))) + (test-case "tab width belongs to the admitted list declaration" + (let* ((source "+ a\n\t+ b\n + c\n") (result (events source))) + (check (starts result 'Sequence) => 3) + (check (covers? result source) => #t))) + (test-case "one blank retains the item and two blanks terminate it" + (for-each (lambda (source) + (check (covers? (events source) source) => #t)) + '("+ a\n\n b\n+ c" "+ a\n\n\nplain\n")) + (check (starts (events "+ a\n\n b\n+ c") 'Sequence) => 1)) + (test-case "bullet span is explicit language policy" + (check (and (member '(token Marker 0 1) (events "+ a")) #t) => #t) + (check (and (member '(token Marker 0 2) (events "+ a" 'with-separator)) #t) => #t) + (check (covers? (events "+ a" 'with-separator) "+ a") => #t)) + (test-case "disabled ordered markers and empty documents stay lossless" + (let (unordered (make-list-line "+" #f 'Sequence 'Entry 'Marker 'Space 4)) + (check (starts (events "1. raw\n+ item" 'marker unordered) 'Sequence) => 1) + (check (covers? (events "1. raw\n+ item" 'marker unordered) "1. raw\n+ item") => #t)) + (check (covers? (events "") "") => #t)) + (test-case "invalid declarations and transition policies fail before execution" + (check-exception (source-list-initial #f scope text) true) + (check-exception (source-list-initial rule #f text) true) + (check-exception (source-list-item-open-forms rule scope 'ordered) true) + (check-exception (source-list-item-open-forms rule scope #f bullet-span: 'unknown) true)))) diff --git a/t/lr-action-preparation-test.ss b/t/lr-action-preparation-test.ss new file mode 100644 index 00000000..e65f5c2a --- /dev/null +++ b/t/lr-action-preparation-test.ss @@ -0,0 +1,138 @@ +;;; Prepared layout projections preserve the former pure action decision. +(import :std/test + (only-in :gerbil-parser/t/benchmarks/lr-request-stack/lookahead profile-lr-lookahead) + (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar records-parser) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/layout + prepare-layout-action-selector make-layout-columns + current-layout-columns current-layout-frames) + (only-in :gerbil-parser/src/runtime/lr-action-index index-action-rows lookup-literal-action-entry + lr-action-row-eof lr-action-row-tokens) + (only-in :gerbil-parser/src/runtime/event-program event-program-value-kind) + (only-in :gerbil-parser/src/runtime/lr-parser lr-checkpoint-state) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in ./fixtures/lr-action-selection reference-lr-action-selector) + (only-in :gerbil-parser/src/runtime/lr-parser/prepare make-lr-action-selector)) +(def lr-action-preparation-test + (test-suite "prepared layout action projections" + (test-case "lookahead diagnostics preserve complete products with scalar inputs and EOF" + (let* ((source "a=1\nb=2\n") (expected (parse-source records-parser source)) + (receipt (profile-lr-lookahead records-parser source expected))) + (check (> (cdr (assq 'actionObservations receipt)) 0) => #t) + (check (> (cdr (assq 'eofObservations receipt)) 0) => #t) + (check (>= (cdr (assq 'actionObservations receipt)) + (cdr (assq 'lookaheads receipt))) => #t))) + (test-case "closed engine record access retains checked rejection" + (let* ((entry '((terminal eof) accept)) + (row (vector-ref (index-action-rows (vector (list entry)) #f) 0))) + (check (eq? (lr-action-row-eof row) entry) => #t) + (for-each + (lambda (invalid) + (check-exception (lr-action-row-eof invalid) true) + (check-exception (lr-action-row-tokens invalid) true) + (check-exception (event-program-value-kind invalid) true) + (check-exception (lr-checkpoint-state invalid) true)) + (list #f 0 '() '#() (make-token 'word "a" 0 1))))) + (test-case "all action forms, frame states, token refinements and EOF agree" + (for-each (lambda (action) + (let* ((entries + (list (cons '(terminal literal "end") action) + (cons '(terminal literal "END") '(layout-guard (reduce 12) (reduce 14))) + (cons '(terminal literal "É") '(reduce 13)) + (cons '(terminal token word) '(fork (shift 4) (reduce 5))) + (cons '(terminal layout-next "end") action) + (cons '(terminal layout-start "end") action) + (cons '(terminal eof) '(accept)))) + (rows (vector entries))) + (for-each (lambda (casefold?) + (let* ((index (index-action-rows rows casefold?)) + (reference (reference-lr-action-selector rows index casefold? #t)) + (candidate (prepare-layout-action-selector rows index casefold?))) + (parameterize ((current-layout-columns (make-layout-columns "end é\nEND"))) + (for-each (lambda (frames) + (parameterize ((current-layout-frames frames)) + (for-each (lambda (tokens) + (check (candidate 0 (and (pair? tokens) (car tokens))) => (reference 0 (and (pair? tokens) (car tokens))))) + (list '() (list (make-token 'word "end" 0 3)) + (list (make-token 'word "é" 4 6)) + (list (make-token 'word "END" 7 10)) + (list (make-token 'word "foo" 7 10)))))) + '(() ((2 . "end")) ((100 . "outer")) ((4 . "é")) + ((2 . "end") (100 . "outer"))))))) '(#f #t)))) + '((shift 1) (reduce 2) (accept) (fork (shift 3) (reduce 4)) + (layout-guard (shift 5) (reduce 15)) (layout-guard (reduce 6) (reduce 16)) + (layout-guard (fork (shift 7) (reduce 8)) (reduce 17)) + (fork (layout-guard (shift 9) (reduce 18)) (fork (shift 10) (reduce 11)))))) + (test-case "denied first literal never falls through to duplicates or token kinds" + (let* ((entries '(((terminal literal "end") shift 1) + ((terminal literal "end") reduce 2) + ((terminal literal "END") reduce 3) + ((terminal token word) reduce 4))) + (rows (vector entries)) + (index (index-action-rows rows #t)) + (candidate (prepare-layout-action-selector rows index #t))) + (parameterize ((current-layout-columns (make-layout-columns "end")) + (current-layout-frames '((100 . "outer")))) + (check (candidate 0 (make-token 'word "end" 0 3)) => #f)))) + (test-case "pure reductions skip column decisions only after marker precedence" + (let* ((entry '((terminal literal "end") reduce 1)) + (rows (vector (list entry '((terminal layout-next "end") shift 2)))) + (index (index-action-rows rows #f)) + (candidate (prepare-layout-action-selector rows index #f)) + (reference (reference-lr-action-selector rows index #f #t)) + (tokens (list (make-token 'word "end" 0 3)))) + (parameterize ((current-layout-columns (make-layout-columns "end"))) + (for-each (lambda (frames) + (parameterize ((current-layout-frames frames)) + (check (candidate 0 (and (pair? tokens) (car tokens))) => (reference 0 (and (pair? tokens) (car tokens)))))) + '(() ((2 . "end")) ((100 . "outer"))))))) + (test-case "missing source context retains original entry identity" + (let* ((entry '((terminal literal "end") layout-guard (reduce 1) (reduce 2))) + (rows (vector (list entry))) (index (index-action-rows rows #f)) + (candidate (prepare-layout-action-selector rows index #f))) + (parameterize ((current-layout-columns #f) + (current-layout-frames '((100 . "outer")))) + (check (eq? (candidate 0 (make-token 'word "end" 0 3)) entry) => #t)))))) +(export lr-action-preparation-test lr-ordinary-action-preparation-test) + +(import (only-in :gerbil-parser/src/runtime/lr-parser lr-prepare)) +(def lr-ordinary-action-preparation-test + (test-suite "prepared ordinary LR state selectors" + (test-case "wide string indexes retain first duplicates and independent source entry identity" + (let* ((names (append (map (lambda (i) (string-append "WORD" (number->string i))) (iota 24)) + '("é" "É" "ß" "SS" "猫" "🦊"))) + (entries (append (map (lambda (name) (cons (list 'terminal 'literal name) '(shift 1))) names) + '(((terminal literal "WORD0") reduce 99)))) + (row (vector-ref (index-action-rows (vector entries) #f) 0))) + (for-each + (lambda (name) + ;; Fresh equal strings must select the original first declaration, + ;; independently of the index implementation and string identity. + (let (expected (find (lambda (entry) (equal? (caddar entry) name)) entries)) + (check (eq? (lookup-literal-action-entry row (string-copy name)) expected) => #t))) + (append names '("word0" "missing" ""))))) + (test-case "token-only, literal priority, casefold, duplicates, wide rows and EOF retain entries" + (let ((factory make-lr-action-selector) + (rows + (vector + (list (cons '(terminal token word) '(shift 1)) + (cons '(terminal token word) '(reduce 2)) + (cons '(terminal eof) '(accept))) + (list (cons '(terminal literal "IF") '(reduce 3)) + (cons '(terminal token word) '(fork (shift 4) (reduce 5))) + (cons '(terminal eof) '(reduce 6))) + (map (lambda (n) (cons (list 'terminal 'token (string->symbol (number->string n))) + (list 'shift n))) (iota 16))))) + (for-each (lambda (casefold?) + (let* ((index (index-action-rows rows casefold?)) + (reference (reference-lr-action-selector rows index casefold? #f)) + (candidate (factory rows index casefold? #f))) + (for-each (lambda (state) + (for-each (lambda (input) + (check (eq? (candidate state (and (pair? input) (car input))) (reference state (and (pair? input) (car input)))) => #t)) + (append (list '() (list (make-token 'word "if" 0 2)) + (list (make-token 'word "IF" 0 2)) + (list (make-token 'word "猫" 0 3)) + (list (make-token 'missing "?" 0 1))) + (map (lambda (n) (list (make-token (string->symbol (number->string n)) "x" 0 1))) + (iota 16))))) '(0 1 2)))) '(#f #t)))))) diff --git a/t/lr-completion-test.ss b/t/lr-completion-test.ss new file mode 100644 index 00000000..7d901b27 --- /dev/null +++ b/t/lr-completion-test.ss @@ -0,0 +1,54 @@ +;;; Completion policy invariants independent of parser exploration order. +(import :std/test + (only-in :gerbil-parser/src/runtime/lr-completion + make-candidate select-candidate candidate-root candidate-rest + candidate-score candidate-ambiguities candidate-winner-reason + candidate-completion-count)) +(export lr-completion-test) + +(def (completion-summary candidate) + (list (candidate-root candidate) (candidate-rest candidate) + (candidate-score candidate) (candidate-ambiguities candidate) + (candidate-winner-reason candidate) (candidate-completion-count candidate))) + +(def (check-selection left right expected merged?) + (let-values (((winner merged) (select-candidate left right))) + (check (completion-summary winner) => expected) + (check merged => merged?))) + +(def lr-completion-test + (test-suite "selective GLR completion policy" + (test-case "first completion preserves identity" + (let (candidate (make-candidate 'Root '(tail) 0)) + (let-values (((winner merged?) (select-candidate #f candidate))) + (check (eq? winner candidate) => #t) + (check merged? => #f)))) + (test-case "dynamic score outranks consumption in either exploration order" + (let ((low (make-candidate 'Low '() 1)) + (high (make-candidate 'High '(a b) 2))) + (check-selection low high '(High (a b) 2 0 dynamic-precedence 2) #f) + (check-selection high low '(High (a b) 2 0 dynamic-precedence 2) #f) + (check (completion-summary high) => '(High (a b) 2 0 unique-completion 1)))) + (test-case "maximal consumption ranks equal-score suffixes" + (let ((partial (make-candidate 'Partial '(a b) 0)) + (complete (make-candidate 'Complete '() 0))) + (check-selection partial complete '(Complete () 0 0 maximal-consumption 2) #f) + (check-selection complete partial '(Complete () 0 0 maximal-consumption 2) #f))) + (test-case "equivalent roots and suffixes merge structural copies" + (check-selection (make-candidate (list 'Root) (list 'tail) 0) + (make-candidate (list 'Root) (list 'tail) 0) + '((Root) (tail) 0 0 equivalent-merge 2) #t) + (check-selection (make-candidate 'Root '(tail) 0 2 'ambiguous 3) + (make-candidate 'Root '(tail) 0 1 'ambiguous 4) + '(Root (tail) 0 3 ambiguous 7) #t)) + (test-case "distinct products retain ambiguity until an outer score wins" + (let-values (((ambiguous merged?) + (select-candidate (make-candidate 'First '(a) 0) + (make-candidate 'Second '(a) 0)))) + (check (completion-summary ambiguous) => '(First (a) 0 1 ambiguous 2)) + (check merged? => #f) + (check-selection ambiguous (make-candidate 'Ranked '(a b) 1) + '(Ranked (a b) 1 0 dynamic-precedence 3) #f)) + (check-selection (make-candidate 'Root '(a) 0) + (make-candidate 'Root '(b) 0) + '(Root (a) 0 1 ambiguous 2) #f)))) diff --git a/t/lr-conflict-origins-test.ss b/t/lr-conflict-origins-test.ss new file mode 100644 index 00000000..8c0c1c33 --- /dev/null +++ b/t/lr-conflict-origins-test.ss @@ -0,0 +1,106 @@ +;;; Effective expression occurrences must survive rejected LR table admission. +(import :std/test + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr lower-rules current-grammar-source-map) + (only-in :gerbil-parser/src/compiler/normalize compile-grammar/context + normalized-grammar-ir normalized-grammar-source-map grammar-ir-ref) + (only-in :gerbil-parser/src/modules/parser/objects make-grammar make-grammar-role)) +(export lr-conflict-origins-test) +(def (admission-result rules (construction 'lalr)) + (with-catch + (lambda (condition) (list (error-message condition) (error-irritants condition))) + (lambda () (compile-lr-spec rules 'start 'reject #f construction) 'accepted))) +(def (rule-role name rules) + (make-grammar-role name '() '() '() rules '() '() '() '() '())) +(def lr-conflict-origins-test + (test-suite "LR conflict source origins" + (test-case "lowering origin replay preserves tables and distinguishes equal helpers" + (let* ((rules '((start (alias Start + (sequence (optional (literal "a")) + (optional (literal "a"))))))) + (origins '()) + (plain (lower-rules rules 'start)) + (replayed + (lower-rules rules 'start #f + (lambda (production owner path expression) + (set! origins (cons (list (car production) owner path (car expression)) origins)))))) + (check replayed => plain) + ;; Both alternatives of each optional helper retain that occurrence. + (check (map caddr (reverse origins)) => '((0 0) (0 0) (0 1) (0 1) (0) ())) + (check (map cadr origins) => '(start start start start start start)))) + (test-case "rejected LR conflicts retain effective POO origins across constructions" + (let* ((base (make-grammar 'base '() + (list (rule-role 'base-rules + '((start (reference expr)) (expr (literal "a"))))))) + (derived (make-grammar 'derived (list base) '() + (list (cons 'override + (rule-role 'ambiguous-expr + '((expr (choice (sequence (reference expr) (literal "+") + (reference expr)) + (literal "a"))))))))) + (normalized (compile-grammar/context derived)) + (rules (grammar-ir-ref (normalized-grammar-ir normalized) 'rules)) + (source-map (normalized-grammar-source-map normalized "unknown-source"))) + (for-each + (lambda (construction) + (let* ((result (parameterize ((current-grammar-source-map source-map)) + (admission-result rules construction))) + (details (car (reverse (cadr result)))) + (conflict (cdr (assq 'lrConflict details))) + (reductions (filter-map (lambda (row) (let (origin (assq 'expressionOrigin row)) + (and origin (cdr origin)))) + (cdr (assq 'actions conflict)))) + (shifts (cdr (assq 'shiftItems conflict)))) + (check (car result) => "unresolved shift/reduce conflict requires precedence") + (check (cdr (assq 'terminal conflict)) => '(terminal literal "+")) + (check (length reductions) => 1) + (check (pair? shifts) => #t) + (for-each + (lambda (origin) + (check (cdr (assq 'rule origin)) => 'expr) + (check (cdr (assq 'expressionPath origin)) => '(0)) + (check (cdr (assq 'loweringOperation origin)) => 'sequence) + (let (source (cdr (assq 'source origin))) + (check (cdr (assq 'componentOwner source)) => 'ambiguous-expr) + (check (cdr (assq 'compositionOperation source)) => 'override))) + (append reductions (map (lambda (row) (cdr (assq 'expressionOrigin row))) shifts))))) + '(lalr canonical-lr1 partitioned-lr1 follow-partition-lr1)) + (check (current-grammar-source-map) => '()))) + (test-case "helper conflicts report expression occurrences without fabricated sources" + (for-each + (lambda (construction) + (let* ((result (admission-result + '((start (sequence (optional (literal "a")) + (optional (literal "a"))))) construction)) + (conflict (cdr (assq 'lrConflict (car (reverse (cadr result))))))) + (check (car result) => "unresolved shift/reduce conflict requires precedence") + (for-each + (lambda (origin) + (check (cdr (assq 'rule origin)) => 'start) + (check (member (cdr (assq 'expressionPath origin)) '((0) (1))) ? values) + (check (cdr (assq 'loweringOperation origin)) => 'optional) + (check (cdr (assq 'source origin)) => '())) + (append (filter-map (lambda (row) (let (entry (assq 'expressionOrigin row)) + (and entry (cdr entry)))) + (cdr (assq 'actions conflict))) + (map (lambda (row) (cdr (assq 'expressionOrigin row))) + (cdr (assq 'shiftItems conflict))))))) + '(lalr canonical-lr1 partitioned-lr1 follow-partition-lr1))) + (test-case "reduce conflicts retain both source occurrences and no shift witnesses" + (for-each + (lambda (construction) + (let* ((result (admission-result + '((start (choice (reference left) (reference right))) + (left (literal "a")) (right (literal "a"))) construction)) + (conflict (cdr (assq 'lrConflict (car (reverse (cadr result)))))) + (origins (map (lambda (row) (cdr (assq 'expressionOrigin row))) + (cdr (assq 'actions conflict))))) + (check (car result) => "unresolved reduce/reduce conflict") + (check (length origins) => 2) + (let (owners (map (lambda (row) (cdr (assq 'rule row))) origins)) + (check (memq 'left owners) ? values) + (check (memq 'right owners) ? values)) + (check (map (lambda (row) (cdr (assq 'expressionPath row))) origins) => '(() ())) + (check (cdr (assq 'shiftItems conflict)) => '()))) + '(lalr canonical-lr1 partitioned-lr1 follow-partition-lr1))) +)) diff --git a/t/lr-empty-action-test.ss b/t/lr-empty-action-test.ss new file mode 100644 index 00000000..3e61e749 --- /dev/null +++ b/t/lr-empty-action-test.ss @@ -0,0 +1,204 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; -*- Gerbil -*- +;;; Empty reductions and identity operands must agree across semantic backends. +(import :std/test + :gerbil-parser/src/grammar/algebra + (only-in :gerbil-parser/src/language/grammar deflanguage) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/lr-parser current-lr-event-program-enabled?) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success? parse-artifact-valid? parse-artifact-roundtrip + parse-artifact-events event-kind token-event? token-event-lexeme) + (only-in :gerbil-parser/src/runtime/incremental + make-incremental-session incremental-session-artifact + incremental-session-project-artifact parse-incremental-session + make-edit apply-edit)) +(export lr-empty-action-test) + +(begin + (deflanguage empty-action-probe + (syntax + (lexical + (root source-file) + (lex (word Word (identifier)) + (punctuation Punctuation (literals ",")) + (space Space (whitespace+))) + (extras space) + (keywords) + (recoveries) + (conflicts selective-glr) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile + (seq (optional (field left word)) + (optional (seq "," (field right (node Item word))))))))) + (bind-fixture-grammar-release empty-action-probe "empty-action-probe" "v1" "empty-action-probe.v1") ) + +(begin + (deflanguage ordered-action-probe + (syntax + (lexical + (root source-file) + (lex (word Word (identifier)) + (punctuation Punctuation (literals ",")) + (space Space (whitespace+))) + (extras space) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile + (seq (field first (node First word)) "," + (optional (field middle (node Middle word))) "," + (field last (node Last word))))))) + (bind-fixture-grammar-release ordered-action-probe "ordered-action-probe" "v1" "ordered-action-probe.v1") ) + +(begin + (deflanguage chained-action-probe + (syntax + (lexical + (root source-file) + (lex (word Word (identifier)) + (space Space (whitespace+))) + (extras space) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile + (field outer (node Renamed (field inner (node Original word)))))))) + (bind-fixture-grammar-release chained-action-probe "chained-action-probe" "v1" "chained-action-probe.v1") ) + +(begin + (deflanguage unary-stack-probe + (syntax + (lexical + (root source-file) + (lex (word Word (identifier)) + (punctuation Punctuation (literals ";")) + (space Space (whitespace+))) + (extras space) + (keywords) + (recoveries) + (conflicts reject) + (case-insensitive #f))) + (rules + (source-file + (node SourceFile + (seq (field head (node Head word)) ";" + (field tail (node Renamed (field inner (node Original word))))))))) + (bind-fixture-grammar-release unary-stack-probe "unary-stack-probe" "v1" "unary-stack-probe.v1") ) + +(def (parse-probe source events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (parse-source empty-action-probe-parser source))) + +(def lr-empty-action-test + (test-suite "LR identity operands and empty reductions" + (test-case "nullable and decorated operands retain identical artifacts" + (for-each + (lambda (source) + (let (artifact (parse-probe source #f)) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source) + (check (parse-probe source #t) => artifact))) + '("" " " "a" ",b" "a, b")) + (for-each + (lambda (source) + (let (artifact (parse-probe source #f)) + (check (parse-artifact-success? artifact) => #f) + (check (parse-probe source #t) => artifact))) + '("a," "a,b,c"))) + (test-case "multi-operand concatenation retains field and token order" + (for-each + (lambda (entry) + (let* ((source (car entry)) + (artifact (parse-source ordered-action-probe-parser source)) + (events (parse-artifact-events artifact))) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source) + (check (map (lambda (event) (vector-ref event 1)) + (filter (lambda (event) (eq? (event-kind event) 'start-field)) events)) + => (cadr entry)) + (check (map token-event-lexeme (filter token-event? events)) => (caddr entry)) + (check (parameterize ((current-lr-event-program-enabled? #t)) + (parse-source ordered-action-probe-parser source)) => artifact))) + '(("a,b,c" (first middle last) ("a" "," "b" "," "c")) + ("a,,c" (first last) ("a" "," "," "c")) + ("a, b, c" (first middle last) ("a" "," " " "b" "," " " "c"))))) + (test-case "chained fields and aliases retain their nesting order" + (for-each + (lambda (source) + (let* ((artifact (parameterize ((current-lr-event-program-enabled? #f)) + (parse-source chained-action-probe-parser source))) + (events (parse-artifact-events artifact))) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source) + (check (map (lambda (event) (vector-ref event 1)) + (filter (lambda (event) (eq? (event-kind event) 'start-field)) events)) + => '(outer inner)) + (check (map (lambda (event) (vector-ref event 2)) + (filter (lambda (event) (eq? (event-kind event) 'start-node)) events)) + => '(SourceFile Renamed Original)) + (check (parameterize ((current-lr-event-program-enabled? #t)) + (parse-source chained-action-probe-parser source)) => artifact))) + '("a" " a "))) + (test-case "unary actions consume only the top value above retained siblings" + (for-each + (lambda (source) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (let* ((artifact (parse-source unary-stack-probe-parser source)) + (events (parse-artifact-events artifact)) + (session (make-incremental-session unary-stack-probe-parser source #t)) + (edit (make-edit 0 1 "alpha"))) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source) + (check (map (lambda (event) (vector-ref event 1)) + (filter (lambda (event) (eq? (event-kind event) 'start-field)) events)) + => '(head tail inner)) + (check (map (lambda (event) (vector-ref event 2)) + (filter (lambda (event) (eq? (event-kind event) 'start-node)) events)) + => '(SourceFile Head Renamed Original)) + (check (parameterize ((current-lr-event-program-enabled? (not events?))) + (parse-source unary-stack-probe-parser source)) => artifact) + (let-values (((next receipt) (parse-incremental-session session edit))) + (check (incremental-session-artifact next) + => (parse-source unary-stack-probe-parser (apply-edit source edit))) + ;; Two successors must retain independent semantics while the + ;; source session and its previously published artifact stay valid. + (let (other-edit (make-edit 0 1 "gamma")) + (let-values (((other other-receipt) + (parse-incremental-session session other-edit))) + (check (incremental-session-artifact other) + => (parse-source unary-stack-probe-parser + (apply-edit source other-edit))) + (check (incremental-session-artifact next) + => (parse-source unary-stack-probe-parser + (apply-edit source edit))) + (check (incremental-session-artifact session) => artifact))))))) + '(#f #t))) + '("a;b" "a ; β"))) + (test-case "editing through empty source matches independent production replay" + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (let loop ((source "a, b") + (session (make-incremental-session empty-action-probe-parser "a, b" #t)) + (edits (list (make-edit 0 4 "") (make-edit 0 0 "a, b")))) + (check (incremental-session-artifact session) => (parse-probe source events?)) + (check (incremental-session-project-artifact session) => (incremental-session-artifact session)) + (when (pair? edits) + (let-values (((next receipt) (parse-incremental-session session (car edits)))) + (loop (apply-edit source (car edits)) next (cdr edits))))))) + '(#f #t))))) diff --git a/t/lr-fragment-transfer-test.ss b/t/lr-fragment-transfer-test.ss new file mode 100644 index 00000000..dfdc50f7 --- /dev/null +++ b/t/lr-fragment-transfer-test.ss @@ -0,0 +1,349 @@ +;;; -*- Gerbil -*- +;;; Exact terminal admission at the authoritative LR transfer boundary. +(import :std/test + (only-in :gerbil-parser/src/runtime/funcs recognition-sequence->list) + (only-in :gerbil-parser/src/runtime/recognition recognition-child-value) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/runtime/token make-token token? token-kind token-lexeme token-start token-end) + (only-in :gerbil-parser/src/runtime/artifact make-success-parse-artifact) + (only-in :gerbil-parser/src/runtime/lr-parser + current-lr-recognition-observer current-lr-transfer-yield-observer lr-prepare lr-parse/prepared + lr-initial-checkpoint lr-checkpoint-before-shift + lr-checkpoint-fragment-compatible? + lr-checkpoint-inject-fragment lr-checkpoint-resume lr-checkpoint-feed + lr-recognition-fragment? lr-recognition-fragment-children lr-recognition-fragment-token-count + lr-recognition-view? lr-recognition-view-base lr-recognition-view-delta + lr-recognition-project lr-recognition-relocate)) +(import (only-in :gerbil-parser/src/runtime/lr-parser/state lr-capture-children? lr-capture-plan-runtime lr-capture-plan-production-id lr-capture-plan-width lr-capture-plan-contexts lr-runtime-reduction-widths lr-recognition-fragment-lookahead lr-recognition-fragment-lookahead-cache lr-capture-context-lookahead-kind lr-recognition-fragment-yield-cache lr-recognition-fragment-children-cache lr-capture-context-runtime lr-capture-context-executor + lr-capture-context-exit-state lr-runtime-actions-set! lr-runtime-capture-plans lr-runtime-goto-columns) + (only-in :gerbil-parser/src/runtime/lr-parser/recognition lr-recognition-fragment-terminal-yield capture-context-from-plan prepare-capture-plans capture-reduction)) +(export lr-fragment-transfer-test) +;;; Frozen f21204f grammar-tree yield traversal: matched admission reference. +(def (reference-transfer-yield fragment delta tokens) + (let loop ((pending (list (cons fragment delta))) (rest tokens) (visited 0)) + (if (null? pending) (values (null? rest) visited) + (let* ((frame (car pending)) (piece (car frame)) (shift (cdr frame)) (visited (+ visited 1))) + (cond + ((lr-recognition-view? piece) + (loop (cons (cons (lr-recognition-view-base piece) (+ shift (lr-recognition-view-delta piece))) + (cdr pending)) rest visited)) + ((lr-recognition-fragment? piece) + (loop (append (map (lambda (child) (cons child shift)) (lr-recognition-fragment-children piece)) + (cdr pending)) rest visited)) + ((and (pair? rest) (token? (car rest)) (token? piece) + (eq? (token-kind (car rest)) (token-kind piece)) + (equal? (token-lexeme (car rest)) (token-lexeme piece)) + (= (token-start (car rest)) (+ shift (token-start piece))) + (= (token-end (car rest)) (+ shift (token-end piece)))) + (loop (cdr pending) (cdr rest) visited)) + (else (values #f visited))))))) +(def (capture-transfer-root runtime tokens) + (let (captured #f) + (parameterize ((current-lr-recognition-observer (lambda (root) (set! captured root)))) + (let-values (((root rest) (lr-parse/prepared runtime tokens))) (check rest => '()))) + captured)) +(def (shift-transfer-tokens tokens delta) + (map (lambda (token) + (make-token (token-kind token) (token-lexeme token) + (+ delta (token-start token)) (+ delta (token-end token)))) tokens)) + +(def (publish-transfer root prefix tokens) + (let* ((source (string-append (make-string prefix #\space) + (string-join (map token-lexeme tokens) " "))) + (stream + (append (if (zero? prefix) '() (list (make-token 'space (make-string prefix #\space) 0 prefix))) + (let loop ((rest tokens)) + (if (null? (cdr rest)) rest + (cons (car rest) + (cons (make-token 'space " " (token-end (car rest)) (token-start (cadr rest))) + (loop (cdr rest))))))))) + (make-success-parse-artifact "yield-test" source stream root (lambda (token) (eq? (token-kind token) 'space))))) + +(def lr-fragment-transfer-test + (test-suite "LR fragment transfer admission" + (test-case "prepared capture plans own exact immutable runtime production and width input" + (let* ((runtime (lr-prepare (compile-lr-spec '((source-file (sequence (token identifier) (token identifier)))) 'source-file))) + (widths (lr-runtime-reduction-widths runtime)) (plans (prepare-capture-plans runtime))) + (check (vector-length plans) => (vector-length widths)) + (for-each (lambda (production) + (let (plan (vector-ref plans production)) + (check (eq? (lr-capture-plan-runtime plan) runtime) => #t) + (check (lr-capture-plan-production-id plan) => production) + (check (lr-capture-plan-width plan) => (vector-ref widths production)))) + (iota (vector-length plans))) + (check (eq? plans (prepare-capture-plans runtime)) => #t))) + (test-case "custom executors cannot pollute canonical prepared contexts" + (let* ((grammar '((source-file (alias SourceFile (token identifier))))) + (runtime (lr-prepare (compile-lr-spec grammar 'source-file))) + (other-runtime (lr-prepare (compile-lr-spec grammar 'source-file))) + (columns (lr-runtime-goto-columns runtime)) + (production (find (lambda (id) (ormap identity (vector->list (vector-ref columns id)))) + (iota (vector-length columns)))) + (plan (vector-ref (prepare-capture-plans runtime) production)) + (column (vector-ref columns production)) + (entry (let loop ((i 0)) (if (vector-ref column i) i (loop (+ i 1))))) + (target (vector-ref column entry)) + (executor (lambda _ #f)) + (first (capture-context-from-plan plan #f entry target 'identifier)) + (changed (capture-context-from-plan plan executor entry target 'identifier)) + (retargeted (capture-context-from-plan plan executor entry (+ target 1) 'identifier))) + (check-exception (capture-context-from-plan plan executor -1 target 'identifier) true) + (check-exception (capture-context-from-plan plan executor (vector-length column) target 'identifier) true) + (check-exception (capture-context-from-plan plan executor 1/2 target 'identifier) true) + (check (eq? first changed) => #f) + (check (lr-capture-context-executor first) => #f) + (check (eq? (lr-capture-context-executor changed) executor) => #t) + (check (lr-capture-context-exit-state changed) => target) + (check (lr-capture-context-exit-state retargeted) => (+ target 1)) + (check (eq? retargeted (capture-context-from-plan plan executor entry (+ target 1) 'identifier)) => #t) + (check (eq? first (capture-context-from-plan plan #f entry target 'identifier)) => #t) + (check (eq? first (vector-ref (lr-capture-plan-contexts plan) entry)) => #t) + (check (eq? retargeted (capture-context-from-plan plan executor entry (+ target 1) 'identifier)) => #t) + (check (<= (* (vector-length (prepare-capture-plans runtime)) + (vector-length (lr-capture-plan-contexts plan))) 65536) => #t) + (check (eq? first (capture-context-from-plan (vector-ref (prepare-capture-plans other-runtime) production) #f entry target 'identifier)) => #f) + (check (eq? (lr-capture-context-runtime first) runtime) => #t))) + (test-case "oversized capture context directories retain exact uncached descriptors" + (let* ((runtime (lr-prepare (compile-lr-spec '((source-file (token identifier))) 'source-file)))) + (lr-runtime-actions-set! runtime (make-vector 65537 #f)) + (let* ((plan (vector-ref (prepare-capture-plans runtime) 0)) + (first (capture-context-from-plan plan #f 0 1 'identifier)) + (second (capture-context-from-plan plan #f 0 1 'identifier))) + (check (lr-capture-plan-contexts (vector-ref (lr-runtime-capture-plans runtime) 0)) => #f) + (check (eq? first second) => #f) + (check (lr-capture-context-exit-state first) => 1) + (check (eq? (lr-capture-context-runtime second) runtime) => #t)))) + (test-case "malformed live boundary values cannot masquerade as EOF or a ready pair" + (let* ((runtime (lr-prepare (compile-lr-spec '((source-file (token identifier))) 'source-file))) + (plan (vector-ref (prepare-capture-plans runtime) 0)) + (stack (list (make-token 'identifier "a" 0 1)))) + (for-each + (lambda (boundary) + (check-exception + (capture-reduction plan #f 0 stack (vector (car boundary) (cdr boundary) 2 3) 2 'value 1) true)) + (list (cons 'identifier #f) (cons 'identifier '(not a lexeme)) (cons 17 "text"))) + (check (andmap not (vector->list (lr-capture-plan-contexts plan))) => #t))) + (test-case "boundary snapshots survive token mutation and materialize stable pairs" + (let* ((runtime (lr-prepare (compile-lr-spec '((source-file (token identifier))) 'source-file))) + (plan (vector-ref (prepare-capture-plans runtime) 0)) + (executor (lambda _ #f)) + (kind 'identifier) (lexeme "next") + (input (make-token kind lexeme 2 6)) + (first (capture-context-from-plan plan executor 0 1 kind)) + (changed (capture-context-from-plan plan executor 0 1 'integer)) + (fragment (car (capture-reduction plan executor 0 + (list (make-token 'identifier "a" 0 1)) input 2 'value 1)))) + (check (eq? first changed) => #f) + (check (lr-capture-context-lookahead-kind first) => kind) + (check (lr-capture-context-lookahead-kind changed) => 'integer) + (check (eq? (lr-recognition-fragment-lookahead-cache fragment) lexeme) => #t) + (vector-set! input 0 'integer) (vector-set! input 1 "changed") + (let (signature (lr-recognition-fragment-lookahead fragment)) + (check signature => (cons kind lexeme)) + (check (eq? signature (lr-recognition-fragment-lookahead fragment)) => #t)))) + (test-case "unary child lists materialize once without losing source order" + (let* ((runtime (lr-prepare (compile-lr-spec '((source-file (token identifier))) 'source-file))) + (token (make-token 'identifier "a" 0 1)) + (root (capture-transfer-root runtime (list token)))) + (check (eq? (lr-recognition-fragment-children-cache root) token) => #t) + (let (first (lr-recognition-fragment-children root)) + (check first => (list token)) + (check (eq? first (lr-recognition-fragment-children root)) => #t) + (check (eq? (car first) token) => #t)))) + (test-case "packed children retain ownership across requests and materialize once" + (let* ((runtime (lr-prepare (compile-lr-spec + '((source-file (alias SourceFile (sequence (token identifier) (token identifier) + (token identifier) (token identifier))))) 'source-file))) + (tokens (map (lambda (i) (make-token 'identifier "a" i (+ i 1))) (iota 4))) + (captured (capture-transfer-root runtime tokens)) + (root (let loop ((piece captured)) + (let (stored (lr-recognition-fragment-children-cache piece)) + (cond ((lr-capture-children? stored) piece) + ((lr-recognition-fragment? stored) (loop stored)) + (else (error "wide production was not retained as packed children"))))))) + (check (lr-capture-children? (lr-recognition-fragment-children-cache root)) => #t) + (check (void? (lr-recognition-fragment-yield-cache root)) => #t) + (let* ((widths (lr-runtime-reduction-widths runtime)) + (id (find (lambda (id) (= (vector-ref widths id) 4)) (iota (vector-length widths)))) + (suffix (list (make-token 'identifier "outside" 20 27))) + (stack (append (reverse tokens) suffix)) + (captured (capture-reduction (vector-ref (prepare-capture-plans runtime) id) + (lambda _ #f) 0 stack #f 4 'value 1))) + (set-car! stack (make-token 'identifier "changed-stack" 0 13)) + (set-car! suffix (make-token 'identifier "changed-suffix" 20 34)) + (check (eq? (cdr captured) suffix) => #t) + (check (lr-recognition-fragment-children (car captured)) => tokens)) + (capture-transfer-root runtime + (map (lambda (i) (make-token 'identifier "next" (* 4 i) (+ 4 (* 4 i)))) (iota 4))) + (let (children (lr-recognition-fragment-children root)) + (check children => tokens) + (check (eq? children (lr-recognition-fragment-children root)) => #t) + (for-each (lambda (left right) (check (eq? left right) => #t)) children tokens)))) + (test-case "fresh capture defers its derived terminal index and repeated transfer queries share it" + (let* ((grammar '((source-file (alias SourceFile (sequence (token identifier) (token identifier)))))) + (runtime (lr-prepare (compile-lr-spec grammar 'source-file))) + (tokens (list (make-token 'identifier "α" 0 2) (make-token 'identifier "b" 3 4))) + (captured (capture-transfer-root runtime tokens))) + (check (void? (lr-recognition-fragment-yield-cache captured)) => #t) + (let-values (((matches visits) (reference-transfer-yield captured 0 tokens))) + (check matches => #t)) + (let (first (lr-recognition-fragment-terminal-yield captured)) + (check (void? first) => #f) + (check (eq? first (lr-recognition-fragment-terminal-yield captured)) => #t) + (check-exception + (lr-checkpoint-inject-fragment + (lr-checkpoint-before-shift (lr-initial-checkpoint runtime '()) (car tokens)) + captured 0 (list (car tokens) (make-token 'identifier "x" 3 4))) true)))) + (test-case "shared yields eliminate unary grammar traversal without relaxing terminal checks" + (let* ((names (map (lambda (n) (string->symbol (string-append "layer" (number->string n)))) (iota 64))) + (grammar + (cons (list 'source-file (list 'alias 'SourceFile (list 'reference (car names)))) + (map (lambda (index) + (list (list-ref names index) + (if (= index 63) '(sequence (token identifier) (token identifier)) + (list 'reference (list-ref names (+ index 1)))))) (iota 64)))) + (runtime (lr-prepare (compile-lr-spec grammar 'source-file))) + (old (list (make-token 'identifier "猫" 0 3) (make-token 'identifier "b" 4 5))) + (captured (capture-transfer-root runtime old)) + (tokens (shift-transfer-tokens old 7)) + (checkpoint (lr-checkpoint-before-shift (lr-initial-checkpoint runtime '()) (car tokens))) + (visits #f)) + (let-values (((matches baseline-visits) (reference-transfer-yield captured 7 tokens))) + (check matches => #t) + (check baseline-visits => 67) + (parameterize ((current-lr-transfer-yield-observer (lambda (count) (set! visits count)))) + (let-values (((root rest) + (lr-checkpoint-resume (lr-checkpoint-inject-fragment checkpoint captured 7 tokens)))) + (let-values (((fresh rest) (lr-parse/prepared runtime tokens))) + (check (publish-transfer root (token-start (car tokens)) tokens) + => (publish-transfer fresh (token-start (car tokens)) tokens))))) + (check visits => 3)) + (let (wrong (list (car tokens) (make-token 'identifier "c" 11 12))) + (let-values (((matches count) (reference-transfer-yield captured 7 wrong))) (check matches => #f)) + (check-exception (lr-checkpoint-inject-fragment checkpoint captured 7 wrong) true)))) + (test-case "terminal yields compose relocated subtrees with fresh sibling tokens" + (let* ((runtime (lr-prepare (compile-lr-spec + '((source-file (alias SourceFile (sequence (reference pair) (token identifier)))) + (pair (sequence (token identifier) (token identifier)))) 'source-file))) + (old (list (make-token 'identifier "猫" 0 3) (make-token 'identifier "b" 4 5) + (make-token 'identifier "c" 6 7))) + (captured (capture-transfer-root runtime old)) + (pair-fragment + (let walk ((pending (list captured))) + (and (pair? pending) + (let (piece (car pending)) + (cond ((not (lr-recognition-fragment? piece)) (walk (cdr pending))) + ((= (lr-recognition-fragment-token-count piece) 2) piece) + (else (walk (append (lr-recognition-fragment-children piece) (cdr pending))))))))) + (moved (shift-transfer-tokens old 7)) + (checkpoint (parameterize ((current-lr-recognition-observer (lambda (_) #f))) + (lr-checkpoint-before-shift (lr-initial-checkpoint runtime '()) (car moved)))) + (with-pair (lr-checkpoint-inject-fragment checkpoint pair-fragment 7 (take moved 2))) + (rebuilt #f)) + (parameterize ((current-lr-recognition-observer (lambda (root) (set! rebuilt root)))) + (let-values (((status next) (lr-checkpoint-feed with-pair (caddr moved)))) + (check status => 'checkpoint) + (let-values (((root rest) (lr-checkpoint-resume next))) (check rest => '())))) + (let* ((later (shift-transfer-tokens moved 5)) + (next (lr-checkpoint-before-shift (lr-initial-checkpoint runtime '()) (car later)))) + (let-values (((matches count) (reference-transfer-yield rebuilt 5 later))) (check matches => #t)) + (let-values (((root rest) (lr-checkpoint-resume (lr-checkpoint-inject-fragment next rebuilt 5 later)))) + (let-values (((fresh rest) (lr-parse/prepared runtime later))) + (check (publish-transfer root 12 later) => (publish-transfer fresh 12 later))) + (check (publish-transfer root 12 later) + => (publish-transfer + (recognition-child-value + (car (recognition-sequence->list + (lr-recognition-project (lr-recognition-relocate rebuilt 5) later)))) 12 later))) + (let (wrong (list (car later) (make-token 'identifier "b" 17 18) (caddr later))) + (check-exception (lr-checkpoint-inject-fragment next rebuilt 5 wrong) true))))) + (test-case "epsilon reductions disappear from the nonempty transfer yield" + (let* ((runtime (lr-prepare (compile-lr-spec + '((source-file (alias SourceFile + (sequence (optional (token identifier)) (literal "!"))))) + 'source-file))) + (tokens (list (make-token 'symbol "!" 0 1))) + (captured (capture-transfer-root runtime tokens)) + (checkpoint (lr-initial-checkpoint runtime tokens)) + (visits #f)) + (parameterize ((current-lr-transfer-yield-observer (lambda (count) (set! visits count)))) + (let-values (((root rest) (lr-checkpoint-resume + (lr-checkpoint-inject-fragment checkpoint captured 0 tokens)))) + (let-values (((fresh rest) (lr-parse/prepared runtime tokens))) + (check (publish-transfer root 0 tokens) => (publish-transfer fresh 0 tokens))))) + (check visits => 1))) + (test-case "long left-recursive yields remain shared and reject a changed last terminal" + (let* ((runtime (lr-prepare (compile-lr-spec + '((source-file (alias SourceFile (repeat1 (token identifier))))) 'source-file))) + (old (map (lambda (n) (make-token 'identifier "α" (* n 3) (+ (* n 3) 2))) (iota 512))) + (captured (capture-transfer-root runtime old)) + (tokens (shift-transfer-tokens old 7)) + (checkpoint (lr-checkpoint-before-shift (lr-initial-checkpoint runtime '()) (car tokens))) + (visits #f)) + (parameterize ((current-lr-transfer-yield-observer (lambda (count) (set! visits count)))) + (let-values (((root rest) (lr-checkpoint-resume + (lr-checkpoint-inject-fragment checkpoint captured 7 tokens)))) + (let-values (((fresh rest) (lr-parse/prepared runtime tokens))) + (check (publish-transfer root 7 tokens) => (publish-transfer fresh 7 tokens))))) + (check visits => 1023) + (let* ((final (last tokens)) + (wrong (append (take tokens 511) + (list (make-token 'identifier "β" (token-start final) (token-end final)))))) + (let-values (((matches count) (reference-transfer-yield captured 7 wrong))) (check matches => #f)) + (check-exception (lr-checkpoint-inject-fragment checkpoint captured 7 wrong) true)) + ;; Rejected transfer retains both checkpoint and shared yield. + (let-values (((root rest) (lr-checkpoint-resume + (lr-checkpoint-inject-fragment checkpoint captured 7 tokens)))) + (check rest => '())))) + (test-case "LR transfer validates terminal yield and the current input boundary" + (let* ((runtime (lr-prepare + (compile-lr-spec + '((source-file (alias SourceFile + (sequence (token identifier) (token identifier))))) + 'source-file))) + (old (list (make-token 'identifier "猫" 0 3) + (make-token 'identifier "b" 4 5))) + (captured #f)) + (parameterize ((current-lr-recognition-observer + (lambda (root) (set! captured root)))) + (let-values (((root rest) (lr-parse/prepared runtime old))) + (check rest => '()))) + (let* ((tokens (list (make-token 'identifier "猫" 7 10) + (make-token 'identifier "b" 11 12))) + (checkpoint (lr-checkpoint-before-shift + (lr-initial-checkpoint runtime '()) (car tokens)))) + (check (lr-checkpoint-fragment-compatible? checkpoint captured) => #t) + (let-values (((root rest) + (lr-checkpoint-resume + (lr-checkpoint-inject-fragment checkpoint captured 7 tokens)))) + (check rest => '()) + (let-values (((fresh fresh-rest) (lr-parse/prepared runtime tokens))) + (def (publish value) + (make-success-parse-artifact + "transfer-test" " 猫 b" + (list (make-token 'space " " 0 7) (car tokens) + (make-token 'space " " 10 11) (cadr tokens)) + value (lambda (token) (eq? (token-kind token) 'space)))) + (check (publish root) => (publish fresh)))) + (for-each + (lambda (wrong) + (check-exception + (lr-checkpoint-inject-fragment checkpoint captured 7 wrong) true)) + (list (list (car tokens) (make-token 'identifier "c" 11 12)) + (list (car tokens) (make-token 'identifier "b" 12 13)) + (list (car tokens) (make-token 'number "b" 11 12)) + (list (car tokens)) + (list (car tokens) (cadr tokens) (cadr tokens)))) + (check-exception + (lr-checkpoint-inject-fragment checkpoint captured 0 old) true) + (check-exception + (lr-checkpoint-inject-fragment + (lr-checkpoint-before-shift (lr-initial-checkpoint runtime '()) + (make-token 'identifier "犬" 7 10)) + captured 7 tokens) true) + ;; Rejection does not consume or mutate the certified continuation. + (let-values (((root rest) + (lr-checkpoint-resume + (lr-checkpoint-inject-fragment checkpoint captured 7 tokens)))) + (check rest => '()))))) +)) diff --git a/t/lr-goto-plan-test.ss b/t/lr-goto-plan-test.ss new file mode 100644 index 00000000..dd3eff42 --- /dev/null +++ b/t/lr-goto-plan-test.ss @@ -0,0 +1,19 @@ +(import :std/test + (only-in :gerbil-parser/src/runtime/lr-parser/prepare prepare-lr-goto-columns)) +(export lr-goto-plan-test) +(def lr-goto-plan-test + (test-suite "bounded prepared LR goto columns" + (test-case "first duplicates, missing cells and shared LHS columns are exact" + (let* ((table '#((production root) (production root) (production other))) + (rows '#(((root . 1) (root . 9) (other . 2)) ((other . 3)))) + (columns (prepare-lr-goto-columns table rows))) + (check columns => '#(#(1 #f) #(1 #f) #(2 3))) + (check (eq? (vector-ref columns 0) (vector-ref columns 1)) => #t) + (check (prepare-lr-goto-columns table rows 3) => #f) + (check (prepare-lr-goto-columns table rows 4) => columns))) + (test-case "cell ceiling admits its boundary and refuses wider grammars" + (let* ((table (list->vector (map (lambda (i) (list 'production (string->symbol (number->string i)))) (iota 256)))) + (columns (prepare-lr-goto-columns table (make-vector 256 '())))) + (check (vector-length columns) => 256) + (check (andmap (lambda (column) (= (vector-length column) 256)) (vector->list columns)) => #t) + (check (prepare-lr-goto-columns table (make-vector 257 '())) => #f))))) diff --git a/t/lr-index-test.ss b/t/lr-index-test.ss new file mode 100644 index 00000000..8561d292 --- /dev/null +++ b/t/lr-index-test.ss @@ -0,0 +1,223 @@ +;;; -*- Gerbil -*- +(import :std/test + (only-in :gerbil-parser/src/compiler/lr base-symbol) + (only-in :gerbil-parser/src/compiler/lr-table validate-lr-tables layout-productions?) + (only-in :gerbil-parser/src/runtime/funcs + association-row-vector->index association-row-index-ref) + (only-in :gerbil-parser/src/runtime/lr-action-index + index-action-row index-action-rows lookup-casefolded-literal-action-entry + lookup-action-entry lr-action-row-tokens + lookup-literal-action-entry lr-action-row-has-literals?) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/runtime/lr-parser lr-prepare lr-parse/prepared) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/recognition recognition-node-kind)) +(export lr-index-test) + +(def (check-table-rejected actions gotos) + (check-exception + (lr-prepare + (list (cons 'productions '((0 root ((terminal token word)) concat #f))) + (cons 'actions actions) (cons 'gotos gotos) + (cons 'case-insensitive? #f))) + (lambda (condition) (string-prefix? "invalid LR table" (error-message condition))))) + +(def lr-index-test + (test-suite "prepared LR indexes preserve association semantics" + (test-case "canonical table admission preserves original rows and entries" + (for-each + (lambda (entry) + (let ((actions (vector (if (equal? (car entry) '(terminal eof)) + (list entry) + (list entry '((terminal eof) accept))))) + (gotos (vector '((root . 0))))) + (let-values (((same-actions same-gotos) + (validate-lr-tables + '((0 root ((terminal token word) (terminal literal "+")) concat #f) + (1 root () layout-end #f)) actions gotos))) + (check (eq? actions same-actions) => #t) + (check (eq? gotos same-gotos) => #t) + (check (eq? entry (car (vector-ref same-actions 0))) => #t)))) + '(((terminal token word) shift 0 #f) + ((terminal token word) shift 0 (left 10)) + ((terminal token word) reduce 0) + ((terminal eof) accept) + ((terminal literal "+") reject-nonassoc (terminal literal "+") 10) + ((terminal token word) fork (shift 0 #f) (reduce 0)) + ((terminal token word) layout-guard (shift 0 #f) (reduce 0)) + ((terminal token word) layout-guard (shift 0 #f) (fork (reduce 0) (reduce 0)))))) + (test-case "layout tables require production-derived request capability" + (for-each + (lambda (entry) (check-table-rejected (vector (list entry)) (vector '()))) + '(((terminal token word) layout-guard (shift 0 #f) (reduce 0)) + ((terminal layout-start "{") shift 0 #f) + ((terminal layout-next ";") shift 0 #f))) + (for-each + (lambda (rhs) + (let ((productions (list (list 0 'root rhs 'concat #f))) + (actions (vector (list (cons (base-symbol (car rhs)) '(shift 0 #f)) + '((terminal eof) accept)))) + (gotos '#(()))) + (check (layout-productions? productions) => #t) + (let-values (((same-actions _) (validate-lr-tables productions actions gotos))) + (check (eq? same-actions actions) => #t)))) + '(((terminal layout-start "{")) + ((marked (terminal layout-next ";") ((field separator))))))) + (test-case "malformed dimensions and rows fail before indexing" + (for-each (lambda (pair) (check-table-rejected (car pair) (cadr pair))) + '((#f #()) (#() #()) (#(()) #()) (#(invalid) #(())) + (#(()) #(invalid)) (#((invalid)) #(())) + (#(()) #((root))) (#(()) #((("root" . 0)))) + (#(()) #(((root . -1)))) (#(()) #(((root . 1)))) + (#(()) #(((root . 0) (root . 0))))))) + (test-case "action targets and closed wrappers are admitted as one algebra" + (for-each + (lambda (action) + (check-table-rejected (vector (list (cons '(terminal token word) action))) (vector '()))) + '((shift -1 #f) (shift 1 #f) (shift 0) (shift 0 (left "10")) + (reduce -1) (reduce 1) (reduce 0 extra) (accept) (accept extra) + (callback 0) (fork) (fork (reduce 0)) + (fork (shift 0 #f) (fork (reduce 0) (reduce 0))) + (fork (shift 0 #f) (reduce 1)) + (layout-guard (reduce 0) (reduce 0)) + (layout-guard (shift 0 #f) (shift 0 #f)) + (reject-nonassoc (terminal token other) 1))) + (check-table-rejected (vector '(((terminal token "word") reduce 0))) (vector '())) + (check-table-rejected (vector '(((terminal token word) reduce 0) + ((terminal token word) shift 0 #f))) (vector '()))) + (test-case "table symbols belong to canonical production vocabulary" + (check-table-rejected '#((((terminal token undeclared) reduce 0) + ((terminal eof) accept))) '#(())) + (check-table-rejected '#((((terminal literal "undeclared") reduce 0) + ((terminal eof) accept))) '#(())) + (check-table-rejected '#((((terminal eof) accept))) '#(((undeclared . 0)))) + (def productions + '((0 root ((marked (terminal token word) ((field value))) + (terminal literal "+")) concat #f) + (1 unused ((terminal token extra)) concat #f))) + (def actions '#((((terminal token word) shift 0 #f) + ((terminal literal "+") reduce 0) + ((terminal token extra) reduce 1) + ((terminal eof) accept)))) + (let-values (((same-actions same-gotos) + (validate-lr-tables productions actions '#(((unused . 0)))))) + (check (eq? actions same-actions) => #t) + (check (vector-ref same-gotos 0) => '((unused . 0))))) + (test-case "absent or disconnected acceptance fails before runtime preparation" + (check-table-rejected '#(() ()) '#(() ())) + ;; An accepting cycle isolated from state zero cannot complete a request. + (check-table-rejected '#(() (((terminal token word) shift 1 #f) + ((terminal eof) accept))) '#(() ()))) + (test-case "state traversal follows gotos, forks and layout guards" + (def productions + '((0 root ((terminal token word)) concat #f) + (1 root () layout-end #f))) + (for-each + (lambda (tables) + (let-values (((same-actions same-gotos) + (validate-lr-tables productions (car tables) (cadr tables)))) + (check (eq? same-actions (car tables)) => #t) + (check (eq? same-gotos (cadr tables)) => #t))) + '((#(() (((terminal eof) accept))) #(((root . 1)) ((root . 0)))) + (#((((terminal token word) fork (shift 1 #f) (shift 2 #f))) () + (((terminal eof) accept))) #(() () ())) + (#((((terminal token word) layout-guard (shift 1 #f) (reduce 0))) + (((terminal eof) accept))) #(() ())) + (#((((terminal token word) shift 1 #f)) + (((terminal token word) shift 0 #f) ((terminal eof) accept))) #(() ())) + (#((((terminal eof) fork (reduce 0) (accept))) ()) #(() ())))) + ;; The second fork edge, not just its first branch, must be traversed. + (check-table-rejected + '#((((terminal token word) fork (shift 0 #f) (reduce 0))) + (((terminal eof) accept))) '#(() ()))) + (test-case "goto list/table boundaries preserve identity, misses and first duplicates" + (for-each + (lambda (width) + (let* ((row (map (lambda (n) (cons (number->string n) n)) (iota width))) + (rows (vector '() row)) + (indexes (association-row-vector->index rows))) + (check (association-row-index-ref indexes 0 "missing") => #f) + (check (association-row-index-ref indexes 1 "missing") => #f) + (for-each + (lambda (entry) + (check (eq? (association-row-index-ref indexes 1 (string-copy (car entry))) entry) => #t)) + row) + (when (pair? row) + (let* ((duplicate (cons (string-copy (caar row)) 'later)) + (indexed (association-row-vector->index (vector (append row (list duplicate)))))) + (check (eq? (association-row-index-ref indexed 0 (caar row)) (car row)) => #t))))) + '(0 1 8 9 32))) + (test-case "action list/table boundaries preserve original entries and first duplicates" + (for-each + (lambda (width) + (let* ((entries (map (lambda (n) (cons (list 'terminal 'token (number->string n)) (list 'shift n))) (iota width))) + (rows (if (pair? entries) + (append entries (list (cons (caar entries) '(shift 999)))) entries)) + (index (lr-action-row-tokens (index-action-row rows)))) + (check (lookup-action-entry index "missing") => #f) + (for-each + (lambda (entry) + (check (eq? (lookup-action-entry index (string-copy (caddar entry))) entry) => #t)) + entries))) + '(0 1 7 8 9 32))) + (test-case "literal capability retains indexed association semantics" + (for-each + (lambda (width) + (let* ((entries (map (lambda (n) + (cons (list 'terminal 'literal (string-append "KEY" (number->string n))) + (list 'shift n))) (iota width))) + (row (index-action-row entries)) (reads 0)) + (check (lr-action-row-has-literals? (begin (set! reads (+ reads 1)) row)) => (> width 0)) + (check reads => 1) + (check (lookup-literal-action-entry row "missing") => #f) + (for-each + (lambda (entry) + (check (eq? (lookup-literal-action-entry row (string-copy (caddar entry))) entry) => #t)) + entries))) + '(0 1 8 9 32)) + (for-each + (lambda (literal) + (let* ((entry (cons (list 'terminal 'literal literal) '(shift 1))) + (row (index-action-row (list entry)))) + (check (lr-action-row-has-literals? row) => #t) + (check (eq? (lookup-literal-action-entry row literal) entry) => #t))) + '("+" "A" "λ" "CAFÉ")) + (for-each + (lambda (terminal) + (check (lr-action-row-has-literals? (index-action-row (list (cons terminal '(shift 1))))) => #f)) + '((terminal token word) (terminal eof) (terminal layout-start line) (terminal layout-next line)))) + (test-case "shared casefold trie agrees with whole-string conversion and row admission" + (def (entry text n) (cons (list 'terminal 'literal text) (list 'shift n))) + (for-each + (lambda (width) + (let* ((entries (append (map (lambda (n) (entry (string-append "KEY" (number->string n)) n)) (iota width)) + (map (lambda (text) (entry text 100)) '("" "A" "AB" "ABC" "CAFÉ" "ZÉ" "+É" "Λ" "SS" "S" "K" "lower")))) + (first (entry "KEY0" 201)) + (rows (index-action-rows (vector (cons first entries) (list (entry "OTHER" 202)) '()) #t))) + (for-each + (lambda (row) + (for-each + (lambda (text) + (check (eq? (lookup-casefolded-literal-action-entry row text) + (lookup-literal-action-entry row (string-upcase text))) => #t)) + '("" "a" "ab" "abc" "abcd" "key0" "key31" "other" "café" "zé" "xé" "+é" "λ" "ß" "ſ" "K" "lower" "missing-long-identifier"))) + (vector->list rows)) + (check (eq? (lookup-casefolded-literal-action-entry (vector-ref rows 0) "key0") first) => #t) + (check (lookup-casefolded-literal-action-entry (vector-ref rows 0) "other") => #f) + (check (lookup-casefolded-literal-action-entry (vector-ref rows 1) "key0") => #f) + (check (lookup-casefolded-literal-action-entry (vector-ref rows 0) 'word) => #f))) + '(0 1 8 9 32))) + (test-case "literal priority and case-sensitive token fallback remain distinct" + (def rules '((source-file (choice (alias Keyword (literal "KEY")) + (alias Word (token word)))))) + (def (check-root grammar insensitive? text expected) + (let* ((runtime (lr-prepare (compile-lr-spec grammar 'source-file 'error insensitive?))) + (token (make-token 'word text 0 (u8vector-length (string->utf8 text))))) + (let-values (((root rest) (lr-parse/prepared runtime (list token)))) + (check rest => '()) + (check (recognition-node-kind root) => expected)))) + (check-root rules #t "keY" 'Keyword) + (check-root rules #f "keY" 'Word) + (check-root rules #f "KEY" 'Keyword) + (check-root rules #t "café" 'Word) + (check-root '((source-file (alias Word (token word)))) #t "café" 'Word)))) diff --git a/t/lr-input-protocol-test.ss b/t/lr-input-protocol-test.ss new file mode 100644 index 00000000..9d43d290 --- /dev/null +++ b/t/lr-input-protocol-test.ss @@ -0,0 +1,105 @@ +;;; Scalar admission is compared with the same executor's borrowed-list policy. +(import :std/test + (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar records-parser) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime parser-machine-trivia parser-machine-with-lr-action-selector + parser-machine-grammar-digest install-parser-machine-direct-step!) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr production-lhs) + (only-in :gerbil-parser/src/runtime/reduction-plan reduce-lr-stack-plan) + (only-in :gerbil-parser/src/runtime/lr-parser/prepare make-lr-action-selector goto-target) + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/parser parse-source parse-source/checkpoints) + (only-in :gerbil-parser/src/runtime/token make-token token-start) + (only-in :gerbil-parser/src/runtime/lr-parser lr-initial-checkpoint lr-prepare lr-lexical-mode-id current-lr-recognition-observer) + (only-in :gerbil-parser/src/runtime/lr-stack-storage make-lr-token-observer) + (only-in :gerbil-parser/src/runtime/lr-parser/state lr-checkpoint-states lr-runtime-table lr-runtime-reduction-widths lr-runtime-reduction-plans) + (only-in :gerbil-parser/src/runtime/lr-parser/checkpoint + lr-run-checkpoint lr-run-persistent-checkpoint)) +(export lr-input-protocol-test) +(def (observe-input-driver driver runtime tokens budget fork?) + (let ((pending tokens) (trace '())) + (def (next-input mode) + (set! trace (cons (list 'scan (lr-lexical-mode-id mode) (and (pair? pending) (car pending))) trace)) + (if (null? pending) #f + (let (token (car pending)) (set! pending (cdr pending)) token))) + (let (result + (call-with-values + (lambda () + (driver (lr-initial-checkpoint runtime '()) budget #f #t #f fork? #f next-input + (lambda (token _states _values actions shifts) + (set! trace (cons (list 'shift token actions shifts) trace))))) list)) + (list result pending (reverse trace))))) +(def (check-input-drivers runtime tokens budget fork?) + (check (observe-input-driver lr-run-checkpoint runtime tokens budget fork?) + => (observe-input-driver lr-run-persistent-checkpoint runtime tokens budget fork?))) +(def (observe-captured-buffers driver tokens) + (let ((pending tokens) (frames '()) (root #f) (shifted '())) + (parameterize ((current-lr-recognition-observer (lambda (value) (set! root value)))) + (let (result + (call-with-values + (lambda () + (driver (lr-initial-checkpoint (parser-machine-runtime records-parser) '()) #f #f #t #f #t #f + (lambda (_mode) + (if (null? pending) #f + (let (token (car pending)) (set! pending (cdr pending)) token))) + (make-lr-token-observer (lambda (token) (set! shifted (cons token shifted)))) + 'installed #f #f 2 (lambda (checkpoint) (set! frames (cons checkpoint frames))))) list)) + ;; Inspect every earlier frozen frontier after all later buffer writes. + (list result root (reverse shifted) (reverse frames)))))) +(def lr-input-protocol-test + (test-suite "LR scalar input protocol and external frontiers" + (test-case "action budgets preserve pending lookahead, scanner mode and callbacks" + (let ((runtime (parser-machine-runtime records-parser)) + (tokens (filter (lambda (token) (not ((parser-machine-trivia records-parser) token))) + (lex-source records-parser "a=1\nb=2\n")))) + (for-each (lambda (budget) (check-input-drivers runtime tokens budget #t)) '(0 1 2 3 4 8 32 #f)))) + (test-case "captured owned buffers preserve every earlier immutable frontier" + (let (tokens (filter (lambda (token) (not ((parser-machine-trivia records-parser) token))) + (lex-source records-parser (apply string-append (make-list 64 "a=1\n"))))) + (let* ((owned (observe-captured-buffers lr-run-checkpoint tokens)) + (persistent (observe-captured-buffers lr-run-persistent-checkpoint tokens)) + (frames (cadddr owned)) + (last-cell (lambda (values) + (let loop ((values values)) + (if (null? (cdr values)) values (loop (cdr values))))))) + (check owned => persistent) + ;; A genuine owned execution copies each frozen state spine. The + ;; reference shares its unchanged root cell across these frontiers. + (check (eq? (last-cell (lr-checkpoint-states (car frames))) + (last-cell (lr-checkpoint-states (cadr frames)))) => #f)))) + (test-case "generated steps retain one borrowed lookahead tail across reductions" + (let* ((machine (parser-machine-with-lr-action-selector records-parser make-lr-action-selector)) + (runtime (parser-machine-runtime machine)) + (table (lr-runtime-table runtime)) (widths (lr-runtime-reduction-widths runtime)) + (plans (lr-runtime-reduction-plans runtime)) + (previous-token #f) (previous-rest #f) (reuses 0) + (source "a=1\nb=2\n")) + (install-parser-machine-direct-step! machine (parser-machine-grammar-digest machine) + (lambda (production states semantic-values rest input-end gotos) + (when (pair? rest) + (when (eq? (car rest) previous-token) + (check (eq? rest previous-rest) => #t) + (set! reuses (+ reuses 1))) + (set! previous-token (car rest)) (set! previous-rest rest)) + (let* ((width (vector-ref widths production)) + (remaining-states (list-tail states width)) + (remaining-values (list-tail semantic-values width)) + (offset (if (pair? rest) (token-start (car rest)) input-end)) + (value (reduce-lr-stack-plan (vector-ref plans production) semantic-values offset)) + (target (goto-target gotos (car remaining-states) (production-lhs (vector-ref table production))))) + (values target (cons target remaining-states) (cons value remaining-values))))) + (let-values (((artifact _tokens _modes _frames) (parse-source/checkpoints machine source 2))) + (check artifact => (parse-source records-parser source)) + (check (> reuses 0) => #t)))) + (test-case "failure and EOF preserve materialized checkpoint rest" + (for-each (lambda (tokens) + (check-input-drivers (parser-machine-runtime records-parser) tokens #f #t)) + (list '() (list (make-token 'missing "?" 0 1))))) + (test-case "fork exposes unconsumed input and fallback completes without replay" + (let* ((rules '((source-file (choice (reference first-path) (reference second-path))) + (first-path (alias SourceFile (field value (token identifier)))) + (second-path (alias SourceFile (field value (token identifier)))))) + (runtime (lr-prepare (compile-lr-spec rules 'source-file 'selective-glr))) + (tokens (list (make-token 'identifier "x" 0 1)))) + (check-input-drivers runtime tokens #f #t) + (check-input-drivers runtime tokens #f #f))))) diff --git a/t/lr-semantic-preparation-test.ss b/t/lr-semantic-preparation-test.ss new file mode 100644 index 00000000..660b1c7f --- /dev/null +++ b/t/lr-semantic-preparation-test.ss @@ -0,0 +1,52 @@ +;;; Prepared backend admission and retained runtime identity. +(import :std/test + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-runtime-for-current-semantic-backend + current-lr-event-program-enabled? lr-runtime-event-program? + lr-runtime-direct-step install-lr-runtime-event-step!)) +(export lr-semantic-preparation-test) + +(def (event-runtime runtime) + (parameterize ((current-lr-event-program-enabled? #t)) + (lr-runtime-for-current-semantic-backend runtime))) + +(def lr-semantic-preparation-test + (test-suite "LR semantic preparation" + (test-case "preparation captures both strategies independently of request preference" + (let* ((spec (compile-lr-spec '((source-file (token word))) 'source-file)) + (runtime (parameterize ((current-lr-event-program-enabled? #t)) + (lr-prepare spec))) + (selected (event-runtime runtime))) + (check (lr-runtime-event-program? runtime) => #f) + (check (lr-runtime-event-program? selected) => #t) + (check (eq? selected runtime) => #f) + (check (eq? selected (event-runtime runtime)) => #t) + (check (eq? selected (event-runtime selected)) => #t) + (parameterize ((current-lr-event-program-enabled? #f)) + (check (eq? runtime (lr-runtime-for-current-semantic-backend runtime)) => #t) + (check (eq? selected (lr-runtime-for-current-semantic-backend selected)) => #t)) + (check (eq? selected (event-runtime (lr-prepare spec))) => #f))) + (test-case "generated step installation changes future selection, not retained executors" + (let* ((runtime (lr-prepare (compile-lr-spec '((source-file (token word))) 'source-file))) + (retained (event-runtime runtime)) + (step (lambda args (error "test step must not execute")))) + (install-lr-runtime-event-step! runtime step) + (let (selected (event-runtime runtime)) + (check (eq? selected retained) => #f) + (check (eq? selected (event-runtime runtime)) => #t) + (check (eq? step (lr-runtime-direct-step selected)) => #t) + (check (lr-runtime-direct-step retained) => #f)) + (check-exception (install-lr-runtime-event-step! runtime step) true))) + (test-case "dynamic, branching and layout grammars retain recognition fallback" + (for-each + (lambda (rules) + (let (runtime (lr-prepare (compile-lr-spec rules 'source-file 'selective-glr))) + (for-each (lambda (_) (check (eq? runtime (event-runtime runtime)) => #t)) + '(first second third)) + (check (lr-runtime-event-program? runtime) => #f))) + '(((source-file (precedence dynamic 1 (token word)))) + ((source-file (choice (reference first-path) (reference second-path))) + (first-path (alias First (token word))) + (second-path (alias Second (token word)))) + ((source-file (sequence (layout-start "|>") (token word) (layout-end "END"))))))))) diff --git a/t/lr-stack-reduction-test.ss b/t/lr-stack-reduction-test.ss new file mode 100644 index 00000000..c2b2d452 --- /dev/null +++ b/t/lr-stack-reduction-test.ss @@ -0,0 +1,436 @@ +;;; Shared deterministic stack reduction versus independent materialized GLR. +(import :std/test + (only-in :std/test/base TestCase test-case-add!) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr operand-actions-valid? validate-production-semantics) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/recognition + make-recognition-child make-recognition-node make-recognition-fragment + recognition-child-value recognition-value-start recognition-value-end + recognition-value-bounds with-recognition-value-bounds recognition-node-children relocate-recognition-value) + (only-in :gerbil-parser/src/runtime/event-program event-program-node-value event-program-fragment-value) + (only-in :gerbil-parser/src/runtime/reduce + recognition-children-field recognition-children-alias + recognition-action-field recognition-action-alias + make-recognition-field-kernel make-recognition-alias-kernel) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-parse/prepared lr-parse/prepared/receipt + lr-initial-checkpoint lr-checkpoint-frontier lr-checkpoint-state + lr-checkpoint-rebind-suffix lr-checkpoint-resume lr-checkpoint-drive + lr-checkpoint-deterministic-actions lr-checkpoint-deterministic-shifts + lr-checkpoint-remaining-token-count + install-lr-runtime-direct-step! lr-rejection-condition? + current-lr-event-program-enabled? lr-runtime-for-current-semantic-backend + current-lr-recognition-observer lr-runtime-with-action-selector) + (only-in :gerbil-parser/src/runtime/funcs association-row-index-ref + recognition-sequence-append recognition-sequence-relocate + recognition-sequence-for-action) + (only-in :gerbil-parser/src/runtime/artifact + make-success-parse-artifact parse-artifact-valid? parse-artifact-roundtrip) + (only-in :gerbil-parser/t/benchmarks/lr-request-stack/derivation + prepare-lr-semantic-replay replay-lr-semantics replay-lr-semantic-control + lr-semantic-replay-count profile-lr-semantic-actions)) +(import (only-in :gerbil-parser/src/runtime/lr-stack-storage make-lr-token-observer)) +(import (only-in :gerbil-parser/src/runtime/lr-parser/state lr-failure-frontier-checkpoint)) +(import (only-in :gerbil-parser/t/fixtures/lr-action-selection reference-lr-action-selector)) +(export lr-stack-reduction-test) + +;;; Installed procedures are arbitrary callbacks. They may change the current +;;; lookahead during a reduction, so their executor must not cache its class. +(def (installed-step-lookahead-case) + (let* ((runtime (lr-prepare + (compile-lr-spec '((start (sequence (reference first) (literal "b"))) + (first (literal "a"))) 'start))) + (tokens (list (make-token 'literal "a" 0 1) (make-token 'literal "b" 1 2))) + (calls 0)) + (install-lr-runtime-direct-step! runtime + (lambda (_production states semantic-values rest _end gotos) + (set! calls (+ calls 1)) + (let (target (cdr (association-row-index-ref gotos (cadr states) 'first))) + (vector-set! (car rest) 1 "c") + (values target (cons target (cdr states)) semantic-values)))) + (check-exception (lr-parse/prepared runtime tokens) lr-rejection-condition?) + (check calls => 1))) + +(def (production-spec productions) + ;; Isolate production admission with an already accepting table graph. + (list (cons 'productions productions) + (cons 'actions (vector '(((terminal eof) accept)))) (cons 'gotos (vector '())) + (cons 'case-insensitive? #f))) + +(def (check-stack-family width decorated?) + (let* ((operand (if decorated? + '(field outer (alias Renamed (field inner (token word)))) + '(token word))) + (rules (list (list 'source-file + (list 'alias 'SourceFile + (cons 'sequence (make-list width operand)))))) + (runtime (lr-prepare (compile-lr-spec rules 'source-file))) + (source (make-string width #\a)) + (tokens (map (lambda (index) (make-token 'word "a" index (+ index 1))) + (iota width))) + ;; This tree is the declaration's semantic oracle, independent of + ;; either LR executor and its prepared action representation. + (expected-root + (make-recognition-node 'SourceFile 0 width + (map (lambda (token index) + (if decorated? + (make-recognition-child 'outer + (make-recognition-node 'Renamed index (+ index 1) + (list (make-recognition-child 'inner token)))) + (make-recognition-child #f token))) + tokens (iota width))))) + (let-values (((reference rest receipt) (lr-parse/prepared/receipt runtime tokens))) + (check rest => '()) + (let* ((digest (string-append "sha256:" (make-string 64 #\0))) + (expected (make-success-parse-artifact digest source tokens expected-root false))) + (check (make-success-parse-artifact digest source tokens reference false) => expected) + (check (parse-artifact-valid? expected) => #t) + (check (parse-artifact-roundtrip expected) => source) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (let (selected (lr-runtime-for-current-semantic-backend runtime)) + (let-values (((root rest) (lr-parse/prepared selected tokens))) + (check rest => '()) + (check (make-success-parse-artifact digest source tokens root false) + => expected))))) + '(#f #t)))))) + +(def lr-stack-reduction-test + (test-suite "shared LR stack reduction" + (test-case "owned failure frontiers remain immutable across later requests and resumption" + (let* ((runtime (lr-prepare (compile-lr-spec + '((start (alias Source (sequence (literal "a") (literal "b"))))) 'start))) + (a (make-token 'literal "a" 0 1)) (b (make-token 'literal "b" 1 2)) + (wrong (make-token 'literal "x" 1 2)) (good (list a b))) + (let-values (((status frontier) (lr-checkpoint-frontier (lr-initial-checkpoint runtime (list a wrong))))) + (check status => 'failure) + (let* ((checkpoint (lr-failure-frontier-checkpoint frontier)) + (before (list (lr-checkpoint-state checkpoint) (lr-checkpoint-deterministic-actions checkpoint) + (lr-checkpoint-deterministic-shifts checkpoint) (lr-checkpoint-remaining-token-count checkpoint))) + (expected (call-with-values (lambda () (lr-parse/prepared runtime good)) list))) + (check (lr-checkpoint-deterministic-shifts checkpoint) => 1) + (let-values (((root rest) (lr-checkpoint-resume (lr-checkpoint-rebind-suffix checkpoint good (list b))))) + (check (list root rest) => expected)) + (check (list (lr-checkpoint-state checkpoint) (lr-checkpoint-deterministic-actions checkpoint) + (lr-checkpoint-deterministic-shifts checkpoint) (lr-checkpoint-remaining-token-count checkpoint)) + => before))))) + (test-case "token-only and arbitrary stack observers preserve their distinct contracts" + (let* ((runtime (lr-prepare (compile-lr-spec + '((start (alias Source (sequence (literal "a") (literal "b"))))) 'start))) + (tokens (list (make-token 'literal "a" 0 1) (make-token 'literal "b" 1 2))) + (expected (call-with-values (lambda () (lr-parse/prepared runtime tokens)) list))) + (for-each + (lambda (owned?) + (let ((remaining tokens) (seen '())) + (def (next-input _mode) + (and (pair? remaining) + (let (token (car remaining)) (set! remaining (cdr remaining)) token))) + (def observer + (if owned? + (make-lr-token-observer (lambda (token) (set! seen (cons token seen)))) + (lambda (token states values _actions _shifts) + (check (list? states) => #t) (check (list? values) => #t) + (set! seen (cons token seen))))) + (let-values (((status actual) (lr-checkpoint-drive (lr-initial-checkpoint runtime '()) next-input observer))) + (check status => 'accepted) (check actual => expected) + (check (reverse seen) => tokens)))) '(#f #t)))) + (test-case "owned unary chains preserve decorated ASTs, boundaries and failure handoff" + (for-each + (lambda (depth) + (def (name index) (string->symbol (string-append "unit" (number->string index)))) + (let* ((rules + (cons '(start (alias Root (sequence (reference unit0) (literal "b")))) + (map (lambda (index) + (let (child (if (= index depth) '(token word) + (list 'reference (name (+ index 1))))) + (list (name index) + (if (even? index) child + (list 'alias (name index) (list 'field 'value child)))))) + (iota (+ depth 1))))) + (selections 0) + (runtime (lr-runtime-with-action-selector + (lr-prepare (compile-lr-spec rules 'start)) + (lambda (rows index casefold? layout?) + (let (select (reference-lr-action-selector rows index casefold? layout?)) + (lambda (state tokens) + (set! selections (+ selections 1)) + (select state tokens))))))) + (for-each + (lambda (offset) + (let* ((tokens (list (make-token 'word "a" offset (+ offset 1)) + (make-token 'literal "b" (+ offset 1) (+ offset 2)))) + (expected (call-with-values + (lambda () (lr-parse/prepared/receipt runtime tokens)) list)) + (owned (begin (set! selections 0) + (call-with-values (lambda () (lr-parse/prepared runtime tokens)) list))) + (owned-selections selections) + (persistent + (begin (set! selections 0) + (parameterize ((current-lr-recognition-observer (lambda (_) (void)))) + (call-with-values (lambda () (lr-parse/prepared runtime tokens)) list))))) + (check owned => (list (car expected) (cadr expected))) + (check persistent => owned) + (check selections => owned-selections) + (check-exception + (lr-parse/prepared runtime + (list (car tokens) (make-token 'literal "c" (+ offset 1) (+ offset 2)))) + lr-rejection-condition?))) '(0 37)))) '(1 16 96))) + (test-case "constructor bounds continuation retains scalar endpoints and evaluates once" + (for-each + (lambda (value) + (let ((evaluations 0) (current 'caller-binding) (delta 999)) + (check + (with-recognition-value-bounds + (begin (set! evaluations (+ evaluations 1)) value) + (start end) (list start end current delta)) + => (list (recognition-value-start value) (recognition-value-end value) + 'caller-binding 999)) + (check evaluations => 1))) + (let ((token (make-token 'word "a" 5 6)) + (node (make-recognition-node 'Node 9 13 '()))) + (list token node (make-recognition-fragment 17 19 '()) + (event-program-node-value 'Event 21 23 #f) + (event-program-fragment-value 25 26 #f) + (relocate-recognition-value token 37) + (relocate-recognition-value (relocate-recognition-value node 11) -4)))) + (for-each + (lambda (value) + (check-exception (with-recognition-value-bounds value (start end) (list start end)) true)) + (list #f 0 '() '#()))) + (test-case "action boundaries precede normalization without changing child ownership" + (let* ((a (list (make-recognition-child #f (make-token 'word "a" 5 6)))) + (b (list (make-recognition-child 'inner (make-token 'word "b" 9 10)))) + (branch (recognition-sequence-append a b)) + (deep (foldl recognition-sequence-append '() (make-list 2048 a)))) + (check (eq? a (recognition-node-children + (recognition-child-value (car (recognition-action-alias 'Node a 0))))) => #t) + (for-each + (lambda (children) + (for-each + (lambda (offset) + (let ((ready (recognition-sequence-for-action children)) (calls '())) + (def (construct start end values) + (check (list start end) + => (if (null? ready) (list offset offset) + (list (recognition-value-start (recognition-child-value (car ready))) + (recognition-value-end (recognition-child-value (last ready)))))) + (set! calls (cons (list start end values) calls)) + (make-recognition-fragment start end values)) + (check (recognition-action-alias 'Node children offset) + => (recognition-children-alias 'Node ready offset)) + (check (recognition-action-field 'outer children offset construct) + => (recognition-children-field 'outer ready offset construct)) + (when (pair? calls) + (check (car calls) => (cadr calls))) + (check ((make-recognition-alias-kernel 'Node) children offset construct) + => (recognition-action-alias 'Node children offset)) + (check ((make-recognition-field-kernel 'outer) children offset construct) + => (recognition-action-field 'outer children offset construct)) + (when (pair? calls) + (check (car calls) => (cadr calls))) + (check (recognition-sequence-for-action children) => ready))) + '(0 37))) + (list '() a b branch deep + (recognition-sequence-relocate a 11) + (recognition-sequence-relocate b -3) + (recognition-sequence-relocate branch 7) + (recognition-sequence-append (recognition-sequence-relocate a 11) + (recognition-sequence-relocate b -3)))))) + (test-case "combined recognition endpoints retain checked value admission" + (for-each + (lambda (value) + (check (call-with-values (lambda () (recognition-value-bounds value)) list) + => (list (recognition-value-start value) (recognition-value-end value)))) + (list (make-token 'word "a" 5 6) + (make-recognition-node 'Node 7 9 '()) + (make-recognition-fragment 11 11 '()) + (event-program-node-value 'Event 17 19 #f) + (event-program-fragment-value 21 22 #f) + (relocate-recognition-value (make-recognition-node 'Moved 31 33 '()) -7))) + (for-each + (lambda (value) (check-exception (recognition-value-bounds value) true)) + (list #f #() (make-token 'word "a" 6 5)))) + (test-case "semantic replay retains decorated operands and request boundaries" + (let* ((spec (compile-lr-spec + '((source-file + (alias SourceFile + (sequence (alias Before (empty)) + (field outer (alias Word (field inner (token word)))) + (alias After (empty)))))) 'source-file)) + (runtime (lr-prepare spec))) + (for-each + (lambda (offset) + (let* ((tokens (list (make-token 'word "a" offset (+ offset 1)))) + (replay (prepare-lr-semantic-replay runtime spec tokens))) + (check (positive? (lr-semantic-replay-count replay)) => #t) + (let (work (profile-lr-semantic-actions replay)) + (check (cdr (assq 'fieldActions work)) => 2) + (check (cdr (assq 'aliasActions work)) => 4) + (check (cdr (assq 'widestActionInput work)) => 3)) + (let-values (((expected rest) (lr-parse/prepared runtime tokens))) + (check rest => '()) + (check (replay-lr-semantics replay) => expected) + (check (replay-lr-semantic-control replay) => expected) + (check (replay-lr-semantics replay) => expected)))) + '(0 17 3)))) + (test-case-add! (TestCase "installed reduction callbacks retain fresh lookahead queries" + installed-step-lookahead-case)) + (test-case "prepared nullable aliases receive each request's actual boundaries" + (let (runtime + (lr-prepare + (compile-lr-spec + '((source-file + (alias SourceFile + (sequence (alias Before (empty)) (token word) + (alias After (empty)))))) 'source-file))) + (for-each + (lambda (offset) + (let* ((end (+ offset 1)) (token (make-token 'word "a" offset end)) + (expected-root + (make-recognition-node 'SourceFile offset end + (list + (make-recognition-child #f (make-recognition-node 'Before offset offset '())) + (make-recognition-child #f token) + (make-recognition-child #f (make-recognition-node 'After end end '()))))) + (source (string-append (make-string offset #\space) "a")) + (digest (string-append "sha256:" (make-string 64 #\0))) + (publish (lambda (root) + (make-success-parse-artifact digest source (list token) root false))) + (expected (publish expected-root))) + (let-values (((root rest) (lr-parse/prepared runtime (list token)))) + (check rest => '()) + (check (publish root) => expected)) + (let-values (((root rest receipt) (lr-parse/prepared/receipt runtime (list token)))) + (check rest => '()) + (check (publish root) => expected)))) + '(0 17 3)))) + (test-case "canonical productions validate numbering, width and scalar metadata" + (for-each + (lambda (production) + (let (productions (list production)) + (check (eq? productions (validate-production-semantics productions)) => #t) + (check (not (not (lr-prepare (production-spec productions)))) => #t))) + '((0 root ((terminal token word)) pass #f) + (0 root () concat (left 10)) + (0 root () layout-end #f) + (0 root () (layout-end "END" ")") (dynamic -1)) + (0 root ((nonterminal root) (terminal literal "+")) concat (none 0)) + (0 root ((terminal layout-start "|>") (terminal layout-next "|>")) concat (right 2))))) + (test-case "malformed canonical productions fail before backend preparation" + (for-each + (lambda (productions) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (check-exception (lr-prepare (production-spec productions)) + (lambda (condition) + (string-prefix? "invalid canonical LR" (error-message condition)))))) + '(#f #t))) + '(#f #() ((0 root () concat #f) . invalid) + ((1 root () concat #f)) + ((0 root () concat #f) (0 other () concat #f)) + ((0 "root" () concat #f)) ((0 root () concat #f extra)) + ((0 root () pass #f)) + ((0 root ((terminal token word) (terminal token word)) pass #f)) + ((0 root ((terminal token word)) layout-end #f)) + ((0 root () (layout-end) #f)) ((0 root () (layout-end "") #f)) + ((0 root () (layout-end END) #f)) ((0 root () (layout-end . "END") #f)) + ((0 root () concat (callback 1))) ((0 root () concat (left "10"))) + ((0 root () concat (left 1 extra))) ((0 root () concat (dynamic . 1))) + ((0 root ((terminal word)) concat #f)) + ((0 root ((terminal literal "")) concat #f)) + ((0 root ((marked (terminal token word))) concat #f)) + ((0 root ((marked (terminal token word) () extra)) concat #f)) + ((0 root ((nonterminal "item")) concat #f))))) + (test-case "reference admission resolves complete owners without pruning dead cycles" + (def productions + '((0 root ((marked (nonterminal later) ((field value)))) pass #f) + (1 later ((terminal token word)) pass #f) + (2 unused ((nonterminal unused)) pass #f))) + (check (eq? productions (validate-production-semantics productions)) => #t) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (check (not (not (lr-prepare (production-spec productions)))) => #t) + (for-each + (lambda (invalid) + (check-exception + (lr-prepare (production-spec invalid)) + (lambda (condition) + (equal? (error-message condition) "invalid canonical LR unresolved reference")))) + '(((0 root ((nonterminal missing)) pass #f)) + ((0 root ((terminal token word)) pass #f) + (1 unused ((marked (nonterminal missing) ((alias Item)))) pass #f)))))) + '(#f #t))) + (test-case "all LR construction routes reject undeclared references including unused rules" + (for-each + (lambda (construction) + (for-each + (lambda (rules) + (check-exception + (compile-lr-spec rules 'root 'reject #f construction) + (lambda (condition) + (equal? (error-message condition) "invalid canonical LR unresolved reference")))) + '(((root (reference missing))) + ((root (token word)) (unused (field value (reference missing))))))) + '(lalr canonical-lr1 partitioned-lr1 follow-partition-lr1 lalr-then-follow))) + (test-case "standalone lowering cannot publish invalid canonical operands" + (for-each + (lambda (construction) + (check-exception + (compile-lr-spec '((root (field "name" (token word)))) 'root 'reject #f construction) + (lambda (condition) (equal? (error-message condition) "invalid LR operand actions"))) + (check-exception + (compile-lr-spec '((root (layout-end ""))) 'root 'reject #f construction) + (lambda (condition) (equal? (error-message condition) "invalid canonical LR production")))) + '(lalr canonical-lr1 partitioned-lr1 follow-partition-lr1 lalr-then-follow))) + (test-case "canonical action admission requires proper symbolic declarations" + (for-each (lambda (actions) (check (operand-actions-valid? actions) => #t)) + '(() ((field left)) ((alias Item)) ((alias Item) (field left)))) + (for-each (lambda (actions) (check (operand-actions-valid? actions) => #f)) + '(#f #() ((field)) ((alias Item extra)) ((callback Item)) + ((field #f)) ((field "left")) ((alias 1)) ((alias (Item))) + ((field . left)) ((field left) . invalid)))) + (test-case "prepared runtimes reject unsupported actions before execution" + (def (prepare action operand-actions) + (lr-prepare + (production-spec + (list (list 0 'source-file + (list (list 'marked '(terminal token word) operand-actions)) + action #f))))) + (check-exception (prepare 'callback '()) true) + (check-exception (prepare 'concat '((callback user))) true) + (check-exception (prepare 'concat '((field))) true) + (check-exception (prepare 'concat '((alias Name extra))) true) + (check-exception (prepare 'concat '((field Name) . invalid)) true) + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (for-each (lambda (actions) + (check-exception (prepare 'concat actions) + (lambda (condition) + (and (equal? (error-message condition) "invalid LR operand actions") + (equal? (error-irritants condition) (list actions)))))) + '(((field #f)) ((field "left")) ((alias 1)) ((alias (Item))))))) + '(#f #t)) + (check (not (not (prepare 'concat '((field inner) (alias Renamed))))) => #t)) + (test-case "unselected productions cannot bypass semantic name admission" + ;; Immediate acceptance does not select either production for reduction. + ;; Preparation still owns admission of the whole canonical production set. + (for-each + (lambda (events?) + (parameterize ((current-lr-event-program-enabled? events?)) + (check-exception + (lr-prepare + (production-spec + '((0 source-file ((terminal token word)) pass #f) + (1 unused ((marked (terminal token word) ((alias "Item")))) concat #f)))) + (lambda (condition) + (equal? (error-message condition) "invalid LR operand actions"))))) + '(#f #t))) + (test-case "empty, unary and wide identity operands agree with GLR" + (for-each (lambda (width) (check-stack-family width #f)) '(0 1 2 8 32 128))) + (test-case "wide ordered field and alias chains agree with GLR" + (for-each (lambda (width) (check-stack-family width #t)) '(1 2 8 32 128))))) diff --git a/t/lr-unary-corridor-test.ss b/t/lr-unary-corridor-test.ss new file mode 100644 index 00000000..0855bb17 --- /dev/null +++ b/t/lr-unary-corridor-test.ss @@ -0,0 +1,63 @@ +;;; Consecutive decorated unary reductions versus independent GLR and pauses. +(import :std/test + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-parse/prepared lr-parse/prepared/receipt + lr-initial-checkpoint lr-checkpoint-drive lr-checkpoint-advance + lr-checkpoint-deterministic-actions lr-checkpoint-deterministic-shifts + current-lr-event-program-enabled? lr-runtime-for-current-semantic-backend) + (only-in :gerbil-parser/src/runtime/artifact make-success-parse-artifact)) +(export lr-unary-corridor-test) + +(def (check-unary-chain depth events? (decorated? #t)) + (def (name n) (string->symbol (string-append "layer" (number->string n)))) + (let* ((rules + (cons '(source-file (alias SourceFile (sequence (reference layer0) (token end)))) + (map (lambda (n) + (list (name n) + (let (operand (if (= n depth) '(token word) + (list 'reference (name (+ n 1))))) + (if decorated? + (list 'field (name n) (list 'alias (name n) operand)) + operand)))) + (iota (+ depth 1))))) + (runtime (lr-prepare (compile-lr-spec rules 'source-file))) + (tokens (list (make-token 'word "a" 0 1) (make-token 'end ";" 1 2))) + (digest (string-append "sha256:" (make-string 64 #\0)))) + (let-values (((root rest receipt) (lr-parse/prepared/receipt runtime tokens))) + (check rest => '()) + (let (expected (make-success-parse-artifact digest "a;" tokens root false)) + (parameterize ((current-lr-event-program-enabled? events?)) + (let* ((selected (lr-runtime-for-current-semantic-backend runtime)) + (initial (lr-initial-checkpoint selected tokens)) + (second-shift-actions #f)) + (let-values (((status result) + (lr-checkpoint-drive initial + (lambda (_mode) #f) + (lambda (_token _states _values actions shifts) + (when (= shifts 2) (set! second-shift-actions actions)))))) + (check status => 'accepted) + (check (make-success-parse-artifact digest "a;" tokens (car result) false) + => expected)) + (let pause ((checkpoint initial)) + (let-values (((status result) (lr-checkpoint-advance checkpoint 1))) + (case status + ((checkpoint) + (check (lr-checkpoint-deterministic-actions result) + => (+ 1 (lr-checkpoint-deterministic-actions checkpoint))) + (if (= (lr-checkpoint-deterministic-shifts result) 2) + (check (lr-checkpoint-deterministic-actions result) => second-shift-actions) + (pause result))) + (else (error "bounded chain failed before second shift" status))))))))))) + +(def lr-unary-corridor-test + (test-suite "shared unary reduction corridors" + (test-case "decorated chains retain canonical artifacts and every action count" + (for-each (lambda (depth) + (for-each (lambda (events?) (check-unary-chain depth events?)) '(#f #t))) + '(0 1 4 16))) + (test-case "identity chains preserve artifacts and every paused action" + (for-each (lambda (depth) + (for-each (lambda (events?) (check-unary-chain depth events? #f)) '(#f #t))) + '(0 1 4 16))))) diff --git a/t/module-source-cases.ss b/t/module-source-cases.ss new file mode 100644 index 00000000..ee53cc7f --- /dev/null +++ b/t/module-source-cases.ss @@ -0,0 +1,14 @@ +;;; Character offsets; generators translate these witnesses to Rust byte offsets. +(export module-source-cases) +(def module-source-cases + '(("preamble\n---- MODULE Demo ----" 0 9) + ("---- MODULE A ----\n====\nfooter" 0 #f) + ("---- MODULE A ----\n====\nfooter" 23 30) + ("---- MODULE A ----\n====\nfooter" 24 30) + ("---- MODULE 字 ----\n====\nλ" 24 25) + ("---- MODULE A ----\n---- MODULE B ----\n====\ninner\n====\nfooter" 43 #f) + ("---- MODULE A ----\n---- MODULE B ----\n====\ninner\n====\nfooter" 54 60) + ("---- MODULE A ----\n\"====\"\n====\nfooter" 25 #f) + ("---- MODULE A ----\n====\nfooter\n---- MODULE B ----" 24 31) + ("preamble" 0 8) + ("preamble" 8 #f))) diff --git a/t/module-source-test.ss b/t/module-source-test.ss new file mode 100644 index 00000000..ffcd9a8a --- /dev/null +++ b/t/module-source-test.ss @@ -0,0 +1,74 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; Source framing is closed lexical data with bounded immutable indexes. +(import :std/test + (only-in :gerbil-parser/language-support deflanguage) + (only-in :gerbil-parser/src/grammar/lexical-algebra lexical-expression?) + (only-in :gerbil-parser/src/runtime/lexical-source + scan-module-text prepare-lexical-source-plan call-with-lexical-source) + (only-in :gerbil-parser/src/language/entry parse-language-source) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success? parse-artifact-roundtrip) + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-sany-candidate-language-grammar) + (only-in :gerbil-parser/src/language/descriptor language-grammar-ir) + (only-in "module-source-cases.ss" module-source-cases)) +(export module-source-test module-framing-language-grammar) +(def module-expression '(module-text "----" "MODULE" "====" "(*" "*)" "\\*" "_")) +(def module-plan (prepare-lexical-source-plan (list (list 'text module-expression)))) +(begin + (deflanguage module-framing + (syntax + (lexical + (root document) + (lex (text ModuleText (module-text "----" "MODULE" "====" "(*" "*)" "\\*" "_")) + (border Border (character-run "-" 4)) (end End (character-run "=" 4)) + (name Name (identifier)) (string String (quoted-string "\"")) + (space Space (whitespace+)) + (comment Comment (choice (line-comment "\\*") (nested-block-comment "(*" "*)")))) + (extras space comment))) + (rules + (document (node Document (seq (optional text) (repeat1 (seq (reference unit) (optional text)))))) + (unit (node Unit (seq border (literal "MODULE") (field name name) border + (repeat (choice name string (reference unit))) end))))) + (bind-fixture-grammar-release module-framing "module-framing" "v1" "module-framing.v1") ) +(def module-source-test + (test-suite "prepared module framing IR" + (test-case "the real TLA+ descriptor publishes closed framing data" + (check (cadr (assq 'module-text (cdr (assq 'lexical-rules (language-grammar-ir tla-plus-sany-candidate-language-grammar))))) + => module-expression)) + (test-case "framing keeps character boundaries, nested depth and module text stops" + (for-each (lambda (row) + (call-with-lexical-source module-plan (car row) + (lambda () (check (scan-module-text (car row) (cadr row) module-expression) => (caddr row))))) + module-source-cases)) + (test-case "nested comments and quoted end markers never close a module" + (for-each (lambda (middle) + (let* ((prefix (string-append "---- MODULE A ----\n" middle "\n")) + (source (string-append prefix "still inside\n====\nfooter"))) + (call-with-lexical-source module-plan source + (lambda () (check (scan-module-text source (string-length prefix) module-expression) => #f))))) + '("(* ---- MODULE B ----\n==== (* nested ==== *) *)" "\"====\"" "\\* ====\nword"))) + (test-case "long repeated borders remain one indexed interval" + (let (source (string-append (make-string 100000 #\-) " MODULE A ----\n====\nfooter")) + (call-with-lexical-source module-plan source + (lambda () + (check (scan-module-text source 0 module-expression) => #f) + (check (scan-module-text source (- (string-length source) 6) module-expression) => (string-length source)))))) + (test-case "the same data executes in an ordinary lossless parser" + (for-each (lambda (source) + (let (artifact (parse-language-source module-framing-language-grammar source)) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + '("preamble\n---- MODULE A ----\n====\nfooter" + "---- MODULE A ----\n---- MODULE B ----\n====\ninner\n====\nfooter" + "---- MODULE A ----\n(* ==== *)\n\"====\"\n====\nfooter"))) + (test-case "an explicit extra name character may itself be numeric" + (let* ((expression '(module-text "~~~" "UNIT" "!!!" "(*" "*)" "\\*" "0")) + (plan (prepare-lexical-source-plan (list (list 'text expression))))) + (for-each (lambda (source expected) + (call-with-lexical-source plan source (lambda () + (check (scan-module-text source 0 expression) => expected)))) + '("~~~ UNIT 000 ~~~" "~~~ UNIT 111 ~~~") '(#f 16)))) + (test-case "malformed framing profiles reject before source preparation" + (for-each (lambda (expression) (check (lexical-expression? expression) => #f)) + '((module-text "-x" "MODULE" "====" "(*" "*)" "\\*" "_") + (module-text "----" "" "====" "(*" "*)" "\\*" "_") + (module-text "----" "MODULE" "====" "(*" "*)" "\\*" "__")))))) diff --git a/t/native-ffi-header-smoke.c b/t/native-ffi-header-smoke.c deleted file mode 100644 index c25f7d9f..00000000 --- a/t/native-ffi-header-smoke.c +++ /dev/null @@ -1,20 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2026 tao3k team and Contributors - * SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - */ - -#include -#include - -int main(void) { - gerbil_parser_result_v1 parse_result; - gerbil_parser_rowan_result_v1 rowan_result; - gerbil_parser_result_v1_init(&parse_result); - gerbil_parser_rowan_result_v1_init(&rowan_result); - (void)gerbil_parser_native_descriptor("gql", &parse_result); - (void)gerbil_parser_native_parse("gql", "RETURN 1", &parse_result); - (void)gerbil_parser_rowan_compile("grammar.ss", &rowan_result); - gerbil_parser_result_v1_release(&parse_result); - gerbil_parser_rowan_result_v1_release(&rowan_result); - return 0; -} diff --git a/t/native-ffi-test.ss b/t/native-ffi-test.ss deleted file mode 100644 index cdf07022..00000000 --- a/t/native-ffi-test.ss +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env gxi -;;; -*- Gerbil -*- - -(import :std/test - (only-in :std/vector/u8vector little u8vector-u32-ref) - :std/encoding/json - (only-in :gerbil-parser/src/ffi/parse-artifact-v1 - native-abi-version - native-descriptor-payload - native-parse-binary-payload) - (only-in ../src/ffi/rust-rowan-aot-v1 - native-rowan-aot-abi-version - native-rust-rowan-source)) - -(def native-ffi-tests - (test-suite "parser-owned native ParseArtifact v1 ABI" - (test-case "descriptor publishes the parser-owned grammar surface" - (check (native-abi-version) => 1) - (let (descriptor (string->json (native-descriptor-payload "gql"))) - (check (hash-get descriptor "schema") - => "gerbil-parser.native-descriptor.v1") - (check (hash-get descriptor "language") => "gql") - (check (positive? (length (hash-get descriptor "syntaxKinds"))) - => #t) - (let (fields (hash-get descriptor "fields")) - (check (not (not (member "operator" fields))) => #t) - (check (not (not (member "sign" fields))) => #t)))) - (test-case "accepted source produces typed binary ParseArtifact v1" - (let (artifact - (native-parse-binary-payload "gql" "MATCH (n) RETURN n")) - (check (subu8vector artifact 0 4) => #u8(71 80 65 49)) - (check (u8vector-u32-ref artifact 4 little) => 1) - (check (u8vector-u32-ref artifact 8 little) => 0) - (check (positive? (u8vector-u32-ref artifact 12 little)) => #t) - (check (u8vector-length artifact) - => (+ 80 (* 24 (u8vector-u32-ref artifact 12 little)))))) - (test-case "openCypher uses its own descriptor and parser" - (let* ((descriptor - (string->json (native-descriptor-payload "cypher"))) - (artifact - (native-parse-binary-payload - "cypher" "MATCH (n:Person) RETURN n\n"))) - (check (hash-get descriptor "language") => "cypher") - (check (positive? (length (hash-get descriptor "syntaxKinds"))) - => #t) - (check (u8vector-u32-ref artifact 8 little) => 0) - (check (positive? (u8vector-u32-ref artifact 12 little)) => #t))) - (test-case "unknown language fails closed" - (check-exception (native-descriptor-payload "unknown") true)) - (test-case "grammar path compiles through the Scheme Rowan backend" - (check (native-rowan-aot-abi-version) => 1) - (let (source - (native-rust-rowan-source - "t/fixtures/rust-rowan-downstream/languages/records/v1/grammar.ss")) - (check (not (not (string-contains - source "language: \"downstream-records\""))) => #t) - (check (not (not (string-contains - source "grammar_digest: \"sha256:9c7e8b0f"))) => #t) - (check (not (not (string-contains - source "pub static LANGUAGE: LanguageSpec"))) => #t))) - (test-case "module without a language descriptor fails closed" - (check-exception - (native-rust-rowan-source - "t/fixtures/rust-rowan-downstream/languages/records/v1/parser.ss") - true)))) - -(export native-ffi-tests) diff --git a/t/opencypher-large-query-benchmark-test.ss b/t/opencypher-large-query-benchmark-test.ss index bc776eb9..edd8e727 100644 --- a/t/opencypher-large-query-benchmark-test.ss +++ b/t/opencypher-large-query-benchmark-test.ss @@ -3,7 +3,8 @@ (import :std/test :asp-gerbil-scheme/src/benchmark/framework - "./scenarios/performance/opencypher-large-query/scenario") + "./scenarios/performance/opencypher-large-query/scenario" + (only-in :gerbil-parser/t/fixtures/progress report-test-progress!)) (def benchmark-path "t/scenarios/performance/opencypher-large-query/benchmark.ss") @@ -11,7 +12,10 @@ (def (run-scenario) (unless (opencypher-large-query-scenario-pass? (opencypher-large-query-scenario)) - (error "large openCypher query scenario failed"))) + (error "large openCypher query scenario failed")) + (when (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1") + (report-test-progress! "SCENARIO-OK opencypher-large-query clauses=100") + (force-output))) (def opencypher-large-query-benchmark-tests (test-suite "openCypher large-query complexity scenario" @@ -20,6 +24,7 @@ (let (algorithm-receipt (opencypher-large-query-scenario)) (write algorithm-receipt) (newline) + (force-output) (check (opencypher-large-query-scenario-pass? algorithm-receipt) => #t)) (run-scenario) @@ -28,6 +33,14 @@ (benchmark-contract-run benchmark-path run-scenario)) (write benchmark-receipt) (newline) + (force-output) (check (benchmark-contract-receipt-pass? benchmark-receipt) => #t))))) -(export opencypher-large-query-benchmark-tests) +(def opencypher-large-query-benchmark-test + opencypher-large-query-benchmark-tests) +(export opencypher-large-query-benchmark-tests + opencypher-large-query-benchmark-test) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/parse-artifact-test.ss b/t/parse-artifact-test.ss index a9f3b012..70f2c54f 100644 --- a/t/parse-artifact-test.ss +++ b/t/parse-artifact-test.ss @@ -3,9 +3,10 @@ (import :std/test :gerbil-parser/src/runtime/artifact + (only-in :gerbil-parser/src/runtime/identity sha256-bytes) :gerbil-parser/src/runtime/cst :gerbil-parser/src/runtime/parser - :gerbil-parser/languages/arithmetic/v1/parser) + :gerbil-parser/languages/arithmetic/parser) (def (artifact-replace artifact key value) (map (lambda (row) @@ -23,6 +24,36 @@ (def parse-artifact-tests (test-suite "ParseArtifact contract" + (test-case "source-bound admission proves lexemes and identity without roundtrip ownership" + (for-each (lambda (source) + (let (artifact (parse-source arithmetic-parser source)) + (check (parse-artifact-valid-for-source? artifact source) => #t) + (check (parse-artifact-valid-for-source? artifact (string-append source "x")) => #f) + (check (parse-artifact-valid-for-source? artifact #f) => #f) + (check (parse-artifact-valid-for-source? + (artifact-replace artifact 'sourceByteLength 4294967296) source) => #f) + (check (parse-artifact-valid-for-source? + (artifact-replace artifact 'sourceDigest (sha256-text "unrelated")) source) => #f))) + '("" "alpha + 2" "λ中文🙂@")) + (let* ((artifact (parse-source arithmetic-parser "λ")) + (forged (artifact-replace artifact 'sourceDigest (sha256-text "β")))) + (check (parse-artifact-valid-for-source? forged "β") => #f) + (check (parse-artifact-valid-for-source? artifact "β") => #f) + (check (parse-artifact-roundtrip artifact) => "λ"))) + (test-case "canonical SHA-256 identities match fixed digest vectors" + (check (sha256-text "") + => "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + (check (sha256-text "abc") + => "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + (check (sha256-text "λ中文🙂") + => "sha256:769868d0dd7575e844f10175e18222eb50c31aacd8f3cce0f3ce55fe2f0e53e2") + (let ((bytes (make-u8vector 256))) + (let fill ((index 0)) + (when (< index 256) + (u8vector-set! bytes index index) + (fill (+ index 1)))) + (check (sha256-bytes bytes) + => "sha256:40aff2e9d2d8922e47afd4648e6967497158785fbd1da870e7110266bf944880"))) (test-case "identity and event replay are deterministic" (let* ((source "alpha + (2 * beta)") (first (parse-source arithmetic-parser source)) @@ -35,6 +66,47 @@ (check (parse-artifact-ref first 'sourceDigest) => (sha256-text source)) (check first-cst => second-cst))) + (test-case "source identity keeps UTF-8 byte extent" + (let* ((source "λ + @") + (artifact (parse-source arithmetic-parser source)) + (bytes (string->utf8 source))) + (check (parse-artifact-ref artifact 'sourceByteLength) + => (u8vector-length bytes)) + (check (parse-artifact-ref artifact 'sourceDigest) + => (sha256-text source)) + (check (sha256-bytes bytes) => (sha256-text source)))) + (test-case "token byte coverage preserves every UTF-8 width boundary" + (let (base (parse-source arithmetic-parser "@")) + (for-each + (lambda (row) + (let* ((text (string (integer->char (car row)))) + (width (cadr row)) + (event (vector 'token 0 'raw text 0 width)) + (artifact + (artifact-replace + (artifact-replace + (artifact-replace base 'sourceDigest (sha256-text text)) + 'sourceByteLength width) 'events (list event)))) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-valid-for-source? artifact text) => #t) + (check (parse-artifact-roundtrip artifact) => text) + ;; Keep advertised coverage self-consistent but falsify UTF-8 width. + (vector-set! event 5 (+ width 1)) + (check (parse-artifact-valid? + (artifact-replace artifact 'sourceByteLength (+ width 1))) => #f))) + '((0 1) (127 1) (128 2) (2047 2) (2048 3) + (55295 3) (57344 3) (65535 3) (65536 4) (1114111 4))))) + (test-case "reconstruction owns its output and ignores untrusted advertised capacity" + (let* ((source "alpha + 2") + (artifact (parse-source arithmetic-parser source)) + (view (parse-artifact-roundtrip artifact))) + (string-set! view 0 #\b) + (check (parse-artifact-roundtrip artifact) => source) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-valid? + (artifact-replace artifact 'sourceByteLength #x100000000)) => #f)) + (let (empty (parse-source arithmetic-parser "")) + (check (parse-artifact-roundtrip empty) => ""))) (test-case "a missing token event fails closed" (let* ((artifact (parse-source arithmetic-parser "1 + 2")) (corrupt @@ -81,3 +153,7 @@ => #t))))) (export parse-artifact-tests) + +;; gxtest discovers only exported names ending in -test. +(def parse-artifact-test parse-artifact-tests) +(export parse-artifact-test) diff --git a/t/parse-cost-test.ss b/t/parse-cost-test.ss new file mode 100644 index 00000000..f790b669 --- /dev/null +++ b/t/parse-cost-test.ss @@ -0,0 +1,28 @@ +;;; Diagnostics must preserve engine results, failure and dynamic scope. +(import :std/test + (only-in :gerbil-parser/src/runtime/parse-cost current-parser-cost-observer with-parser-cost-stage admit-parser-allocation)) +(export parse-cost-test) +(def parse-cost-test + (test-suite "request-local parser cost observations" + (test-case "allocation admission preserves signed counter and GC boundaries" + (check (admit-parser-allocation 128. 0.) => 128.) + (check (admit-parser-allocation 0. 0.) => 0.) + (check (admit-parser-allocation -64. 0.) => #f) + (check (admit-parser-allocation 128. 1.) => #f) + (check (admit-parser-allocation -64. 1.) => #f)) + (test-case "disabled observation preserves zero and multiple values" + (check (current-parser-cost-observer) => #f) + (check (call-with-values (lambda () (with-parser-cost-stage 'zero (values))) list) => '()) + (check (call-with-values (lambda () (with-parser-cost-stage 'pair (values 1 2))) list) => '(1 2))) + (test-case "nested observation preserves values and thrown conditions" + (let (rows '()) + (parameterize ((current-parser-cost-observer (lambda (name row) (set! rows (cons (cons name row) rows))))) + (check (call-with-values + (lambda () (with-parser-cost-stage 'outer (with-parser-cost-stage 'inner (values 3 4)))) list) + => '(3 4)) + (check (with-catch (lambda (e) e) + (lambda () (with-parser-cost-stage 'failed (raise 'original-condition)))) => 'original-condition)) + (check (current-parser-cost-observer) => #f) + (check (map car rows) => '(failed outer inner)) + (check (cdr (assq 'completed (cdar rows))) => #f) + (check (cdr (assq 'completed (cdadr rows))) => #t))))) diff --git a/t/parser-ast-syntax-test.ss b/t/parser-ast-syntax-test.ss index 8fe16257..cbda9fa0 100644 --- a/t/parser-ast-syntax-test.ss +++ b/t/parser-ast-syntax-test.ss @@ -2,25 +2,40 @@ ;;; Canonical parser tests assert exact AST, fields, spans and source bytes. (import (only-in :std/test check test-case test-suite) - (only-in :gerbil-parser/languages/arithmetic/v1/parser - parse-arithmetic-v1) + (only-in :gerbil-parser/languages/arithmetic/parser + parse-arithmetic) (only-in :gerbil-parser/src/runtime/cst parse-artifact->cst) (only-in :gerbil-parser/src/testing/parser-ast - check-parser-ast parser-ast-diff parser-ast-pattern + check-parser-ast check-accepted-parse + parser-ast-diff parser-ast-pattern parser-artifact-ast-diff)) (export parser-ast-syntax-test) (def parser-ast-syntax-test (test-suite "parser-owned AST test syntax" + (test-case "accepted artifact macro is hygienic and evaluates once" + (let ((result "1") + (expected-source 'caller-binding) + (source-calls 0) + (candidate-calls 0)) + (check-accepted-parse + (begin (set! source-calls (fx+ source-calls 1)) result) + (begin + (set! candidate-calls (fx+ candidate-calls 1)) + (parse-arithmetic result)) + (parse-arithmetic result)) + (check source-calls => 1) + (check candidate-calls => 1) + (check expected-source => 'caller-binding))) (test-case "one source proves exact kinds, fields, spans, token and roundtrip" - (check-parser-ast (parse-arithmetic-v1 "1") "1" + (check-parser-ast (parse-arithmetic "1") "1" (node SourceFile 0 1 (field expression 0 1 (node NumberExpression 0 1 (field value 0 1 (token number "1" 0 1))))))) (test-case "a wrong field cannot pass as a matching source roundtrip" - (let (actual (parse-artifact->cst (parse-arithmetic-v1 "1"))) + (let (actual (parse-artifact->cst (parse-arithmetic "1"))) (check (parser-ast-diff actual (parser-ast-pattern @@ -31,7 +46,7 @@ (token number "1" 0 1))))))) => '((0) kind wrong expression)))) (test-case "byte span and token text differences name the failed leaf" - (let (actual (parse-artifact->cst (parse-arithmetic-v1 "1"))) + (let (actual (parse-artifact->cst (parse-arithmetic "1"))) (check (parser-ast-diff actual (parser-ast-pattern @@ -52,7 +67,7 @@ => '(() end 2 1)))) (test-case "source mismatch is part of the same AST acceptance contract" (check (parser-artifact-ast-diff - (parse-arithmetic-v1 "1") "2" + (parse-arithmetic "1") "2" (parser-ast-pattern (node SourceFile 0 1 (field expression 0 1 diff --git a/t/parser-runtime-test.ss b/t/parser-runtime-test.ss index 1c1585c7..7c6c9baa 100644 --- a/t/parser-runtime-test.ss +++ b/t/parser-runtime-test.ss @@ -2,12 +2,18 @@ ;;; -*- Gerbil -*- (import :std/test + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) :gerbil-parser/src/runtime/artifact :gerbil-parser/src/runtime/cst :gerbil-parser/src/runtime/parser :gerbil-parser/src/runtime/scan + (only-in :gerbil-parser/src/runtime/lexer lex-source) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-parse/prepared/receipt) + (only-in :gerbil-parser/src/runtime/significant + parser-significant-tokens parser-significant-joined) :gerbil-parser/src/runtime/token - :gerbil-parser/languages/arithmetic/v1/parser) + :gerbil-parser/languages/arithmetic/parser) (def (artifact-token-events artifact) (let loop ((rest (parse-artifact-events artifact)) (found '())) @@ -39,6 +45,18 @@ (def parser-runtime-tests (test-suite "parser runtime" + (test-case "fork projection preserves significant token order" + (let* ((tokens (lex-source arithmetic-parser "1 + 2")) + (prefix-reversed (list (caddr tokens) (cadr tokens) + (car tokens))) + (suffix (cdddr tokens))) + (let-values (((significant rest) + (parser-significant-joined + arithmetic-parser prefix-reversed suffix))) + (check significant + => (parser-significant-tokens arithmetic-parser tokens)) + (check rest + => (parser-significant-tokens arithmetic-parser suffix))))) (test-case "ParseArtifact is lossless and CST is an event projection" (let* ((source " 1 + (2 * value) ") (artifact (parse-source arithmetic-parser source)) @@ -159,6 +177,41 @@ (check (scan-number-literal/profile "1." 0 prefixes "_" suffixes #t #f) => 1))) + (test-case "many distinct GLR completions remain ambiguous" + (let* ((names + (map (lambda (n) + (string->symbol + (string-append "path" (number->string n)))) + (iota 12))) + (rules + (cons + (list 'source-file + (cons 'choice + (map (lambda (name) (list 'reference name)) + names))) + (map (lambda (name) + (list name + (list 'alias name + (list 'field 'value + (list 'token 'identifier))))) + names))) + (runtime + (lr-prepare + (compile-lr-spec rules 'source-file 'selective-glr))) + (failure + (with-catch + (lambda (condition) + (let (irritants (error-irritants condition)) + (and (pair? irritants) (car irritants)))) + (lambda () + (call-with-values + (lambda () + (lr-parse/prepared/receipt + runtime (list (make-token 'identifier "x" 0 1)))) + (lambda _ #f)))))) + (check (diagnostic-ref failure 'failureKind) + => 'selective-glr-ambiguity) + (check (diagnostic-ref failure 'distinctCompletions) => 12))) (test-case "failure emits one typed terminal and no partial CST" (let* ((source "1 + @") (artifact (parse-source arithmetic-parser source)) @@ -185,4 +238,6 @@ => #t) (check-exception (parse-artifact->cst artifact) true))))) -(export parser-runtime-tests) +(def parser-runtime-test parser-runtime-tests) + +(export parser-runtime-tests parser-runtime-test) diff --git a/t/parser-throughput-test.ss b/t/parser-throughput-test.ss new file mode 100644 index 00000000..390de800 --- /dev/null +++ b/t/parser-throughput-test.ss @@ -0,0 +1,33 @@ +;;; Protocol controls, not performance samples. +(import (only-in :std/test check check-exception test-case test-suite) + (only-in ./benchmarks/parser-throughput/protocol + parser-throughput-percentile parser-throughput-gate + parser-throughput-batch run-parser-throughput)) +(export parser-throughput-test) + +(def parser-throughput-test + (test-suite "Parser throughput protocol" + (test-case "nearest-rank uses request samples not batch totals" + (let (samples (list->vector (iota 100 1))) + (check (parser-throughput-percentile samples 95) => 95) + (check (parser-throughput-percentile samples 99) => 99)) + (check (parser-throughput-percentile '#(1 2 3) 95) => 3) + (check-exception (parser-throughput-percentile '#() 95) true) + (check-exception (parser-throughput-percentile '#(1) 0) true)) + (test-case "wall gates scale to each declared load" + (for-each (lambda (count) + (let (limit (/ count 100000)) + (check (parser-throughput-gate count limit) => #t) + (check (parser-throughput-gate count (+ limit 1/1000000)) => #f))) + '(1000 10000 100000)) + (check (parser-throughput-gate 100000 32) => #f) + (check (parser-throughput-gate 100000 0) => #f) + (check (parser-throughput-gate 0 1) => #f)) + (test-case "semantic mismatch cannot produce a passing sample" + (check-exception (parser-throughput-batch (lambda (_) 'wrong) + '#(changing) '#(expected)) true) + (check-exception (parser-throughput-batch values '#(1) '#()) true)) + (test-case "reject unsupported modes before starting a matrix" + (check-exception (run-parser-throughput "test" "sha256:test" values values values + 'partial-pass) true) + (check-exception (run-parser-throughput "test" "" values values values 'full) true)))) diff --git a/t/part-profile-test.ss b/t/part-profile-test.ss new file mode 100644 index 00000000..ec3c76cd --- /dev/null +++ b/t/part-profile-test.ss @@ -0,0 +1,190 @@ +;;; Recognition rules are executable POO values, including independent dialects. +(import :std/test + (only-in :clan/poo/object .o .ref) + :gerbil-parser/src/language/part-profile + (only-in :gerbil-parser/src/language/result-profile compile-result-profile ResultProfile. defresult-profile result-plan-recipe) + (only-in :gerbil-parser/src/runtime/shell-word make-shell-word-parser) + (only-in :gerbil-parser/src/runtime/scan make-bounded-text-profile-scanner make-bounded-literal-end-scanner) + (only-in :gerbil-parser/src/runtime/recognition recognition-node-kind recognition-node-children recognition-child-field) + (only-in :gerbil-parser/src/runtime/token make-token token-kind token-lexeme token-start token-end) + (only-in :gerbil-parser/src/language/source declare-source-language source-language-digest) + (only-in :gerbil-parser/src/runtime/source-engines ShellSourceStrategy.) + (only-in :gerbil-parser/t/fixtures/bash-products bash-commands bash-parts bash-simple-binding bash-parameter-binding bash-assignment-binding bash-results bash-word-regions bash-command-scanner bash-source-language)) +(export part-profile-test) +(def (rejects? thunk) (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(defpart-profile (template-parts :: self PartProfile.) + (contexts word HereDocument) + (bindings (template (.o (:: self BindingProfile.)))) + (rules ((word HereDocument) (prefix "@" next) (name SimpleParameter template))) + (literal LiteralPart)) +(defpart-profile (literal-parts :: self template-parts) (rules)) +(defresult-profile (template-results :: self ResultProfile.) + (projections + (Word TemplateWord ((parts parts)) (many fragment parts)) + (HereDocumentLine TemplateBody ((parts parts)) (many fragment parts)) + (LiteralPart TemplateLiteral ((text span)) (token content text text required)) + (SimpleParameter TemplateName ((text span)) (token identifier name text required)))) +(def (components profile text) + (let-values (((word assignment here) + (make-shell-word-parser bash-word-regions (compile-result-profile bash-results) + (admit-part-plan (compile-part-profile profile) (compile-result-profile bash-results) bash-word-regions)))) + (word (make-token 'word text 0 (u8vector-length (string->utf8 text)))))) +(def (assignment-components profile text) + (let-values (((word assignment here) + (make-shell-word-parser bash-word-regions (compile-result-profile bash-results) + (admit-part-plan (compile-part-profile profile) (compile-result-profile bash-results) bash-word-regions)))) + (assignment (make-token 'word text 0 (u8vector-length (string->utf8 text)))))) +(def part-profile-test + (test-suite "closed part recognition plans" + (test-case "independent template projection replaces every public word field and terminal through POO" + (let* ((results (compile-result-profile template-results)) + (parts (admit-part-plan (compile-part-profile template-parts) results bash-word-regions))) + (let-values (((word assignment here) (make-shell-word-parser bash-word-regions results parts))) + (let-values (((node tokens) (word (make-token 'raw "α@name😀" 7 18)))) + (check (recognition-node-kind node) => 'TemplateWord) + (check (map recognition-child-field (recognition-node-children node)) => '(fragment fragment fragment)) + (check (map token-kind tokens) => '(text name text)) + (check (map token-start tokens) => '(7 9 14)) + (check (map token-end tokens) => '(9 14 18))) + (let-values (((node tokens) (here (make-token 'raw "中@name" 0 8)))) + (check (recognition-node-kind node) => 'TemplateBody) + (check (map token-kind tokens) => '(text name)))))) + (test-case "missing word projection rejects and projection-only change alters Source identity" + (let* ((projection-declarations (caddr (result-plan-recipe (compile-result-profile bash-results)))) + (replacement (map (lambda (row) (if (eq? (car row) 'Word) + (cons 'Word (cons 'HereDocumentLine (cddr row))) row)) projection-declarations)) + (result-override (.o (:: self bash-results) projections: replacement))) + (check (rejects? (lambda () (admit-part-plan (compile-part-profile bash-parts) + (compile-result-profile (.o (:: self bash-results) projections: '())) bash-word-regions))) => #t) + (let (changed (declare-source-language "bash" "5.3" "bash-5.3-structured-source.v1" + (.o (:: self ShellSourceStrategy.) regions: bash-word-regions scanner: bash-command-scanner + results: result-override parts: bash-parts commands: bash-commands))) + (check (equal? (source-language-digest changed) (source-language-digest bash-source-language)) => #f)))) + (test-case "independent template uses the same execution operations and exact UTF-8 spans" + (let-values (((node tokens) (components template-parts "α@name😀"))) + (check (recognition-node-kind node) => 'Word) + (check (map token-kind tokens) => '(LiteralPart SimpleParameter LiteralPart)) + (check (map token-lexeme tokens) => '("α" "@name" "😀")) + (check (map token-start tokens) => '(0 2 7)) + (check (map token-end tokens) => '(2 7 11))) + (let-values (((node tokens) (components literal-parts "α@name😀"))) + (check (map token-kind tokens) => '(LiteralPart)) + (check (map token-lexeme tokens) => '("α@name😀")))) + (test-case "non-ASCII prefixes use the same ordered recognition and byte projection" + (let-values (((node tokens) + (components (.o (:: self template-parts) + rules: '(((word HereDocument) (prefix "λ" next) (name SimpleParameter template)))) + "中λname😀"))) + (check (map token-lexeme tokens) => '("中" "λname" "😀")) + (check (map token-start tokens) => '(0 3 9)) + (check (map token-end tokens) => '(3 9 13)))) + (test-case "one matcher controls literal boundary and recognition precedence" + (let (plan (compile-part-profile bash-parts)) + (check (part-step-kind (part-plan-match plan "$((1))" 0 6 'word)) => 'ArithmeticExpansion) + (check (part-step-kind (part-plan-match plan "$(x)" 0 4 'word)) => 'CommandSubstitution) + (check (part-plan-match plan "$x" 0 2 'SingleQuoted) => #f) + (check (part-plan-literal-end plan "a$x" 0 3 'word) => 1) + (check (part-plan-literal-end plan "a$x" 0 3 'SingleQuoted) => 3) + (check (part-plan-match plan "\\q" 0 2 'HereDocument) => #f) + (check (part-step-operation (part-plan-match plan "\\$" 0 2 'HereDocument)) => 'escape) + (check (rejects? (lambda () (part-plan-match plan "x" 0 1 'undeclared))) => #t))) + (test-case "literal runs agree with complete matching at every candidate boundary" + (let (plan (compile-part-profile bash-parts)) + (for-each (lambda (context) + (for-each (lambda (text) + (let (end (string-length text)) + (let starts ((start 0)) + (when (< start end) + (check (part-plan-literal-end plan text start end context) + => (let reference ((offset (+ start 1))) + (if (or (= offset end) (part-plan-match plan text offset end context)) + offset (reference (+ offset 1))))) + (starts (+ start 1)))))) + (list "plain" "中λ😀" "a$" "a$%b$x" "a\\qb\\$c" "a$(x)b" "a$((1))b" "a'α'b"))) + '(word HereDocument SingleQuoted))) + (let* ((profile (.o (:: self template-parts) + rules: '(((word HereDocument) (prefix "λ" next) (name SimpleParameter template))))) + (plan (compile-part-profile profile))) + (check (part-plan-literal-end plan "中λ" 0 2 'word) => 2) + (check (part-plan-literal-end plan "中λx" 0 3 'word) => 1))) + (test-case "literal scanning terminates on empty ranges and rejects invalid bounds" + (let (plan (compile-part-profile bash-parts)) + (check (part-plan-literal-end plan "" 0 0 'word) => 0) + (check (part-plan-literal-end plan "αx" 1 1 'word) => 1) + (for-each (lambda (range) + (check (rejects? (lambda () (part-plan-literal-end plan "αx" (car range) (cadr range) 'word))) => #t)) + '((-1 2) (2 1) (0 3) (0 1/2) (1/2 2))))) + (test-case "unknown dollars preserve token grouping instead of dropping boundaries" + (let-values (((node tokens) (components bash-parts "a$%b$x"))) + (check (map token-kind tokens) => '(LiteralPart LiteralPart SimpleParameter)) + (check (map token-lexeme tokens) => '("a" "$%b" "$x")))) + (test-case "owned declarations and returned recipes cannot mutate executable dispatch" + (let* ((declarations (list (list '(word HereDocument) (list 'prefix (string-copy "@") 'next) '(name SimpleParameter template)))) + (profile (.o (:: self template-parts) rules: declarations)) + (plan (compile-part-profile profile)) (recipe (part-plan-recipe plan))) + (string-set! (cadadr (car declarations)) 0 #\!) + (string-set! (cadadr (caadr recipe)) 0 #\!) + (check (part-step-kind (part-plan-match plan "@x" 0 2 'word)) => 'SimpleParameter) + (check (part-plan-match plan "!x" 0 2 'word) => #f))) + (test-case "malformed contexts, matchers, operations and ambiguous duplicates reject" + (for-each (lambda (profile) (check (rejects? (lambda () (compile-part-profile profile))) => #t)) + (list (.o (:: self template-parts) contexts: '(word word)) + (.o (:: self template-parts) rules: '(((other) (prefix "@" next) (name SimpleParameter template)))) + (.o (:: self template-parts) rules: '(((word) (prefix "" any) (escape EscapeSequence)))) + (.o (:: self template-parts) rules: '(((word) (prefix "@" any) (callback arbitrary)))) + (.o (:: self template-parts) rules: '(((word) (prefix "@" any) (pair CommandSubstitution 2 1)))) + (.o (:: self template-parts) rules: (append (.ref template-parts 'rules) (.ref template-parts 'rules)))))) + (test-case "loader admission rejects result and region disagreement before source execution" + (for-each (lambda (profile) + (check (rejects? (lambda () (admit-part-plan (compile-part-profile profile) + (compile-result-profile bash-results) bash-word-regions))) => #t)) + (list (.o (:: self template-parts) literal: 'Undeclared) + (.o (:: self template-parts) rules: '(((word) (prefix "@" next) (name Undeclared template)))) + (.o (:: self template-parts) rules: '(((word) (prefix "@(" any) (pair CommandSubstitution 2 1)))) + (.o (:: self bash-parts) rules: '(((word) (prefix "$((" any) (pair ArithmeticExpansion 3 1))))))) + (test-case "bounded lexical profiles and tries cannot see outside the current body" + (let ((name (make-bounded-text-profile-scanner '(run (alphabetic) 1 #f))) + (operator (make-bounded-literal-end-scanner '(":" ":-")))) + (check (name "αβgamma" 0 2) => 2) + (check (name "αβgamma" 2 2) => #f) + (check (operator ":-x" 0 1) => 1) + (check (operator ":-x" 0 2) => 2) + (check (operator ":-x" 0 0) => #f) + (check (name "x" 0 2) => #f))) + (test-case "parameter, simple and assignment names execute the shared text algebra" + (let* ((plan (compile-part-profile bash-parts)) (parameter (part-plan-binding plan 'parameter)) + (simple (part-plan-binding plan 'simple)) (assignment (part-plan-binding plan 'assignment))) + (check (binding-plan-name-end parameter "123x" 0 4) => 3) + (check (binding-plan-name-end simple "123x" 0 4) => 1) + (check (binding-plan-name-end parameter "α_1:-x" 0 6) => 3) + (check (binding-plan-name-end assignment "1x=" 0 3) => 0) + (check (binding-plan-prefix-end parameter "#" 0 1) => #f) + (check (binding-plan-prefix-end parameter "#α" 0 2) => 1) + (check (binding-plan-operator-end parameter "##rest" 0 6) => 2))) + (test-case "independent assignment syntax inherits the same projection without callbacks" + (let* ((binding (.o (:: self BindingProfile.) operators: '("=" ":="))) + (profile (.o (:: self template-parts) bindings: (list (cons 'template (.o (:: name BindingProfile.))) + (cons 'assignment binding))))) + (let-values (((node tokens) (assignment-components profile "α:=中@name"))) + (check (recognition-node-kind node) => 'Assignment) + (check (map token-lexeme tokens) => '("α" ":=" "中" "@name")) + (check (map token-start tokens) => '(0 2 4 7))) + (let-values (((node tokens) (assignment-components profile "1α:=中"))) (check node => #f)))) + (test-case "binding recipes own value overrides and reject nullable names or missing references" + (let* ((operator-declarations (list (string-copy ":="))) + (binding (.o (:: self BindingProfile.) operators: operator-declarations)) + (profile (.o (:: self template-parts) bindings: (list (cons 'template binding)))) + (plan (compile-part-profile profile)) (recipe (part-plan-recipe plan))) + (string-set! (car operator-declarations) 0 #\!) + (string-set! (car (caddr (cadar (list-ref recipe 3)))) 0 #\!) + (check (binding-plan-operator-end (part-plan-binding plan 'template) ":=x" 0 3) => 2)) + (for-each (lambda (profile) (check (rejects? (lambda () (compile-part-profile profile))) => #t)) + (list (.o (:: self template-parts) bindings: '()) + (.o (:: self template-parts) bindings: (list (cons 'template (.o (:: binding BindingProfile.) + name: '(run (alphabetic) 0 #f))))) + (.o (:: self template-parts) bindings: (list (cons 'template (.o (:: binding BindingProfile.) operators: '("=" "=")))))))) + (test-case "recognition values participate in source identity without changing source schema" + (let (changed (declare-source-language "bash" "5.3" "bash-5.3-structured-source.v1" + (.o (:: self ShellSourceStrategy.) regions: bash-word-regions scanner: bash-command-scanner + results: bash-results parts: literal-parts commands: bash-commands))) + (check (equal? (source-language-digest changed) (source-language-digest bash-source-language)) => #f))))) diff --git a/t/pure-function-fixture.ss b/t/pure-function-fixture.ss index 397b24dd..67ab60d2 100644 --- a/t/pure-function-fixture.ss +++ b/t/pure-function-fixture.ss @@ -5,7 +5,9 @@ (only-in :gerbil-parser/src/compiler/rust-pure-aot define-rust-pure string-first-word string-words string-rest-after-first-word string-last-word - string-before-last-word string-prefix? string-suffix? + string-before-last-word string-before + string-trim-start string-replace string-lowercase + string-prefix? string-suffix? string-in?)) (export normalized_title normalized_title_rust classify_first_word classify_first_word_rust @@ -14,12 +16,30 @@ owned_rest_after_first_word owned_rest_after_first_word_rust last_word last_word_rust before_last_word before_last_word_rust boundary_token? boundary_token_rust - count_words count_words_rust) + count_words count_words_rust + strip_word_annotation strip_word_annotation_rust + map_word_annotations map_word_annotations_rust + trim_start_only trim_start_only_rust + replace_and_append replace_and_append_rust + slug_words slug_words_rust) (define-rust-pure normalized_title normalized_title_rust ((input "&str")) "String" (string-trim input)) +(define-rust-pure trim_start_only trim_start_only_rust + ((input "&str")) "String" + (string-trim-start input)) + +(define-rust-pure replace_and_append replace_and_append_rust + ((template "&str") (value "&str")) "String" + (string-join (list (string-replace template "%s" value) value) "")) + +(define-rust-pure slug_words slug_words_rust + ((text "&str")) "String" + (let* ((lower (string-lowercase text))) + (string-join (string-words lower) "-"))) + (define-rust-pure classify_first_word classify_first_word_rust ((input "&str") (active "&[&str]") (complete "&[&str]")) "&'static str" @@ -60,3 +80,12 @@ ((input "&str")) "u64" (foldl (lambda (_word count) (+ count 1)) 0 (string-words input))) + +(define-rust-pure strip_word_annotation strip_word_annotation_rust + ((word "&str")) "String" + (string-before word "(")) + +(define-rust-pure map_word_annotations map_word_annotations_rust + ((input "&str")) "Vec" + (using ((strip_word_annotation "&str")) + (map strip_word_annotation (string-words input)))) diff --git a/t/qualification-output-test.ss b/t/qualification-output-test.ss new file mode 100644 index 00000000..0c1e8fd4 --- /dev/null +++ b/t/qualification-output-test.ss @@ -0,0 +1,49 @@ +;;; Qualification must finish large logs and preserve streaming error framing. +(import :std/test :std/io/tempfile + :gerbil-parser/t/support/qualification/process) +(export qualification-output-test) +(def (with-output-result mode proc) + (let ((log (make-temporary-file-name "parser-output-test-")) + (console (open-output-u8vector))) + (try + ;; The process protocol tees raw bytes; the Suite captures text output. + ;; Supply a byte sink only while the real child executes. + (let (result (parameterize ((current-output-port console)) + (run-observed-process ["gxi" "t/fixtures/qualification/output.ss" mode] + log timeout: 30 idle-timeout: 5))) + (when (equal? mode "large") + (write (list 'QUALIFICATION-OUTPUT-RESULT + 'bytes (u8vector-length (process-result-output result)) + 'elapsed (process-result-elapsed result) + 'status (process-result-status result) + 'reason (process-result-reason result))) + (newline) (force-output)) + (proc result)) + (finally + (close-output-port console) + (when (file-exists? log) (delete-file log)))))) +(def qualification-output-test + (test-suite "qualification output framing" + (test-case "ordinary identifier prefixes do not become errors" + (with-output-result "ordinary" + (lambda (result) + (check (process-result-status result) => 0) + (check (process-result-reason result) => #f)))) + (test-case "line, punctuation and Gambit errors fail closed" + (for-each + (lambda (mode) + (with-output-result mode + (lambda (result) (check (process-result-reason result) => 'error-output)))) + '("error" "punctuation" "gambit"))) + (test-case "bare error at EOF survives a buffer boundary" + (with-output-result "boundary" + (lambda (result) (check (process-result-reason result) => 'error-output)))) + (test-case "completed eight MiB log has bounded finalization" + (let (started (##current-time-point)) + (with-output-result "large" + (lambda (result) + (check (process-result-status result) => 0) + (check (process-result-reason result) => #f) + (check (u8vector-length (process-result-output result)) => (* 8 1024 1024)) + ;; Include result construction after the child monitor stops. + (check (< (- (##current-time-point) started) 30) => #t))))))) diff --git a/t/qualification-plans-test.ss b/t/qualification-plans-test.ss new file mode 100644 index 00000000..a52a0ffd --- /dev/null +++ b/t/qualification-plans-test.ss @@ -0,0 +1,27 @@ +(import :std/test :gerbil-parser/t/support/qualification/plans + (only-in :gerbil-parser/t/support/qualification/process process-child-arguments)) +(export qualification-plans-test) +(def qualification-plans-test + (test-suite "qualification plans" + (test-case "child argv is quoted data, without source imports" + (let (args (process-child-arguments "session" "" '("fixture" "λ a;$(not-a-command)"))) + (check (car args) => "-e") + (check (cadr args) + => "(load-module \"gerbil-parser/t/support/qualification/process-child\")") + (check (read (open-input-string (list-ref args 3))) + => '(apply gerbil-parser/t/support/qualification/process-child#main + (quote ("session" "" "fixture" "λ a;$(not-a-command)")))))) + (test-case "diagnostic modules are unique and ordered" + (check (diagnostic-modules '("gql-profile" "gql-actors" "gql-profile")) + => '("reduction-counts" "execution-counts" "matched-stages" "actors"))) + (test-case "one driver and one diagnostic build" + (let (plan (local-diagnostic-plan '("gql-profile" "gql-actors") #f)) + (check (length plan) => 2) + (check (length (qualification-stage-command (cadr plan))) => 7) + (check (list-ref (qualification-stage-command (cadr plan)) 2) + => "t/benchmarks/parser-stage-cost/benchmark.ss"))) + (test-case "prepared diagnostics do not rebuild" + (check (local-diagnostic-plan '("gql-profile" "gql-actors") #t) => '())) + (test-case "invalid prepared scope and unknown suites are rejected" + (check-exception (local-diagnostic-plan '("ffi") #t) true) + (check-exception (local-diagnostic-plan '("unknown") #f) true)))) diff --git a/t/quoted-string-profile-test.ss b/t/quoted-string-profile-test.ss new file mode 100644 index 00000000..ba799b1e --- /dev/null +++ b/t/quoted-string-profile-test.ss @@ -0,0 +1,50 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; Strict quoted text is shared lexical data, independent of language callbacks. +(import :std/test + (only-in :gerbil-parser/language-support deflanguage) + (only-in :gerbil-parser/src/grammar/lexical-algebra lexical-expression?) + (only-in :gerbil-parser/src/runtime/scan scan-quoted-string/profile) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success? parse-artifact-valid? parse-artifact-roundtrip)) +(export quoted-string-profile-test quoted-profile-language-grammar) + +(begin + (deflanguage quoted-profile + (syntax + (lexical + (root document) + (lex (string String (quoted-string-profile "'" "`'\\/fnrt" 4)) + (name Name (quoted-string-profile "`" "`'\\/fnrt" 4)) + (unknown Unknown (fallback))))) + (rules (document (node SourceFile (field item (choice string name)))))) + (bind-fixture-grammar-release quoted-profile "quoted-profile" "v1" "quoted-profile.v1") ) + +(def quoted-string-profile-test + (test-suite "shared quoted string profiles" + (test-case "strict escape profiles retain valid and invalid endpoints" + (for-each + (lambda (row) + (check (scan-quoted-string/profile (car row) 0 "'" "`'\\/fnrt" 4) => (cadr row))) + '(("'α'next" 3) ("'a''b'" 3) ("'a\nb'" 5) ("'\\n'" 4) + ("'\\u0041'" 8) ("'\\u١٢٣٤'" 8) ("'\\u00411'" 9) + ("'\\x'" #f) ("'\\u123'" #f) ("'\\uGGGG'" #f) ("'\\u²Ⅻ½４'" #f) ("'\\" #f) ("'" #f))) + (check (scan-quoted-string/profile "x'α'" 1 "'" "" 0) => 4) + (check (scan-quoted-string/profile "§α§tail" 0 "§" "" 0) => 3) + (check (scan-quoted-string/profile "'\\u0041'" 0 "'" "" 0) => #f)) + (test-case "compiler admission rejects malformed profiles" + (for-each + (lambda (row) (check (lexical-expression? row) => #f)) + '((quoted-string-profile "" "" 4) (quoted-string-profile "''" "" 4) + (quoted-string-profile "'" () 4) (quoted-string-profile "'" "" -1) + (quoted-string-profile "'" "" 9) (quoted-string-profile "'" "" 1.5) + (quoted-string-profile "'" ""))) + (check (lexical-expression? '(quoted-string-profile "§" "" 0)) => #t)) + (test-case "generated Scheme grammar keeps source and rejection lossless" + (for-each + (lambda (row) + (let* ((source (car row)) (artifact (parse-source quoted-profile-parser source))) + (check (parse-artifact-success? artifact) => (cdr row)) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + '(("'α'" . #t) ("`name`" . #t) ("'\\u0041'" . #t) + ("'\\q'" . #f) ("'\\u123'" . #f) ("'a''b'" . #f)))))) diff --git a/t/ranked-literal-scanner-test.ss b/t/ranked-literal-scanner-test.ss new file mode 100644 index 00000000..245e0ba0 --- /dev/null +++ b/t/ranked-literal-scanner-test.ss @@ -0,0 +1,125 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +;;; Shared literal trie preserves generated lexer's longest/rank/order contract. + +(import :std/test + (only-in :gerbil-parser/src/runtime/scan + make-ranked-literal-scanner make-bounded-literal-end-scanner + make-literal-trie literal-trie-child)) + +;;; Independent linear catalog oracle; no trie or candidate publication reuse. +(def (literal-reference entries source start admitted) + (let loop ((rest entries) (best #f)) + (if (null? rest) + (and best (list (cadr best) (+ start (string-length (car best))) (caddr best) (cadddr best))) + (let* ((entry (car rest)) (text (car entry)) (width (string-length text))) + (loop (cdr rest) + (if (and (or (not admitted) (vector-ref admitted (cadddr entry))) + (<= (+ start width) (string-length source)) + (string=? text (substring source start (+ start width))) + (or (not best) (> width (string-length (car best))) + (and (= width (string-length (car best))) + (or (> (caddr entry) (caddr best)) + (and (= (caddr entry) (caddr best)) (< (cadddr entry) (cadddr best))))))) + entry best)))))) + +(def ranked-literal-scanner-tests + (test-suite "ranked literal scanner" + (test-case "longest match precedes rank, then declaration order" + (let (scan + (make-ranked-literal-scanner + '(("a" short 99 0) + ("ab" first 2 4) + ("ab" later 2 5) + ("ab" higher 3 6) + ("α" unicode 0 7)))) + (check (scan "ab!" 0) => '(higher 2 3 6)) + (check (scan "a!" 0) => '(short 1 99 0)) + (check (scan "!α" 1) => '(unicode 2 0 7)) + (let (admitted (make-vector 8 #f)) + (vector-set! admitted 0 #t) + (vector-set! admitted 4 #t) + (check (scan "ab!" 0 admitted) => '(first 2 2 4)) + (vector-set! admitted 4 #f) + (check (scan "ab!" 0 admitted) => '(short 1 99 0))) + (check (scan "!" 0) => #f))) + (test-case "accepting prefixes agree with linear catalog under every mode mask" + (let* ((entries '(("a" short 9 0) ("ab" medium 1 1) ("abc" long 0 2) + ("ab" ranked 2 3) ("α" wide 1 4) ("α😀" wider 0 5))) + (scan (make-ranked-literal-scanner entries))) + (for-each (lambda (source) + (let offsets ((at 0)) + (when (<= at (string-length source)) + (check (scan source at) => (literal-reference entries source at #f)) + (let masks ((mask 0)) + (when (< mask 64) + (let (admitted (list->vector (map (lambda (index) (odd? (quotient mask (expt 2 index)))) (iota 6)))) + (check (scan source at admitted) => (literal-reference entries source at admitted))) + (masks (+ mask 1)))) + (offsets (+ at 1))))) + '("" "a" "ab" "abc" "abcd" "!abc!α😀?" "α" "α😀" "x")))) + (test-case "prepared widths and fresh results are isolated from caller mutation" + (let* ((text (string-copy "ab")) + (scan (make-ranked-literal-scanner (list (list text 'word 1 0)))) + (result (scan "ab!" 0))) + (string-set! text 0 #\x) + (set-car! result 'foreign) + (set-car! (cdr result) 999) + (check (scan "ab!" 0) => '(word 2 1 0)) + (check (scan "xab" 1) => '(word 3 1 0)) + (check (scan "xb" 0) => #f))) + (test-case "rank catalogs retain filtered winners and fresh terminal publication" + (let* ((entries (map (lambda (rank) (list "word" rank rank rank)) (iota 16))) + (scan (make-ranked-literal-scanner entries)) + (mask (make-vector 16 #f))) + (check (scan "word!" 0) => '(15 4 15 15)) + (for-each (lambda (rank) + (vector-set! mask rank #t) + (check (scan "word!" 0 mask) => (list rank 4 rank rank)) + (vector-set! mask rank #f)) (iota 16)) + (check (scan "word!" 0 mask) => #f) + (let (result (scan "word!" 0)) + (set-car! (cddr result) -100) + (check (scan "word!" 0) => '(15 4 15 15))))) + (test-case "dense root directory retains every ASCII edge and Unicode fallback" + (let* ((entries (append (map (lambda (code) + (list (string (integer->char code)) 'branch 0 (- code 32))) (iota 96 32)) + '(("λ😀" unicode 0 96)))) + (scan (make-ranked-literal-scanner entries)) + (mask (make-vector 97 #t))) + (for-each (lambda (source) + (let offsets ((start 0)) + (when (<= start (string-length source)) + (check (scan source start) => (literal-reference entries source start #f)) + (check (scan source start mask) => (literal-reference entries source start mask)) + (offsets (+ start 1))))) + (append (map (lambda (code) (string (integer->char code))) (iota 128)) + '("!λ😀?" "λ" "λ😀" ""))) + (vector-set! mask 96 #f) + (check (scan "λ😀" 0 mask) => #f))) + (test-case "deep prepared paths preserve bounded endpoints without preparation stack growth" + (let* ((text (make-string 8192 #\a)) + (scan (make-bounded-literal-end-scanner (list text "a" "β😀")))) + (check (scan text 0 8192) => 8192) + (check (scan text 0 8191) => 1) + (check (scan "!β😀!" 1 3) => 3) + (check (scan "!β😀!" 1 2) => #f) + (check (scan text -1 8192) => #f) + (check (scan text 0 8193) => #f))) + (test-case "child access evaluates supplied node and character once" + (let ((root (make-literal-trie '(("ab" . terminal)))) (calls 0)) + (check (not (not (literal-trie-child + (begin (set! calls (+ calls 1)) root) + (begin (set! calls (+ calls 1)) #\a)))) => #t) + (check calls => 2) + (check (literal-trie-child root #\z) => #f))) + (test-case "equal rank retains the earlier rule" + (let (scan + (make-ranked-literal-scanner + '(("word" first 1 3) + ("word" second 1 4)))) + (check (scan "word!" 0) => '(first 4 1 3)))))) + +;; gxtest discovers only exported names ending in -test. +(def ranked-literal-scanner-test ranked-literal-scanner-tests) +(export ranked-literal-scanner-test) diff --git a/t/ranked-parser-request-test.ss b/t/ranked-parser-request-test.ss new file mode 100644 index 00000000..f37eb8aa --- /dev/null +++ b/t/ranked-parser-request-test.ss @@ -0,0 +1,53 @@ +;;; Scanner substitution retains the canonical public parser request graph. +(import :std/test + (only-in :gerbil-parser/src/compiler/machine + parser-machine-with-ranked-scanner parser-machine-with-lr-action-selector parser-machine-lex + parser-machine-runtime parser-machine-grammar-digest) + (only-in :gerbil-parser/src/runtime/scan make-ranked-lexical-scanner-factory) + (only-in :gerbil-parser/src/runtime/parser parse-source) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success? parse-artifact-valid-for-source?) + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine) + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-layout-language-grammar tla-plus-core-language-grammar) + (only-in ./fixtures/ranked-publication reference-ranked-lexical-scanner-factory)) + +(import (only-in ./fixtures/lr-action-selection reference-lr-action-selector)) + +(def ranked-parser-request-test + (test-suite "ranked scanner public parser requests" + (test-case "complete fresh artifacts retain source and runtime ownership" + (let* ((machine (language-grammar-machine tla-plus-layout-language-grammar)) + (lexer (parser-machine-lex machine)) + (old-lexical (parser-machine-with-ranked-scanner machine reference-ranked-lexical-scanner-factory)) + (reference (parser-machine-with-lr-action-selector old-lexical reference-lr-action-selector)) + (candidate (parser-machine-with-ranked-scanner machine make-ranked-lexical-scanner-factory))) + (check (eq? lexer (parser-machine-lex machine)) => #t) + (check (eq? (parser-machine-runtime candidate) (parser-machine-runtime machine)) => #t) + (check (eq? (parser-machine-runtime old-lexical) (parser-machine-runtime machine)) => #t) + (check (parser-machine-grammar-digest candidate) => (parser-machine-grammar-digest machine)) + (for-each (lambda (source) + (let (expected (parse-source machine source)) + (check (parse-artifact-success? expected) => #t) + (check (parse-artifact-valid-for-source? expected source) => #t) + (check (parse-source reference source) => expected) + (check (parse-source candidate source) => expected) + (check (parse-source (parser-machine-with-ranked-scanner reference make-ranked-lexical-scanner-factory) source) => expected))) + '("---- MODULE Ranked ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" + "---- MODULE Unicode ----\n\\* é中😀\nInit == <<1, 2>>\n====\n")))) + (test-case "ordinary LR retains complete source artifacts after mode specialization" + (let* ((machine (language-grammar-machine tla-plus-core-language-grammar)) + (reference (parser-machine-with-ranked-scanner machine reference-ranked-lexical-scanner-factory)) + (candidate (parser-machine-with-ranked-scanner machine make-ranked-lexical-scanner-factory))) + (for-each (lambda (source) + (let (expected (parse-source machine source)) + (check (parse-artifact-success? expected) => #t) + (check (parse-artifact-valid-for-source? expected source) => #t) + (check (parse-source reference source) => expected) + (check (parse-source candidate source) => expected))) + '("---- MODULE Ranked ----\nInit == TRUE /\\ FALSE\n====\n" + "---- MODULE Unicode ----\n\\* é中😀\nInit == <<1, 2>>\n====\n")))) + (test-case "invalid factories fail before lexical publication" + (let (machine (language-grammar-machine tla-plus-layout-language-grammar)) + (check-exception (parser-machine-with-ranked-scanner machine #f) true) + (check-exception (parser-machine-with-lr-action-selector machine #f) true) + (check-exception (parser-machine-with-lr-action-selector machine (lambda (_rows _index _casefold _layout) 42)) true))))) +(export ranked-parser-request-test) diff --git a/t/ranked-regular-scanner-test.ss b/t/ranked-regular-scanner-test.ss new file mode 100644 index 00000000..b177ce37 --- /dev/null +++ b/t/ranked-regular-scanner-test.ss @@ -0,0 +1,165 @@ +;;; -*- Gerbil -*- +;;; Differential witness for the mode-local closed-run lexical DFA. + +(import (only-in :std/test check test-case test-suite) + (only-in :gerbil-parser/src/compiler/machine lexical-dispatch/ranked) + (only-in :gerbil-parser/src/runtime/scan + make-ranked-regular-scanner make-ranked-lexical-scanner-factory scan-whitespace scan-horizontal-whitespace + scan-newline scan-decimal-digits scan-identifier scan-number-literal)) + +(def regular-scanner + (make-ranked-regular-scanner + '((whitespace+ white 0 0) + (horizontal-whitespace+ horizontal 3 1) + (newline+ newline 2 2) + (decimal-digit+ digits 1 3) + (identifier name 0 4) + (number number 0 5)))) + +(def (legacy-regular-match source offset) + (lexical-dispatch/ranked + source offset + ((white (whitespace+)) + (horizontal (precedence 3 (horizontal-whitespace+))) + (newline (precedence 2 (newline+))) + (digits (precedence 1 (decimal-digit+))) + (name (identifier)) + (number (number))))) + +(def regular-entries + '((whitespace+ white 0 0) (horizontal-whitespace+ horizontal 3 1) + (newline+ newline 2 2) (decimal-digit+ digits 1 3) + (identifier name 0 4) (number number 0 5))) +(def independent-runs + (list scan-whitespace scan-horizontal-whitespace scan-newline + scan-decimal-digits scan-identifier scan-number-literal)) +(def (masked-run-reference source at mask) + (and (< at (string-length source)) + (let loop ((entries regular-entries) (runs independent-runs) (index 0) (best #f)) + (if (null? entries) best + (let* ((entry (car entries)) + (end (and (odd? (quotient mask (expt 2 index))) ((car runs) source at))) + (candidate (and end (list (cadr entry) end (caddr entry) index)))) + (loop (cdr entries) (cdr runs) (+ index 1) + (if (and candidate + (or (not best) (> end (cadr best)) + (and (= end (cadr best)) + (or (> (caddr candidate) (caddr best)) + (and (= (caddr candidate) (caddr best)) + (< index (cadddr best))))))) + candidate best))))))) + +(def fused-literals + '(("if" keyword 2 6) ("ifelse" longer 2 7) ("a" short 9 8) + ("ab" exact 0 9) ("α😀" wide 2 10) ("1e" prefix 6 11) ("1.2" fraction 10 12))) +(def (prefer-oracle left right) + (if (and right + (or (not left) (> (cadr right) (cadr left)) + (and (= (cadr right) (cadr left)) + (or (> (caddr right) (caddr left)) + (and (= (caddr right) (caddr left)) (< (cadddr right) (cadddr left))))))) right left)) +(def (fused-literal-oracle source at admissions) + (fold (lambda (entry best) + (let (end (+ at (string-length (car entry)))) + (prefer-oracle best + (and (vector-ref admissions (cadddr entry)) (<= end (string-length source)) + (string=? (car entry) (substring source at end)) + (list (cadr entry) end (caddr entry) (cadddr entry)))))) #f fused-literals)) + +(def ranked-regular-scanner-tests + (test-suite "ranked regular scanner" + (test-case "DFA agrees with individual scanners at every source offset" + (for-each + (lambda (source) + (let loop ((offset 0)) + (when (< offset (string-length source)) + (let (match (regular-scanner source offset)) + (check (and match (list (car match) (cadr match) + (caddr match))) + => (legacy-regular-match source offset))) + (loop (+ offset 1))))) + '("alpha-27 42\r\n" "αβ-٣\t\n?" "7.2 foo" " \t\n" + "_name-1+other" "1." "1e" "1e+" "1.2e+3" + "٧.٢E-٣" "8e+4z"))) + (test-case "start-state specialization agrees with independent runs in every rule subset" + (for-each (lambda (mask) + (let (scan (make-ranked-regular-scanner + (filter (lambda (entry) (odd? (quotient mask (expt 2 (cadddr entry))))) regular-entries))) + (for-each (lambda (source) + (let offsets ((at 0)) + (when (< at (string-length source)) + (check (scan source at) => (masked-run-reference source at mask)) + (offsets (+ at 1))))) + '("alpha-27 42\r\n" "αβ-٣\t\n?" "1." "1e" "1e+" "1.2e+3" + "٧.٢E-٣" "Ⅳ¼²名字" "8e+4z" "e.E+-1" "_name-1+other" "" "?")))) + (iota 64))) + (test-case "product automaton agrees with independent scans in every regular subset" + (let (factory (make-ranked-lexical-scanner-factory fused-literals)) + (for-each (lambda (mask) + (for-each (lambda (literal-mask) + (let* ((admissions (list->vector (map (lambda (index) + (odd? (quotient literal-mask (expt 2 index)))) (iota 13)))) + (entries (filter (lambda (entry) (odd? (quotient mask (expt 2 (cadddr entry))))) regular-entries)) + (scan (factory entries admissions))) + (for-each (lambda (source) + (let offsets ((at 0)) + (when (<= at (string-length source)) + (check (scan source at) + => (prefer-oracle (masked-run-reference source at mask) + (fused-literal-oracle source at admissions))) + (offsets (+ at 1))))) + '("if ifelse ifelseif" "a ab abc" "α😀 αβ-٣ \t\r\n" + "1.2e+3 1e+ 1." "Ⅳ¼² 名字 ?")))) + '(0 64 8191 4096))) (iota 64)))) + (test-case "mode preparation freezes masks and publishes fresh complete results" + (let* ((factory (make-ranked-lexical-scanner-factory fused-literals)) + (admissions (make-vector 13 #t)) + (scan (factory regular-entries admissions)) + (result (scan "if!" 0))) + (vector-set! admissions 6 #f) + (set-car! result 'foreign) + (check (scan "if!" 0) => '(keyword 2 2 6)) + (check (scan "ifelsewhere!" 0) => '(name 11 0 4)))) + (test-case "closed singleton modes own their literal snapshot" + (let* ((text (string-copy "α😀")) (entry (list text 'wide 2 0)) + (factory (make-ranked-lexical-scanner-factory (list entry)))) + (string-set! text 0 #\x) + (set-car! (cdr entry) 'foreign) + (let (scan (factory '() (vector #t))) + (check (scan "α😀!" 0) => '(wide 2 2 0)) + (check (scan "x😀!" 0) => #f) + (check (scan "α" 0) => #f) + (check (scan "" 0) => #f)))) + (test-case "regular scanning retains live winner payloads and fresh result cells" + (let* ((entry (list 'identifier 'name 0 0)) + (scan (make-ranked-regular-scanner (list entry))) + (first (scan "alpha!" 0))) + (check first => '(name 5 0 0)) + (set-car! first 'foreign) + (set-car! (cdr entry) 'updated) + (check (scan "a!" 0) => '(updated 1 0 0)) + (check (scan "α!" 0) => '(updated 1 0 0)) + (check (scan "!a" 1) => '(updated 2 0 0)) + (check (scan "" 0) => #f))) + (test-case "product scanning compare live regular priority against literal candidates" + (let* ((entry (list 'identifier 'name 0 0)) + (factory (make-ranked-lexical-scanner-factory '(("a" keyword 2 1)))) + (scan (factory (list entry) (vector #t #t)))) + (check (scan "a!" 0) => '(keyword 1 2 1)) + (set-car! (cddr entry) 9) + (check (scan "a!" 0) => '(name 1 9 0)) + (check (scan "alpha!" 0) => '(name 5 9 0)) + (check (scan "α!" 0) => '(name 1 9 0)))) + (test-case "longest match precedes rank and declaration order" + (check (regular-scanner " \t\n" 0) => '(white 3 0 0)) + (check (regular-scanner "\t\tx" 0) => '(horizontal 2 3 1)) + (check (regular-scanner "α-2+" 0) => '(name 3 0 4)) + (check (regular-scanner "12.3e+4!" 0) => '(number 7 0 5)) + (check (regular-scanner "12e+!" 0) => '(digits 2 1 3)) + (check (regular-scanner "?" 0) => #f)))) + +(export ranked-regular-scanner-tests) + +;; gxtest discovers only exported names ending in -test. +(def ranked-regular-scanner-test ranked-regular-scanner-tests) +(export ranked-regular-scanner-test) diff --git a/t/recognition-prepend-test.ss b/t/recognition-prepend-test.ss new file mode 100644 index 00000000..512db726 --- /dev/null +++ b/t/recognition-prepend-test.ss @@ -0,0 +1,58 @@ +#!/usr/bin/env gxi +(import :std/test + :gerbil-parser/src/runtime/funcs + :gerbil-parser/src/runtime/recognition + :gerbil-parser/src/runtime/reduce + (only-in :gerbil-parser/src/runtime/token make-token)) +(def recognition-prepend-test + (test-suite "borrowed singleton recognition composition" + (test-case "ordinary suffix sharing preserves both borrowed inputs" + (let* ((a (make-recognition-child #f (make-token 'number "1" 0 1))) + (b (make-recognition-child 'value (make-token 'number "2" 2 3))) + (prefix (list a)) (suffix (list b)) + (result (recognition-sequence-append prefix suffix))) + (check result => (append prefix suffix)) + (check (eq? (cdr result) suffix) => #t) + (check prefix => (list a)) + (check suffix => (list b)) + (check (recognition-sequence-start result 99) => 0) + (check (recognition-sequence-end result 99) => 3))) + (test-case "shared suffix accepts independent prefixes without mutation" + (let* ((child (make-recognition-child #f (make-token 'number "3" 4 5))) + (suffix (list child)) + (prefixes (map (lambda (n) + (list (make-recognition-child #f (make-token 'number "1" n (+ n 1))))) + '(0 1 2))) + (products (map (lambda (prefix) (recognition-sequence-append prefix suffix)) prefixes))) + (for-each (lambda (prefix product) + (check product => (append prefix suffix)) + (check (eq? (cdr product) suffix) => #t)) prefixes products) + (check suffix => (list child)))) + (test-case "relocation retains its context across an ordinary prefix" + (let* ((a (make-recognition-child #f (make-token 'number "1" 0 1))) + (b (make-recognition-child 'value (make-token 'number "2" 2 3))) + (suffix (recognition-sequence-relocate (list b) 7)) + (result (recognition-sequence-append (list a) suffix)) + (observed '())) + (recognition-sequence-for-each + (lambda (child delta moved?) (set! observed (cons (list child delta moved?) observed))) result) + (check (reverse observed) => (list (list a 0 #f) (list b 7 #t))) + (check (recognition-value-start (recognition-child-value b)) => 2) + (check (recognition-sequence-end result 99) => 10))) + (test-case "field and alias actions retain ordered canonical children" + (let* ((a (make-recognition-child #f (make-token 'number "1" 0 1))) + (b (make-recognition-child #f (make-token 'number "2" 2 3))) + (children (recognition-sequence-append (list a) (list b))) + (field (recognition-action-field 'value children 99)) + (alias (recognition-action-alias 'pair children 99))) + (check field => (recognition-action-field 'value (list a b) 99)) + (check alias => (recognition-action-alias 'pair (list a b) 99)))) + (test-case "deep singleton prepending remains ordered without branch flattening" + (let* ((children (map (lambda (n) (make-recognition-child #f (make-token 'number "1" n (+ n 1)))) + (iota 10000))) + (result (foldr (lambda (child suffix) (recognition-sequence-append (list child) suffix)) + '() children))) + (check (recognition-sequence->list result) => children) + (check (recognition-sequence-arity result) => 2) + (check (recognition-sequence-end result 99) => 10000))))) +(export recognition-prepend-test) diff --git a/t/recognition-sequence-test.ss b/t/recognition-sequence-test.ss new file mode 100644 index 00000000..7875ad25 --- /dev/null +++ b/t/recognition-sequence-test.ss @@ -0,0 +1,119 @@ +#!/usr/bin/env gxi +(import :std/test + :gerbil-parser/src/runtime/funcs + :gerbil-parser/src/runtime/recognition + :gerbil-parser/src/runtime/reduce + (only-in :gerbil-parser/src/runtime/token make-token) + "./scenarios/performance/selective-glr/scenario") +(def recognition-sequence-tests + (test-suite "ordered semantic sequence publication" + (test-case "mixed branch views isolate offsets and preserve untouched children" + (let* ((a (make-recognition-child #f (make-token 'number "1" 0 1))) + (b (make-recognition-child 'item (make-token 'number "2" 2 3))) + (c (make-recognition-child #f (make-token 'number "3" 4 5))) + (sequence (recognition-sequence-append + (recognition-sequence-relocate (list a) 10) + (recognition-sequence-append (list b) + (recognition-sequence-relocate (list c) -2)))) + (result (recognition-sequence->list sequence))) + (check (map (lambda (child) (recognition-value-start (recognition-child-value child))) result) + => '(10 2 2)) + (check (eq? (cadr result) b) => #t) + (check (recognition-child-field (cadr result)) => 'item) + (check (recognition-value-start (recognition-child-value a)) => 0) + (check (recognition-value-start (recognition-child-value c)) => 4) + (let (leaf (list b)) + (check (eq? (recognition-sequence->list leaf) leaf) => #t)))) + (test-case "deep concatenation preserves order and moved bounds without recursion" + (let* ((children (map (lambda (n) (make-recognition-child #f (make-token 'number "1" n (+ n 1)))) (iota 10000))) + (sequence (recognition-sequence-concatenate (map list children))) + (moved (recognition-sequence-relocate (recognition-sequence-relocate sequence 13) -5))) + (check (recognition-sequence->list sequence) => children) + (check (recognition-sequence-start moved 0) => 8) + (check (recognition-sequence-end moved 0) => 10008) + (let (values (recognition-sequence->list moved)) + (check (recognition-value-start (recognition-child-value (car values))) => 8) + (check (recognition-value-end (recognition-child-value (last values))) => 10008)))) + (test-case "pending branches retain independent relocation contexts" + (let* ((one (make-recognition-child 'a (make-token 'number "1" 0 1))) + (two (make-recognition-child 'b (make-token 'number "2" 1 2))) + (three (make-recognition-child 'c (make-token 'number "3" 2 3))) + (four (make-recognition-child 'd (make-token 'number "4" 3 4))) + (left (recognition-sequence-relocate + (recognition-sequence-append (list one) (list two)) 7)) + (right (recognition-sequence-append (list three) + (recognition-sequence-relocate (list four) -2))) + (sequence (recognition-sequence-append left right)) + (observed '())) + (recognition-sequence-for-each + (lambda (child delta moved?) (set! observed (cons (list child delta moved?) observed))) sequence) + (check (reverse observed) => (list (list one 7 #t) (list two 7 #t) + (list three 0 #f) (list four -2 #t))) + (let* ((nested (recognition-sequence-relocate + (recognition-sequence-append + (recognition-sequence-relocate (list one) -7) (list two)) 7)) + (observed '())) + (recognition-sequence-for-each + (lambda (child delta moved?) (set! observed (cons (list child delta moved?) observed))) nested) + (check (reverse observed) => (list (list one 0 #t) (list two 7 #t)))))) + (test-case "shared subtrees restore every nested view boundary" + (let* ((child (make-recognition-child 'value (make-token 'number "1" 1000 1001))) + (product + (let build ((depth 8)) + (if (zero? depth) (cons (list child) (list (list child 0 #f))) + (let* ((nested (build (- depth 1))) (sequence (car nested)) (expected (cdr nested))) + (cons (recognition-sequence-append + (recognition-sequence-relocate sequence depth) + (recognition-sequence-relocate sequence (- depth))) + (append (map (lambda (row) (list (car row) (+ (cadr row) depth) #t)) expected) + (map (lambda (row) (list (car row) (- (cadr row) depth) #t)) expected))))))) + (observed '())) + (recognition-sequence-for-each + (lambda (value delta moved?) (set! observed (cons (list value delta moved?) observed))) (car product)) + (check (reverse observed) => (cdr product)) + (check (map (lambda (value) (recognition-value-start (recognition-child-value value))) + (recognition-sequence->list (car product))) + => (map (lambda (row) (+ 1000 (cadr row))) (cdr product))))) + (test-case "visitor failures propagate before later branches" + (let* ((one (make-recognition-child #f (make-token 'number "1" 0 1))) + (sequence (recognition-sequence-append (list one) (list one))) + (condition (cons 'visitor 'stopped)) (visited 0)) + (check (with-catch (lambda (caught) (eq? caught condition)) + (lambda () + (recognition-sequence-for-each + (lambda (_child _delta _moved?) (set! visited (+ visited 1)) (raise condition)) sequence) + #f)) => #t) + (check visited => 1))) + (test-case "fusion can append a prefix retained by a materialized session" + (let (bare + (parameterize ((current-recognition-sequence-fusion-enabled? #f)) + (recognition-sequence-append + (list (make-recognition-child #f (make-token 'number "1" 1 2))) + (list (make-recognition-child #f (make-token 'number "2" 3 4)))))) + (parameterize ((current-recognition-sequence-fusion-enabled? #t)) + (let (combined (recognition-sequence-append bare + (list (make-recognition-child #f (make-token 'number "3" 5 6))))) + (check (recognition-sequence-arity combined) => 2) + (check (recognition-sequence-start combined 0) => 1) + (check (recognition-sequence-end combined 0) => 6) + (check (length (recognition-sequence->list combined)) => 3))))) + (test-case "field nesting and alias ranges survive deferred materialization" + (let* ((one (list (make-recognition-child 'inner (make-token 'number "1" 3 4)))) + (two (recognition-sequence-append one (list (make-recognition-child #f (make-token 'number "2" 5 6))))) + (moved (recognition-sequence-relocate two 7)) + (field (car (recognition-children-field 'outer moved 0))) + (alias (recognition-child-value (car (recognition-children-alias 'pair moved 0))))) + (check (recognition-children-field 'empty '() 99) => '()) + (check (recognition-sequence-start (recognition-sequence-relocate '() 7) 99) => 99) + (check (recognition-sequence-arity + (recognition-sequence-append (recognition-sequence-relocate '() 7) one)) => 1) + (check (recognition-fragment? (recognition-child-value (car (recognition-children-field 'outer one 0)))) => #t) + (check (recognition-child-field field) => 'outer) + (check (recognition-value-start (recognition-child-value field)) => 10) + (check (recognition-value-end alias) => 13) + (check (recognition-child-field (car (recognition-sequence->list (recognition-node-children alias)))) => 'inner))) + (test-case "GLR completion evidence remains canonical with fusion requested" + (parameterize ((current-recognition-sequence-fusion-enabled? #t)) + (check (selective-glr-scenario-pass? (selective-glr-scenario)) => #t))))) +(def recognition-sequence-test recognition-sequence-tests) +(export recognition-sequence-tests recognition-sequence-test) diff --git a/t/recognition-snapshot-test.ss b/t/recognition-snapshot-test.ss new file mode 100644 index 00000000..b5e77295 --- /dev/null +++ b/t/recognition-snapshot-test.ss @@ -0,0 +1,400 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- + +(import :std/test + (only-in :gerbil-parser/languages/arithmetic/parser arithmetic-parser parse-arithmetic) + (only-in :gerbil-parser/languages/hcl/parser hcl-parser parse-hcl) + (only-in :gerbil-parser/src/runtime/artifact + parse-artifact-success?) + (only-in :gerbil-parser/src/runtime/funcs recognition-sequence->list current-recognition-sequence-fusion-enabled?) + (only-in :gerbil-parser/src/runtime/recognition + recognition-child-value recognition-node? recognition-node-kind recognition-node-children recognition-fragment? recognition-fragment-children) + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-layout-language-grammar) + (only-in :gerbil-parser/src/language/descriptor language-grammar-machine) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/lr-parser + current-lr-event-program-enabled? current-lr-recognition-observer lr-prepare lr-parse/prepared + install-lr-runtime-direct-step! lr-runtime-fragment-reuse-safe? lr-initial-checkpoint + lr-checkpoint-feed lr-checkpoint-before-shift lr-checkpoint-fragment-compatible? + lr-recognition-view? + lr-recognition-view-base lr-recognition-view-delta + lr-recognition-fragment? lr-recognition-fragment-children lr-recognition-fragment-end + lr-recognition-fragment-token-count lr-recognition-fragment-value + lr-recognition-project) + (only-in :gerbil-parser/src/runtime/incremental + current-lr-probe-reuse-enabled? current-lr-source-index-enabled? incremental-session-source-index incremental-session-source-modes make-incremental-session incremental-session-artifact + incremental-session-recognition-root incremental-session-project-artifact + parse-incremental-session apply-edit make-edit)) + +(import (only-in :gerbil-parser/src/runtime/source-index source-index->lists source-index-audit source-index-storage source-index-count)) +(def (index-lists session) + (call-with-values (lambda () (source-index->lists (incremental-session-source-index session))) list)) +(def (check-source-index source session) + (check (source-index-audit (incremental-session-source-index session)) => #t) + (if (incremental-session-source-modes session) + (check (cadr (index-lists session)) => (incremental-session-source-modes session)) + (check (incremental-session-source-index session) => #f)) + (check (car (index-lists session)) => (car (index-lists (make-incremental-session hcl-parser source #t))))) + +(def (check-projection machine source session) + (let (root (incremental-session-recognition-root session)) + (check (or (lr-recognition-fragment? root) (lr-recognition-view? root)) => #t) + ;; Independent production replay publishes the full canonical artifact, + ;; including current token identity and coordinates after relative moves. + (check (incremental-session-project-artifact session) + => (incremental-session-artifact session)))) + +(def (check-edits source edits) + (parameterize ((current-lr-source-index-enabled? #t)) + (let (session (make-incremental-session hcl-parser source #t)) + (for-each + (lambda (edit) + (let-values (((next receipt) (parse-incremental-session session edit))) + (set! source (apply-edit source edit)) + (set! session next) + (check-source-index source next) + (check (incremental-session-artifact next) => (parse-hcl source)) + (when (incremental-session-recognition-root next) + (check-projection hcl-parser source next)))) + edits)))) + +(def (retained-prefix-piece root bound) + (let loop ((pending (list root))) + (and (pair? pending) + (let (piece (car pending)) + (cond + ((not (lr-recognition-fragment? piece)) (loop (cdr pending))) + ((and (<= (lr-recognition-fragment-end piece) bound) + (> (lr-recognition-fragment-token-count piece) 1)) piece) + (else (loop (append (lr-recognition-fragment-children piece) + (cdr pending))))))))) + +(def (contains-identical-piece? root expected) + (let loop ((pending (list root))) + (and (pair? pending) + (or (eq? (car pending) expected) + (loop (append + (cond ((lr-recognition-view? (car pending)) + (list (lr-recognition-view-base (car pending)))) + ((lr-recognition-fragment? (car pending)) + (lr-recognition-fragment-children (car pending))) + (else '())) + (cdr pending))))))) + +(def recognition-snapshot-test + (test-suite "private grammar recognition snapshots" + (test-case "source provenance shares old chunks and matches vector control" + (parameterize ((current-lr-source-index-enabled? #t)) + (let* ((source (apply string-append (make-list 80 "value = 001\n"))) + (session (make-incremental-session hcl-parser source #t)) + (before (index-lists session)) (edit (make-edit 0 0 "other = 002\n"))) + (let-values (((next receipt) (parse-incremental-session session edit))) + (check-source-index (apply-edit source edit) next) + (check (index-lists session) => before) + (check (> (cdr (assq 'sourceIndexSharedTokenCount receipt)) 300) => #t) + (check (< (cdr (assq 'sourceIndexFreshTokenCount receipt)) 100) => #t) + (check (+ (cdr (assq 'sourceIndexFreshTokenCount receipt)) (cdr (assq 'sourceIndexSharedTokenCount receipt))) + => (source-index-count (incremental-session-source-index next))) + (check (> (length (filter (lambda (storage) (memq storage (source-index-storage (incremental-session-source-index session)))) + (source-index-storage (incremental-session-source-index next)))) 5) => #t) + (parameterize ((current-lr-source-index-enabled? #f)) + (let-values (((control ignored) (parse-incremental-session session edit))) + (check (incremental-session-source-index control) => #f) + (check (incremental-session-artifact control) => (incremental-session-artifact next)) + (check (incremental-session-project-artifact control) => (incremental-session-project-artifact next)))))))) + (test-case "event window splices preserve index when grammar capture is dropped" + (check-edits "value = 1\nother = 2\n" + (list (make-edit 8 1 "9") (make-edit 9 0 " /* λ中😀 */") + (make-edit 9 17 "") (make-edit 8 1 "1")))) + (test-case "full artifact capture defaults to the qualified vector path" + (check (current-lr-source-index-enabled?) => #f) + (check (incremental-session-source-index (make-incremental-session hcl-parser "value = 1\n" #t)) => #f)) + (test-case "fused semantic publication matches original replay across edits" + (parameterize ((current-recognition-sequence-fusion-enabled? #t)) + (let* ((session (make-incremental-session hcl-parser "a = 1\nb = 2\n" #t)) + (root (incremental-session-recognition-root session)) + (value (recognition-child-value (car (recognition-sequence->list (lr-recognition-fragment-value root)))))) + (check (recognition-node? value) => #t) + (check + (let loop ((pending (list value))) + (and (pair? pending) + (let* ((value (car pending)) + (children (cond ((recognition-node? value) (recognition-node-children value)) + ((recognition-fragment? value) (recognition-fragment-children value)) + (else '())))) + (or (not (list? children)) + (loop (append (map recognition-child-value children) (cdr pending))))))) + => #t)) + (check-edits "value = \"λ中😀\" /* trivia */\nother = { nested = [1, 2] }\n" + (list (make-edit 0 0 "first = 001\n") + (make-edit 0 12 "") + (make-edit 0 0 "block { value = 3 }\n") + (make-edit 0 (string-length "block { value = 3 }\n") ""))))) + (test-case "ordinary sessions allocate no grammar snapshot" + (check (incremental-session-recognition-root + (make-incremental-session arithmetic-parser "1 + 2")) => #f)) + (test-case "grammar structure independently projects complete artifacts" + (for-each + (lambda (row) + (check-projection (car row) (cdr row) + (make-incremental-session (car row) (cdr row) #t))) + (list (cons arithmetic-parser "001 + 002 * (003 + 004)") + (cons hcl-parser "value = 001\nother = { nested = [1, 2] }\n") + (cons hcl-parser "value = \"λ中😀\" /* retained trivia */\n") + (cons hcl-parser "")))) + (test-case "captured prefix and successive topology changes remain projectable" + (let ((source (apply string-append (make-list 80 "value = 001\n"))) + (session (make-incremental-session + hcl-parser + (apply string-append (make-list 80 "value = 001\n")) #t))) + (for-each + (lambda (edit) + (let-values (((next ignored) (parse-incremental-session session edit))) + (set! source (apply-edit source edit)) + (set! session next) + (check (incremental-session-artifact session) => (parse-hcl source)) + (check-projection hcl-parser source session))) + (list (make-edit (* 40 12) 0 "other = 002\n") + (make-edit (* 40 12) 12 "") + (make-edit 0 0 "other = 002\n") + (make-edit 0 12 ""))))) + (test-case "resume physically shares unchanged grammar prefix fragments" + (let* ((source (apply string-append (make-list 80 "value = 001\n"))) + (session (make-incremental-session hcl-parser source #t)) + (piece (retained-prefix-piece (incremental-session-recognition-root session) 120))) + (check (lr-recognition-fragment? piece) => #t) + (let-values (((next ignored) + (parse-incremental-session session (make-edit 480 0 "other = 002\n")))) + (check (contains-identical-piece? (incremental-session-recognition-root next) piece) + => #t)))) + (test-case "nonterminal transfers skip unchanged declarations" + (let* ((source (apply string-append (make-list 80 "value = 001\n"))) + (session (make-incremental-session hcl-parser source #t))) + (let-values (((next receipt) + (parse-incremental-session session (make-edit 0 0 "other = 002\n")))) + (check (> (cdr (assq 'reusedRecognitionFragmentCount receipt)) 0) => #t) + (check (> (cdr (assq 'reusedSignificantTokenCount receipt)) 200) => #t) + (check (incremental-session-artifact next) + => (parse-hcl (string-append "other = 002\n" source))) + (check-projection hcl-parser source next)))) + (test-case "probe reuse preserves the entire captured transfer result" + (let* ((source (apply string-append (make-list 80 "value = 001\n"))) + (session (make-incremental-session hcl-parser source #t)) + (edit (make-edit 0 0 "other = 002\n"))) + (let-values (((cached receipt) (parse-incremental-session session edit))) + (parameterize ((current-lr-probe-reuse-enabled? #f)) + (let-values (((control control-receipt) (parse-incremental-session session edit))) + (check (incremental-session-artifact cached) + => (incremental-session-artifact control)) + (check (cdr (assq 'remainingSignificantTokenCount receipt)) + => (cdr (assq 'remainingSignificantTokenCount control-receipt))) + (check (> (cdr (assq 'fragmentProbeReuseTokenCount receipt)) 0) => #t) + (check (cdr (assq 'fragmentProbeReuseTokenCount control-receipt)) => 0) + (check-projection hcl-parser source cached) + (check-projection hcl-parser source control)))))) + (test-case "relative positions compose across edits inside reused suffixes" + (check-edits (apply string-append (make-list 12 "value = 001\n")) + (list (make-edit 0 0 "other = 002\n") + (make-edit 36 0 "third = 003\n") + (make-edit 0 12 "") + (make-edit 24 12 "") + (make-edit 0 0 "other = 002\n") + (make-edit 0 12 "")))) + (test-case "cursor maps bulk deletion and empty-source transitions" + (let (line "value = 001\n") + (check-edits (apply string-append (make-list 80 line)) + (list (make-edit 240 84 "") + (make-edit 0 0 (apply string-append (make-list 5 "other = 002\n"))) + (make-edit 0 60 "") + (make-edit 240 0 (apply string-append (make-list 7 line))) + (make-edit 0 960 "") + (make-edit 0 0 (apply string-append (make-list 80 line))))))) + (test-case "unchanged nested blocks skip their complete grammar interior" + (let* ((body (apply string-append (make-list 80 " value = 001\n"))) + (source (string-append "group {\n" body "}\n")) + (session (make-incremental-session hcl-parser source #t))) + (check (parse-artifact-success? (incremental-session-artifact session)) => #t) + (let-values (((next receipt) + (parse-incremental-session session (make-edit 0 0 "top = 0\n")))) + (check (incremental-session-artifact next) + => (parse-hcl (string-append "top = 0\n" source))) + (check (> (cdr (assq 'reusedSignificantTokenCount receipt)) 300) => #t) + (check (< (cdr (assq 'fragmentCursorVisitCount receipt)) 30) => #t) + (check-projection hcl-parser source next)) + (check-edits source + (list (make-edit 568 0 " other = 002\n") + (make-edit 568 14 "") + (make-edit 0 0 "top = 0\n") + (make-edit 0 8 ""))))) + (test-case "UTF8 and lexical boundary changes retain canonical ownership" + (check-edits "value = \"λ中😀\"\nother = [1, 2]\n" + (list (make-edit 0 0 "prefix = 0\n") + (make-edit 0 11 "") + (make-edit 0 0 "/* λ */ ") + (make-edit 0 9 ""))) + (check-edits "value = 1\nother = 2\nthird = 3\n" + (list (make-edit 9 0 " + 4") + (make-edit 9 4 "") + (make-edit 9 0 " /* comment */") + (make-edit 9 14 "") + (make-edit 0 0 "value = ") + (make-edit 0 8 "")))) + (test-case "UTF8 token leaves move inside actually reused structures" + (check-edits (apply string-append (make-list 30 "value = \"λ中😀\"\n")) + (list (make-edit 0 0 "prefix = 0\n") + (make-edit 0 11 "")))) + (test-case "failed parse clears transferred-work receipts" + ;; Trivia makes this checkpoint tail large enough to admit the catalog; + ;; unchanged declarations before the bad final identifier can transfer. + (let* ((line "value /* a */ = /* b */ 001 /* c */\n") + (source (apply string-append (make-list 80 line))) + (edit (make-edit (* 79 (string-length line)) 1 "@")) + (session (make-incremental-session hcl-parser source #t))) + (let-values (((next receipt) (parse-incremental-session session edit))) + (check (incremental-session-artifact next) + => (parse-hcl (apply-edit source edit))) + (check (incremental-session-recognition-root next) => #f) + (check (cdr (assq 'freshFallback? receipt)) => #t) + (check (assq 'reusedRecognitionFragmentCount receipt) => #f)))) + (test-case "an equivalent grammar in another prepared runtime is rejected" + (let* ((spec (compile-lr-spec + '((source-file (alias SourceFile (token identifier)))) + 'source-file)) + (first (lr-prepare spec)) (second (lr-prepare spec)) + (token (make-token 'identifier "x" 0 1)) (captured #f)) + (parameterize ((current-lr-recognition-observer + (lambda (root) (set! captured root)))) + (let-values (((root rest) (lr-parse/prepared first (list token)))) + (check rest => '()))) + (check (lr-recognition-fragment? captured) => #t) + (check (lr-checkpoint-fragment-compatible? + (lr-checkpoint-before-shift (lr-initial-checkpoint first '()) token) + captured) => #t) + (install-lr-runtime-direct-step! first + (lambda args (error "rejection test must not execute replacement reducer"))) + (check (lr-checkpoint-fragment-compatible? + (lr-checkpoint-before-shift (lr-initial-checkpoint first '()) token) + captured) => #f) + (check (lr-checkpoint-fragment-compatible? + (lr-checkpoint-before-shift (lr-initial-checkpoint second '()) token) + captured) => #f))) + (test-case "Wagner context-dependent repetition rejects identical public Item kinds" + ;; A c+ / B d+ shares the X X yield and semantic Item kind, while + ;; the distant marker selects a different grammar nonterminal/context. + (let* ((runtime + (lr-prepare + (compile-lr-spec + '((source-file + (alias SourceFile + (choice + (sequence (literal "A") (repeat1 (reference c))) + (sequence (literal "B") (repeat1 (reference d)))))) + (c (alias Item (sequence (token identifier) (token identifier)))) + (d (alias Item (sequence (token identifier) (token identifier))))) + 'source-file))) + (tokens (map (lambda (index) (make-token 'identifier "X" index (+ index 1))) + (iota 80 1))) + (captured #f)) + (check (lr-runtime-fragment-reuse-safe? runtime) => #t) + (parameterize ((current-lr-recognition-observer + (lambda (root) (set! captured root)))) + (let-values (((root rest) + (lr-parse/prepared runtime + (cons (make-token 'identifier "A" 0 1) tokens)))) + (check rest => '()))) + (let (item + (let walk ((pending (list captured))) + (and (pair? pending) + (let (piece (car pending)) + (if (not (lr-recognition-fragment? piece)) (walk (cdr pending)) + (let (children (recognition-sequence->list + (lr-recognition-fragment-value piece))) + (if (and (= (lr-recognition-fragment-token-count piece) 2) + (pair? children) (null? (cdr children)) + (recognition-node? (recognition-child-value (car children))) + (eq? (recognition-node-kind + (recognition-child-value (car children))) 'Item)) + piece + (walk (append (lr-recognition-fragment-children piece) + (cdr pending)))))))))) + (check (lr-recognition-fragment? item) => #t) + (let-values (((status after-marker) + (lr-checkpoint-feed (lr-initial-checkpoint runtime '()) + (make-token 'identifier "B" 0 1)))) + (check status => 'checkpoint) + (check (lr-checkpoint-fragment-compatible? + (lr-checkpoint-before-shift after-marker (car tokens)) item) => #f))) + (let-values (((root rest) + (lr-parse/prepared runtime + (cons (make-token 'identifier "B" 0 1) tokens)))) + (check (recognition-node-kind root) => 'SourceFile) + (check rest => '())))) + (test-case "short tail edits retain earlier checkpoints for inverse edits on both backends" + (for-each + (lambda (event?) + (parameterize ((current-lr-event-program-enabled? event?)) + (for-each + (lambda (units) + (let* ((source (string-join (make-list units "a = 1\n") "")) + (session (make-incremental-session hcl-parser source #t)) + (start (* 6 (- units 1)))) + (for-each + (lambda (insert?) + (let* ((edit (make-edit start (if insert? 0 6) (if insert? "b = 2\n" ""))) + (inverse (make-edit start (if insert? 6 0) (if insert? "" "a = 1\n")))) + (let-values (((next receipt) (parse-incremental-session session edit))) + (check (incremental-session-artifact next) => (parse-hcl (apply-edit source edit))) + (check-projection hcl-parser (apply-edit source edit) next) + (let-values (((restored receipt) (parse-incremental-session next inverse))) + (check (assq 'freshFallback? receipt) => #f) + (check (> (cdr (assq 'checkpointReusedShiftCount receipt)) 0) => #t) + (check (< (cdr (assq 'remainingSignificantTokenCount receipt)) units) => #t) + (check (incremental-session-artifact restored) => (parse-hcl source)) + (check-projection hcl-parser source restored))))) '(#t #f)))) '(400 800)))) + '(#f #t))) + (test-case "deep left recursion projects without recursive Scheme traversal" + (let* ((source (string-join (make-list 1600 "001") " + ")) + (session (make-incremental-session arithmetic-parser source #t))) + (check-projection arithmetic-parser source session) + (check (lr-recognition-fragment-token-count + (incremental-session-recognition-root session)) => 3199))) + (test-case "event-only fast paths discard stale grammar snapshots" + (let ((session (make-incremental-session arithmetic-parser "001 + 002" #t))) + (let-values (((next ignored) (parse-incremental-session session (make-edit 6 3 "003")))) + (check (incremental-session-artifact next) => (parse-arithmetic "001 + 003")) + (check (incremental-session-recognition-root next) => #f) + (let-values (((resumed ignored) + (parse-incremental-session next (make-edit 0 0 "004 + ")))) + (check-projection arithmetic-parser "004 + 001 + 003" resumed))))) + (test-case "selective GLR cannot publish a deterministic reuse root" + (let ((observed 'not-called) + (runtime + (lr-prepare + (compile-lr-spec + '((source-file (choice (reference first-path) (reference second-path))) + (first-path (alias SourceFile (field value (token identifier)))) + (second-path (alias SourceFile (field value (token identifier))))) + 'source-file 'selective-glr)))) + (parameterize ((current-lr-recognition-observer + (lambda (root) (set! observed root)))) + (let-values (((root rest) + (lr-parse/prepared runtime (list (make-token 'identifier "x" 0 1))))) + (check rest => '()))) + (check observed => #f) + (check (lr-runtime-fragment-reuse-safe? runtime) => #f))) + (test-case "layout execution retains only its authoritative artifact" + (let (session + (make-incremental-session + (language-grammar-machine tla-plus-layout-language-grammar) + "---- MODULE J ----\nInit ==\n /\\ TRUE\n /\\ FALSE\n====\n" #t)) + (check (parse-artifact-success? (incremental-session-artifact session)) => #t) + (check (incremental-session-recognition-root session) => #f) + (check (incremental-session-source-index session) => #f))) + (test-case "invalid input cannot retain a reusable root" + (let (session (make-incremental-session arithmetic-parser "1 +" #t)) + (check (parse-artifact-success? (incremental-session-artifact session)) => #f) + (check (incremental-session-recognition-root session) => #f) + (check (incremental-session-source-index session) => #f))))) + +(export recognition-snapshot-test) diff --git a/t/recovery-frontier-benchmark-test.ss b/t/recovery-frontier-benchmark-test.ss index cf2d2a93..8919408e 100644 --- a/t/recovery-frontier-benchmark-test.ss +++ b/t/recovery-frontier-benchmark-test.ss @@ -20,6 +20,7 @@ (let (algorithm-receipt (recovery-frontier-scenario)) (write algorithm-receipt) (newline) + (force-output) (check (recovery-frontier-scenario-pass? algorithm-receipt) => #t)) (run-scenario) (##gc) @@ -27,6 +28,12 @@ (benchmark-contract-run benchmark-path run-scenario)) (write benchmark-receipt) (newline) + (force-output) (check (benchmark-contract-receipt-pass? benchmark-receipt) => #t))))) -(export recovery-frontier-benchmark-tests) +(def recovery-frontier-benchmark-test recovery-frontier-benchmark-tests) +(export recovery-frontier-benchmark-tests recovery-frontier-benchmark-test) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/recovery-incremental-test.ss b/t/recovery-incremental-test.ss index 0660d9e6..7df8fbec 100644 --- a/t/recovery-incremental-test.ss +++ b/t/recovery-incremental-test.ss @@ -3,53 +3,76 @@ (import (only-in :std/test check check-exception test-case test-suite) - (only-in :gerbil-parser/languages/arithmetic/v1/parser - arithmetic-parser parse-arithmetic-v1) + (only-in :gerbil-parser/languages/arithmetic/parser + arithmetic-parser parse-arithmetic) + (only-in :gerbil-parser/languages/gql/parser + +gql-representative-query+ + gql-parser parse-gql) + (only-in :gerbil-parser/languages/hl7/parser + hl7-parser parse-hl7) + (only-in :gerbil-parser/languages/hcl/parser + hcl-parser parse-hcl) (only-in :gerbil-parser/src/runtime/artifact - parse-artifact-success? parse-artifact-valid? + parse-artifact-events parse-artifact-success? parse-artifact-valid? parse-artifact-roundtrip) (only-in :gerbil-parser/src/runtime/incremental - apply-edit make-edit parse-source/incremental) + apply-edit make-edit parse-source/incremental + make-incremental-session incremental-session-artifact + parse-incremental-session) + (only-in :gerbil-parser/src/runtime/parser parse-source) (only-in :gerbil-parser/src/runtime/recovery parse-source/recover)) (def (row-ref row key) (let (entry (assq key row)) (and entry (cdr entry)))) -(def recovery-incremental-tests +(def recovery-incremental-test (test-suite "recovery and incremental v1 sidecars" + (test-case "streamed fresh parse equals checkpointed source driver" + (for-each + (lambda (case) + (let ((machine (car case)) (source (cdr case))) + (check (parse-source machine source) + => (incremental-session-artifact + (make-incremental-session machine source))))) + (list + (cons arithmetic-parser "1 + 2 * (3 + 4)") + (cons arithmetic-parser "1 +") + (cons gql-parser +gql-representative-query+) + (cons hl7-parser + "MSH|^~\\&|LEGACY|AU|FHIR|AU|202609170900||ADT^A08|1|P|2.5.1\r")))) (test-case "UTF-8 edits are byte-bound and reject split characters" (check (apply-edit "λx" (make-edit 0 2 "a")) => "ax") - (check-exception (apply-edit "λx" (make-edit 0 1 "a")) true)) + (check (apply-edit "aλ中😀z" (make-edit 3 7 "b")) => "aλbz") + (check-exception (apply-edit "λx" (make-edit 0 1 "a")) true) + (check-exception (apply-edit "aλ中😀z" (make-edit 4 0 "b")) true)) (test-case "incremental prefix reuse publishes fresh-equivalent v1" (let* ((source "1 + 2") - (base (parse-arithmetic-v1 source)) + (base (parse-arithmetic source)) (source-edit (make-edit 4 1 "30"))) (let-values (((artifact receipt) (parse-source/incremental arithmetic-parser source base source-edit))) - (let (fresh (parse-arithmetic-v1 "1 + 30")) + (let (fresh (parse-arithmetic "1 + 30")) (check artifact => fresh) (check (parse-artifact-valid? artifact) => #t) (check (parse-artifact-roundtrip artifact) => "1 + 30") (check (row-ref receipt 'schema) => "gerbil-parser.incremental-receipt.v1") - (check (> (row-ref receipt 'reusedTokenCount) 0) => #t) - (check (> (row-ref receipt 'resumedSignificantTokenCount) 0) - => #t) + (check (row-ref receipt 'freshFallback?) => #f) (check (row-ref receipt 'publicationSchema) => "gerbil-parser.parse-artifact.v1"))))) (test-case "equal-width middle edits converge and reuse the token suffix" (let* ((source (string-append "(" (string-join (make-list 100 "001") " + ") ")")) - (base (parse-arithmetic-v1 source)) + (base (parse-arithmetic source)) (edit-start (+ 1 (* 50 6))) (source-edit (make-edit edit-start 3 "002"))) (let-values (((artifact receipt) (parse-source/incremental arithmetic-parser source base source-edit))) (let (fresh - (parse-arithmetic-v1 + (parse-arithmetic (apply-edit source source-edit))) (check artifact => fresh) (check (row-ref receipt 'suffixByteDelta) => 0) @@ -57,26 +80,437 @@ (check (row-ref receipt 'reusedSuffixTokenCount) => (row-ref receipt 'convergedSuffixTokenCount)) (check (row-ref receipt 'relocatedSuffixTokenCount) => 0) - (check (> (row-ref receipt 'resumedSignificantTokenCount) 90) - => #t) + (check (< (row-ref receipt 'relexedByteCount) 128) => #t) + (check (row-ref receipt 'resumedSignificantTokenCount) => 0) + (check (> (row-ref receipt + 'reusedRecognitionEventCount) 0) => #t) (check (< (row-ref receipt 'remainingSignificantTokenCount) 120) => #t))))) + (test-case "persistent checkpoints resume sequential middle edits" + (let* ((source (string-join (make-list 160 "001") " + ")) + (session (make-incremental-session arithmetic-parser source)) + (first-edit (make-edit (* 80 6) 3 "0002 + 003"))) + (let-values (((next first-receipt) + (parse-incremental-session session first-edit))) + (let* ((first-source (apply-edit source first-edit)) + (second-edit (make-edit (+ (* 120 6) 7) 3 "0003"))) + (check (incremental-session-artifact next) + => (parse-arithmetic first-source)) + (check (> (row-ref first-receipt + 'checkpointReusedShiftCount) 0) + => #t) + (let-values (((final second-receipt) + (parse-incremental-session next second-edit))) + (check (incremental-session-artifact final) + => (parse-arithmetic + (apply-edit first-source second-edit))) + (check (> (row-ref second-receipt + 'reusedRecognitionEventCount) 0) + => #t)))))) + (test-case "checkpoint byte cursors cover late tokens and trivia edits" + (let* ((source (string-join (make-list 128 "001") " + ")) + (session (make-incremental-session arithmetic-parser source)) + (late-edit + (make-edit (- (string-length source) 3) 3 "0002 + 003"))) + (let-values (((next receipt) + (parse-incremental-session session late-edit))) + (let* ((late-source (apply-edit source late-edit)) + (trivia-edit (make-edit 5 1 " "))) + (check (incremental-session-artifact next) + => (parse-arithmetic late-source)) + (check (> (row-ref receipt 'checkpointReusedShiftCount) 200) + => #t) + (let-values (((final _receipt) + (parse-incremental-session next trivia-edit))) + (check (incremental-session-artifact final) + => (parse-arithmetic + (apply-edit late-source trivia-edit)))))))) + (test-case "same-width trivia edits reuse recognition across sessions" + (let* ((source "001 + 002 + 003") + (session (make-incremental-session arithmetic-parser source)) + (first-edit (make-edit 3 1 "\t"))) + (let-values (((next first-receipt) + (parse-incremental-session session first-edit))) + (let* ((first-source (apply-edit source first-edit)) + (second-edit (make-edit 9 1 "\t"))) + (check (incremental-session-artifact next) + => (parse-arithmetic first-source)) + (check (eq? (car (parse-artifact-events + (incremental-session-artifact session))) + (car (parse-artifact-events + (incremental-session-artifact next)))) + => #t) + (check (> (row-ref first-receipt + 'reusedRecognitionEventCount) 0) => #t) + (check (row-ref first-receipt 'relexedByteCount) => 1) + (check (row-ref first-receipt 'relexStopByte) => 4) + (check (row-ref first-receipt + 'remainingSignificantTokenCount) => 0) + (let-values (((again second-receipt) + (parse-incremental-session next second-edit))) + (check (incremental-session-artifact again) + => (parse-arithmetic + (apply-edit first-source second-edit))) + (check (> (row-ref second-receipt + 'reusedRecognitionEventCount) 0) => #t)))))) + (test-case "generic significant edits reuse events and retire later checkpoints" + (let* ((source (string-join (make-list 160 "001") " + ")) + (session (make-incremental-session arithmetic-parser source)) + (first-edit (make-edit (* 80 6) 3 "002"))) + (let-values (((next first-receipt) + (parse-incremental-session session first-edit))) + (let* ((first-source (apply-edit source first-edit)) + (second-edit (make-edit (* 120 6) 3 "003"))) + (check (incremental-session-artifact next) + => (parse-arithmetic first-source)) + (check (> (row-ref first-receipt + 'reusedRecognitionEventCount) 0) => #t) + (check (row-ref first-receipt + 'remainingSignificantTokenCount) => 1) + (check (row-ref first-receipt 'relexedByteCount) => 3) + (check (eq? (car (parse-artifact-events + (incremental-session-artifact session))) + (car (parse-artifact-events + (incremental-session-artifact next)))) + => #t) + (let-values (((again second-receipt) + (parse-incremental-session next second-edit))) + (let* ((second-source + (apply-edit first-source second-edit)) + (third-edit (make-edit (* 150 6) 3 "0004 + 005"))) + (check (incremental-session-artifact again) + => (parse-arithmetic second-source)) + (check (> (row-ref second-receipt + 'reusedRecognitionEventCount) 0) => #t) + (let-values (((final third-receipt) + (parse-incremental-session again third-edit))) + (check (incremental-session-artifact final) + => (parse-arithmetic + (apply-edit second-source third-edit))) + (check (> (row-ref third-receipt + 'checkpointReusedShiftCount) 0) + => #t)))))))) + (test-case "width-changing token edits relocate events without suffix LR" + (for-each + (lambda (case) + (let* ((source (car case)) + (source-edit (cadr case)) + (session (make-incremental-session arithmetic-parser source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-arithmetic + (apply-edit source source-edit))) + (check (parse-artifact-valid? + (incremental-session-artifact next)) => #t) + (check (> (row-ref receipt + 'reusedRecognitionEventCount) 0) => #t) + (check (row-ref receipt 'resumedSignificantTokenCount) => 0) + (check (row-ref receipt 'suffixByteDelta) => 1)))) + (list (list "001 + 002 + 003" (make-edit 6 3 "0002")) + (list "001 + 002 + 003" (make-edit 3 1 " ")) + (list "λ + 2 + 3" (make-edit 2 1 " "))))) + (test-case "certified multi-token window preserves LR event topology" + (let* ((source (string-join (make-list 160 "001") " + ")) + (session (make-incremental-session arithmetic-parser source)) + (first-edit (make-edit (* 80 6) 9 "0002 + 0003"))) + (let-values (((next first-receipt) + (parse-incremental-session session first-edit))) + (let* ((first-source (apply-edit source first-edit)) + (second-edit (make-edit (+ (* 120 6) 2) 9 + "004 + 005"))) + (check (incremental-session-artifact next) + => (parse-arithmetic first-source)) + (check (parse-artifact-valid? + (incremental-session-artifact next)) => #t) + (check (> (row-ref first-receipt + 'reusedRecognitionEventCount) 0) => #t) + (check (row-ref first-receipt 'suffixByteDelta) => 2) + (check (row-ref first-receipt + 'remainingSignificantTokenCount) => 3) + (let-values (((again second-receipt) + (parse-incremental-session next second-edit))) + (check (incremental-session-artifact again) + => (parse-arithmetic + (apply-edit first-source second-edit))) + (check (> (row-ref second-receipt + 'reusedRecognitionEventCount) 0) + => #t)))))) + (test-case "multi-token literal action change falls back to LR" + (let* ((source "001 + 002 + 003") + (session (make-incremental-session arithmetic-parser source)) + (source-edit (make-edit 0 9 "004 * 005"))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-arithmetic + (apply-edit source source-edit))) + (check (row-ref receipt 'reusedRecognitionEventCount) => #f)))) + (test-case "contracted and Unicode token windows equal fresh artifacts" + (for-each + (lambda (case) + (let* ((source (car case)) + (source-edit (cadr case)) + (session (make-incremental-session arithmetic-parser source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-arithmetic + (apply-edit source source-edit))) + (check (parse-artifact-valid? + (incremental-session-artifact next)) => #t) + (check (> (row-ref receipt + 'reusedRecognitionEventCount) 0) => #t)))) + (list (list "0001 + 0002 + 0003" + (make-edit 0 11 "004 + 005")) + (list "λ + 2 + 3" + (make-edit 0 6 "λ + 4"))))) + (test-case "trivia token count changes preserve HCL recognition" + (let* ((source "x = 1 /*a*/\ny = 2\n") + (session (make-incremental-session hcl-parser source))) + (for-each + (lambda (source-edit) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-hcl + (apply-edit source source-edit))) + (check (parse-artifact-valid? + (incremental-session-artifact next)) => #t) + (check (> (row-ref receipt + 'reusedRecognitionEventCount) 0) => #t) + (check (row-ref receipt + 'remainingSignificantTokenCount) => 0))) + (list (make-edit 6 5 "/*a*/ /*b*/") + (make-edit 6 5 "")))) + (let* ((source "x = 1 /*a*/ /*b*/\ny = 2\n") + (session (make-incremental-session hcl-parser source)) + (source-edit (make-edit 6 11 "/*c*/"))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-hcl (apply-edit source source-edit))) + (check (parse-artifact-valid? + (incremental-session-artifact next)) => #t) + (check (> (row-ref receipt + 'reusedRecognitionEventCount) 0) => #t)))) + (test-case "significant tokens align across changed trivia counts" + (for-each + (lambda (case) + (let* ((source (car case)) + (source-edit (cadr case)) + (session (make-incremental-session hcl-parser source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-hcl + (apply-edit source source-edit))) + (check (parse-artifact-valid? + (incremental-session-artifact next)) => #t) + (check (> (row-ref receipt + 'reusedRecognitionEventCount) 0) => #t) + (check (row-ref receipt + 'remainingSignificantTokenCount) => 1)))) + (list + (list "x = 1 /*a*/\ny = 2\n" + (make-edit 4 7 "3 /*a*/ /*b*/")) + (list "x = 3 /*a*/ /*b*/\ny = 2\n" + (make-edit 4 13 "4 /*c*/"))))) + (test-case "changed LR literal action rejects mixed window reuse" + (let* ((source "x = 1 /*a*/ + 2\n") + (source-edit (make-edit 4 11 "3 /*a*/ /*b*/ *")) + (session (make-incremental-session hcl-parser source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-hcl (apply-edit source source-edit))) + (check (row-ref receipt 'reusedRecognitionEventCount) => #f)))) + (test-case "literal action edits require LR re-execution" + (let* ((source "001 + 002") + (session (make-incremental-session arithmetic-parser source)) + (source-edit (make-edit 4 1 "*"))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-arithmetic + (apply-edit source source-edit))) + (check (row-ref receipt 'reusedRecognitionEventCount) => #f)))) + (test-case "trivia fast path rejects a lexical kind change" + (let* ((source "001 + 002") + (session (make-incremental-session arithmetic-parser source)) + (source-edit (make-edit 3 1 "+"))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-arithmetic + (apply-edit source source-edit))) + (check (row-ref receipt 'reusedRecognitionEventCount) => #f)))) + (test-case "Unicode and contextual trivia retain fresh parse products" + (for-each + (lambda (case) + (let* ((machine (car case)) + (source (cadr case)) + (source-edit (caddr case)) + (session (make-incremental-session machine source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-source machine + (apply-edit source source-edit))) + (check (> (row-ref receipt + 'reusedRecognitionEventCount) 0) => #t)))) + (list (list arithmetic-parser "λ + 2" (make-edit 2 1 "\t")) + (list gql-parser "CREATE GRAPH mygraph ANY" + (make-edit 6 1 "\t"))))) + (test-case "session falls back on a rejected edit" + (let* ((source (string-join (make-list 96 "001") " + ")) + (session (make-incremental-session arithmetic-parser source)) + (source-edit (make-edit (* 48 6) 3 "???"))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-arithmetic + (apply-edit source source-edit))) + (check (parse-artifact-success? + (incremental-session-artifact next)) + => #f) + (check (row-ref receipt 'checkpointReusedShiftCount) => 0)))) + (test-case "rejected session can accept a repairing edit" + (let* ((source "001 + ???") + (session (make-incremental-session arithmetic-parser source)) + (source-edit (make-edit 6 3 "002"))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-arithmetic "001 + 002")) + (check (row-ref receipt 'checkpointReusedShiftCount) => 0)))) + (test-case "session checkpoints survive inserted tokens and byte offsets" + (let* ((source (string-join (make-list 96 "001") " + ")) + (session (make-incremental-session arithmetic-parser source)) + (first-edit (make-edit (* 48 6) 3 "001 + 002"))) + (let-values (((next _receipt) + (parse-incremental-session session first-edit))) + (let* ((first-source (apply-edit source first-edit)) + (second-edit (make-edit (+ (* 72 6) 6) 3 "0003"))) + (let-values (((final receipt) + (parse-incremental-session next second-edit))) + (check (incremental-session-artifact final) + => (parse-arithmetic + (apply-edit first-source second-edit))) + (check (> (row-ref receipt + 'reusedRecognitionEventCount) 0) + => #t)))))) + (test-case "GQL session preserves contextual parse after an identifier edit" + (let* ((source "CREATE GRAPH mygraph ANY") + (session (make-incremental-session gql-parser source)) + (source-edit (make-edit 13 7 "newgraph"))) + (let-values (((artifact receipt) + (parse-source/incremental + gql-parser source + (parse-gql source) source-edit))) + (check artifact + => (parse-gql + (apply-edit source source-edit))) + (check (row-ref receipt 'freshFallback?) => #f)) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-gql + (apply-edit source source-edit))) + (check (row-ref receipt 'freshFallback?) => #f) + (check (row-ref receipt 'checkpointReusedShiftCount) => 0)))) + (test-case "GQL edit after a GLR fork retains the ordinary path" + (let* ((source "CREATE GRAPH mygraph ANY") + (session (make-incremental-session gql-parser source)) + (source-edit (make-edit 13 7 "yrgraph"))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-gql + (apply-edit source source-edit))) + (check (row-ref receipt 'reusedRecognitionEventCount) => #f)))) + (test-case "GQL retains deterministic LR checkpoints before a GLR fork" + (let* ((source +gql-representative-query+) + (source-edit + (make-edit (- (string-length source) 7) 6 "res")) + (session (make-incremental-session gql-parser source))) + (let-values (((next receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) + => (parse-gql + (apply-edit source source-edit))) + (check (> (row-ref receipt 'reusedRecognitionEventCount) 0) => #t) + (check (row-ref receipt 'freshFallback?) => #f) + (let (next-edit (make-edit 0 5 "match")) + (let-values (((again _receipt) + (parse-incremental-session next next-edit))) + (check (incremental-session-artifact again) + => (parse-gql + (apply-edit + (apply-edit source source-edit) + next-edit)))))))) + (test-case "GQL contextual edits agree with fresh parsing across positions" + (for-each + (lambda (case) + (let* ((source (car case)) + (source-edit (cdr case)) + (edited (apply-edit source source-edit)) + (fresh (parse-gql edited)) + (session (make-incremental-session gql-parser source))) + (let-values (((artifact _receipt) + (parse-source/incremental + gql-parser source + (parse-gql source) source-edit))) + (check artifact => fresh)) + (let-values (((next _receipt) + (parse-incremental-session session source-edit))) + (check (incremental-session-artifact next) => fresh)))) + (list + (cons "CREATE GRAPH mygraph ANY" (make-edit 0 6 "create")) + (cons "CREATE GRAPH mygraph ANY" (make-edit 7 5 "GRAPH")) + (cons "CREATE GRAPH mygraph ANY" (make-edit 13 7 "return")) + (cons "match (n) return n\n" (make-edit 7 1 "CREATE")) + (cons "match (n) return n\n" (make-edit 10 6 "RETURN"))))) (test-case "UTF-8 byte shifts relocate rather than falsely reuse a suffix" (let* ((source "λ + 2") - (base (parse-arithmetic-v1 source)) + (base (parse-arithmetic source)) (source-edit (make-edit 0 2 "name"))) (let-values (((artifact receipt) (parse-source/incremental arithmetic-parser source base source-edit))) (check artifact - => (parse-arithmetic-v1 + => (parse-arithmetic (apply-edit source source-edit))) (check (row-ref receipt 'suffixByteDelta) => 2) (check (> (row-ref receipt 'convergedSuffixTokenCount) 0) => #t) (check (row-ref receipt 'reusedSuffixTokenCount) => 0) (check (row-ref receipt 'relocatedSuffixTokenCount) => (row-ref receipt 'convergedSuffixTokenCount))))) + (test-case "token merge scans through the changed boundary" + (let* ((source "name + 2 + 3") + (base (parse-arithmetic source)) + (source-edit (make-edit 4 0 "x"))) + (let-values (((artifact receipt) + (parse-source/incremental + arithmetic-parser source base source-edit))) + (check artifact => (parse-arithmetic "namex + 2 + 3")) + (check (> (row-ref receipt 'relexedByteCount) 4) => #t) + (check (< (row-ref receipt 'relexedByteCount) + (string-length (apply-edit source source-edit))) + => #t)))) + (test-case "external lexical scanners retain complete suffix re-lexing" + (let* ((source + "MSH|^~\\&|LEGACY|AU|FHIR|AU|202609170900||ADT^A08|1|P|2.5.1\r") + (base (parse-hl7 source)) + (source-edit (make-edit 11 6 "NEWAPP"))) + (let-values (((artifact receipt) + (parse-source/incremental + hl7-parser source base source-edit))) + (check artifact + => (parse-hl7 (apply-edit source source-edit))) + (check (row-ref receipt 'relexStopByte) + => (string-length (apply-edit source source-edit)))))) (test-case "missing literal recovery stays a rejected v1 publication" (let-values (((artifact receipt) (parse-source/recover arithmetic-parser "(1"))) @@ -104,4 +538,4 @@ (check (row-ref (car (row-ref receipt 'operations)) 'kind) => 'SKIPPED))))) -(export recovery-incremental-tests) +(export recovery-incremental-test) diff --git a/t/region-scanner-test.ss b/t/region-scanner-test.ss new file mode 100644 index 00000000..23ed495b --- /dev/null +++ b/t/region-scanner-test.ss @@ -0,0 +1,173 @@ +;;; Closed declared regions: shared matching, source isolation and admission. +(import :std/test + (only-in :gerbil-parser/src/runtime/region-scanner + prepare-region-plan valid-region-specification? region-plan-end + region-plan-specification region-plan-pair-end region-plan-quote-end + region-plan-operator prepare-region-source prepare-scoped-region-source region-source-pair-end region-source-quote-end) + (only-in :gerbil-parser/t/fixtures/bash-products bash-word-regions)) +(export region-scanner-test) +(def (rejects? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def (collision-rows root count) + (let loop ((n 0) (rows '())) + (if (= n count) rows + (loop (+ n 1) (cons (list (string-append root (number->string n) "{") #\{ #\} 1) rows))))) +(def region-scanner-test + (test-suite "closed region plans" + (test-case "longest openers and initial depth retain word boundaries" + (for-each + (lambda (word) + (check (region-plan-end bash-word-regions (string-append word " ;tail") 0) + => (string-length word))) + '("α$((1+(2*3)))" "<(printf 'α')" ">(cat)" "${x:-$(printf '}')}")) + (check (region-plan-operator bash-word-regions "x;;&y" 1) => ";;&")) + (test-case "quote scopes admit declared nesting and shield literal openers" + (let (source "\"${x:-$(printf '%s' '}')}\" tail") + (check (region-plan-quote-end bash-word-regions source 0 #\") + => (region-plan-end bash-word-regions source 0))) + (check (region-plan-end bash-word-regions "'${opaque' rest" 0) => 10) + (check (region-plan-end bash-word-regions "`$(opaque` rest" 0) => 10) + (check (region-plan-end bash-word-regions "a\\ b rest" 0) => 4)) + (test-case "deep nesting uses explicit frames with exact endpoints" + (let* ((depth 5000) + (word (string-append + (apply string-append (make-list depth "$(")) "α" + (make-string depth #\)))) + (source (string-append "xx" word " rest"))) + (check (region-plan-end bash-word-regions source 2) => (+ 2 (string-length word))) + (check (region-plan-pair-end bash-word-regions source 2) => (+ 2 (string-length word))))) + (test-case "counter frames retain endpoints across reuse and independent scans" + (let* ((plan (prepare-region-plan '((";") ((#\" #t ())) (("@{" #\{ #\} 1) ("%(" #\( #\) 1)) #f))) + (text (string-append "@{" (make-string 2048 #\{) "%(a(b)c)\"}\"" + (make-string 2048 #\}) "}")) + (end (string-length text))) + (for-each (lambda (_) (check (region-plan-pair-end plan text 0) => end)) (iota 3)) + (let ((first (prepare-region-source plan text)) + (second (prepare-region-source plan text))) + (check (region-source-pair-end first 0) => end) + (check (region-source-pair-end second 0) => end) + (check (region-source-pair-end first 0) => end)) + (let (workers (map (lambda (_) (spawn (lambda () (region-plan-pair-end plan text 0)))) (iota 4))) + (for-each (lambda (worker) (check (thread-join! worker) => end)) workers)) + (check (rejects? (lambda () (region-plan-pair-end plan (substring text 0 (- end 1)) 0))) => #t) + (check (region-plan-pair-end plan text 0) => end))) + (test-case "source-local boundaries retain all nested frame endpoints" + (let* ((depth 5000) + (text (string-append (apply string-append (make-list depth "${")) + "α" (make-string depth #\}))) + (source (prepare-region-source bash-word-regions text))) + (check (region-source-pair-end source 0) => (string-length text)) + (check (let loop ((at 0)) + (or (= at depth) + (and (= (region-source-pair-end source (* 2 at)) + (- (string-length text) at)) + (loop (+ at 1))))) => #t))) + (test-case "independent declared regions own text and preserve quote boundaries" + (let* ((plan (prepare-region-plan '((";") ((#\" #t ("@{"))) (("@{" #\{ #\} 1)) #f))) + (text (string-copy "@{\"@{α}\"}")) + (source (prepare-region-source plan text))) + (string-set! text 0 #\x) + (check (region-source-pair-end source 0) => 9) + (check (region-source-quote-end source 2 #\") => 8) + (check (region-source-pair-end source 3) => 7) + (check (rejects? (lambda () (region-source-quote-end source 2 #\'))) => #t)) + (let (source (prepare-region-source bash-word-regions "'literal ${x}")) + ;; Here-document literals do not require matching the leading quote. + (check (region-source-pair-end source 9) => 13) + (check (rejects? (lambda () (region-source-quote-end source 0 #\'))) => #t))) + (test-case "overlapping quote and pair entries keep separate match identities" + (let* ((plan (prepare-region-plan '((";") ((#\" #t ())) (("\"(" #\( #\) 1)) #f))) + (first (prepare-region-source plan "\"(α)\"")) + (second (prepare-region-source plan "\"(α)\""))) + (check (region-source-quote-end first 0 #\") => 5) + (check (region-source-pair-end first 0) => 4) + (check (region-source-pair-end second 0) => 4) + (check (region-source-quote-end second 0 #\") => 5))) + (test-case "unterminated scopes fail rather than publish a partial word" + (for-each + (lambda (source) + (check (rejects? (lambda () (region-plan-end bash-word-regions source 0))) => #t)) + '("${x" "$((1)" "<(cat" "\"${x}" "`x"))) + (test-case "prepared plans own their declaration data across sources" + (let* ((spec (region-plan-specification bash-word-regions)) + (plan (prepare-region-plan spec)) + (view (region-plan-specification plan))) + (string-set! (caar spec) 0 #\x) + (string-set! (caar view) 0 #\y) + (check (region-plan-operator plan ";;&" 0) => ";;&") + (check (region-plan-end plan "${a} " 0) => 4) + (check (region-plan-end plan "β " 0) => 1))) + (test-case "scoped overlapping pair openers retain independent cached endpoints" + (let (plan (prepare-region-plan '((";") () (("#{" #\{ #\} 1) ("@" #\@ #\! 1) ("@(" #\( #\) 1)) #f))) + (for-each (lambda (first) + (let (source (prepare-scoped-region-source plan "#{@(x)!}" '(("#{" "@")))) + (check (region-source-pair-end source first) => (if (= first 0) 8 6)) + (check (region-source-pair-end source 0) => 8) + (check (region-source-pair-end source 2) => 6))) '(0 2)) + (check (rejects? (lambda () (prepare-scoped-region-source plan "x" '(("unknown" "@"))))) => #t))) + (test-case "candidate indexes retain Unicode and declaration ownership" + (let* ((spec (list '(";") '() + (list (list (string-copy "α{") #\{ #\} 1) + (list (string-copy "😀[") #\[ #\] 1)) #f)) + (plan (prepare-region-plan spec)) + (view (region-plan-specification plan)) + (source (prepare-region-source plan "α{😀[x]}"))) + (string-set! (car (car (caddr spec))) 0 #\x) + (string-set! (car (car (caddr view))) 0 #\y) + (check (region-source-pair-end source 0) => 7) + (check (region-source-pair-end source 2) => 6) + (check (region-plan-end plan "βx rest" 0) => 2) + (for-each + (lambda (at) (check (rejects? (lambda () (region-source-pair-end source at))) => #t)) + '(-1 1 7 8 0.5)))) + (test-case "overlapping buckets preserve longest admitted opener in either order" + (for-each + (lambda (pairs) + (let* ((plan (prepare-region-plan (list '(";") '() pairs #f))) + (outer (prepare-scoped-region-source plan "#{@(x)!}" '(("#{" "@")))) + (unrestricted (prepare-region-source plan "@(x)!"))) + (check (region-source-pair-end unrestricted 0) => 4) + (check (region-source-pair-end outer 0) => 8) + (check (region-source-pair-end outer 2) => 6) + (let (opaque (prepare-scoped-region-source plan "#{@(x}" '(("#{")))) + (check (region-source-pair-end opaque 0) => 6)))) + '((("#{" #\{ #\} 1) ("@" #\@ #\! 1) ("@(" #\( #\) 1)) + (("@(" #\( #\) 1) ("@" #\@ #\! 1) ("#{" #\{ #\} 1))))) + (test-case "shared prefix paths retain shorter admitted terminals in sparse and dense catalogs" + (for-each + (lambda (extra) + (let* ((plan (prepare-region-plan + (list '(";") '() + (append '(("#{" #\{ #\} 1) ("@" #\@ #\! 1) + ("@a" #\a #\! 1) ("@ab{" #\{ #\} 1)) extra) #f))) + (text "#{@ab{x!}") + (source (prepare-scoped-region-source plan text '(("#{" "@a"))))) + (check (region-source-pair-end source 0) => (string-length text)) + ;; A direct query is unrestricted and cannot reuse the shorter scoped row. + (check (region-source-pair-end source 2) => 9) + (for-each + (lambda (text) (check (rejects? (lambda () (region-plan-pair-end plan text 0))) => #t)) + '("@" "@a" "@ab{")))) + (list '() (collision-rows "@" 16)))) + (test-case "Unicode continuation edges discriminate dense ASCII and Unicode roots" + (for-each + (lambda (root) + (let* ((texts (map (lambda (tail) (string-append root tail)) '("α{x}" "中{x}" "😀{x}"))) + (pairs (map (lambda (text) (list (substring text 0 3) #\{ #\} 1)) texts)) + (plan (prepare-region-plan (list '(";") '() (append pairs (collision-rows root 16)) #f)))) + (for-each + (lambda (text) (check (region-plan-pair-end plan text 0) => (string-length text))) texts) + (check (rejects? (lambda () (region-plan-pair-end plan (string-append root "β{x}") 0))) => #t))) + '("@" "α"))) + (test-case "malformed rows and foreign quote prefixes reject at admission" + (for-each + (lambda (spec) + (check (valid-region-specification? spec) => #f) + (check (rejects? (lambda () (prepare-region-plan spec))) => #t)) + '(((";") (bad-row) () #f) + ((";") () (bad-row) #f) + ((";" ";") () () #f) + ((";") ((#\" #t ("missing"))) () #f) + ((";") () (("$(" #\( #\) 0)) #f) + ((";") () (("$(" #\( #\) 3)) #f) + ((";") () (("α(" #\( #\) 3)) #f)))))) diff --git a/t/resolved-grammar-test.ss b/t/resolved-grammar-test.ss new file mode 100644 index 00000000..abd568e5 --- /dev/null +++ b/t/resolved-grammar-test.ss @@ -0,0 +1,758 @@ +;;; Resolved grammar semantics must be checked before target state construction. +(import :std/test + (only-in :std/test/base TestCase test-case-add!) + (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) + (only-in :gerbil-parser/src/compiler/lr lr-spec-ref current-grammar-source-map compute-nullable compute-first compute-productive compute-completion terminal-symbol? + lower-rules production-lhs production-rhs production-table production-terminal-catalog + sequence-first sequence-nullable? base-symbol nonterminal-symbol? nonterminal-name) + (only-in :gerbil-parser/src/compiler/funcs compiler-index-set-union) + (only-in :gerbil-parser/src/compiler/lr-automaton make-item-layout make-core-symbol-catalog + make-core-item core-item-count core-item-production-id core-item-dot) + (only-in :gerbil-parser/src/compiler/lr-lookahead make-core-suffix-catalog) + (only-in :gerbil-parser/src/compiler/lr-conflict-candidates initial-backward-follow-partitions) + (only-in :gerbil-parser/src/compiler/normalize compile-grammar grammar-ir-ref compile-grammar/context normalized-grammar-ir normalized-grammar-source-map) + (only-in :gerbil-parser/src/modules/parser/objects make-grammar make-grammar-role) + (only-in :gerbil-parser/src/runtime/lr-parser lr-parse lr-rejection-condition?) + (only-in :gerbil-parser/src/runtime/token make-token) + (only-in :gerbil-parser/src/runtime/recognition + recognition-node-kind recognition-node-start recognition-node-end + recognition-node-children recognition-child-field recognition-child-value + recognition-value-start recognition-value-end)) +(import (only-in :gerbil-parser/t/fixtures/lr1-construction + lr1-not-lalr-rules shared-lookahead-rules precedence-expression-rules + mixed-context-rules lr1-context-family-rules + state-local-candidate-family-rules mixed-context-family-rules + acyclic-mixed-context-family-rules inactive-core-conflict-rules skew-width-rules)) +(export resolved-grammar-test) + +;;; Independent coordinates include epsilon, unary and uneven-width rules. +;;; The expected enumeration is explicit, not decoded from the packed value. +(def (dense-item-coordinates-case) + (let* ((productions (lower-rules '((start (sequence (literal "a") (literal "b") (literal "c"))) + (empty (sequence)) (short (literal "z"))) 'start)) + (table (production-table productions))) + (let-values (((terminals _index) (production-terminal-catalog productions))) + (let* ((layout (make-item-layout table terminals)) + (symbols (make-core-symbol-catalog table layout)) + (coordinates '((0 0) (0 1) (1 0) (1 1) (1 2) (1 3) (2 0) (3 0) (3 1)))) + (check (core-item-count layout) => 9) + (check (vector->list symbols) + => '((nonterminal start) #f (terminal literal "a") (terminal literal "b") + (terminal literal "c") #f #f (terminal literal "z") #f)) + (for-each + (lambda (coordinate item) + (check (make-core-item (car coordinate) (cadr coordinate) layout) => item) + (check (core-item-production-id item layout) => (car coordinate)) + (check (core-item-dot item layout) => (cadr coordinate))) + coordinates (iota 9)))))) + +(def (skew-item-space-case) + (let* ((productions (lower-rules (skew-width-rules 128 64) 'source-file)) + (table (production-table productions))) + (let-values (((terminals _index) (production-terminal-catalog productions))) + (let* ((layout (make-item-layout table terminals)) + (symbols (make-core-symbol-catalog table layout))) + ;; Two unary entry rules, 128 unary composition members and one wide + ;; rule. The old padded domain had 8,515 positions for this grammar. + (check (vector-length symbols) => 325) + (check (make-core-item 130 0 layout) => 260) + (check (make-core-item 130 64 layout) => 324) + (check (vector-ref symbols 324) => #f))) + (check-suffix-catalog productions) + (let-values (((_names nullable) (compute-nullable productions))) + (let-values (((_rows first) (compute-first productions nullable))) + (let-values (((partitions _candidates _terminals) + (initial-backward-follow-partitions productions table first nullable))) + (check (vector-length partitions) => 325)))))) + +(def (admission-result rules (construction 'lalr) (policy 'reject)) + (with-catch + (lambda (condition) (list (error-message condition) (error-irritants condition))) + (lambda () (compile-lr-spec rules 'start policy #f construction) 'accepted))) + +(def (nullable-repeat-result owner operand (kind 'repeat)) + (list "resolved repetition operand accepts empty input" (list owner kind operand))) + +(def (parse-literals rules text (return-root? #f)) + (let (spec (compile-lr-spec + (map (lambda (row) + (if (eq? (car row) 'start) + (list 'start (list 'alias 'Start (cadr row))) row)) rules) + 'start)) + (with-catch + (lambda (condition) + (if (lr-rejection-condition? condition) 'rejected (raise condition))) + (lambda () + (let-values (((root rest) + (lr-parse spec + (map (lambda (index) + (make-token 'punctuation (string (string-ref text index)) + index (+ index 1))) + (iota (string-length text)))))) + (unless (null? rest) (error "unconsumed test input" rest)) + (if return-root? root 'accepted)))))) + +(def (rule-role name rules) + (make-grammar-role name '() '() '() rules '() '() '() '() '())) + +;; Independent repeated-scan oracle retained only in this semantic control. +(def (reference-nullable productions) + (let (index (make-table test: eq?)) + (let converge () + (let (changed? #f) + (for-each + (lambda (production) + (when (and (not (table-ref index (production-lhs production) #f)) + (every (lambda (value) + (let (symbol (base-symbol value)) + (and (nonterminal-symbol? symbol) + (table-ref index (nonterminal-name symbol) #f)))) + (production-rhs production))) + (table-set! index (production-lhs production) #t) + (set! changed? #t))) productions) + (when changed? (converge)))) + index)) + +(def (nullable-production name rhs) (list 0 name rhs #f #f)) + +(def (check-nullable-reference productions) + (let (reference (reference-nullable productions)) + (let-values (((names index) (compute-nullable productions))) + (check names => (filter (lambda (name) (table-ref reference name #f)) '(a b c))) + (for-each (lambda (name) + (check (table-ref index name #f) => (table-ref reference name #f))) + '(a b c missing))))) + +(def (reference-first productions nullable) + (let ((index (make-table test: eq?)) (names '()) (terminals '((terminal eof)))) + (def (union left right) + (foldl (lambda (value found) (if (member value found) found (append found (list value)))) left right)) + (for-each + (lambda (production) + (set! names (if (memq (production-lhs production) names) names + (append names (list (production-lhs production))))) + (for-each (lambda (value) + (let (symbol (base-symbol value)) + (when (terminal-symbol? symbol) + (set! terminals (union terminals (list symbol)))))) + (production-rhs production))) productions) + (def (prefix rhs) + (if (null? rhs) '() + (let (symbol (base-symbol (car rhs))) + (if (terminal-symbol? symbol) (list symbol) + (let* ((name (nonterminal-name symbol)) (found (table-ref index name '()))) + (if (table-ref nullable name #f) (union found (prefix (cdr rhs))) found)))))) + (let converge () + (let (changed? #f) + (for-each + (lambda (production) + (let* ((name (production-lhs production)) (before (table-ref index name '())) + (after (union before (prefix (production-rhs production))))) + (unless (equal? before after) + (table-set! index name after) (set! changed? #t)))) productions) + (when changed? (converge)))) + (map (lambda (name) + (cons name (filter (lambda (terminal) (member terminal (table-ref index name '()))) terminals))) names))) + +(def (check-first-reference productions) + (let* ((nullable (reference-nullable productions)) + (expected (reference-first productions nullable))) + (let-values (((rows index) (compute-first productions nullable))) + (check rows => expected) + (for-each (lambda (row) (check (table-ref index (car row)) => (cdr row))) expected)))) + +(def (reference-productive productions) + (let (index (make-table test: eq?)) + (let converge () + (let (changed? #f) + (for-each + (lambda (production) + (when (and (not (table-ref index (production-lhs production) #f)) + (every (lambda (value) + (let (symbol (base-symbol value)) + (or (terminal-symbol? symbol) + (table-ref index (nonterminal-name symbol) #f)))) + (production-rhs production))) + (table-set! index (production-lhs production) #t) + (set! changed? #t))) productions) + (when changed? (converge)))) + index)) + +(def (check-productive-reference productions) + (let ((expected (reference-productive productions)) + (nullable (reference-nullable productions)) + (names '())) + (for-each (lambda (production) + (unless (memq (production-lhs production) names) + (set! names (append names (list (production-lhs production)))))) productions) + (let-values (((nullable-found nullable-index found index) (compute-completion productions))) + (check nullable-found => (filter (lambda (name) (table-ref nullable name #f)) names)) + (for-each (lambda (name) (check (table-ref nullable-index name #f) => (table-ref nullable name #f))) (cons 'missing names)) + (check found => (filter (lambda (name) (table-ref expected name #f)) names)) + (for-each (lambda (name) + (check (table-ref index name #f) => (table-ref expected name #f)) + (when (table-ref nullable name #f) + (check (table-ref index name #f) => #t))) (cons 'missing names))))) + +;; Independent forward suffix scans check the entire catalog, including +;; unused/padded slots. The optimized builder folds backwards and uses masks. +(def (check-suffix-catalog productions) + (let (table (production-table productions)) + (let-values (((terminals terminal-index) (production-terminal-catalog productions)) + ((names nullable) (compute-nullable productions))) + (let-values (((first-rows first) (compute-first productions nullable))) + (let* ((layout (make-item-layout table terminals)) + (symbols (make-core-symbol-catalog table layout)) + (expected-masks (make-vector (vector-length symbols) 0)) + (expected-nullable (make-vector (vector-length symbols) #f))) + (let production-loop ((id 0)) + (when (< id (vector-length table)) + (let suffix-loop ((rest (production-rhs (vector-ref table id))) (dot 0)) + (let (item (make-core-item id dot layout)) + (vector-set! expected-masks item + (foldl (lambda (terminal mask) + (compiler-index-set-union mask + (arithmetic-shift 1 (table-ref terminal-index terminal)))) + 0 (sequence-first rest first nullable))) + (vector-set! expected-nullable item (sequence-nullable? rest nullable))) + (unless (null? rest) + (suffix-loop (cdr rest) (+ dot 1)))) + (production-loop (+ id 1)))) + (let-values (((masks nullable-tails) + (make-core-suffix-catalog table layout symbols first nullable terminal-index))) + (check masks => expected-masks) + (check nullable-tails => expected-nullable))))))) + +;;; Named case procedures retain independent assertions and separate native C hosts. + +(def (suffix-facts-case) + (for-each (lambda (rules) (check-suffix-catalog (lower-rules rules 'source-file))) + [shared-lookahead-rules lr1-not-lalr-rules precedence-expression-rules + inactive-core-conflict-rules (mixed-context-family-rules 8)]) + ;; Empty FIRST is not nullable; a missing reference blocks a suffix. + ;; Marked symbols use the same underlying terminal/nonterminal facts. + (let* ((a '(nonterminal a)) (b '(nonterminal b)) + (dead '(nonterminal dead)) (missing '(nonterminal missing)) + (x '(terminal literal "x")) + (marked-a (list 'marked a '((alias A)))) + (marked-x (list 'marked x '((alias X)))) + (operands [a b dead missing x marked-a marked-x])) + (for-each + (lambda (tail) + (check-suffix-catalog + (list (list 0 'root (append operands (list tail)) #f #f) + (list 1 'a '() #f #f) + (list 2 'a (list x) #f #f) + (list 3 'b (list x) #f #f) + (list 4 'dead (list dead) #f #f)))) operands) + (check-suffix-catalog + (list (list 0 'root (make-list 256 marked-a) #f #f) + (list 1 'a '() #f #f) (list 2 'a (list x) #f #f))))) + +(def (completion-publications-case) + (let ((prefix (list (nullable-production 'c '((nonterminal a) (nonterminal b))) + (nullable-production 'a '((nonterminal d))) + (nullable-production 'b '((nonterminal a) (nonterminal a))) + (nullable-production 'd '((nonterminal b))) + (nullable-production 'blocked '((terminal literal "x") (nonterminal missing))))) + (seeds (list (nullable-production 'a '((marked (terminal literal "x") (field value)))) + (nullable-production 'b '())))) + (check-productive-reference (append prefix seeds)) + (check-productive-reference (append prefix (reverse seeds)))) + (check-productive-reference '()) + (check-productive-reference + (list (nullable-production 'a '((nonterminal missing) (nonterminal missing))) + (nullable-production 'b '((terminal literal "x") (nonterminal a))) + (nullable-production 'c '((marked (nonterminal b) (field value))))))) + +(def (cyclic-productivity-case) + (let (operands '(() ((terminal literal "x")) ((nonterminal a)) + ((nonterminal b)) ((nonterminal c)) + ((nonterminal a) (nonterminal a)) + ((terminal literal "x") (nonterminal b)) + ((marked (nonterminal b) (field value))) + ((nonterminal missing)))) + (for-each (lambda (a) + (for-each (lambda (b) + (for-each (lambda (c) + (check-productive-reference + (list (nullable-production 'a a) + (nullable-production 'b b) + (nullable-production 'c c)))) operands)) operands)) operands))) + +(def (family-productivity-case) + (for-each (lambda (rules) (check-productive-reference (lower-rules rules 'source-file))) + (list lr1-not-lalr-rules shared-lookahead-rules precedence-expression-rules + mixed-context-rules (lr1-context-family-rules 8) + (state-local-candidate-family-rules 4) (mixed-context-family-rules 4) + (acyclic-mixed-context-family-rules 4))) + (check-productive-reference + (list (nullable-production 'a '((nonterminal b) (nonterminal b))) + (nullable-production 'b '((nonterminal c))) + (nullable-production 'c '((nonterminal a))) + (nullable-production 'c '((marked (terminal literal "x") (field value)))) + (nullable-production 'a '((nonterminal missing))) + (nullable-production 'unused '((nonterminal unused)))))) + +(def (reverse-productivity-case) + (let* ((names (map (lambda (n) (string->symbol (string-append "productive-chain-" (number->string n)))) (iota 4096))) + (productions (map (lambda (name next) + (nullable-production name (if next (list (list 'nonterminal next)) + '((terminal literal "x"))))) + names (append (cdr names) '(#f))))) + (let-values (((found index) (compute-productive productions))) + (check found => names) + (check (every (lambda (name) (table-ref index name #f)) names) => #t)))) + +(def (dead-start-case) + (for-each + (lambda (rules) + (for-each + (lambda (route) + (for-each + (lambda (policy) + (let (result (admission-result rules route policy)) + (check (car result) => "LR start rule has no finite terminal derivation") + (check (car (cadr result)) => 'start))) + '(reject selective-glr probe))) + '(lalr canonical-lr1 partitioned-lr1 follow-partition-lr1 lalr-then-follow))) + '(((start (reference start))) + ((start (sequence (literal "a") (reference tail))) (tail (reference tail)))))) + +(def (finite-exits-case) + (check (parse-literals '((start (empty)) (dead (reference dead))) "") => 'accepted) + (check (parse-literals '((start (choice (sequence (reference start) (literal "a")) + (literal "b"))) + (dead (reference dead))) "baa") => 'accepted) + (check (parse-literals '((start (choice (literal "b") (reference dead))) + (dead (sequence (literal "a") (reference dead)))) "b") => 'accepted) + (check (parse-literals '((start (choice (literal "b") (reference dead))) + (dead (sequence (literal "a") (reference dead)))) "a") => 'rejected)) + +(def (blocked-productivity-case) + (let* ((rules '((start (alias Start (choice (reference a) + (sequence (literal "x") (reference b))))) + (a (reference start)) (b (reference b)))) + (result (parameterize ((current-grammar-source-map '((rule)))) + (admission-result rules))) + (details (cadr (cadr result))) + (blockers (cdr (assq 'unproductiveRuleBlockers details))) + (start (car blockers)) + (references (cdr (assq 'blockedReferences start)))) + (check (map (lambda (row) (cdr (assq 'rule (cdr (assq 'ruleOrigin row))))) blockers) + => '(start b a)) + (check (map (lambda (row) (cdr (assq 'rule row))) references) => '(a b)) + (check (map (lambda (row) (cdr (assq 'expressionPath (cdr (assq 'referenceOrigin row))))) references) + => '((0 0) (0 1 1))) + (check (length blockers) => 3))) + +(def (authored-productivity-case) + (let* ((rules '((start (alias Start (repeat1 (field value (reference dead))))) + (dead (reference dead)))) + (result (admission-result rules)) + (details (cadr (cadr result))) + (blockers (cdr (assq 'unproductiveRuleBlockers details))) + (reference (car (cdr (assq 'blockedReferences (car blockers)))))) + (check (map (lambda (row) (cdr (assq 'rule (cdr (assq 'ruleOrigin row))))) blockers) + => '(start dead)) + (check (cdr (assq 'expressionPath (cdr (assq 'referenceOrigin reference)))) => '(0 0 0))) + (for-each (lambda (kind) + (check (parse-literals (list (list 'start (list kind '(reference dead))) + '(dead (sequence (literal "a") (reference dead)))) "") => 'accepted)) + '(optional repeat)) + ;; Finite epsilon admission does not discharge independent LR conflicts. + (check (car (admission-result '((start (optional (reference dead))) + (dead (reference dead))))) + => "unresolved reduce/reduce conflict")) + +(def (poo-productivity-case) + (let* ((base (make-grammar 'base '() + (list (rule-role 'base-rules '((start (reference item)) (item (literal "a"))))))) + (dead (make-grammar 'dead (list base) '() + (list (cons 'override (rule-role 'dead-item '((item (reference item)))))))) + (normalized (compile-grammar/context dead)) + (result (parameterize ((current-grammar-source-map (normalized-grammar-source-map normalized "unknown-source"))) + (admission-result (grammar-ir-ref (normalized-grammar-ir normalized) 'rules)))) + (blockers (cdr (assq 'unproductiveRuleBlockers (cadr (cadr result))))) + (item-source (cdr (assq 'source (cdr (assq 'ruleOrigin (cadr blockers))))))) + (check (cdr (assq 'componentOwner item-source)) => 'dead-item) + (check (cdr (assq 'compositionOperation item-source)) => 'override) + (for-each + (lambda (exit) + (let (restored (make-grammar 'restored (list dead) '() + (list (cons 'override (rule-role 'restored-item (list (list 'item exit))))))) + (check (admission-result (grammar-ir-ref (compile-grammar restored) 'rules)) => 'accepted))) + '((literal "b") (empty))) + (check (current-grammar-source-map) => '()))) + +(def (cyclic-first-case) + (let (operands '(() ((terminal literal "x")) ((terminal literal "y")) + ((nonterminal a)) ((nonterminal b)) ((nonterminal c)) + ((nonterminal a) (terminal literal "x")) + ((marked (nonterminal b) (field value))))) + (for-each (lambda (a) + (for-each (lambda (b) + (for-each (lambda (c) + (check-first-reference + (list (nullable-production 'a a) + (nullable-production 'b b) + (nullable-production 'c c)))) operands)) operands)) operands))) + +(def (family-first-case) + (for-each (lambda (rules) (check-first-reference (lower-rules rules 'source-file))) + (list lr1-not-lalr-rules shared-lookahead-rules precedence-expression-rules + mixed-context-rules (lr1-context-family-rules 8) + (state-local-candidate-family-rules 4) (mixed-context-family-rules 4) + (acyclic-mixed-context-family-rules 4)))) + +(def (first-prefix-case) + (check-first-reference + (list (nullable-production 'c '((marked (nonterminal a) (field value)))) + (nullable-production 'a '((nonterminal b) (terminal literal "z"))) + (nullable-production 'b '()) + (nullable-production 'a '((terminal literal "x"))) + (nullable-production 'a '((nonterminal b) (nonterminal b) (terminal literal "z"))) + (nullable-production 'c '((nonterminal c) (terminal eof))))) + (check-first-reference + (list (nullable-production 'a '((nonterminal b) (terminal literal "blocked"))) + (nullable-production 'b '((marked (terminal literal "y") (field value)))) + (nullable-production 'c '((nonterminal missing) (terminal literal "blocked")))))) + +(def (incomplete-prefix-case) + (let (productions + (list (nullable-production 'a '((terminal literal "x") (nonterminal b))) + (nullable-production 'b '((nonterminal b))) + (nullable-production 'c '((nonterminal a))))) + (check-first-reference productions) + (let-values (((rows index) (compute-first productions (reference-nullable productions)))) + (check (table-ref index 'a) => '((terminal literal "x"))) + (check (table-ref index 'b) => '()) + (check (table-ref index 'c) => '((terminal literal "x")))))) + +(def (bignum-first-case) + (let* ((terminals (map (lambda (n) (list 'terminal 'literal (number->string n))) (iota 128))) + (productions + (append (list (nullable-production 'a '((nonterminal b))) + (nullable-production 'b '((nonterminal c))) + (nullable-production 'c '((nonterminal a)))) + (map (lambda (terminal) (nullable-production 'b (list terminal))) terminals)))) + (check-first-reference productions) + (let-values (((rows index) (compute-first productions (reference-nullable productions)))) + (check (table-ref index 'a) => terminals)))) + +(def (reverse-first-case) + (let* ((names (map (lambda (n) (string->symbol (string-append "first-chain-" (number->string n)))) (iota 4096))) + (productions (map (lambda (name next) + (nullable-production name (if next (list (list 'nonterminal next)) + '((terminal literal "x"))))) + names (append (cdr names) '(#f))))) + (let-values (((rows index) (compute-first productions (reference-nullable productions)))) + (check (map car rows) => names) + (check (every (lambda (name) (equal? (table-ref index name) '((terminal literal "x")))) names) => #t)))) + +(def (cyclic-nullability-case) + (let (operands '(() ((terminal literal "x")) ((nonterminal a)) + ((nonterminal b)) ((nonterminal c)) + ((nonterminal a) (nonterminal a)) + ((nonterminal a) (nonterminal b)) + ((marked (nonterminal b) (field value))))) + (for-each (lambda (a) + (for-each (lambda (b) + (for-each (lambda (c) + (check-nullable-reference + (list (nullable-production 'a a) + (nullable-production 'b b) + (nullable-production 'c c)))) operands)) operands)) operands))) + +(def (nullable-occurrences-case) + (check-nullable-reference + (list (nullable-production 'a '((nonterminal b) (nonterminal b))) + (nullable-production 'a '((nonterminal missing))) + (nullable-production 'b '((nonterminal c) (terminal literal "x"))) + (nullable-production 'b '()) + (nullable-production 'c '((nonterminal c))))) + (check-nullable-reference + (list (nullable-production 'a '((nonterminal missing))) + (nullable-production 'b '((nonterminal a))) + (nullable-production 'c '()))) + (check-nullable-reference + (list (nullable-production 'a '((marked (terminal literal "x") (field value)))) + (nullable-production 'b '()) (nullable-production 'c '()))) + (let-values (((names index) + (compute-nullable + (list (nullable-production 'c '()) (nullable-production 'b '()) + (nullable-production 'c '()) (nullable-production 'a '()))))) + (check names => '(c b a)))) + +(def (reverse-nullability-case) + (let* ((names (map (lambda (n) (string->symbol (string-append "chain-" (number->string n)))) (iota 4096))) + (productions (map (lambda (name next) + (nullable-production name (if next (list (list 'nonterminal next)) '()))) + names (append (cdr names) '(#f))))) + (let-values (((found index) (compute-nullable productions))) + (check found => names) + (check (every (lambda (name) (table-ref index name #f)) names) => #t)))) + +(def (empty-reference-case) + (check (admission-result '((start (repeat (reference item))) (item (empty)))) + => (nullable-repeat-result 'start '(reference item)))) + +(def (optional-reference-case) + (for-each + (lambda (kind) + (check (admission-result + `((start (,kind (reference item))) + (item (reference tail)) (tail (optional (literal "a"))))) + => (nullable-repeat-result 'start '(reference item) kind))) + '(repeat repeat1))) + +(def (wrapped-nullability-case) + (let (operand '(precedence left 1 + (alias Item (field value + (sequence (reference left) (reference right)))))) + (check (admission-result + `((start (repeat ,operand)) (left (empty)) + (right (optional (literal "b"))))) + => (nullable-repeat-result 'start operand)))) + +(def (mutual-nullability-case) + (check (admission-result + '((start (repeat (reference item))) + (item (reference tail)) + (tail (choice (reference item) (empty))))) + => (nullable-repeat-result 'start '(reference item)))) + +(def (nested-repetition-case) + (check (admission-result + '((start (reference container)) + (container (alias Container + (sequence (literal "[") + (field items (repeat1 (reference item))) (literal "]")))) + (item (optional (literal "a"))))) + => (nullable-repeat-result 'container '(reference item) 'repeat1))) + +(def (obligation-traversal-case) + (for-each + (lambda (expression) + (check (admission-result `((start ,expression) (item (empty)))) + => (nullable-repeat-result 'start '(reference item)))) + '((sequence (literal "a") (repeat (reference item))) + (choice (empty) (repeat (reference item))) + (optional (repeat (reference item)))))) + +(def (construction-admission-case) + (for-each + (lambda (construction) + (check (admission-result + '((start (repeat (reference item))) (item (empty))) construction) + => (nullable-repeat-result 'start '(reference item)))) + '(lalr canonical-lr1 partitioned-lr1 follow-partition-lr1 lalr-then-follow))) + +(def (glr-repetition-case) + (for-each + (lambda (policy) + (check (admission-result + '((start (repeat (reference item))) (item (empty))) 'lalr policy) + => (nullable-repeat-result 'start '(reference item)))) + '(reject selective-glr probe))) + +(def (raw-nullability-case) + (for-each + (lambda (operand) + (check (admission-result `((start (repeat ,operand)))) + => (nullable-repeat-result 'start operand))) + '((empty) (layout-end)))) + +(def (consuming-suffix-case) + (let (rules '((start (repeat (reference item))) + (item (sequence (reference prefix) (literal "b"))) + (prefix (optional (literal "a"))))) + (check (parse-literals rules "") => 'accepted) + (check (parse-literals rules "babb") => 'accepted) + (check (parse-literals rules "a") => 'rejected))) + +(def (productive-recursion-case) + (let (rules '((start (repeat (reference item))) + (item (choice (sequence (reference item) (literal "a")) + (literal "b"))))) + (check (parse-literals rules "baabaa") => 'accepted) + (check (parse-literals rules "a") => 'rejected))) + +(def (effective-override-case) + (let* ((base (make-grammar 'base '() + (list (rule-role 'base-rules + '((start (repeat (reference item))) (item (literal "a"))))))) + (nullable-role (rule-role 'nullable-item '((item (empty))))) + (derived (make-grammar 'derived (list base) '() + (list (cons 'override nullable-role)))) + (restored (make-grammar 'restored (list derived) '() + (list (cons 'override (rule-role 'consuming-item + '((item (literal "b"))))))))) + (check (admission-result (grammar-ir-ref (compile-grammar derived) 'rules)) + => (nullable-repeat-result 'start '(reference item))) + (check (parse-literals (grammar-ir-ref (compile-grammar restored) 'rules) "bb") + => 'accepted))) + +(def (nullable-origin-case) + (let* ((base (make-grammar 'base '() + (list (rule-role 'base-rules + '((start (alias Start (sequence (repeat (reference item)) + (literal "z")))) + (item (literal "a"))))))) + (derived (make-grammar 'derived (list base) '() + (list (cons 'override (rule-role 'nullable-item '((item (empty)))))))) + (normalized (compile-grammar/context derived)) + (source-map (normalized-grammar-source-map normalized "unknown-source")) + (result (parameterize ((current-grammar-source-map source-map)) + (admission-result (grammar-ir-ref (normalized-grammar-ir normalized) 'rules)))) + (details (list-ref (cadr result) 3)) + (origin (cdr (assq 'expressionOrigin details))) + (references (cdr (assq 'nullableReferencePath details)))) + (check (cdr (assq 'rule origin)) => 'start) + (check (cdr (assq 'expressionPath origin)) => '(0 0)) + (check (cdr (assq 'componentOwner (cdr (assq 'source origin)))) => 'base-rules) + (check (map (lambda (row) (cdr (assq 'rule row))) references) => '(item)) + (let (source (cdr (assq 'source (car references)))) + (check (cdr (assq 'componentOwner source)) => 'nullable-item) + (check (cdr (assq 'compositionOperation source)) => 'override)) + ;; The dynamic context ends with this compile, never contaminating another. + (check (current-grammar-source-map) => '()))) + +(def (finite-witness-case) + (let* ((rules '((start (repeat (reference a))) + (a (choice (reference b) (reference epsilon))) + (b (reference a)) (epsilon (empty)))) + (result (parameterize ((current-grammar-source-map '((rule)))) + (admission-result rules))) + (details (list-ref (cadr result) 3))) + (check (map (lambda (row) (cdr (assq 'rule row))) + (cdr (assq 'nullableReferencePath details))) => '(a epsilon)))) + +(def (shared-witness-case) + (let* ((names (map (lambda (index) (string->symbol (string-append "nullable-" (number->string index)))) (iota 32))) + (rules (cons (list 'start (list 'repeat (list 'reference (car names)))) + (map (lambda (name index) + (list name (if (= index 31) '(empty) + (list 'sequence (list 'reference (list-ref names (+ index 1))) + (list 'reference (list-ref names (+ index 1))))))) + names (iota 32)))) + (result (parameterize ((current-grammar-source-map '((rule)))) (admission-result rules))) + (details (list-ref (cadr result) 3))) + (check (map (lambda (row) (cdr (assq 'rule row))) + (cdr (assq 'nullableReferencePath details))) => names))) + +(def (choice-acceptance-case) + (for-each + (lambda (alternatives) + (let (rules `((start (alias Start + (sequence (field prefix (choice ,@alternatives)) + (literal "c")))))) + (check (parse-literals rules "abc") => 'accepted) + (check (parse-literals rules "ac") => 'accepted) + (check (parse-literals rules "ab") => 'rejected))) + '(((literal "a") (sequence (literal "a") (literal "b"))) + ((sequence (literal "a") (literal "b")) (literal "a"))))) + +(def (choice-span-case) + (let* ((short '(literal "a")) + (long '(sequence (literal "a") (literal "b"))) + (forward `((start (sequence (field prefix (choice ,short ,long)) + (literal "c"))))) + (reverse `((start (sequence (field prefix (choice ,long ,short)) + (literal "c"))))) + (root (parse-literals forward "abc" #t))) + (check root => (parse-literals reverse "abc" #t)) + (check (recognition-node-kind root) => 'Start) + (check (list (recognition-node-start root) (recognition-node-end root)) => '(0 3)) + (check (map recognition-child-field (recognition-node-children root)) => '(prefix #f)) + (let (prefix (recognition-child-value (car (recognition-node-children root)))) + (check (list (recognition-value-start prefix) (recognition-value-end prefix)) => '(0 2))))) + +(def (helper-continuation-case) + (let (rules '((start (sequence (reference prefix) (literal "c"))) + (prefix (choice (literal "a") (reference longer))) + (longer (sequence (literal "a") (literal "b"))))) + (check (parse-literals rules "abc") => 'accepted) + (check (parse-literals rules "ac") => 'accepted) + (check (parse-literals rules "abbc") => 'rejected))) + +(def resolved-grammar-test + (test-suite "resolved grammar admission and CFG alternatives" + (test-case-add! (TestCase "dense item coordinates preserve epsilon and production boundaries" + dense-item-coordinates-case)) + (test-case-add! (TestCase "uneven rule widths allocate only actual dotted positions" + skew-item-space-case)) + (test-case-add! (TestCase "suffix facts preserve forward scans and marked operands" + suffix-facts-case)) + (test-case-add! (TestCase "joint completion preserves delayed and coalesced domain publications" + completion-publications-case)) + (test-case-add! (TestCase "productive facts match exhaustive cyclic fixed-point controls" + cyclic-productivity-case)) + (test-case-add! (TestCase "productive facts match actual LR families and finite alternative exits" + family-productivity-case)) + (test-case-add! (TestCase "reverse productivity chain completes in canonical name order" + reverse-productivity-case)) + (test-case-add! (TestCase "LR targets reject a dead start before state construction for every route" + dead-start-case)) + (test-case-add! (TestCase "finite epsilon and LR recursive exits survive unreachable dead rules" + finite-exits-case)) + (test-case-add! (TestCase "productivity diagnostics retain every blocked choice and finite cyclic graph" + blocked-productivity-case)) + (test-case-add! (TestCase "productivity explains authored wrappers rather than generated helpers" + authored-productivity-case)) + (test-case-add! (TestCase "productivity sees selected POO exits and their declaration owners" + poo-productivity-case)) + (test-case-add! (TestCase "FIRST delta facts match exhaustive cyclic grammar controls" + cyclic-first-case)) + (test-case-add! (TestCase "FIRST facts match an independent oracle across actual LR grammar families" + family-first-case)) + (test-case-add! (TestCase "FIRST prefix blockers and duplicate edges preserve canonical rows" + first-prefix-case)) + (test-case-add! (TestCase "FIRST keeps a terminal prefix without claiming finite completion" + incomplete-prefix-case)) + (test-case-add! (TestCase "FIRST bignum deltas preserve terminal catalog order across a cycle" + bignum-first-case)) + (test-case-add! (TestCase "long reverse FIRST chain propagates its terminal without full rescans" + reverse-first-case)) + (test-case-add! (TestCase "dependency nullable facts match exhaustive cyclic grammar controls" + cyclic-nullability-case)) + (test-case-add! (TestCase "nullable alternatives retain duplicate occurrences and terminal blockers" + nullable-occurrences-case)) + (test-case-add! (TestCase "long reverse nullable chain publishes every name in production order" + reverse-nullability-case)) + (test-case-add! (TestCase "direct reference to an empty rule rejects before LR conflicts" + empty-reference-case)) + (test-case-add! (TestCase "indirect optional reference rejects both repetition forms" + optional-reference-case)) + (test-case-add! (TestCase "nullable sequence through field alias and precedence rejects" + wrapped-nullability-case)) + (test-case-add! (TestCase "nullable alternative propagates through a mutual reference cycle" + mutual-nullability-case)) + (test-case-add! (TestCase "nested repetitions retain the original rule owner" + nested-repetition-case)) + (test-case-add! (TestCase "boolean analysis never skips nested repetition obligations" + obligation-traversal-case)) + (test-case-add! (TestCase "all LR constructions share resolved repetition admission" + construction-admission-case)) + (test-case-add! (TestCase "selective GLR cannot admit a zero-width repetition cycle" + glr-repetition-case)) + (test-case-add! (TestCase "raw IR empty and layout operands reject before state construction" + raw-nullability-case)) + (test-case-add! (TestCase "a nullable prefix followed by a consuming suffix remains valid" + consuming-suffix-case)) + (test-case-add! (TestCase "productive left recursion remains valid inside repeat" + productive-recursion-case)) + (test-case-add! (TestCase "effective POO override is analyzed rather than its parent declaration" + effective-override-case)) + (test-case-add! (TestCase "nullable diagnostics retain selected POO override and expression occurrence" + nullable-origin-case)) + (test-case-add! (TestCase "finite nullable witness skips cycles and retains the effective empty exit" + finite-witness-case)) + (test-case-add! (TestCase "nullable explanation follows a shared DAG without expanding derivation trees" + shared-witness-case)) + (test-case-add! (TestCase "CFG choice reordering preserves acceptance with shared token prefixes" + choice-acceptance-case)) + (test-case-add! (TestCase "choice ordering preserves node fields and exact ranges" + choice-span-case)) + (test-case-add! (TestCase "CFG helper-reference alternatives preserve enclosing continuation" + helper-continuation-case)))) diff --git a/t/result-profile-test.ss b/t/result-profile-test.ss new file mode 100644 index 00000000..3d91bae1 --- /dev/null +++ b/t/result-profile-test.ss @@ -0,0 +1,351 @@ +;;; Result declarations constrain construction and are ordinary inheritable POO slots. +(import :std/test + (only-in :gerbil-parser/src/compiler/result-projection result-profile-scheme-source) + (only-in :clan/poo/object .o .ref) + :gerbil-parser/src/language/result-profile + (only-in :gerbil-parser/src/language/scanner-profile ScannerProfile. defscanner-profile) + :gerbil-parser/src/runtime/recognition + (only-in :gerbil-parser/src/runtime/token make-token token-kind token-lexeme token-start token-end) + (only-in :gerbil-parser/src/runtime/funcs recognition-sequence->list) + (only-in :gerbil-parser/src/language/source declare-source-language source-language-digest source-language-result-catalog) + (only-in :gerbil-parser/src/runtime/source-engines LineSourceStrategy.)) +(export result-profile-test) +(def (rejects? thunk) (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(defresult-profile (base-results :: self ResultProfile.) + (nodes (File item) (Item text)) (tokens text)) +(defresult-profile (extended-results :: self base-results) + (nodes (File item extra) (Item text) (Extra text))) +(defscanner-profile (independent-scanner :: self ScannerProfile.) + (modes main) (initial-mode main) (tokens raw) + (rules (value main raw (literal "x") 0 keep))) +(defresult-profile (scanner-results :: self base-results) (scanner independent-scanner)) +(defresult-profile (token-results :: self base-results) (tokens other)) +(defresult-profile (projected-results :: self ResultProfile.) + (nodes (Atom content) (Pair first second)) (tokens glyph) + (projections + (leaf Atom ((text span)) (token content glyph text required)) + (pair Pair ((left node) (right parts)) (one first left optional) (many second right)))) +(defresult-profile (renamed-results :: self projected-results) + (nodes (Atom value) (Pair first second))) +(defresult-profile (renamed-projection-results :: self renamed-results) + (projections + (leaf Atom ((text span)) (token value glyph text required)) + (pair Pair ((left node) (right parts)) (one first left optional) (many second right)))) +(defresult-profile (inferred-results :: self ResultProfile.) + (projections + (leaf Atom ((text span)) (token content glyph text required)) + (pair Pair ((left node) (right parts)) (one first left optional) (many second right)))) +(defresult-profile (mixed-results :: self ResultProfile.) + (projections + (mixed Mixed ((text span) (head node) (tail parts)) + (token text glyph text optional) (one head head required) (many tail tail)) + (empty Empty ()))) +(defresult-profile (producer-results :: self ResultProfile.) + (nodes (List item) (Empty)) (tokens) + (projections (items List ((items parts)) (many item items)))) +(def (generated-result-plan plan) + ;; Evaluate the generator's complete forms, not a separately reimplemented + ;; constructor. Native compilation is an additional producer qualification. + (call-with-input-string (result-profile-scheme-source plan) + (lambda (port) + (let loop ((form (read port))) + (unless (eof-object? form) + (unless (eq? (car form) 'export) (eval form)) + (loop (read port)))))) + ((eval 'bind-generated-result-projections) plan)) +(def (build-parts projection source start end parts tokens) + (let (field (result-projection-parts-field projection)) + (result-projection-build-parts projection source start end + (if field (map (lambda (part) (make-recognition-child field part)) parts) parts) tokens))) +(def result-profile-test + (test-suite "declarative result construction" + (test-case "generated direct executors preserve mixed projection products and rejection" + (let* ((original (compile-result-profile mixed-results)) + (bound (generated-result-plan original)) + (source (prepare-recognition-source (make-token 'raw "α😀" 11 17))) + (a (make-recognition-node 'Empty 11 13 '())) + (b (make-recognition-node 'Empty 13 17 '())) + (signature '((text span optional) (head node required) (tail parts)))) + (check (result-plan-recipe bound) => (result-plan-recipe original)) + (check (eq? bound original) => #f) + (for-each (lambda (captures) + (check (call-with-values (lambda () (result-projection-build + (result-plan-projection bound 'mixed signature) source 0 2 captures)) list) + => (call-with-values (lambda () (result-projection-build + (result-plan-projection original 'mixed signature) source 0 2 captures)) list))) + (list (vector (cons 0 1) (cons a '()) (cons (list b b) '())) + (vector #f (cons a '()) (cons '() '())))) + (let-values (((node tokens) (result-projection-build + (result-plan-projection bound 'empty '()) source 0 2 (vector)))) + (check (recognition-node-kind node) => 'Empty) + (check (recognition-node-children node) => '())) + (for-each (lambda (captures) + (check (rejects? (lambda () (result-projection-build + (result-plan-projection bound 'mixed signature) source 0 2 captures))) => #t)) + (list (vector #f #f (cons '() '())) + (vector (cons 0 3) (cons a '()) (cons '() '())) + (vector #f (cons (make-recognition-node 'Foreign 11 13 '()) '()) (cons '() '())) + (vector #f (cons a '()) (cons (list (make-recognition-node 'Empty 13 18 '())) '())))))) + (test-case "mixed publication preserves long capture spines across repeated builds" + (let* ((original (compile-result-profile mixed-results)) + (bound (generated-result-plan original)) + (source (prepare-recognition-source (make-token 'raw "α😀" 11 17))) + (head (make-recognition-node 'Empty 11 13 '())) + (tail (make-recognition-node 'Empty 13 17 '())) + (parts (map (lambda (_) tail) (iota 1024))) + (captures (vector (cons 0 1) (cons head '()) (cons parts '()))) + (signature '((text span optional) (head node required) (tail parts)))) + (for-each (lambda (plan) + (let (projection (result-plan-projection plan 'mixed signature)) + (for-each (lambda (_) + (let-values (((node tokens) (result-projection-build projection source 0 2 captures))) + (let (children (recognition-node-children node)) + (check (map recognition-child-field children) + => (cons 'text (cons 'head (map (lambda (_) 'tail) parts)))) + (check (eq? (recognition-child-value (cadr children)) head) => #t) + (check (andmap (lambda (child) (eq? (recognition-child-value child) tail)) + (cddr children)) => #t)) + (check (eq? (car (vector-ref captures 2)) parts) => #t) + (check (length parts) => 1024) + (check (andmap (lambda (part) (eq? part tail)) parts) => #t))) '(first second)))) + (list original bound)))) + (test-case "profile syntax retains clause combinations and rejects malformed declarations" + (eval '(import :gerbil-parser/src/language/result-profile)) + (for-each (lambda (clauses) + (check (rejects? (lambda () (eval + `(defresult-profile (invalid-profile :: self ResultProfile.) ,@clauses)))) => #t)) + '(((tokens a) (tokens b)) ((tokens a) (nodes (A field))) + ((scanner #f) (projections)) ((nodes malformed)) ((unknown value)) ())) + (for-each (lambda (clauses) + (check (with-catch (lambda (_) #f) + (lambda () (eval `(defresult-profile (valid-profile :: self ResultProfile.) ,@clauses)) #t)) => #t)) + '(((scanner #f) (nodes) (tokens) (projections)) + ((nodes) (tokens) (projections)) ((projections)) + ((scanner #f) (nodes) (tokens)) ((nodes) (tokens)) + ((nodes)) ((tokens)) ((scanner #f))))) + (test-case "direct parts frames match capture publication for empty and long input" + (let* ((original (compile-result-profile producer-results)) + (bound (generated-result-plan original)) + (source (prepare-recognition-source (make-token 'raw "α😀" 11 17))) + (a (make-recognition-node 'Empty 11 13 '())) + (b (make-recognition-node 'Empty 13 17 '())) + (tokens (list (make-token 'raw "α😀" 11 17)))) + (for-each (lambda (parts) + (for-each (lambda (plan) + (let* ((projection (result-plan-projection plan 'items '((items parts)))) + (expected (call-with-values (lambda () (result-projection-build + projection source 0 2 (vector (cons parts tokens)))) list)) + (produced (call-with-values (lambda () (build-parts projection source 0 2 parts tokens)) list))) + (check produced => expected) + (check (eq? (cadr produced) tokens) => #t) + (check (map recognition-child-value (recognition-node-children (car produced))) => parts))) + (list original bound))) + (list '() (list a b) (map (lambda (n) (if (even? n) a b)) (iota 1024)))) + (let* ((projection (result-plan-projection original 'items '((items parts)))) + (children (list (make-recognition-child 'item a) (make-recognition-child 'item b)))) + (let-values (((node produced) (result-projection-build-parts projection source 0 2 children tokens))) + (check (eq? (recognition-node-children node) children) => #t) + (check (eq? produced tokens) => #t)) + (check (rejects? (lambda () (result-projection-build-parts projection source 0 2 + (list (make-recognition-child 'other a)) tokens))) => #t)))) + (test-case "direct parts frames retain bound executor authority" + (let* ((original (compile-result-profile producer-results)) + (source (prepare-recognition-source (make-token 'raw "α" 11 13))) + (node (make-recognition-node 'Empty 11 13 '())) + (calls 0) + (bound (result-plan-bind-projections original (result-plan-recipe original) + (list (cons 'items (lambda (given start end captures) + (set! calls (+ calls 1)) + (check (eq? given source) => #t) + (check (car (vector-ref captures 0)) => (list node node)) + (values node 'override-tokens))))))) + (check (call-with-values (lambda () (build-parts (result-plan-projection bound 'items '((items parts))) source 0 1 + (list node node) 'producer-tokens)) list) + => (list node 'override-tokens)) + (check calls => 1))) + (test-case "direct parts frames reject invalid nodes and non-unary recipes" + (let* ((plan (compile-result-profile producer-results)) + (projection (result-plan-projection plan 'items '((items parts)))) + (source (prepare-recognition-source (make-token 'raw "α" 11 13)))) + (for-each (lambda (value) + (check (rejects? (lambda () (build-parts projection source 0 1 (list value) '()))) => #t)) + (list (make-token 'raw "α" 11 13) + (make-recognition-node 'Foreign 11 13 '()) + (make-recognition-node 'Empty 11 14 '()))) + (let ((mixed (result-plan-projection (compile-result-profile mixed-results) 'mixed + '((text span optional) (head node required) (tail parts))))) + (check (rejects? (lambda () (result-projection-parts-field mixed))) => #t) + (check (rejects? (lambda () (result-projection-build-parts mixed source 0 1 '() '()))) => #t)))) + (test-case "generated bindings reject POO recipe drift and malformed producer tables" + (let* ((plan (compile-result-profile projected-results)) + (recipe (result-plan-recipe plan)) + (changed (compile-result-profile renamed-projection-results)) + (stub (lambda args (error "must not execute")))) + (generated-result-plan plan) + (check (rejects? (lambda () ((eval 'bind-generated-result-projections) changed))) => #t) + (for-each (lambda (rows) + (check (rejects? (lambda () (result-plan-bind-projections plan recipe rows))) => #t)) + (list '() (list (cons 'leaf stub)) + (list (cons 'leaf stub) (cons 'leaf stub)) + (list (cons 'pair stub) (cons 'leaf stub)) + (list (cons 'leaf stub) (cons 'pair #f)))) + (let* ((regenerated (generated-result-plan changed)) + (source (prepare-recognition-source (make-token 'raw "α" 0 2)))) + (let-values (((node tokens) (result-projection-build + (result-plan-projection regenerated 'leaf '((text span required))) source 0 1 (vector (cons 0 1))))) + (check (map recognition-child-field (recognition-node-children node)) => '(value)))) + (let* ((bound (generated-result-plan plan)) + (source (prepare-recognition-source (make-token 'raw "α" 0 2)))) + (let-values (((node tokens) (result-projection-build + (result-plan-projection bound 'leaf '((text span required))) source 0 1 (vector (cons 0 1))))) + (check (map token-lexeme (recognition-sequence->list tokens)) => '("α")))))) + (test-case "prepared mixed projections preserve order and reject foreign or escaping children" + (let* ((plan (compile-result-profile mixed-results)) + (source (prepare-recognition-source (make-token 'raw "α😀" 11 17))) + (mixed (result-plan-projection plan 'mixed '((text span optional) (head node required) (tail parts)))) + (empty (result-plan-projection plan 'empty '())) + (a (make-recognition-node 'Empty 11 13 '())) + (b (make-recognition-node 'Empty 13 17 '()))) + (let-values (((node tokens) (result-projection-build mixed source 0 2 + (vector (cons 0 1) (cons a '()) (cons (list b b) '()))))) + (check (map recognition-child-field (recognition-node-children node)) => '(text head tail tail)) + (check (map token-lexeme (recognition-sequence->list tokens)) => '("α"))) + (let-values (((node tokens) (result-projection-build mixed source 0 2 + (vector #f (cons a '()) (cons '() '()))))) + (check (map recognition-child-field (recognition-node-children node)) => '(head)) + (check (recognition-sequence->list tokens) => '())) + (let-values (((node tokens) (result-projection-build empty source 0 2 (vector)))) + (check (recognition-node-start node) => 11) + (check (recognition-node-end node) => 17) + (check (recognition-node-children node) => '())) + (for-each (lambda (captures) + (check (rejects? (lambda () (result-projection-build mixed source 0 2 captures))) => #t)) + (list (vector #f #f (cons '() '())) + (vector (cons 0 3) (cons a '()) (cons '() '())) + (vector #f (cons (make-recognition-node 'Foreign 11 13 '()) '()) (cons '() '())) + (vector #f (cons a '()) (cons (list (make-recognition-node 'Empty 10 13 '())) '())) + (vector #f (cons a '()) (cons (list (make-recognition-node 'Empty 13 18 '())) '())) + (vector #f (cons a '()) (cons (list #f) '())) + (vector #f (cons a '()) (cons (cons b b) '())))) + (check (rejects? (lambda () (result-projection-build empty + (prepare-recognition-source (make-token 'raw "α" -2 0)) 0 1 (vector)))) => #t))) + (test-case "projection-only profile derives constructors, fields and terminals without repetition" + (check (result-plan-catalog (compile-result-profile inferred-results)) + => '((syntax-kinds (Atom result (content)) (Pair result (first second))) (terminals (glyph token))))) + (test-case "POO extension shares a derived node with a new field and terminal" + (let* ((projection-declarations (append (.ref inferred-results 'projections) + '((annotation Atom ((text span)) (token label annotation text required))))) + (profile (.o (:: self inferred-results) projections: projection-declarations)) + (plan (compile-result-profile profile)) + (source (prepare-recognition-source (make-token 'raw "中" 0 3)))) + (check (result-plan-catalog plan) + => '((syntax-kinds (Atom result (content label)) (Pair result (first second))) + (terminals (glyph token) (annotation token)))) + (let-values (((node tokens) (result-projection-build + (result-plan-projection plan 'annotation '((text span required))) source 0 1 (vector (cons 0 1))))) + (check (map recognition-child-field (recognition-node-children node)) => '(label)) + (check (map token-kind (recognition-sequence->list tokens)) => '(annotation))))) + (test-case "generic projection executes inherited Unicode captures and ordered shared tokens" + (let* ((plan (compile-result-profile renamed-projection-results)) + (source (prepare-recognition-source (make-token 'raw "α😀" 11 17))) + (leaf (result-plan-projection plan 'leaf '((text span required)))) + (pair (result-plan-projection plan 'pair '((left node optional) (right parts))))) + (let-values (((a at) (result-projection-build leaf source 0 1 (vector (cons 0 1)))) + ((b bt) (result-projection-build leaf source 1 2 (vector (cons 1 2))))) + (let-values (((node tokens) (result-projection-build pair source 0 2 + (vector (cons a at) (cons (list b) bt))))) + (check (recognition-node-kind node) => 'Pair) + (check (map recognition-child-field (recognition-node-children node)) => '(first second)) + (check (map recognition-child-field (recognition-node-children a)) => '(value)) + (let (published (recognition-sequence->list tokens)) + (check (map token-kind published) => '(glyph glyph)) + (check (map token-lexeme published) => '("α" "😀")) + (check (map token-start published) => '(11 13)) + (check (map token-end published) => '(13 17)))) + (let-values (((node tokens) (result-projection-build pair source 0 2 + (vector #f (cons (list a b) (append at bt)))))) + (check (map recognition-child-field (recognition-node-children node)) => '(second second)))))) + (test-case "projection rejects invalid declarations, frames and incompatible engine signatures" + (for-each (lambda (declaration) + (check (rejects? (lambda () (compile-result-profile + (.o (:: self projected-results) projections: declaration)))) => #t)) + '(((leaf Atom ((text span)) (token missing glyph text required))) + ((leaf Atom ((text span)) (callback content text))) + ((leaf Atom ((text span)) (token content glyph text optional extra))) + ((leaf Atom ((text span)) (token content glyph other required))) + ((leaf Atom ((text span)) (many content text))) + ((leaf Atom ((text span) (text span)) (token content glyph text required) (token content glyph text required))) + ((leaf Atom ((text span)))) + ((leaf Atom ()) (leaf Atom ())) + ((leaf Atom ((text span)) (token))))) + (let* ((plan (compile-result-profile projected-results)) + (source (prepare-recognition-source (make-token 'raw "α" 0 2))) + (leaf (result-plan-projection plan 'leaf '((text span required))))) + (for-each (lambda (thunk) (check (rejects? thunk) => #t)) + (list (lambda () (result-plan-projection plan 'leaf '((text span optional)))) + (lambda () (result-plan-projection plan 'leaf '((text node)))) + (lambda () (result-plan-projection plan 'missing '((text span)))) + (lambda () (result-projection-build leaf source 0 1 (vector))) + (lambda () (result-projection-build leaf source 0 1 (vector #f))) + (lambda () (result-projection-build leaf source 0 1 (vector (cons -1 1)))) + (lambda () (result-projection-build leaf source 0 1 (vector (cons 0 2)))) + (lambda () (result-projection-build leaf source 0 1 (vector "not a span"))))))) + (test-case "projection recipes own input data and copies cannot mutate execution" + (let* ((projection-declarations (list (list 'leaf 'Atom (list (list 'text 'span)) (list 'token 'content 'glyph 'text 'required)))) + (profile (.o (:: self projected-results) projections: projection-declarations)) + (plan (compile-result-profile profile)) (recipe (result-plan-recipe plan)) + (source (prepare-recognition-source (make-token 'raw "x" 0 1)))) + (set-car! (cdddar projection-declarations) '(token missing unknown text required)) + (set-car! (cdddar (caddr recipe)) '(token missing unknown text required)) + (let-values (((node tokens) (result-projection-build + (result-plan-projection plan 'leaf '((text span required))) source 0 1 (vector (cons 0 1))))) + (check (map recognition-child-field (recognition-node-children node)) => '(content)) + (check (map token-kind (recognition-sequence->list tokens)) => '(glyph))))) + (test-case "inheritance compiles execution operations and the same catalog" + (let* ((plan (compile-result-profile extended-results)) (token (make-token 'text "α" 0 2)) + (node (result-plan-node plan 'Extra 0 2 (list (make-recognition-child 'text token))))) + (check (recognition-node-kind node) => 'Extra) + (check (result-plan-catalog plan) + => '((syntax-kinds (File result (item extra)) (Item result (text)) (Extra result (text))) + (terminals (text token)))))) + (test-case "single value slots override and scanner terminals derive without repeated declarations" + (check (result-plan-catalog (compile-result-profile scanner-results)) + => '((syntax-kinds (File result (item)) (Item result (text))) (terminals (text token) (raw token)))) + (check (result-plan-catalog (compile-result-profile token-results)) + => '((syntax-kinds (File result (item)) (Item result (text))) (terminals (other token))))) + (test-case "undeclared kinds, fields and terminals reject at construction" + (let ((plan (compile-result-profile base-results)) (token (make-token 'text "x" 0 1))) + (for-each (lambda (thunk) (check (rejects? thunk) => #t)) + (list (lambda () (result-plan-node plan 'Other 0 1 '())) + (lambda () (result-plan-node plan 'Item 0 1 (list (make-recognition-child 'other token)))) + (lambda () (result-plan-token plan (make-token 'other "x" 0 1))) + (lambda () (result-plan-node plan 'Item 0 1 (list (make-recognition-child 'text (make-recognition-node 'Other 0 1 '()))))))))) + (test-case "invalid bounds and nonrecognition children reject" + (let (plan (compile-result-profile base-results)) + (for-each (lambda (thunk) (check (rejects? thunk) => #t)) + (list (lambda () (result-plan-node plan 'Item -1 1 '())) + (lambda () (result-plan-node plan 'Item 0 0 (list (make-recognition-child 'text (make-token 'text "x" 0 1))))) + (lambda () (result-plan-node plan 'Item 0 1 (list (make-recognition-child 'text #f)))))))) + (test-case "duplicate declarations reject before binding" + (for-each (lambda (profile) (check (rejects? (lambda () (compile-result-profile profile))) => #t)) + (list (.o (:: self base-results) nodes: '((File (item item)))) + (.o (:: self base-results) nodes: '((File ()) (File ()))) + (.o (:: self base-results) tokens: '(text text))))) + (test-case "input and returned catalogs cannot change an admitted plan" + (let* ((declarations (list (list 'File (list 'item)))) + (plan (compile-result-profile (.o (:: self ResultProfile.) nodes: declarations tokens: '(text)))) + (recipe (result-plan-recipe plan)) (catalog (result-plan-catalog plan))) + (set-car! (cadar declarations) 'other) + (check (result-plan-catalog plan) => '((syntax-kinds (File result (item))) (terminals (text token)))) + (set-car! (cadar (car recipe)) 'other) + (check (result-plan-catalog plan) => '((syntax-kinds (File result (item))) (terminals (text token)))) + (set-car! (caddr (cadar catalog)) 'other) + (check (result-plan-catalog plan) => '((syntax-kinds (File result (item))) (terminals (text token)))))) + (test-case "source identity includes construction operations, independent line language derives catalog" + (let* ((strategy (.o (:: self LineSourceStrategy.) root-kind: 'RecordFile token-kind: 'Record)) + (source (declare-source-language "records" "1" "records.v1" strategy)) + (changed (declare-source-language "records" "1" "records.v1" + (.o (:: self strategy) results: (.o (:: result ResultProfile.) nodes: '((RecordFile (line extra))) tokens: '(Record)))))) + (check (source-language-result-catalog source) + => '((syntax-kinds (RecordFile result (line))) (terminals (Record token)))) + (check (equal? (source-language-digest source) (source-language-digest changed)) => #f))))) diff --git a/t/runtime-host.c b/t/runtime-host.c new file mode 100644 index 00000000..17c63086 --- /dev/null +++ b/t/runtime-host.c @@ -0,0 +1,96 @@ +/* Independent process: real SDK setup/cleanup, never a gxi child exit shim. */ +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE 1 +#endif +#include +#include +#include +#include +#include +#ifdef __APPLE__ +#include +#endif +#include +#include +#include "t/fixtures/shared-scanner/records-abi.h" +extern int gerbil_parser_rust_native_host_probe(void); +extern int gerbil_parser_rust_tokio_host_probe(void); +extern int gerbil_parser_rust_tokio_cpu_probe(int unicode); +int gerbil_parser_benchmark_prepare_thread(void) { +#ifdef __APPLE__ + qos_class_t before, after; int relative; + if (pthread_get_qos_class_np(pthread_self(), &before, &relative)) return -1; + if (pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0)) return -1; + if (pthread_get_qos_class_np(pthread_self(), &after, &relative)) return -1; + printf("RUST-TOKIO-CPU-QOS before=%u after=%u relative=%d\n", before, after, relative); + fflush(stdout); + return after == QOS_CLASS_USER_INTERACTIVE && relative == 0 ? 0 : -1; +#else + return 0; +#endif +} +double gerbil_parser_benchmark_cpu_seconds(void) { + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage)) return -1; + return usage.ru_utime.tv_sec + usage.ru_utime.tv_usec * 1e-6 + + usage.ru_stime.tv_sec + usage.ru_stime.tv_usec * 1e-6; +} +extern int gerbil_parser_rust_native_probe(void); +static double seconds(void) { + struct timespec ts; clock_gettime(CLOCK_MONOTONIC,&ts); + return ts.tv_sec + ts.tv_nsec * 1e-9; +} +static void *foreign_shutdown(void *argument) { + (void)argument; + return gerbil_parser_runtime_shutdown() == -1 ? NULL : (void *)1; +} +int main(int argc, char **argv) { + if (argc == 2 && !strcmp(argv[1],"rust")) return gerbil_parser_rust_native_host_probe(); + if (argc == 2 && !strcmp(argv[1],"tokio")) return gerbil_parser_rust_tokio_host_probe(); + if (argc == 2 && !strcmp(argv[1],"tokio-cpu-ascii")) return gerbil_parser_rust_tokio_cpu_probe(0); + if (argc == 2 && !strcmp(argv[1],"tokio-cpu-unicode")) return gerbil_parser_rust_tokio_cpu_probe(1); + puts("NATIVE-HOST-SETUP");fflush(stdout); + double started = seconds(); + if (gerbil_parser_runtime_init()) return 1; + printf("NATIVE-HOST-READY setup-ms=%.3f\n",1000*(seconds()-started));fflush(stdout); + if (argc == 2 && !strcmp(argv[1],"borrowed")) { + if (gerbil_parser_rust_native_probe()) return 12; + if (gerbil_parser_runtime_shutdown()) return 13; + puts("RUST-BORROWED-HOST-OK");fflush(stdout); + return 0; + } + puts("NATIVE-HOST-DUPLICATE-INIT");fflush(stdout); + if (gerbil_parser_runtime_init() != -3) return 2; + puts("NATIVE-HOST-FOREIGN-THREAD");fflush(stdout); + pthread_t foreign; void *thread_status; + if (pthread_create(&foreign,NULL,foreign_shutdown,NULL) || + pthread_join(foreign,&thread_status) || thread_status) return 3; + puts("NATIVE-HOST-LANGUAGE-CREATE");fflush(stdout); + uint64_t handle = records_language_create(); + if (!handle) return 4; + puts("NATIVE-HOST-HANDLE-BARRIER");fflush(stdout); + if (gerbil_parser_runtime_shutdown() != -2) return 4; + gerbil_parser_result result; gerbil_parser_result_init(&result); + const uint8_t source[] = "α=1\n"; + started = seconds(); + for (int i=0;i<100;i++) { + if (gerbil_parser_language_parse(handle,source,sizeof(source)-1,&result) || + result.length<80 || result.payload[8]) return 5; + if ((i+1)%25 == 0) {printf("NATIVE-HOST-PARSE-OK calls=%d\n",i+1);fflush(stdout);} + } + printf("NATIVE-HOST-100-CALLS wall-ms=%.3f\n",1000*(seconds()-started));fflush(stdout); + if (gerbil_parser_language_release(handle)) return 6; + /* The result alone must keep shutdown busy after the handle is released. */ + if (gerbil_parser_runtime_shutdown() != -2) return 7; + gerbil_parser_result_release(&result); + puts("NATIVE-HOST-CLEANUP");fflush(stdout); + if (gerbil_parser_runtime_shutdown()) return 8; + if (gerbil_parser_language_is_owner_thread() || records_language_create()) return 9; + if (gerbil_parser_language_parse(handle,source,sizeof(source)-1,&result) != -1 || + gerbil_parser_language_descriptor(handle,&result) != -1 || + gerbil_parser_language_release(handle) != -1) return 10; + gerbil_parser_result_release(&result); + if (gerbil_parser_runtime_shutdown() != -3 || gerbil_parser_runtime_init() != -3) return 11; + puts("NATIVE-HOST-CLEANUP-OK");fflush(stdout); + return 0; +} diff --git a/t/runtime-module-organization-test.ss b/t/runtime-module-organization-test.ss new file mode 100644 index 00000000..ed83e959 --- /dev/null +++ b/t/runtime-module-organization-test.ss @@ -0,0 +1,136 @@ +;;; -*- Gerbil -*- +;;; Source-form correspondence for the runtime split, with one admitted +;;; capture algorithm delta covered by the independent frozen semantic control. +(import (only-in :std/test check test-case test-suite) + (only-in :std/misc/process run-process)) +(export runtime-module-organization-test) +(def base "f57d2aa6502bd39b62119cdd2e5a95469ef6fd9f") +(def (read-forms port) + (let loop ((forms '())) + (let (form (read port)) + (if (eof-object? form) (reverse forms) (loop (cons form forms)))))) +(def (behavior forms) + (filter (lambda (form) (not (memq (car form) '(import export)))) forms)) +(def (same-forms? expected actual) + (let loop ((expected expected) (actual actual)) + (if (null? expected) (null? actual) + (let remove ((before '()) (rest actual)) + (cond + ((null? rest) #f) + ((equal? (car expected) (car rest)) + (loop (cdr expected) (append (reverse before) (cdr rest)))) + (else (remove (cons (car rest) before) (cdr rest)))))))) +(def (capture-definition? form name) + (and (pair? form) (memq (car form) '(def definline)) + (pair? (cadr form)) (eq? (caadr form) name))) +(def (admit-capture-delta original actual) + (let* ((old (filter (lambda (form) (capture-definition? form 'capture-reduction)) original)) + (current (filter (lambda (form) (capture-definition? form 'capture-reduction)) actual)) + (reference (filter (lambda (form) (capture-definition? form 'reference-capture-reduction)) + (call-with-input-file "t/benchmarks/lr-request-stack/capture-reference.ss" read-forms)))) + ;; The historical body stays pinned. Renaming the frozen control must be + ;; its only difference; every other owner form still compares exactly. + (check (length old) => 1) + (check (length current) => 1) + (check (length reference) => 1) + (let (form (car reference)) + (check (cons (car form) (cons (cons 'capture-reduction (cdadr form)) (cddr form))) + => (car old))) + (let* ((getters '(lr-recognition-fragment-lookahead lr-recognition-fragment-children lr-recognition-fragment-runtime lr-recognition-fragment-executor + lr-recognition-fragment-production-id lr-recognition-fragment-entry-state + lr-recognition-fragment-exit-state)) + (introduced? (lambda (form) + (or (capture-definition? form 'lr-recognition-fragment-terminal-yield) + (capture-definition? form 'prepare-capture-plans) + (and (eq? (car form) 'definline) + (eq? (caadr form) 'capture-context-from-plan)) + (and (eq? (car form) 'defrule) (pair? (cadr form)) + (eq? (caadr form) 'with-capture-bounds)) + (and (eq? (car form) 'defstruct) (memq (cadr form) '(lr-capture-context lr-capture-plan lr-capture-children))) + (and (eq? (car form) 'definline) (pair? (cadr form)) + (memq (caadr form) getters))))) + (fragment? (lambda (form) (and (eq? (car form) 'defstruct) + (eq? (cadr form) 'lr-recognition-fragment))))) + (check (length (filter introduced? actual)) => 14) + (for-each (lambda (name) + (check (length (filter (lambda (form) + (and (eq? (car form) 'definline) (pair? (cadr form)) + (eq? (caadr form) name))) actual)) => 1)) getters) + (check (caddr (find (lambda (form) + (and (eq? (car form) 'defstruct) (eq? (cadr form) 'lr-capture-plan))) actual)) + => '(runtime production-id width contexts)) + (check (caddr (find (lambda (form) + (and (eq? (car form) 'defstruct) (eq? (cadr form) 'lr-capture-children))) actual)) + => '(items)) + (check (caddr (find (lambda (form) + (and (eq? (car form) 'defstruct) (eq? (cadr form) 'lr-capture-context))) actual)) + => '(runtime executor production-id entry-state exit-state lookahead-kind)) + (let ((old-record (find fragment? original)) (new-record (find fragment? actual))) + (check (caddr new-record) => '(context children-cache lookahead-cache value start end token-count offset yield-cache)) + (check (cdddr new-record) => (cdddr old-record)) + (map (lambda (form) + (cond ((capture-definition? form 'capture-reduction) (car old)) + ((fragment? form) old-record) (else form))) + (filter (lambda (form) (not (introduced? form))) actual)))))) +(def (admit-stream-deltas original actual) + ;; Only these reviewed execution definitions differ. Other split forms keep + ;; exact historical correspondence; independent controls own semantics. + (let ((names '(define-lr-driver lr-run-checkpoint lr-checkpoint-drive + make-lr-action-selector lr-parse/prepared/receipt/materialized + lr-prepare lr-runtime-for-current-semantic-backend lr-runtime-with-action-selector)) + (inputs '(lr-input-start lr-input-empty? lr-input-token lr-input-tail lr-input-admit lr-input-list))) + (def (named? form name) + (and (pair? form) (memq (car form) '(def defrule)) + (pair? (cadr form)) (eq? (caadr form) name))) + (def (driver? form) (eq? (car form) 'define-lr-driver)) + (def (input-form? form) (and (eq? (car form) 'defrules) (memq (cadr form) inputs))) + (def (runtime-form? form) (and (eq? (car form) 'defstruct) (eq? (cadr form) 'lr-runtime))) + (def (introduced? form) (or (input-form? form) (driver? form) (named? form 'prepare-lr-goto-columns))) + (for-each (lambda (name) + (check (length (filter (lambda (form) (named? form name)) original)) => 1) + (check (length (filter (lambda (form) (named? form name)) actual)) => 1)) names) + (for-each (lambda (name) + (check (length (filter (lambda (form) (and (input-form? form) (eq? (cadr form) name))) actual)) => 1)) inputs) + (check (length (filter (lambda (form) (named? form 'prepare-lr-goto-columns)) actual)) => 1) + (check (filter driver? actual) => + '((define-lr-driver #f #f lr-run-persistent-checkpoint) + (define-lr-driver #f #t lr-run-stream-checkpoint) + (define-lr-driver #t #f lr-run-owned-checkpoint) + (define-lr-driver #t #t lr-run-owned-stream-checkpoint))) + (let ((old (filter runtime-form? original)) (new (filter runtime-form? actual))) + (check (length old) => 1) + (check (length new) => 1) + (check (car new) => (cons 'defstruct (cons 'lr-runtime + (cons (append (caddar old) '(goto-columns capture-plans)) (cdddar old))))) + (append (filter driver? original) + (map (lambda (form) + (let (name (find (lambda (name) (named? form name)) names)) + (cond (name (find (lambda (old) (named? old name)) original)) + ((runtime-form? form) (car old)) + (else form)))) + (filter (lambda (form) (not (introduced? form))) actual)))))) +(def (check-family family) + (let* ((facade (string-append "src/runtime/" family ".ss")) + (directory (string-append "src/runtime/" family)) + (original (run-process (list "git" "show" (string-append base ":" facade)) + coprocess: read-forms)) + (current (call-with-input-file facade read-forms)) + (owners (map (lambda (name) + (call-with-input-file (path-expand name directory) read-forms)) + (filter (lambda (name) (equal? (path-extension name) ".ss")) + (directory-files directory))))) + (check (behavior current) => '()) + (check (filter (lambda (f) (eq? (car f) 'export)) current) + => (filter (lambda (f) (eq? (car f) 'export)) original)) + (let (actual (apply append (map behavior owners))) + (check (same-forms? (behavior original) + (if (equal? family "lr-parser") + (admit-stream-deltas (behavior original) (admit-capture-delta (behavior original) actual)) actual)) => #t)))) +(def runtime-module-organization-test + (test-suite "Runtime module organization correspondence" + (test-case "LR facade and private owners retain every execution form exactly once" + (check-family "lr-parser")) + (test-case "incremental facade and owners retain every execution form exactly once" + (check-family "incremental")) + (test-case "scan facade and pure lexical owners retain every execution form exactly once" + (check-family "scan")))) diff --git a/t/runtime-organization-closure.ss b/t/runtime-organization-closure.ss new file mode 100644 index 00000000..6184e605 --- /dev/null +++ b/t/runtime-organization-closure.ss @@ -0,0 +1,30 @@ +;;; -*- Gerbil -*- +;;; Existing gxtest values linked ahead of time; no request-time expansion. +(import (only-in :std/test/base TestHarness TestConfig TestModule test-run! test-result-ok?) + (only-in :std/test test-suite test-case check) + "runtime-module-organization-test.ss" + (only-in "test-style-contract-test.ss" test-style-contract-tests) + (only-in "benchmarks/parser-throughput/link-locks-test.ss" link-locks-test) + "lr-semantic-preparation-test.ss" + "lr-stack-reduction-test.ss" + "lr-action-preparation-test.ss" + "lr-fragment-transfer-test.ss" + "recovery-incremental-test.ss") +(export main) +(def (main . arguments) + (unless (or (null? arguments) (equal? arguments '("--failure-control"))) + (error "runtime organization closure accepts no suite filters")) + (let* ((suites (if (null? arguments) + (list runtime-module-organization-test test-style-contract-tests link-locks-test lr-semantic-preparation-test + lr-stack-reduction-test lr-action-preparation-test lr-ordinary-action-preparation-test + lr-fragment-transfer-test recovery-incremental-test) + (list (test-suite "failure exit control" + (test-case "intentional failure reaches the process exit" + (check #f => #t)))))) + (result (test-run! + (TestHarness "native runtime organization" + (TestConfig verbosity: 5 capture-output?: #f) + (list (TestModule "native runtime organization" suites '() void void)))))) + (if (test-result-ok? result) + (begin (displayln "OK") (force-output) (exit 0)) + (begin (displayln "FAILED") (force-output) (exit 42))))) diff --git a/t/rust-rowan-generation-test.ss b/t/rust-rowan-generation-test.ss deleted file mode 100644 index f6a443df..00000000 --- a/t/rust-rowan-generation-test.ss +++ /dev/null @@ -1,97 +0,0 @@ -;;; -*- Gerbil -*- -;;; Parser IR v1 is the sole authority for deterministic Rust/Rowan output. - -(import (only-in :std/io/tempfile make-temporary-file-name) - (only-in :std/test check test-case test-suite) - (only-in :gerbil-parser/languages/arithmetic/v1/grammar - +arithmetic-language-version+ - +arithmetic-syntax-contract+ - arithmetic-language-grammar - arithmetic-parser-ir - arithmetic-parser) - (only-in :gerbil-parser/languages/gql/iso-39075-2024/grammar - +gql-syntax-contract+ - gql-iso-parser-ir - gql-iso-parser) - (only-in :gerbil-parser/src/compiler/machine - parser-machine-grammar-digest) - (only-in :gerbil-parser/rust-rowan-support - generate-language-rust-rowan-module) - (only-in :gerbil-parser/src/compiler/rust-rowan - rust-rowan-module-source)) -(export rust-rowan-generation-test) - -(def (arithmetic-rust-source) - (rust-rowan-module-source - "arithmetic" - +arithmetic-language-version+ - +arithmetic-syntax-contract+ - (parser-machine-grammar-digest arithmetic-parser) - arithmetic-parser-ir)) - -(def rust-rowan-generation-test - (test-suite "Rust/Rowan AOT generation" - (test-case "one Parser IR produces one deterministic Rust module" - (let ((first (arithmetic-rust-source)) - (second (arithmetic-rust-source))) - (check first => second) - (check (and - (string-contains first - "Source authority: canonical Grammar IR v1") - (string-contains first - "use gerbil_parser_rowan::{") - (string-contains first - (parser-machine-grammar-digest arithmetic-parser)) - (string-contains first "ParserAction::Shift(9)") - (string-contains first "OperandAction::Alias(0)") - (string-contains first - "LexicalExpr::Literals(&[\"+\", \"-\", \"*\"") - #t) - => #t))) - (test-case "GQL lowers its lexical algebra and selective-GLR tables" - (let (source - (rust-rowan-module-source - "gql" - "edition-1-2024-04" - +gql-syntax-contract+ - (parser-machine-grammar-digest gql-iso-parser) - gql-iso-parser-ir)) - (check (and (string-contains source "LexicalExpr::NumberLiteral") - (string-contains source "LexicalExpr::Choice") - (string-contains source "ParserAction::Fork(&[") - (string-contains source "dynamic_precedence:") - #t) - => #t))) - (test-case "whole-line lexical IR lowers to the generic Rowan engine" - (let* ((line-ir - (map (lambda (entry) - (if (eq? (car entry) 'lexical-rules) - (cons 'lexical-rules - (map (lambda (row) - (if (eq? (car row) 'number) - '(number (line)) - row)) - (cdr entry))) - entry)) - arithmetic-parser-ir)) - (source - (rust-rowan-module-source - "arithmetic" +arithmetic-language-version+ - +arithmetic-syntax-contract+ - (parser-machine-grammar-digest arithmetic-parser) - line-ir))) - (check (and (string-contains source - "LexicalRule { terminal: \"number\", expression: LexicalExpr::Line") - #t) - => #t))) - (test-case "the language descriptor is the complete generation input" - (let (path (make-temporary-file-name "gerbil-parser-rowan")) - (try - (begin - (generate-language-rust-rowan-module - path arithmetic-language-grammar) - (check (call-with-input-file - path (lambda (port) (read-line port))) - => "// @generated by gerbil-parser/src/compiler/rust-rowan.ss")) - (finally - (when (file-exists? path) (delete-file path)))))))) diff --git a/t/rust-runtime-generation-test.ss b/t/rust-runtime-generation-test.ss new file mode 100644 index 00000000..05c41215 --- /dev/null +++ b/t/rust-runtime-generation-test.ss @@ -0,0 +1,295 @@ +;;; -*- Gerbil -*- +;;; Parser IR v1 is the sole authority for deterministic Rust output. + +(import (only-in :std/io/tempfile make-temporary-file-name) + (only-in :std/test check check-exception test-case test-suite) + (only-in :gerbil-parser/languages/arithmetic/parser + +arithmetic-language-version+ + +arithmetic-syntax-contract+ + arithmetic-language-grammar + arithmetic-parser-ir + arithmetic-parser) + (only-in :gerbil-parser/languages/gql/parser + +gql-syntax-contract+ + gql-parser-ir + gql-parser) + (only-in :gerbil-parser/src/compiler/machine + parser-machine-grammar-digest) + (only-in :gerbil-parser/src/runtime/lr-parser lr-prepare) + (only-in :gerbil-parser/rust-runtime-support + generate-language-rust-runtime-module) + (only-in :gerbil-parser/src/compiler/rust-runtime + rust-runtime-module-source generate-rust-runtime-module)) +(export rust-runtime-generation-test test-worker-exclusive?) +;;; GQL emission builds a large transient Rust string in the shared VM. Keep +;;; its allocation phase separate from other large compiler/index owners. +(def test-worker-exclusive? #t) + +(def (arithmetic-rust-source) + (rust-runtime-module-source + "arithmetic" + +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) + arithmetic-parser-ir)) + +;; Append a dead canonical rule while preserving the real language tables. +(def (arithmetic-ir-with-production tail (id #f)) + (map (lambda (entry) + (if (eq? (car entry) 'lr-spec) + (cons 'lr-spec + (map (lambda (row) + (if (eq? (car row) 'productions) + (cons 'productions + (append (cdr row) + (list (cons (or id (length (cdr row))) tail)))) + row)) + (cdr entry))) + entry)) + arithmetic-parser-ir)) + +(def rust-runtime-generation-test + (test-suite "Rust AOT generation" + (test-case "Rust target admits exact i32 precedence boundaries" + (for-each + (lambda (value) + (let (source (rust-runtime-module-source + "arithmetic" +arithmetic-language-version+ +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) + (arithmetic-ir-with-production (list 'unused '() 'concat (list 'dynamic value))))) + (check (and (string-contains source + (string-append "dynamic_precedence: " (number->string value))) #t) => #t))) + '(-2147483648 0 2147483647))) + (test-case "Rust target failures preserve an existing output before emission" + (for-each + (lambda (tail) + (let ((path (make-temporary-file-name "unrepresentable-parser")) + (ir (arithmetic-ir-with-production tail))) + ;; Target limits do not redefine canonical Scheme legality. + (lr-prepare (cdr (assq 'lr-spec ir))) + (try + (begin + (call-with-output-file path (lambda (port) (display "preserved\n" port))) + (check-exception + (generate-rust-runtime-module path "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) + ir) + (lambda (condition) (string-prefix? "Rust LR" (error-message condition)))) + (check (call-with-input-file path read-line) => "preserved")) + (finally (when (file-exists? path) (delete-file path)))))) + '((unused () concat (dynamic -2147483649)) + (unused () concat (dynamic 2147483648)) + (unused () concat (dynamic 1267650600228229401496703205376)) + (unused () layout-end #f) + (unused ((terminal layout-start "{")) concat #f) + (unused ((marked (terminal layout-next ";") ((field separator)))) concat #f)))) + (test-case "Rust kind capacity is checked before file creation" + (let ((path (make-temporary-file-name "oversized-parser-catalog")) + (ir (map (lambda (entry) + (if (eq? (car entry) 'syntax-kinds) + (cons 'syntax-kinds (make-list 65537 (car (cdr entry)))) + entry)) arithmetic-parser-ir))) + (try + (begin + (check-exception + (generate-rust-runtime-module path "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) ir) + (lambda (condition) + (equal? (error-message condition) "Rust LR kind catalog exceeds u16 capacity"))) + (check (file-exists? path) => #f)) + (finally (when (file-exists? path) (delete-file path)))))) + (test-case "raw Rust publication rejects invalid table domains and layout capability" + (for-each + (lambda (tables) + (let ((path (make-temporary-file-name "invalid-parser-table")) + (ir (map (lambda (entry) + (if (eq? (car entry) 'lr-spec) + (cons 'lr-spec + (map (lambda (row) + (case (car row) + ((actions) (cons 'actions (car tables))) + ((gotos) (cons 'gotos (cadr tables))) + (else row))) + (cdr entry))) + entry)) arithmetic-parser-ir))) + (try + (begin + (def (reject!) + (check-exception + (generate-rust-runtime-module path "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) ir) + (lambda (condition) (string-prefix? "invalid LR table" (error-message condition))))) + (reject!) + (check (file-exists? path) => #f) + (call-with-output-file path (lambda (port) (display "preserve-existing-output" port))) + (reject!) + (check (call-with-input-file path read-line) => "preserve-existing-output")) + (finally (when (file-exists? path) (delete-file path)))))) + '((#((((terminal token number) shift 1 #f))) #(())) + (#((((terminal token undeclared) reduce 0) ((terminal eof) accept))) #(())) + (#((((terminal eof) accept))) #(((undeclared . 0)))) + (#(() ()) #(() ())) + (#(() (((terminal token number) shift 1 #f) ((terminal eof) accept))) #(() ())) + (#(()) #(((root . 1)))) + (#((((terminal token number) shift 0 #f) ((terminal token number) reduce 0))) #(())) + (#((((terminal token number) layout-guard (shift 0 #f) (reduce 0)))) #(())) + (#((((terminal layout-start "{") shift 0 #f))) #(())) + (#((((terminal layout-next ";") shift 0 #f))) #(()))))) + (test-case "unresolved dead references cannot create or overwrite Rust products" + (for-each + (lambda (operand) + (let ((ir (arithmetic-ir-with-production (list 'unused (list operand) 'pass #f))) + (path (make-temporary-file-name "unresolved-parser-reference"))) + (try + (begin + (def (reject!) + (check-exception + (generate-rust-runtime-module path "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) ir) + (lambda (condition) + (equal? (error-message condition) "invalid canonical LR unresolved reference")))) + (reject!) + (check (file-exists? path) => #f) + (call-with-output-file path (lambda (port) (display "preserved" port))) + (reject!) + (check (call-with-input-file path read-line) => "preserved")) + (finally (when (file-exists? path) (delete-file path)))))) + '((nonterminal missing) (marked (nonterminal missing) ((field value) (alias Item)))))) + (test-case "malformed dead productions preserve an existing output file" + (for-each + (lambda (tail) + (let (path (make-temporary-file-name "invalid-parser-production")) + (try + (begin + (call-with-output-file path (lambda (port) (display "preserved\n" port))) + (check-exception + (generate-rust-runtime-module path "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) + (arithmetic-ir-with-production tail)) + (lambda (condition) + (equal? (error-message condition) "invalid canonical LR production"))) + (check (call-with-input-file path read-line) => "preserved")) + (finally (when (file-exists? path) (delete-file path)))))) + '((unused () pass #f) + (unused ((terminal token number)) layout-end #f) + (unused () (layout-end "") #f) + (unused () concat (left "10")) + ("unused" () concat #f))) + (check-exception + (rust-runtime-module-source "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) + (arithmetic-ir-with-production '(unused () concat #f) 0)) + (lambda (condition) (equal? (error-message condition) "invalid canonical LR production")))) + (test-case "invalid action names in unused rules fail before file publication" + (for-each + (lambda (actions) + (let* ((ir (arithmetic-ir-with-production + (list 'unused (list (list 'marked '(terminal token number) actions)) 'concat #f))) + (path (make-temporary-file-name "invalid-parser-actions"))) + (try + (begin + (check-exception + (generate-rust-runtime-module path "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) ir) + (lambda (condition) + (and (equal? (error-message condition) "invalid LR operand actions") + (equal? (error-irritants condition) (list actions))))) + (check (file-exists? path) => #f)) + (finally (when (file-exists? path) (delete-file path)))))) + '(((field "left")) ((field #f)) ((alias 1)) ((alias (Item))) + ((field Name extra)) ((field Name) . invalid)))) + (test-case "one Parser IR produces one deterministic Rust module" + (let ((first (arithmetic-rust-source)) + (second (arithmetic-rust-source))) + (check first => second) + (check (and + (string-contains first + "Source authority: canonical Grammar IR v1") + (string-contains first + "use gerbil_parser_runtime::{") + (string-contains first + (parser-machine-grammar-digest arithmetic-parser)) + (string-contains first "ParserAction::Shift(9)") + (string-contains first "OperandAction::Alias(0)") + (string-contains first + "LexicalExpr::Literals(&[\"+\", \"-\", \"*\"") + #t) + => #t))) + (test-case "GQL lowers its lexical algebra and selective-GLR tables" + (let (source + (rust-runtime-module-source + "gql" + "edition-1-2024-04" + +gql-syntax-contract+ + (parser-machine-grammar-digest gql-parser) + gql-parser-ir)) + (check (and (string-contains source "LexicalExpr::NumberLiteral") + (string-contains source "LexicalExpr::Choice") + (string-contains source "ParserAction::Fork(&[") + (string-contains source "dynamic_precedence:") + #t) + => #t))) + (test-case "whole-line lexical IR lowers to the generic Rust engine" + (let* ((line-ir + (map (lambda (entry) + (if (eq? (car entry) 'lexical-rules) + (cons 'lexical-rules + (map (lambda (row) + (if (eq? (car row) 'number) + '(number (line)) + row)) + (cdr entry))) + entry)) + arithmetic-parser-ir)) + (source + (rust-runtime-module-source + "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) + line-ir))) + (check (and (string-contains source + "LexicalRule { terminal: \"number\", expression: LexicalExpr::Line") + #t) + => #t))) + (test-case "bounded character runs lower without language-specific code" + (let* ((run-ir + (map (lambda (entry) + (if (eq? (car entry) 'lexical-rules) + (cons 'lexical-rules + (map (lambda (row) + (if (eq? (car row) 'number) + '(number (character-run "-" 4)) + row)) + (cdr entry))) + entry)) + arithmetic-parser-ir)) + (source + (rust-runtime-module-source + "arithmetic" +arithmetic-language-version+ + +arithmetic-syntax-contract+ + (parser-machine-grammar-digest arithmetic-parser) + run-ir))) + (check (and + (string-contains + source + "LexicalExpr::CharacterRun { character: \"-\", minimum: 4 }") + #t) + => #t))) + (test-case "the language descriptor is the complete generation input" + (let (path (make-temporary-file-name "gerbil-parser-runtime")) + (try + (begin + (generate-language-rust-runtime-module + path arithmetic-language-grammar) + (check (call-with-input-file + path (lambda (port) (read-line port))) + => "// @generated by gerbil-parser/src/compiler/rust-runtime.ss")) + (finally + (when (file-exists? path) (delete-file path)))))))) diff --git a/t/rust-syntax-test.ss b/t/rust-syntax-test.ss index 69bc418f..6bab71ee 100644 --- a/t/rust-syntax-test.ss +++ b/t/rust-syntax-test.ss @@ -10,24 +10,34 @@ rust-function-value rust-block rust-function-ir-json rust-function-form-name rust-function-form-parameters + rust-function-form-result rust-struct-form? rust-struct-form-name rust-struct-form-fields rust-field-name rust-module-form-item + rust-module-form-origin rust-static-form-value) (only-in "pure-function-fixture.ss" normalized_title normalized_title_rust classify_first_word classify_first_word_rust) + (only-in "pure-function-fixture.ss" + trim_start_only trim_start_only_rust) + (only-in "pure-function-fixture.ss" + replace_and_append replace_and_append_rust) + (only-in "pure-function-fixture.ss" slug_words slug_words_rust) (only-in "pure-function-fixture.ss" classify_or_unknown classify_or_unknown_rust) (only-in "pure-function-fixture.ss" owned_first_word owned_first_word_rust owned_rest_after_first_word owned_rest_after_first_word_rust) (only-in "pure-function-fixture.ss" count_words count_words_rust) + (only-in "pure-function-fixture.ss" + map_word_annotations map_word_annotations_rust) (only-in "pure-function-fixture.ss" last_word last_word_rust before_last_word before_last_word_rust boundary_token? boundary_token_rust) (only-in :gerbil-parser/src/compiler/rust-pure-aot - scheme-pure->rust ascii-ci=? string-words string-after) + scheme-pure->rust ascii-ci=? string-words string-after + string-single-ascii-uppercase? string-unsigned-at-most?) (only-in "rust-aot-test-syntax.ss" check-rust-aot-function check-rust-aot-conditional check-rust-aot-any check-rust-aot-artifact @@ -44,13 +54,19 @@ (table (rust-none)) (blocks (rust-array '())))) (module (rust-module '("LineStructureSpec") - (rust-static STRUCTURE LineStructureSpec value)))) + (rust-static STRUCTURE LineStructureSpec value) + "line-structure-runtime.ss"))) (check (rust-struct-form? value) => #t) (check (rust-struct-form-name value) => 'LineStructureSpec) (check (map rust-field-name (rust-struct-form-fields value)) => '(grammar_digest paragraph_node table blocks)) (check (rust-static-form-value (rust-module-form-item module)) - => value))) + => value) + (check (rust-module-form-origin module) + => "line-structure-runtime.ss") + (check-exception + (rust-module '("LineStructureSpec") value "invalid\norigin") + true))) (test-case "pure function body is a structured syntax value" (check-rust-aot-function normalized_title_rust 'normalized_title '((input . "&str")) @@ -59,7 +75,25 @@ (check (normalized_title "\tBeta\n") => "Beta") (check-rust-aot-artifact normalized_title_rust - "rust/gerbil-parser-rowan/tests/unit/pure_function_generated.rs")) + "rust/gerbil-parser-runtime/tests/unit/pure_function_generated.rs")) + (test-case "left trim keeps source trailing whitespace" + (check (trim_start_only " Alpha ") => "Alpha ") + (check (trim_start_only "\tβ ") => "β ") + (check-rust-aot-function + trim_start_only_rust 'trim_start_only '((input . "&str")) + "String" 'to_owned)) + (test-case "pure replacement and append lower as typed string operations" + (check (replace_and_append "a/%s" "b") => "a/bb") + (check (replace_and_append "a/" "b") => "a/b") + (check (rust-function-form-name replace_and_append_rust) + => 'replace_and_append) + (check (rust-function-form-result replace_and_append_rust) + => "String")) + (test-case "pure lowercase word joins retain Scheme and Rust syntax parity" + (check (slug_words " Mixed Case ") => "mixed-case") + (check (slug_words "RÉSUMÉ Notes") => "résumé-notes") + (check (rust-function-form-name slug_words_rust) => 'slug_words) + (check (rust-function-form-result slug_words_rust) => "String")) (test-case "pure string boundaries retain Scheme and AOT parity" (check (last_word "α beta :tag:") => ":tag:") (check (before_last_word "α beta :tag:") => "α beta") @@ -71,6 +105,30 @@ => 'before_last_word) (check (rust-function-form-name boundary_token_rust) => 'boundary_token_p)) + (test-case "uppercase ASCII and bounded unsigned checks stay typed" + (check (string-single-ascii-uppercase? "A") => #t) + (check (string-single-ascii-uppercase? "a") => #f) + (check (string-single-ascii-uppercase? "É") => #f) + (check (string-single-ascii-uppercase? "AB") => #f) + (check (string-unsigned-at-most? "064" 64) => #t) + (check (string-unsigned-at-most? "65" 64) => #f) + (check (string-unsigned-at-most? "+1" 64) => #f) + (let* ((single + (string->json + (rust-function-ir-json + (scheme-pure->rust 'single '((value . "&str")) "bool" + '(string-single-ascii-uppercase? value))) + (JSONReadOptions object-as-hash: #t))) + (bounded + (string->json + (rust-function-ir-json + (scheme-pure->rust 'bounded '((value . "&str")) "bool" + '(string-unsigned-at-most? value 64))) + (JSONReadOptions object-as-hash: #t)))) + (check (hash-ref (hash-ref (hash-ref single "body") "result") "kind") + => "single_ascii_uppercase") + (check (hash-ref (hash-ref (hash-ref bounded "body") "result") "kind") + => "unsigned_at_most"))) (test-case "pure conditional and membership stay structural" (check-rust-aot-conditional classify_first_word_rust 'classify_first_word @@ -85,7 +143,7 @@ => "") (check-rust-aot-artifact classify_first_word_rust - "rust/gerbil-parser-rowan/tests/unit/classify_first_word_generated.rs")) + "rust/gerbil-parser-runtime/tests/unit/classify_first_word_generated.rs")) (test-case "typed pure function composition shares Scheme algorithm" (check (classify_or_unknown "WAIT Task" '("WAIT") '("DONE")) => "active") @@ -93,7 +151,7 @@ => "unknown") (check-rust-aot-artifact classify_or_unknown_rust - "rust/gerbil-parser-rowan/tests/unit/classify_or_unknown_generated.rs") + "rust/gerbil-parser-runtime/tests/unit/classify_or_unknown_generated.rs") (check-exception (scheme-pure->rust 'invalid '((input . "&str")) "&str" '(classify_first_word input) @@ -104,13 +162,26 @@ (check (owned_first_word " WAIT Task" #f) => "") (check-rust-aot-artifact owned_first_word_rust - "rust/gerbil-parser-rowan/tests/unit/owned_first_word_generated.rs") + "rust/gerbil-parser-runtime/tests/unit/owned_first_word_generated.rs") (check (owned_rest_after_first_word " WAIT [#A] Head :tag: ") => "[#A] Head :tag:") (check (owned_rest_after_first_word "WAIT") => "") (check-rust-function-ir owned_rest_after_first_word_rust "t/fixtures/owned_rest_after_first_word.ir.json")) + (test-case "named pure maps collect typed Rust vectors" + (check (map_word_annotations "NEXT(n) WAIT(w@/!)") + => '("NEXT" "WAIT")) + (check (rust-function-form-result map_word_annotations_rust) + => "Vec") + (check-rust-aot-function + map_word_annotations_rust 'map_word_annotations + '((input . "&str")) "Vec" 'collect) + (check-exception + (scheme-pure->rust + 'invalid '((input . "&str")) "Vec" + '(map display (string-words input))) + true)) (test-case "unsupported Scheme effects and free names fail closed" (check (ascii-ci=? "seq_todo" "SEQ_TODO") => #t) (check (ascii-ci=? "ＴＯＤＯ" "TODO") => #f) diff --git a/t/scanner-profile-test.ss b/t/scanner-profile-test.ss new file mode 100644 index 00000000..6e57aff0 --- /dev/null +++ b/t/scanner-profile-test.ss @@ -0,0 +1,47 @@ +;;; Independent profiles exercise POO inheritance and closed scanner admission. +(import :std/test + (only-in :clan/poo/object .cc) + (only-in :clan/poo/mop element?) + (only-in :gerbil-parser/src/language/scanner-profile + ScannerProfile. ScannerProfileContract defscanner-profile compile-scanner-profile) + (only-in :gerbil-parser/src/runtime/source-scanner make-contextual-source-scanner + source-scanner-tokens source-scanner-initial-state source-scanner-step + source-scan-state-context source-scan-state-with-context) + (only-in :gerbil-parser/src/runtime/token token-lexeme token-start token-end)) +(export scanner-profile-test) +(defscanner-profile (letters :: self ScannerProfile.) + (modes main) (initial-mode main) (tokens letter) + (rules (letter main letter (literal "α") 0 keep))) +(defscanner-profile (digits :: self letters) + (rules (digit main letter (literal "1") 0 keep))) +(def (worker profile source) + (make-contextual-source-scanner (compile-scanner-profile profile) source 'source)) +(def (rejects? thunk) (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def scanner-profile-test + (test-suite "reusable POO scanner profiles" + (test-case "independent declarations inherit slots and replace values" + (check (map token-lexeme (source-scanner-tokens (worker letters "αα") 'source)) => '("α" "α")) + (check (map token-lexeme (source-scanner-tokens (worker digits "11") 'source)) => '("1" "1")) + (check (rejects? (lambda () (source-scanner-tokens (worker digits "α") 'source))) => #t) + (let (extension (.cc letters 'rules '((word main letter (literals ("α" "β")) 0 keep)))) + (check (map token-lexeme (source-scanner-tokens (worker extension "αβ") 'source)) => '("α" "β")))) + (test-case "contracts reject callbacks, missing modes and recursive guard data" + (for-each (lambda (row) + (check (element? ScannerProfileContract (.cc letters (car row) (cdr row))) => #f)) + (list (cons 'rules (lambda () '())) + (cons 'initial-mode 'missing) + (cons 'tokens '(other)) + (cons 'rules '((bad main letter (literal "") 0 keep))) + (cons 'rules '((bad main letter (line-prefix "#\n" "\n") 0 keep))) + (cons 'rules '((bad main letter (literal "α") 0 (expect-marker-in #f missing)))) + (cons 'rules '((bad main letter (unless-prefix ("#") () (unless-prefix ("#") () (literal "α"))) 0 keep)))))) + (test-case "contextual workers reject foreign and displaced internal checkpoints" + (let* ((source "αα") (left (worker letters source)) (right (worker letters source)) + (initial (source-scanner-initial-state left))) + (let-values (((token next) (source-scanner-step left initial 'source))) + (check (list (token-start token) (token-end token)) => '(0 2)) + (check (rejects? (lambda () (source-scanner-step right initial 'source))) => #t) + (check (rejects? (lambda () (source-scanner-step left + (source-scan-state-with-context initial (source-scan-state-context next)) 'source))) => #t) + (let-values (((again replay) (source-scanner-step left initial 'source))) + (check (list (token-start again) (token-end again)) => '(0 2)))))))) diff --git a/t/scenarios/ffi/native-parse-artifact-v1/contract.ss b/t/scenarios/ffi/language-abi/contract.ss similarity index 50% rename from t/scenarios/ffi/native-parse-artifact-v1/contract.ss rename to t/scenarios/ffi/language-abi/contract.ss index c3474f30..6c977dca 100644 --- a/t/scenarios/ffi/native-parse-artifact-v1/contract.ss +++ b/t/scenarios/ffi/language-abi/contract.ss @@ -2,12 +2,12 @@ ((schema . gerbil-parser.native-ffi-scenario.v1) (owner . gerbil-parser) - (abiVersion . 1) + (abiVersion . 2) (transport . caller-owned-result-struct) (forbiddenTransport . process-json-lines) (requiredSymbols - "gerbil_parser_result_v1_init" - "gerbil_parser_result_v1_release" - "gerbil_parser_native_abi_version" - "gerbil_parser_native_descriptor" - "gerbil_parser_native_parse")) + "gerbil_parser_result_init" + "gerbil_parser_result_release" + "gerbil_parser_language_abi_version" + "gerbil_parser_language_descriptor" + "gerbil_parser_language_parse")) diff --git a/t/scenarios/observability/opencypher-grammar-phases/scenario.ss b/t/scenarios/observability/opencypher-grammar-phases/scenario.ss index c06db2ca..84e10971 100644 --- a/t/scenarios/observability/opencypher-grammar-phases/scenario.ss +++ b/t/scenarios/observability/opencypher-grammar-phases/scenario.ss @@ -2,9 +2,9 @@ ;;; Grammar-owned Core phase observation without global instrumentation. (import :core/observability/debug - (only-in :gerbil-parser/languages/cypher/opencypher-2024-1/parser - opencypher-2024-1-language-grammar - parse-opencypher-2024-1) + (only-in :gerbil-parser/languages/cypher/parser + opencypher-language-grammar + parse-opencypher) (only-in :gerbil-parser/src/language/descriptor language-grammar-observability language-grammar-with-observability) @@ -25,11 +25,11 @@ (def (opencypher-grammar-observability-scenario) (let* ((source "RETURN count(*) AS n\n") - (plain (parse-opencypher-2024-1 source)) + (plain (parse-opencypher source)) (policy (poo-flow-debug-call-policy 'gerbil-parser/runtime 4)) (observed-grammar (language-grammar-with-observability - opencypher-2024-1-language-grammar policy)) + opencypher-language-grammar policy)) (port (open-output-string)) (observed (parameterize ((current-error-port port)) @@ -39,7 +39,7 @@ (cons 'schema "gerbil-parser.grammar-observability.v1") (cons 'defaultUnobserved (not (language-grammar-observability - opencypher-2024-1-language-grammar))) + opencypher-language-grammar))) (cons 'observedPhases (filter (lambda (phase) (string-contains output (symbol->string phase))) diff --git a/t/scenarios/performance/incremental-suffix/benchmark.ss b/t/scenarios/performance/incremental-suffix/benchmark.ss index 191c477b..3403516c 100644 --- a/t/scenarios/performance/incremental-suffix/benchmark.ss +++ b/t/scenarios/performance/incremental-suffix/benchmark.ss @@ -7,9 +7,9 @@ (sampleCount . 40) (rule . GERBIL-PARSER-INCREMENTAL-SUFFIX) (feature . incremental-suffix-convergence) - (optimizationFocus . "shift-bound LR checkpoints plus byte-shift-aware maximal token suffix convergence") - (inputShape . "one hundred fixed-width arithmetic operands with an equal-width edit at the midpoint") - (expectedOutcome . "fresh-equivalent publication with more than ninety prefix shifts resumed and more than ninety suffix tokens reused") + (optimizationFocus . "shift-bound LR checkpoints plus lexical-mode-certified suffix token reuse") + (inputShape . "one hundred fixed-width arithmetic operands with a topology-changing equal-width edit at the midpoint") + (expectedOutcome . "fresh-equivalent publication with more than ninety significant tokens resumed or structurally reused, more than ninety suffix tokens reused, and conserved significant-token counts") (measurementPhases collect-before collect-after policy-before policy-after assert-time-gate observe-runtime-memory) (tags performance integration parser incremental checkpoint suffix)) diff --git a/t/scenarios/performance/incremental-suffix/scenario.ss b/t/scenarios/performance/incremental-suffix/scenario.ss index 086d7fd2..646557e0 100644 --- a/t/scenarios/performance/incremental-suffix/scenario.ss +++ b/t/scenarios/performance/incremental-suffix/scenario.ss @@ -1,7 +1,7 @@ ;;; -*- Gerbil -*- -;;; Full-size incremental LR checkpoint and suffix-convergence workload. +;;; Full-size incremental LR checkpoint and mode-certified suffix workload. -(import :gerbil-parser/languages/arithmetic/v1/parser +(import :gerbil-parser/languages/arithmetic/parser :gerbil-parser/src/runtime/incremental) (export incremental-suffix-scenario incremental-suffix-scenario-pass?) @@ -11,10 +11,10 @@ (string-append "(" (string-join (make-list +operand-count+ "001") " + ") ")")) (def +edit-start+ (+ 1 (* 50 6))) -(def +edit+ (make-edit +edit-start+ 3 "002")) -(def +base-artifact+ (parse-arithmetic-v1 +source+)) +(def +edit+ (make-edit +edit-start+ 3 "0+2")) +(def +base-artifact+ (parse-arithmetic +source+)) (def +fresh-artifact+ - (parse-arithmetic-v1 (apply-edit +source+ +edit+))) + (parse-arithmetic (apply-edit +source+ +edit+))) (def (row-ref row key) (let (entry (assq key row)) (and entry (cdr entry)))) @@ -36,10 +36,15 @@ (row-ref receipt 'relocatedSuffixTokenCount)) (cons 'resumedSignificantTokenCount (row-ref receipt 'resumedSignificantTokenCount)) + (cons 'reusedSignificantTokenCount + (row-ref receipt 'reusedSignificantTokenCount)) (cons 'remainingSignificantTokenCount (row-ref receipt 'remainingSignificantTokenCount))))) (def (incremental-suffix-scenario-pass? receipt) + (let ((resumed (row-ref receipt 'resumedSignificantTokenCount)) + (reused (row-ref receipt 'reusedSignificantTokenCount)) + (remaining (row-ref receipt 'remainingSignificantTokenCount))) (and (equal? (row-ref receipt 'schema) "gerbil-parser.incremental-suffix.v1") (= (row-ref receipt 'operandCount) +operand-count+) @@ -49,5 +54,11 @@ (= (row-ref receipt 'reusedSuffixTokenCount) (row-ref receipt 'convergedSuffixTokenCount)) (zero? (row-ref receipt 'relocatedSuffixTokenCount)) - (> (row-ref receipt 'resumedSignificantTokenCount) 90) - (< (row-ref receipt 'remainingSignificantTokenCount) 120))) + (integer? resumed) (>= resumed 0) + (integer? reused) (>= reused 0) + (integer? remaining) (>= remaining 0) + (> (+ resumed reused) 90) + ;; Replacing one number with "0+2" adds a second number and operator. + ;; Count the edited stream (203), not the original 201-token stream. + (= (+ resumed reused remaining) (+ (* 2 +operand-count+) 3)) + (< remaining 120)))) diff --git a/t/scenarios/performance/opencypher-large-query/scenario.ss b/t/scenarios/performance/opencypher-large-query/scenario.ss index b9deaeae..015f8713 100644 --- a/t/scenarios/performance/opencypher-large-query/scenario.ss +++ b/t/scenarios/performance/opencypher-large-query/scenario.ss @@ -1,7 +1,7 @@ ;;; -*- Gerbil -*- ;;; Full-size openCypher repetition workload for LR complexity regression. -(import :gerbil-parser/languages/cypher/opencypher-2024-1/parser +(import :gerbil-parser/languages/cypher/parser :gerbil-parser/src/runtime/artifact) (export opencypher-large-query-scenario opencypher-large-query-scenario-pass?) @@ -16,7 +16,7 @@ (let (entry (assq key row)) (and entry (cdr entry)))) (def (opencypher-large-query-scenario) - (let (artifact (parse-opencypher-2024-1 +source+)) + (let (artifact (parse-opencypher +source+)) (list (cons 'schema "gerbil-parser.opencypher-large-query.v1") (cons 'clauseCount +clause-count+) diff --git a/t/scenarios/performance/recovery-frontier/scenario.ss b/t/scenarios/performance/recovery-frontier/scenario.ss index 265e7805..10adf160 100644 --- a/t/scenarios/performance/recovery-frontier/scenario.ss +++ b/t/scenarios/performance/recovery-frontier/scenario.ss @@ -1,7 +1,7 @@ ;;; -*- Gerbil -*- ;;; Full-size deterministic failure-frontier recovery workload. -(import :gerbil-parser/languages/arithmetic/v1/parser +(import :gerbil-parser/languages/arithmetic/parser :gerbil-parser/src/runtime/artifact :gerbil-parser/src/runtime/recovery) (export recovery-frontier-scenario diff --git a/t/scenarios/performance/selective-glr/benchmark.ss b/t/scenarios/performance/selective-glr/benchmark.ss index 2716ca0a..a2dfd3d2 100644 --- a/t/scenarios/performance/selective-glr/benchmark.ss +++ b/t/scenarios/performance/selective-glr/benchmark.ss @@ -8,7 +8,7 @@ (rule . GERBIL-PARSER-SELECTIVE-GLR-COMPLETION) (feature . selective-glr-completion) (optimizationFocus . "request-local configuration, completion, and recognition-fragment interning across complete GLR branch evaluation") - (inputShape . "one hundred repetitions of four two-branch selective-GLR grammars per sample") + (inputShape . "one hundred repetitions of four prepared two-branch selective-GLR grammars per sample") (expectedOutcome . "every admitted branch completes; equivalent configurations and fragments are reused, dynamic scores rank, and distinct static ties fail closed") (measurementPhases collect-before collect-after policy-before policy-after assert-time-gate observe-runtime-memory) diff --git a/t/scenarios/performance/selective-glr/scenario.ss b/t/scenarios/performance/selective-glr/scenario.ss index decdbe6d..487510f5 100644 --- a/t/scenarios/performance/selective-glr/scenario.ss +++ b/t/scenarios/performance/selective-glr/scenario.ss @@ -2,7 +2,8 @@ ;;; Selective-GLR completion, merge, ranking, and ambiguity receipts. (import (only-in :gerbil-parser/src/compiler/lr-compiler compile-lr-spec) - (only-in :gerbil-parser/src/runtime/lr-parser lr-parse/receipt) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-parse/prepared/receipt) (only-in :gerbil-parser/src/runtime/recognition recognition-node-kind) (only-in :gerbil-parser/src/runtime/token make-token)) (export selective-glr-scenario @@ -47,6 +48,10 @@ (compile-lr-spec dynamic-rules 'source-file 'selective-glr)) (def fragment-spec (compile-lr-spec fragment-rules 'source-file 'selective-glr)) +(def equivalent-runtime (lr-prepare equivalent-spec)) +(def distinct-runtime (lr-prepare distinct-spec)) +(def dynamic-runtime (lr-prepare dynamic-spec)) +(def fragment-runtime (lr-prepare fragment-spec)) (def input-token (make-token 'identifier "x" 0 1)) (def fragment-input (list input-token (make-token 'identifier "y" 2 3))) @@ -54,9 +59,9 @@ (def (row-ref row key) (let (entry (assq key row)) (and entry (cdr entry)))) -(def (parse-receipt spec (tokens (list input-token))) +(def (parse-receipt runtime (tokens (list input-token))) (let-values (((root rest receipt) - (lr-parse/receipt spec tokens))) + (lr-parse/prepared/receipt runtime tokens))) (list (cons 'rootKind (recognition-node-kind root)) (cons 'remainingTokenCount (length rest)) (cons 'receipt receipt)))) @@ -68,15 +73,16 @@ (and (pair? irritants) (car irritants)))) (lambda () (call-with-values - (lambda () (lr-parse/receipt distinct-spec (list input-token))) + (lambda () + (lr-parse/prepared/receipt distinct-runtime (list input-token))) (lambda _ #f))))) (def (selective-glr-scenario) (list (cons 'schema "gerbil-parser.selective-glr-correctness-receipt.v1") - (cons 'equivalent (parse-receipt equivalent-spec)) - (cons 'dynamic (parse-receipt dynamic-spec)) - (cons 'interned (parse-receipt fragment-spec fragment-input)) + (cons 'equivalent (parse-receipt equivalent-runtime)) + (cons 'dynamic (parse-receipt dynamic-runtime)) + (cons 'interned (parse-receipt fragment-runtime fragment-input)) (cons 'ambiguous (ambiguity-receipt)))) (def (parse-case-pass? case expected-root expected-distinct expected-reason) diff --git a/t/selective-glr-benchmark-test.ss b/t/selective-glr-benchmark-test.ss index e3b5a2d6..02713f18 100644 --- a/t/selective-glr-benchmark-test.ss +++ b/t/selective-glr-benchmark-test.ss @@ -3,7 +3,8 @@ (import :std/test :asp-gerbil-scheme/src/benchmark/framework - "./scenarios/performance/selective-glr/scenario") + "./scenarios/performance/selective-glr/scenario" + (only-in :gerbil-parser/t/fixtures/progress report-test-progress!)) (def benchmark-path "t/scenarios/performance/selective-glr/benchmark.ss") @@ -13,7 +14,10 @@ (unless (zero? remaining) (unless (selective-glr-scenario-pass? (selective-glr-scenario)) (error "selective GLR correctness scenario failed")) - (loop (- remaining 1))))) + (loop (- remaining 1)))) + (when (equal? (getenv "GERBIL_PARSER_LR_TRACE" #f) "1") + (report-test-progress! "SCENARIO-BATCH-OK selective-glr groups=100") + (force-output))) (def selective-glr-benchmark-tests (test-suite "selective GLR completion scenario" @@ -22,6 +26,7 @@ (let (algorithm-receipt (selective-glr-scenario)) (write algorithm-receipt) (newline) + (force-output) (check (selective-glr-scenario-pass? algorithm-receipt) => #t)) (run-scenario-batch) (##gc) @@ -29,6 +34,13 @@ (benchmark-contract-run benchmark-path run-scenario-batch)) (write benchmark-receipt) (newline) + (force-output) (check (benchmark-contract-receipt-pass? benchmark-receipt) => #t))))) -(export selective-glr-benchmark-tests) +(def selective-glr-benchmark-test selective-glr-benchmark-tests) + +(export selective-glr-benchmark-tests selective-glr-benchmark-test) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/source-boundary-test.ss b/t/source-boundary-test.ss new file mode 100644 index 00000000..87d1a863 --- /dev/null +++ b/t/source-boundary-test.ss @@ -0,0 +1,86 @@ +;;; Non-Org native recovery controls through admitted POO declarations. +(import :std/test + (only-in :gerbil-parser/src/modules/parser/interface + make-heading-line make-block-line make-source-block-boundary + make-source-named-boundary make-source-boundary-parent + source-boundary-query? source-boundary-parent? + source-container-boundary-condition) + (only-in :gerbil-parser/src/compiler/event-fold-runtime run-event-fold)) +(export source-boundary-test) +(def heading (make-heading-line "*" " " 'Section 'Heading 'Marker)) +(def fixed + (make-source-block-boundary + (make-block-line "OPEN" "CLOSE" #t #t 'Block 'Begin 'Body 'End + 'recover-as-text #t #f #f 'elements) + heading)) +(def named (make-source-named-boundary "OPEN:" "CLOSE:" "" heading)) +(def parents (list (make-source-boundary-parent 1 "PARENT-END"))) +(def named-parents + (list (make-source-boundary-parent 2 "PARENT:" 'parent-from 'parent-until))) +(def (accepted? query source (parent 0) (bindings parents)) + (let* ((condition (source-container-boundary-condition query bindings 'frames)) + (events + (run-event-fold source 'Document + '((done #f) (frames (uint-stack)) (parent-from 0) (parent-until 4)) + `((if (not (state done)) + ((push-frame frames (uint ,parent)) + (if ,condition ((token Accepted start end)) + ((token Rejected start end))) + (set-bool done (bool #t))) + ((token Raw start end)))) + '()))) + (and (ormap (lambda (event) + (and (eq? (car event) 'token) (eq? (cadr event) 'Accepted))) + events) #t))) +(def source-boundary-test + (test-suite "native parent-aware boundary composition" + (test-case "declarations are admitted POO objects" + (check (source-boundary-query? fixed) => #t) + (check (source-boundary-query? named) => #t) + (check (source-boundary-parent? (car parents)) => #t)) + (test-case "fixed root recovery observes headings, CRLF and EOF" + (check (accepted? fixed "OPEN\r\nλ\r\nCLOSE") => #t) + (check (accepted? fixed "OPEN\n* heading\nCLOSE\n") => #f) + (check (accepted? fixed "OPEN\nbody") => #f)) + (test-case "literal parent stop precedes child close" + (check (accepted? fixed "OPEN\nPARENT-END\nCLOSE\n" 1) => #f) + (check (accepted? fixed "OPEN\nCLOSE\nPARENT-END\n" 1) => #t) + (check (accepted? fixed "OPEN\nCLOSE\n" 7) => #f)) + (test-case "named recovery matches source name, not unrelated closers" + (check (accepted? named "OPEN:child\nCLOSE:other\nCLOSE:child\n") => #t) + (check (accepted? named "OPEN:child\nCLOSE:other\n") => #f) + (check (accepted? named "OPEN:child\nPARENT-END\nCLOSE:child\n" 1) => #f)) + (test-case "named parent boundary uses saved source name" + (check (accepted? named "OPEN:child\nPARENT:OPEN\nCLOSE:child\n" + 2 named-parents) => #f) + (check (accepted? named "OPEN:child\nPARENT:other\nCLOSE:child\n" + 2 named-parents) => #t)) + (test-case "heading and indentation policies are explicit" + (check (accepted? named "OPEN:child\n* heading\nCLOSE:child\n") => #f) + (check (accepted? (make-source-named-boundary "OPEN:" "CLOSE:" "" heading + #f #t #f #t) + "OPEN:child\n* heading\n CLOSE:CHILD\n") => #t) + (check (accepted? (make-source-named-boundary "OPEN:" "CLOSE:" "" heading + #f #f #f #f) + "OPEN:child\n CLOSE:child\n") => #f)) + (test-case "terminated names and suffixes are declaration driven" + (check (accepted? (make-source-named-boundary "IN{" "OUT{" "}" heading + "}" #t #f #t) + "IN{child}\nOUT{child}\n") => #t)) + (test-case "invalid shapes and ambiguous parent bindings fail admission" + (check-exception (make-source-named-boundary "" "END" "" heading) true) + (check-exception (make-source-boundary-parent 0 "END") true) + (check-exception (make-source-boundary-parent 1 "END" 'from #f) true) + (check-exception (make-source-boundary-parent 1 "END" #f #f "}") true) + (check-exception (make-source-boundary-parent 1 "END" #f #f "" #f) true) + (check-exception + (source-container-boundary-condition + (make-source-block-boundary + (make-block-line "OPEN" "CLOSE" #f #t 'Block 'Begin 'Body 'End + 'recover-as-text #t #f) + heading) + parents 'frames) true) + (check-exception (source-container-boundary-condition named + (append parents parents) 'frames) true) + (check-exception (source-container-boundary-condition fixed named-parents 'frames) true) + (check-exception (source-container-boundary-condition #f parents 'frames) true)))) diff --git a/t/source-event-scope-test.ss b/t/source-event-scope-test.ss new file mode 100644 index 00000000..e82104e7 --- /dev/null +++ b/t/source-event-scope-test.ss @@ -0,0 +1,83 @@ +;;; Distinct rule instances share a native request without private-state aliases. +(import :std/test + (only-in :gerbil-parser/src/modules/parser/interface + make-source-event-scope source-event-name make-text-line + make-table-line make-inline-link source-table-row-initial + source-table-row-forms source-paragraph-initial + source-paragraph-span-forms source-paragraph-close-form + source-inline-link-initial source-inline-link-scan-forms) + (only-in :gerbil-parser/src/compiler/event-fold-runtime run-event-fold rust-state-name)) +(export source-event-scope-test) +(def a (make-source-event-scope 'first)) +(def b (make-source-event-scope 'second)) +(def table (make-table-line ";" 'Table 'Row 'Rule 'Cell 'Separator 'Value 'Trivia 'RuleText)) +(def paragraph (make-text-line 'Blank 'Text #f 'Paragraph)) +(def (has? events event) (and (member event events) #t)) +(def source-event-scope-test + (test-suite "native rule-instance state ownership" + (test-case "allocation is deterministic and delimiter-safe" + (check (source-event-name a 'open) => (source-event-name (make-source-event-scope 'first) 'open)) + (check (eq? (source-event-name a 'open) (source-event-name b 'open)) => #f) + (check (eq? (source-event-name (make-source-event-scope 'a:b) 'c) + (source-event-name (make-source-event-scope 'a) 'b:c)) => #f) + (check (equal? (rust-state-name (source-event-name (make-source-event-scope 'a-b) 'open)) + (rust-state-name (source-event-name (make-source-event-scope 'a_b) 'open))) => #f)) + (test-case "two paragraph instances retain separate spans in one frame" + (let (events + (run-event-fold "α\nβ" 'Document + (append (source-paragraph-initial paragraph a) + (source-paragraph-initial paragraph b)) + `((if (uint-equal? (offset start) (uint 0)) + ,(source-paragraph-span-forms paragraph a 'start 'end) + ,(source-paragraph-span-forms paragraph b 'start 'end))) + (list (source-paragraph-close-form paragraph b) + (source-paragraph-close-form paragraph a)))) + (check (has? events '(token Text 0 3)) => #t) + (check (has? events '(token Text 3 5)) => #t) + (check (length (filter (lambda (event) (equal? event '(start Paragraph))) events)) => 2))) + (test-case "table instance does not read or reset another instance's state" + (let (events + (run-event-fold ";a;b;" 'Document + (append (source-table-row-initial table a) (source-table-row-initial table b)) + `((set-bool ,(source-event-name a 'table-escaped) (bool #t)) + ,@(source-table-row-forms table b) + (if (state ,(source-event-name a 'table-escaped)) + ((start-node Preserved) (finish-node)) + ((start-node Lost) (finish-node)))) + '())) + (check (has? events '(start Preserved)) => #t) + (check (length (filter (lambda (event) (equal? event '(start Cell))) events)) => 2))) + (test-case "truncated first link cannot capture a second rule's target" + (let* ((first (make-text-line 'Text 'Raw + (make-inline-link "<<" "::" ">>" 'First 'FirstTarget 'Label 'Boundary))) + (second (make-text-line 'Text 'Raw + (make-inline-link "<{" "|" "}>" 'Second 'SecondTarget 'Label 'Boundary))) + (events + (run-event-fold "<" 'Document + (append (source-inline-link-initial first a) + (source-inline-link-initial second b) '((scan-cursor 0))) + `((for-line-bytes scan-index start (line-content-end) + ,(append (source-inline-link-scan-forms first a index: 'scan-index cursor: 'scan-cursor) + (source-inline-link-scan-forms second b index: 'scan-index cursor: 'scan-cursor))) + (token Raw (state-offset scan-cursor) end)) '()))) + (check (has? events '(token SecondTarget 11 13)) => #t) + (check (has? events '(start First)) => #f))) + (test-case "external helper parameter may match a private role name" + (let (events + (run-event-fold ";x;" 'Document + (append (source-table-row-initial table a) '((table-escaped 7))) + (source-table-row-forms table a 'content '(table-escaped)) '() + '((content ((table-escaped 0)) + ((if (uint-equal? (state table-escaped) (uint 7)) + ((token Value start end)) ((token Wrong start end)))) + (table-escaped))))) + (check (has? events '(token Value 1 2)) => #t))) + (test-case "invalid scopes are rejected at projection boundaries" + (check-exception (make-source-event-scope "first") true) + (check-exception (source-event-name #f 'open) true) + (check-exception (source-event-name a "open") true) + (check-exception (source-table-row-initial table #f) true) + (check-exception (source-paragraph-initial paragraph #f) true) + (check-exception (source-inline-link-initial + (make-text-line 'Text 'Raw + (make-inline-link "<<" "::" ">>" 'Link 'Target 'Label 'Boundary)) #f) true)))) diff --git a/t/source-fragment-test.ss b/t/source-fragment-test.ss new file mode 100644 index 00000000..93bb3f90 --- /dev/null +++ b/t/source-fragment-test.ss @@ -0,0 +1,112 @@ +;;; -*- Gerbil -*- +;;; POO source fragments execute and AOT-lower one event algorithm. + +(import (only-in :std/test check check-exception test-case test-suite) + (only-in :std/encoding/json JSONReadOptions string->json) + (only-in "event-strategy-fixture.ss" event-lines-language-grammar) + (only-in :gerbil-parser/src/modules/parser/source-fragment-objects + make-source-delimited-fragment make-source-first-split + make-source-reference-scan) + (only-in :gerbil-parser/src/modules/parser/source-fragment-types + source-delimited-fragment? source-first-split? + source-reference-scan?) + (only-in :gerbil-parser/src/modules/parser/source-fragment-funs + source-delimited-fragment-initial source-delimited-fragment-forms + source-first-split-initial source-first-split-forms + source-reference-scan-initial source-reference-scan-forms) + (only-in :gerbil-parser/rust-runtime-event-support + run-event-fold event-fold-ir-json)) +(export source-fragment-test) + +(def fragment (make-source-delimited-fragment "::" 'Line 'source-item 'Line)) +(def helpers + (list (list 'source-segments + (source-delimited-fragment-initial fragment) + (source-delimited-fragment-forms fragment)) + '(source-item () + ((start-node Text) (token Line start end) (finish-node))))) +(def line-forms '((call-source-helper source-segments start end))) +(def first-split (make-source-first-split "=" 'Line 'source-before 'source-after)) +(def split-helpers + (list (list 'source-split + (source-first-split-initial first-split) + (source-first-split-forms first-split)) + '(source-before () + ((start-node Text) (token Line start end) (finish-node))) + '(source-after () + ((start-node Heading) (token Line start end) (finish-node))))) +(def split-forms '((call-source-helper source-split start end))) +(def reference-rule + (make-source-reference-scan '((#\$ . Line) (#\@ . Line)) + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789$@#.<>_+-" + "remote(" 'Line 'Text 'Line)) +(def reference-helpers + (list (list 'source-references + (source-reference-scan-initial reference-rule) + (source-reference-scan-forms reference-rule)))) +(def reference-forms '((call-source-helper source-references start end))) + +(def source-fragment-test + (test-suite "POO source fragment event AOT" + (test-case "admitted delimiter partitions source without copying text" + (check (source-delimited-fragment? fragment) => #t) + (check (run-event-fold "a::b::c" 'Document '() line-forms '() helpers) + => '((start Document) + (start Text) (token Line 0 1) (finish) + (token Line 1 3) + (start Text) (token Line 3 4) (finish) + (token Line 4 6) + (start Text) (token Line 6 7) (finish) + (finish)))) + (test-case "the same policy lowers to typed AOT helper IR" + (let* ((wire (event-fold-ir-json + 'parse_source_segments event-lines-language-grammar + 'Document '() line-forms '() helpers)) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (hash-ref ir "schema") + => "gerbil-scheme-rust.event-function-ir.v1") + (check (vector-length (hash-ref ir "helpers")) => 2))) + (test-case "first split composes two source-bounded helpers" + (check (source-first-split? first-split) => #t) + (check (run-event-fold "a=b=c" 'Document '() split-forms '() + split-helpers) + => '((start Document) + (start Text) (token Line 0 1) (finish) + (token Line 1 2) + (start Heading) (token Line 2 5) (finish) + (finish))) + (let* ((wire (event-fold-ir-json + 'parse_first_split event-lines-language-grammar + 'Document '() split-forms '() split-helpers)) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (vector-length (hash-ref ir "helpers")) => 3))) + (test-case "marker and balanced-call references share one bounded policy" + (check (source-reference-scan? reference-rule) => #t) + (check (run-event-fold "$2 remote(x,$1) @3" 'Document '() + reference-forms '() reference-helpers) + => '((start Document) + (start Text) (token Line 0 2) (finish) + (token Line 2 3) + (start Text) (token Line 3 15) (finish) + (token Line 15 16) + (start Text) (token Line 16 18) (finish) + (finish))) + (let* ((wire (event-fold-ir-json + 'parse_source_references event-lines-language-grammar + 'Document '() reference-forms '() reference-helpers)) + (ir (string->json wire + (JSONReadOptions object-as-hash: #t + array-as-vector: #t)))) + (check (vector-length (hash-ref ir "helpers")) => 1))) + (test-case "POO admission rejects unsupported delimiters" + (check-exception (make-source-delimited-fragment + ":::" 'Line 'source-item 'Line) + true) + (check-exception (make-source-reference-scan + '((#\$ . Line)) "abc" "remote" + 'Line 'Text 'Line) + true)))) diff --git a/t/source-grammar-test.ss b/t/source-grammar-test.ss new file mode 100644 index 00000000..1e75a9e5 --- /dev/null +++ b/t/source-grammar-test.ss @@ -0,0 +1,132 @@ +;;; Public author-entry and rule admission contracts. +(import :std/test + (only-in :clan/poo/object .o .ref object?) + (only-in :gerbil-parser/language-support/entry deflanguage-parser-loader language-parser-entry-ref language-metadata-ref) + (only-in :gerbil-parser/src/language/source declare-source-syntax source-language-digest) + (only-in :gerbil-parser/src/runtime/source-engines LineSourceStrategy.) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-events parse-artifact-roundtrip) + (only-in :gerbil-parser/language-support/grammar deflanguage defgrammar-syntax) + (only-in :gerbil-parser/language-support/shell shell) + (only-in :gerbil-parser/language-support/command-grammar lower-command-grammar) + (for-syntax (only-in :gerbil/runtime/syntax SyntaxError-irritants) (only-in :gerbil/expander core-expand stx-source))) +(export source-grammar-test) +(defsyntax (source-syntax-error stx) + (syntax-case stx () + ((_ declaration) + (let (message + (with-catch + (lambda (condition) + (unless (syntax-error? condition) (raise condition)) (error-message condition)) + (lambda () (core-expand #'declaration) (error "invalid author declaration admitted")))) + (datum->syntax #'source-syntax-error (list 'quote message)))))) +(defsyntax (source-syntax-blame stx) + (syntax-case stx () + ((_ declaration) + (let (location (with-catch + (lambda (condition) + (unless (syntax-error? condition) (raise condition)) + (let (where (car (SyntaxError-irritants condition))) + (unless (stx-source where) (error "source grammar blame lost its source location")) + (syntax->datum where))) + (lambda () (core-expand #'declaration) (error "invalid author declaration admitted")))) + (datum->syntax #'source-syntax-blame (list 'quote location)))))) +(defgrammar-syntax (nullable-body text) + (repeat (optional (field keyword text)))) +(def (rejects? thunk) (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def metadata-syntax + (declare-source-syntax (.o (:: self LineSourceStrategy.) root-kind: 'Notes required-prefix: "#"))) +(def input-version (string-copy "v1")) +(def input-metadata (list (cons 'language "notes") (cons 'version input-version) + (cons 'contract "notes.metadata.v1") (cons 'edition 'local))) +(deflanguage-parser-loader value-entry + (source metadata-syntax) (parse parse-value-entry) + (slots metadata: input-metadata)) +(def value-descriptor (language-parser-entry-ref value-entry 'descriptor)) +(def MetadataBase. (.o language: "notes" version: "wrong" contract: "notes.metadata.v1" edition: 'base)) +(deflanguage-parser-loader object-entry + (source metadata-syntax) (parse parse-object-entry) + (slots metadata: (.o (:: self MetadataBase.) version: "v1" edition: 'local))) +(def object-descriptor (language-parser-entry-ref object-entry 'descriptor)) +(def source-grammar-test + (test-suite "deflanguage source syntax and rule contract" +(test-case "source errors identify the direct expression or author macro invocation" + (check (source-syntax-blame + (deflanguage invalid-shell (syntax (shell (operators "(") (regions) (bindings) (words) + (commands (roles) (descriptor (literal "x"))))) + (rules (bad (node WhileCommand (repeat (optional (field keyword "while")))))))) + => '(repeat (optional (field keyword "while")))) + (check (source-syntax-blame + (deflanguage invalid-shell (syntax (shell (operators "(") (regions) (bindings) (words) + (commands (roles) (descriptor (literal "x"))))) + (rules (bad (node WhileCommand (nullable-body "while")))))) + => '(nullable-body "while"))) + (test-case "selectors depend on syntax rather than rule and field names" + (let ((left (lower-command-grammar '((function-header (node FunctionDefinition (field name word) (field open (operator "(")) (field close (operator ")")) (field body command)))))) + (right (lower-command-grammar '((renamed (node FunctionDefinition (field name word) (field open (operator "(")) (field close (operator ")")) (field body command))))))) + (check (cdar left) => (cdar right))) + (let ((left (lower-command-grammar '((a (node WhileCommand (field keyword "while")))))) + (right (lower-command-grammar '((b (node WhileCommand (field renamed-field "while"))))))) + (check (cadar left) => (cadar right)) + (check (caddar left) => (caddar right)))) + (test-case "normal metadata values and native POO overrides bind equivalent source identity" + (let ((value (language-parser-entry-ref value-entry 'metadata)) + (object (language-parser-entry-ref object-entry 'metadata))) + (check (list? value) => #t) (check (object? object) => #t) + (for-each (lambda (key) + (check (language-metadata-ref value key) => (language-metadata-ref object key))) + '(language version contract digest digest-kind edition)) + (check (source-language-digest value-descriptor) => (source-language-digest object-descriptor)) + (check (parse-artifact-events (parse-value-entry "# alpha\n")) + => (parse-artifact-events (parse-object-entry "# alpha\n"))) + (check (parse-artifact-roundtrip (parse-value-entry "# alpha\n")) => "# alpha\n") + (string-set! input-version 0 #\x) + (check (language-parser-entry-ref value-entry 'version) => "v1"))) + (test-case "metadata rejects missing identity, duplicate keys and invalid value shapes" + (for-each (lambda (value) + (check (rejects? (lambda () + (let () (deflanguage-parser-loader invalid-entry + (source metadata-syntax) (parse invalid-parse) (slots metadata: value)) + invalid-entry))) => #t)) + '(((language . "notes") (version . "v1")) + ((language . "notes") (version . "v1") (version . "v2") (contract . "notes.v1")) + ((language . "notes") (version . "") (contract . "notes.v1")) + ((language . "notes") (version . "v1") (contract . #f)) + ((1 . "notes")) #f "metadata"))) + (test-case "public entry rejects duplicate blocks and execution-recipe syntax" + (check (string? (source-syntax-error + (deflanguage bad (syntax (shell)) (syntax (shell)) (rules)))) => #t) + (check (string? (source-syntax-error + (deflanguage bad (syntax (shell (strategy arbitrary))) (rules)))) => #t)) + (test-case "retired author declarations fail before parser publication" + (check (source-syntax-error + (deflanguage retired (root source-file) + (lex (identifier Identifier (identifier))) + (rules (source-file (alias SourceFile (token identifier)))))) + => "deflanguage requires exactly one syntax vocabulary and one rules block; flat declarations and release identity are not author syntax") + (check (string? (source-syntax-error + (deflanguage retired (identity "retired" "1" "retired.v1") + (root source-file) (lex (identifier Identifier (identifier))) + (rules (source-file (node SourceFile identifier)))))) => #t) + (check (string? (source-syntax-error + (deflanguage retired + (syntax (lexical (root name) (lex (identifier Identifier (identifier))))) + (rules (name (alias Name identifier)))))) => #t) + (check (string? (source-syntax-error + (deflanguage retired + (syntax (lexical (root name) (lex (identifier Identifier (identifier))) + (flow (source lexical)))) + (rules (name (node Name identifier)))))) => #t) + (check (string? (source-syntax-error + (deflanguage retired + (syntax (lexical (root name) (lex (identifier Identifier (identifier))) + (backends (source digest procedure)))) + (rules (name (node Name identifier)))))) => #t)) + (test-case "command admission rejects duplicates, dangling references and non-consuming repetition" + (for-each (lambda (rules) + (check (rejects? (lambda () (lower-command-grammar rules))) => #t)) + '(((same (node WhileCommand (field keyword "while"))) (same (node UntilCommand (field keyword "until")))) + ((missing (node WhileCommand (field body (reference absent))))) + ((nullable (node WhileCommand (repeat (optional (field keyword "while")))))) + ((recipe (node WhileCommand (take keyword (word "while"))))) + ((unterminated (node WhileCommand (field condition command-list))))))) +)) diff --git a/t/source-index-test.ss b/t/source-index-test.ss new file mode 100644 index 00000000..32706535 --- /dev/null +++ b/t/source-index-test.ss @@ -0,0 +1,124 @@ +#!/usr/bin/env gxi +(import :std/test :std/iter + (only-in :gerbil-parser/src/runtime/token make-token token-kind token-lexeme token-start token-end) + (only-in :gerbil-parser/src/runtime/source-index + source-index-count source-index-height source-index-start source-index-end + source-index-shift source-index-concat source-index-slice source-index->lists + source-index-audit source-index-storage make-source-index-builder + source-index-builder-token! source-index-builder-shared! source-index-builder-finish + source-index-builder-fresh source-index-builder-shared source-index-builder-chunks source-index-builder-mode-chunks + make-source-index-cursor source-index-cursor-copy source-index-cursor-token + source-index-cursor-mode source-index-cursor-rank source-index-cursor-next! source-index-cursor-seek!)) +(def (build tokens modes) + (let (builder (make-source-index-builder)) + (for-each (lambda (token mode) (source-index-builder-token! builder token mode)) tokens modes) + (source-index-builder-finish builder))) +(def (tokens n) + (let loop ((at 0) (found '())) + (if (= at n) (reverse found) + (loop (+ at 1) (cons (make-token 'identifier "x" at (+ at 1)) found))))) +(def (materialize tree) + (call-with-values (lambda () (source-index->lists tree)) list)) +(def (relocate token delta) + (make-token (token-kind token) (token-lexeme token) (+ delta (token-start token)) (+ delta (token-end token)))) +(def source-index-test + (test-suite "persistent measured source index" + (test-case "empty streams and token/mode ownership" + (let* ((source (tokens 130)) (modes (map (lambda (at) (modulo at 7)) (iota 130))) + (tree (build source modes))) + (check (source-index-count #f) => 0) + (check (materialize #f) => '(() ())) + (check (source-index-audit tree) => #t) + (check (materialize tree) => (list source modes)) + (check (<= (source-index-height tree) 4) => #t) + (let (cursor (make-source-index-cursor tree 63)) + (check (eq? (source-index-cursor-token cursor) (list-ref source 63)) => #t) + (check (source-index-cursor-mode cursor) => 0) + (source-index-cursor-next! cursor) + (check (source-index-cursor-token cursor) => (list-ref source 64)) + (check (source-index-cursor-mode cursor) => 1)))) + (test-case "UTF8 byte positions remain independent of character measures" + (let* ((source (list (make-token 'identifier "λ中😀" 0 9) (make-token 'trivia " " 9 10) + (make-token 'identifier "é" 10 12))) (modes '(2 2 4)) + (tree (build source modes)) (shifted (source-index-shift tree 5))) + (check (source-index-audit shifted) => #t) + (check (materialize shifted) => (list (map (cut relocate <> 5) source) modes)) + (check (eq? (source-index-shift shifted -5) tree) => #t) + (let (cursor (make-source-index-cursor shifted)) + (source-index-cursor-seek! cursor 6) + (check (source-index-cursor-rank cursor) => 1) + (check (token-start (source-index-cursor-token cursor)) => 14) + (source-index-cursor-seek! cursor 15) + (check (source-index-cursor-mode cursor) => 4) + (source-index-cursor-seek! cursor 17) + (check (source-index-cursor-token cursor) => #f)))) + (test-case "slice/concat retain backing chunks and rejected trials do not consume" + (let* ((source (tokens 1024)) (modes (make-list 1024 3)) (tree (build source modes)) + (middle (source-index-slice tree 71 831))) + (check (source-index-audit middle) => #t) + (check (materialize middle) => (list (take (drop source 71) 831) (make-list 831 3))) + (check (every (lambda (storage) (and (memq storage (source-index-storage tree)) #t)) + (source-index-storage middle)) => #t) + (let* ((cursor (make-source-index-cursor middle 63)) (trial (source-index-cursor-copy cursor))) + (source-index-cursor-seek! trial 800) + (check (source-index-cursor-rank trial) => 729) + (check (source-index-cursor-rank cursor) => 63) + (check (token-start (source-index-cursor-token cursor)) => 134)) + (check (materialize (source-index-concat (source-index-slice tree 0 512) + (source-index-slice tree 512 512))) + => (list source modes)))) + (test-case "origins coalesce and fresh allocations are confined to changed chunks" + (let* ((tree (build (tokens 1024) (make-list 1024 1))) + (builder (make-source-index-builder (source-index-slice tree 0 100)))) + (source-index-builder-token! builder (make-token 'identifier "zz" 100 102) 6) + (source-index-builder-shared! builder tree 100 400 2) + (source-index-builder-shared! builder tree 500 524 2) + (let (next (source-index-builder-finish builder)) + (check (source-index-count next) => 1025) + (check (source-index-end next) => 1026) + (check (source-index-builder-fresh builder) => 1) + (check (source-index-builder-shared builder) => 1024) + (check (source-index-builder-chunks builder) => 1) + (check (source-index-audit next) => #t) + (check (> (length (filter (lambda (storage) (memq storage (source-index-storage tree))) + (source-index-storage next))) 10) => #t)))) + (test-case "sparse mode changes retain token chunks and old mode versions" + (let* ((original (tokens 130)) (old-modes (make-list 130 1)) (tree (build original old-modes)) + (builder (make-source-index-builder))) + (source-index-builder-shared! builder tree 0 63 0) + (source-index-builder-shared! builder tree 63 1 0 0) + (source-index-builder-shared! builder tree 64 1 0 9) + (source-index-builder-shared! builder tree 65 65 0) + (let (next (source-index-builder-finish builder)) + (check (source-index-audit next) => #t) + (check (materialize tree) => (list original old-modes)) + (check (materialize next) => (list original (append (make-list 63 1) '(0 9) (make-list 65 1)))) + (check (every eq? (source-index-storage next) (source-index-storage tree)) => #t) + (check (source-index-builder-chunks builder) => 0) + (check (source-index-builder-mode-chunks builder) => 2)))) + (test-case "deterministic sequential splices preserve old versions and AVL measures" + (let* ((original (tokens 1024)) (modes (make-list 1024 2)) (initial (build original modes))) + (let loop ((step 0) (tree initial) (expected original) (expected-modes modes)) + (check (source-index-audit tree) => #t) + (check (materialize tree) => (list expected expected-modes)) + (check (materialize initial) => (list original modes)) + (check (<= (source-index-height tree) 16) => #t) + (when (< step 180) + (let* ((size (length expected)) (at (modulo (+ (* step 37) 5) size)) + (deleted (min (modulo step 5) (- size at))) (added (modulo (+ step 2) 7)) + (delta (- added deleted)) + (middle (map (lambda (n) (make-token 'identifier "y" (+ at n) (+ at n 1))) (iota added))) + (middle-modes (make-list added (modulo step 9))) + (builder (make-source-index-builder (source-index-slice tree 0 at)))) + (for-each (lambda (token mode) (source-index-builder-token! builder token mode)) middle middle-modes) + (source-index-builder-shared! builder tree (+ at deleted) (- size at deleted) delta) + (loop (+ step 1) (source-index-builder-finish builder) + (append (take expected at) middle (map (cut relocate <> delta) (drop expected (+ at deleted)))) + (append (take expected-modes at) middle-modes (drop expected-modes (+ at deleted))))))))) + (test-case "invalid gaps and out-of-range slices fail before sharing" + (let (tree (build (tokens 10) (make-list 10 0))) + (check-exception (source-index-slice tree 9 2) true) + (check-exception (source-index-slice tree -1 1) true) + (check-exception (source-index-concat tree (source-index-shift tree 11)) true) + (check-exception (build (list (make-token 'trivia "" 0 0)) '(0)) true))))) +(export source-index-test) diff --git a/t/source-priority-test.ss b/t/source-priority-test.ss new file mode 100644 index 00000000..0b56b7bb --- /dev/null +++ b/t/source-priority-test.ss @@ -0,0 +1,22 @@ +;;; Ordered-choice lowering stays pure and preserves empty matched branches. +(import :std/test + (only-in "../src/modules/parser/source-pattern-funs.ss" + source-priority-forms)) +(export source-priority-test) +(def source-priority-test + (test-suite "source priority lowering" + (test-case "priority and empty branches retain exact event-fold structure" + (check (source-priority-forms '((bool #t) (bool #f)) + '(() ((token Match start end))) + '((token Fallback start end))) + => '((if (bool #t) () + ((if (bool #f) ((token Match start end)) + ((token Fallback start end))))))) + (check (source-priority-forms '() '() '((token Fallback start end))) + => '((token Fallback start end))) + (check (source-priority-forms '((bool #t)) '(())) => '((if (bool #t) () ())))) + (test-case "invalid branch shapes are rejected before lowering" + (for-each + (lambda (args) + (check-exception (apply source-priority-forms args) true)) + '(((a) ()) ((a) (b)) ((a . b) (())) (() () #f)))))) diff --git a/t/source-probe-test.ss b/t/source-probe-test.ss new file mode 100644 index 00000000..ddfbc683 --- /dev/null +++ b/t/source-probe-test.ss @@ -0,0 +1,72 @@ +#!/usr/bin/env gxi +;;; -*- Gerbil -*- +(import :std/test + (only-in :gerbil-parser/languages/hcl/parser hcl-parser) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime parser-machine-ir) + (only-in :gerbil-parser/src/runtime/lr-parser + lr-prepare lr-initial-checkpoint lr-checkpoint-feed + lr-checkpoint-lexical-mode lr-runtime-lexical-mode-catalog lr-lexical-mode-id) + (only-in :gerbil-parser/src/runtime/token make-token token-lexeme token-end) + (only-in :gerbil-parser/src/runtime/probe + make-source-probe-cache source-probe-scan source-probe-take!)) + +(def (value-mode) + (let (initial (lr-initial-checkpoint (parser-machine-runtime hcl-parser) '())) + (let-values (((status identifier) + (lr-checkpoint-feed initial (make-token 'identifier "value" 0 5)))) + (check status => 'checkpoint) + (let-values (((status value) + (lr-checkpoint-feed identifier (make-token 'punctuation "=" 6 7)))) + (check status => 'checkpoint) + (lr-checkpoint-lexical-mode value))))) + +(def source-probe-test + (test-suite "request-owned lexical probes" + (test-case "UTF8 byte and character positions both bind the canonical token" + (let* ((source " \"λ中😀\"") (mode (value-mode)) + (cache (make-source-probe-cache hcl-parser source))) + (let-values (((token next-character) (source-probe-scan cache 1 1 mode))) + (check (token-end token) => 12) + (check next-character => 6) + (check (source-probe-take! cache 1 0 mode) => #f) + (check (source-probe-take! cache 0 1 mode) => #f) + (let (taken (source-probe-take! cache 1 1 mode)) + (check (eq? (car taken) token) => #t) + (check (cdr taken) => next-character)) + (check (source-probe-take! cache 1 1 mode) => #f)))) + (test-case "equal numeric mode IDs from another runtime cannot share probes" + (let* ((mode (value-mode)) + (other (lr-prepare (cdr (assq 'lr-spec (parser-machine-ir hcl-parser))))) + (other-mode (vector-ref (lr-runtime-lexical-mode-catalog other) + (lr-lexical-mode-id mode))) + (cache (make-source-probe-cache hcl-parser "001"))) + (check (lr-lexical-mode-id other-mode) => (lr-lexical-mode-id mode)) + (let-values (((token next) (source-probe-scan cache 0 0 mode))) + (check (source-probe-take! cache 0 0 other-mode) => #f) + (check (eq? (car (source-probe-take! cache 0 0 mode)) token) => #t)))) + (test-case "source requests own distinct bounded slots" + (let* ((mode (value-mode)) + (first (make-source-probe-cache hcl-parser "001")) + (second (make-source-probe-cache hcl-parser "002"))) + (let-values (((token next) (source-probe-scan first 0 0 mode))) + (check (source-probe-take! second 0 0 mode) => #f)) + (let-values (((token next) (source-probe-scan second 0 0 mode))) + (check (token-lexeme (car (source-probe-take! second 0 0 mode))) => "002")) + (check (token-lexeme (car (source-probe-take! first 0 0 mode))) => "001"))) + (test-case "future probes survive trivia and passed probes expire" + (let* ((mode (value-mode)) + (cache (make-source-probe-cache hcl-parser " 001"))) + (let-values (((token next) (source-probe-scan cache 1 1 mode))) + (check (source-probe-take! cache 0 0 mode) => #f) + (check (eq? (car (source-probe-take! cache 1 1 mode)) token) => #t)) + (let-values (((token next) (source-probe-scan cache 1 1 mode))) + (check (source-probe-take! cache 4 4 mode) => #f) + (check (source-probe-take! cache 1 1 mode) => #f)))) + (test-case "scanner exceptions propagate and clear earlier successful probes" + (let* ((mode (value-mode)) + (cache (make-source-probe-cache hcl-parser "001"))) + (let-values (((token next) (source-probe-scan cache 0 0 mode))) + (check (with-catch (lambda (_) #t) + (lambda () (source-probe-scan cache -1 -1 mode) #f)) => #t) + (check (source-probe-take! cache 0 0 mode) => #f)))))) +(export source-probe-test) diff --git a/t/source-recognition-test.ss b/t/source-recognition-test.ss new file mode 100644 index 00000000..6b5fb6eb --- /dev/null +++ b/t/source-recognition-test.ss @@ -0,0 +1,91 @@ +;;; Source-backed recognition spans are independent of language semantics. +(import :std/test + :gerbil-parser/src/runtime/recognition + (only-in :gerbil-parser/src/runtime/token make-token token? token-kind token-lexeme token-start token-end)) +(export source-recognition-test) +(def (rejects? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def source-recognition-test + (test-suite "source-backed recognition spans" + (test-case "token inline calls retain first-class procedures and live checked vector reads" + (let* ((token (make-token 'word "x" 4 5)) + (readers (list token-kind token-lexeme token-start token-end))) + (check (every procedure? readers) => #t) + (check (map (lambda (read) (read token)) readers) => '(word "x" 4 5)) + (vector-set! token 0 'changed) + (vector-set! token 1 "y") + (check (list (token-kind token) (token-lexeme token) + (token-start token) (token-end token)) => '(changed "y" 4 5)) + (check (map (lambda (read) (read token)) readers) => '(changed "y" 4 5)) + (check (token? token) => #t) + (for-each (lambda (read) (check-exception (read '#()) true)) readers) + (check-exception (token-kind '#()) true) + (check-exception (token-lexeme #f) true)) + (for-each (lambda (value) (check (token? value) => #f)) + (list #f 0 '#() '#(word "x" -1 2) '#(word "x" 3 2))) + (check (token? (make-token 0 "x" 0 1)) => #f)) + (test-case "private recognition variants reject extension and wrong-kind access" + (let* ((token (make-token 'value "x" 4 5)) + (child (make-recognition-child #f token)) + (node (make-recognition-node 'Expression 4 5 (list child))) + (fragment (make-recognition-fragment 4 5 (list child))) + (view (relocate-recognition-value node 3))) + (for-each + (lambda (value) + (check (rejects? + (lambda () + (make-class-type 'recognition-extension 'recognition-extension + (list (class-of value)) '() '() #f))) => #t)) + (list node fragment child view)) + (check (rejects? (lambda () (recognition-node-start fragment))) => #t) + (check (rejects? (lambda () (recognition-child-value node))) => #t) + (recognition-child-field-set! child 'value) + (check (recognition-child-field child) => 'value) + (check (recognition-child-value child) => token) + (let-values (((start end) (recognition-value-bounds view))) + (check (list start end) => '(7 8))))) + (test-case "all Unicode widths and nonmonotonic endpoints agree with UTF-8" + (let* ((text "aα中😀\n") + (source (prepare-recognition-source (make-token 'source text 13 24)))) + (for-each (lambda (at) + (check (recognition-source-offset source at) + => (+ 13 (u8vector-length (string->utf8 (substring text 0 at)))))) + '(5 0 4 1 3 2)) + (let* ((token (recognition-source-token source 'value 1 4)) + (node (recognition-source-node source 'Expression 1 4 + (list (make-recognition-child 'value token))))) + (check (token-lexeme token) => "α中😀") + (check (token-start token) => 14) + (check (token-end token) => 23) + (check (recognition-value-start node) => 14) + (check (recognition-value-end node) => 23)))) + (test-case "ASCII and a late multibyte scalar admit exact boundaries" + (for-each + (lambda (text) + (let (source (prepare-recognition-source + (make-token 'source text 7 (+ 7 (u8vector-length (string->utf8 text)))))) + (let loop ((at 0)) + (when (<= at (string-length text)) + (check (recognition-source-offset source at) + => (+ 7 (u8vector-length (string->utf8 (substring text 0 at))))) + (loop (+ at 1)))))) + '("" "abc" "abcα" "abcα中😀"))) + (test-case "original text and returned views cannot invalidate admitted offsets" + (let* ((text (string-copy "αx")) + (source (prepare-recognition-source (make-token 'source text 2 5))) + (view (recognition-source-text source))) + (string-set! text 0 #\a) + (string-set! view 0 #\b) + (check (recognition-source-text source) => "αx") + (check (token-lexeme (recognition-source-token source 'value 0 1)) => "α") + (check (recognition-source-offset source 1) => 4))) + (test-case "malformed byte spans and out-of-source boundaries reject" + (for-each (lambda (token) + (check (rejects? (lambda () (prepare-recognition-source token))) => #t)) + (list #f (make-token 'source "α" 0 1) (make-token 'source "" 3 4))) + (let (source (prepare-recognition-source (make-token 'source "α" 0 2))) + (for-each (lambda (at) + (check (rejects? (lambda () (recognition-source-offset source at))) => #t)) + '(-1 2 0.5)) + (check (rejects? (lambda () (recognition-source-token source 'value 1 0))) => #t) + (check (rejects? (lambda () (recognition-source-node source 'Value 0 2 '()))) => #t))))) diff --git a/t/source-scanner-test.ss b/t/source-scanner-test.ss new file mode 100644 index 00000000..38ed8ea7 --- /dev/null +++ b/t/source-scanner-test.ss @@ -0,0 +1,190 @@ +;;; Engine checkpoints are owned by the creating worker, including shared text. +(import :std/test + (only-in :gerbil-parser/src/runtime/source-scanner + make-source-scanner source-scanner-tokens source-scanner-initial-state source-scanner-step + source-scan-state-with-context source-scan-state-byte-offset + make-source-scan-session source-scan-session-tokens + source-scan-session-scanned-token-count source-scan-session-reused-token-count) + (only-in :gerbil-parser/src/runtime/token token-start token-end token-kind token-lexeme) + (only-in :gerbil-parser/src/runtime/source-scanner source-scan-state-context) + (only-in :gerbil-parser/src/runtime/contextual-scanner + +empty-delimiter-queue+ delimiter-queue-enqueue delimiter-queue-take + delimiter-queue-list delimiter-obligation-marker delimiter-obligation-quoted? + contextual-scan-state-canonical decode-marker) + (only-in :gerbil-parser/src/language/source source-language-scanner-factory + parse-source-language parse-source-language/session source-language-session-artifact + source-language-session? source-language-history? source-language-session-history + source-language-session-scanned-token-count source-language-session-reused-token-count) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-success?)) +(import (only-in :gerbil-parser/languages/bash/parser bash-source-language)) +(def (pending-markers state) + (map car (cdr (assq 'pending (contextual-scan-state-canonical state))))) +(def (rejected? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def source-scanner-test + (test-suite "source scanner worker ownership" + (test-case "history retains scanner ownership without retaining the published artifact" + (let* ((initial (parse-source-language/session bash-source-language "cat < #t) + (check (source-language-session? history) => #f) + (check (source-language-session-artifact next) => (parse-source-language bash-source-language source)) + (check (source-language-session-scanned-token-count next) => 1) + (check (> (source-language-session-reused-token-count next) 0) => #t))) + (test-case "Bash sessions reuse lexical suffixes and publish full fresh-parser parity" + (let* ((old (parse-source-language/session bash-source-language "cat < #t) + (check (source-language-session-artifact next) => (parse-source-language bash-source-language source)) + (check (source-language-session-scanned-token-count next) => 1) + (check (source-language-session-reused-token-count next) + => (- (source-language-session-scanned-token-count old) 1)) + ;; The second edit consumes checkpoints rebound by the first edit. + (check (source-language-session-artifact later) => (parse-source-language bash-source-language later-source)) + (check (> (source-language-session-reused-token-count later) 0) => #t))) + (test-case "changed quoted delimiter waits for the obligation to close before reuse" + (let* ((old (parse-source-language/session bash-source-language "cat < (parse-source-language bash-source-language source)) + (check (source-language-session-scanned-token-count next) => 7) + (check (source-language-session-reused-token-count next) => 4))) + (test-case "source session cache owns tokens and rejects ordinary callback reuse" + (let* ((factory (source-language-scanner-factory bash-source-language)) + (text (string-copy "cat < "cat") + (let* ((next (make-source-scan-session (factory "echo猫 < fresh) + (check (source-scan-session-scanned-token-count next) => 1) + (check (> (source-scan-session-reused-token-count next) 0) => #t)) + (let* ((calls 0) + (worker (make-source-scanner "xy" #f + (lambda (_source start context _mode) + (set! calls (+ calls 1)) + (if (= start 2) (values #f start context) + (values 'character (+ start 1) context))))) + (first (make-source-scan-session worker 'ordinary)) + (second (make-source-scan-session worker 'ordinary first))) + (check calls => 6) + (check (source-scan-session-reused-token-count second) => 0)))) + (test-case "long session histories rebind prefix middle tail and EOF edits" + (let* ((factory (source-language-scanner-factory bash-source-language)) + (prefix (string-join (make-list 64 "echo prefix\n") "")) + (tail (string-join (make-list 64 "echo tail\n") "")) + (original (string-append prefix "cat < fresh) + (check (source-scan-session-tokens again) => fresh) + (check (source-scan-session-scanned-token-count again) => 0) + (check (source-scan-session-reused-token-count again) => (length fresh)) + (when (pair? published) + (string-set! (token-lexeme (car published)) 0 #\X) + (vector-set! (car published) 2 999)) + (check (source-scan-session-tokens next) => fresh) + (check (source-scan-session-tokens first) + => (source-scanner-tokens (factory original) 'command)))) + (list (string-append "猫 " original) + (string-append prefix "cat <<'A'\nβ\nA\n" tail) + (string-append original "echo 終\n") + (string-append original "\n") + "")))) + (test-case "lexical rejection publishes the original diagnostic and retains no cache" + (let* ((bad-source "echo 'unterminated") + (bad (parse-source-language/session bash-source-language bad-source)) + (good (parse-source-language/session bash-source-language "echo ok\n" bad))) + (check (source-language-session-artifact bad) => (parse-source-language bash-source-language bad-source)) + (check (parse-artifact-success? (source-language-session-artifact bad)) => #f) + (check (source-language-session-scanned-token-count bad) => #f) + (check (source-language-session-reused-token-count good) => 0) + (check (source-language-session-artifact good) => (parse-source-language bash-source-language "echo ok\n")))) + (test-case "shared FIFO preserves order and branches without copying the pending prefix" + (let* ((a (decode-marker 'raw "A" #f)) (b (decode-marker 'raw "B" #f)) + (first (delimiter-queue-enqueue +empty-delimiter-queue+ a)) + (second (delimiter-queue-enqueue first b))) + (check (map delimiter-obligation-marker (delimiter-queue-list first)) => '("A")) + (let-values (((head rest) (delimiter-queue-take second))) + (check (delimiter-obligation-marker head) => "A") + (check (map delimiter-obligation-marker (delimiter-queue-list rest)) => '("B")) + (check (map delimiter-obligation-marker (delimiter-queue-list second)) => '("A" "B"))) + (check (rejected? (lambda () (delimiter-queue-enqueue first 'callback))) => #t) + (check (rejected? (lambda () (delimiter-queue-take +empty-delimiter-queue+))) => #t))) + (test-case "Bash uses engine quote removal including preserved double-quote escapes" + (for-each + (lambda (row) + (let (marker (decode-marker 'shell-quote-removal (car row) #f)) + (check (delimiter-obligation-marker marker) => (cadr row)) + (check (delimiter-obligation-quoted? marker) => (caddr row)))) + '(("A" "A" #f) ("'A'" "A" #t) ("\"a\\q\"" "a\\q" #t) + ("\"a\\$\"" "a$" #t) ("'a\\q'" "a\\q" #t) + ("A\\\nB" "AB" #f) ("''" "" #t))) + (check (rejected? (lambda () (decode-marker 'shell-quote-removal "" #f))) => #t) + (check (rejected? (lambda () (decode-marker 'shell-quote-removal "'A" #f))) => #t) + (check (rejected? (lambda () (decode-marker 'arbitrary "A" #f))) => #t)) + (test-case "Bash queued checkpoint replay retains its own pending obligations" + (let* ((worker ((source-language-scanner-factory bash-source-language) "< '("A")) + (let-values (((again replay) (source-scanner-step worker after-open 'command))) + (check (token-lexeme again) => "A") + (check (pending-markers (source-scan-state-context replay)) => '("A"))) + (check (pending-markers (source-scan-state-context initial)) => '()))))) + (test-case "large Bash deferred batches drain in declaration order" + (let* ((markers (map (lambda (n) (string-append "END" (number->string n))) (iota 128))) + (source (string-append "cat " + (apply string-append (map (lambda (name) (string-append "<<" name " ")) markers)) + "\n" (apply string-append (map (lambda (name) (string-append "α\n" name "\n")) markers)))) + (tokens (source-scanner-tokens ((source-language-scanner-factory bash-source-language) source) 'command))) + (check (map token-lexeme (filter (lambda (token) (eq? (token-kind token) 'heredoc-marker)) tokens)) => markers) + (check (map token-lexeme (filter (lambda (token) (eq? (token-kind token) 'heredoc-end)) tokens)) + => (map (lambda (name) (string-append name "\n")) markers)) + (check (apply string-append (map token-lexeme tokens)) => source))) + (test-case "another worker cannot use a checkpoint over the exact same string" + (let* ((source "αx") (calls 0) + (step (lambda (_source offset context _mode) + (set! calls (+ calls 1)) + (values 'character (+ offset 1) context))) + (left (make-source-scanner source 'left step)) + (right (make-source-scanner source 'right step)) + (initial (source-scanner-initial-state left))) + (check (rejected? (lambda () (source-scanner-step right initial 'test))) => #t) + (check calls => 0) + (check (rejected? (lambda () + (source-scanner-step right (source-scan-state-with-context initial 'changed) 'test))) => #t) + (check calls => 0))) + (test-case "owned checkpoint replay and context updates preserve UTF-8 spans" + (let* ((worker (make-source-scanner "αx" 'initial + (lambda (_source offset context _mode) + (if (= offset 2) (values #f offset context) + (values 'character (+ offset 1) context))))) + (initial (source-scanner-initial-state worker))) + (let-values (((first next) (source-scanner-step worker initial 'test))) + (check (list (token-start first) (token-end first)) => '(0 2)) + (check (source-scan-state-byte-offset initial) => 0) + (let-values (((again replay) (source-scanner-step worker initial 'test))) + (check (list (token-start again) (token-end again)) => '(0 2)) + (check (source-scan-state-byte-offset replay) => 2)) + (let-values (((second end) (source-scanner-step worker (source-scan-state-with-context next 'updated) 'test))) + (check (list (token-start second) (token-end second)) => '(2 3)) + (let-values (((eof final) (source-scanner-step worker end 'test))) + (check eof => #f) + (check (source-scan-state-byte-offset final) => 3)))))))) +(export source-scanner-test) diff --git a/t/source-strategy-test.ss b/t/source-strategy-test.ss new file mode 100644 index 00000000..454f81a5 --- /dev/null +++ b/t/source-strategy-test.ss @@ -0,0 +1,126 @@ +;;; A second source family shares admission, identity, workers and publication. +(import :std/test + (only-in :clan/poo/object .o .cc) + (only-in :gerbil-parser/src/language/source-strategy SourceStrategy. bind-source-strategy declare-source-strategy-provider make-source-engine source-engine-scanner source-engine-factory source-engine-parse source-engine-receipt source-engine-results source-engine-program) + (only-in :gerbil-parser/src/runtime/source-engines LineSourceStrategy.) + (only-in :gerbil-parser/src/compiler/contextual-program contextual-program-ir) + (only-in :gerbil-parser/src/runtime/contextual-ir contextual-parser-ir-valid? contextual-ir-ref) + (only-in :gerbil-parser/src/compiler/rust-scanner command-source-rust-module-source) + (only-in :gerbil-parser/t/fixtures/bash-products bash-results bash-parts bash-word-regions bash-commands bash-command-scanner) + (only-in :gerbil-parser/src/runtime/source-engines ShellSourceStrategy.) + (only-in :gerbil-parser/src/language/source declare-source-language parse-source-language source-language-digest source-language-contextual-ir + parse-source-language/session source-language-session-artifact + source-language-session-scanned-token-count source-language-session-reused-token-count) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-valid? parse-artifact-success? parse-artifact-roundtrip) + (only-in :gerbil-parser/language-support/development deflanguage-development-loader LanguageDevelopmentLoader. declare-language-source-scan-worker make-language-scan-worker) + (only-in :gerbil-parser/src/runtime/source-scanner source-scanner-tokens) + (only-in :gerbil-parser/src/runtime/token token-lexeme)) +(export source-strategy-test) +(def prefix (string-copy "#")) +(def notes-source + (declare-source-language "notes" "v1" "notes.v1" + (.o (:: self LineSourceStrategy.) root-kind: 'Notes required-prefix: prefix))) +(deflanguage-development-loader (notes-language :: self LanguageDevelopmentLoader.) + (source notes-source) (parse parse-notes) + (slots scan-workers: (list (cons 'lines (declare-language-source-scan-worker notes-source))))) +(def (rejects? thunk) (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def source-strategy-test + (test-suite "source recipe reuse and ownership" + (test-case "ordinary Source sessions preserve the declared scanner and do not admit reuse" + (let* ((old (parse-source-language/session notes-source "#old\n")) + (next (parse-source-language/session notes-source "#new\n" old))) + (check (source-language-session-artifact next) => (parse-source-language notes-source "#new\n")) + (check (source-language-session-scanned-token-count next) => #f) + (check (source-language-session-reused-token-count next) => 0) + (check (rejects? (lambda () + (parse-source-language/session + (declare-source-language "foreign" "v1" "foreign.v1" LineSourceStrategy.) + "#new\n" old))) => #t))) + (test-case "one compilation supplies both identity and executable source product" + (let ((classifications 0) (compilations 0)) + (let-values (((recipe engine) (bind-source-strategy LineSourceStrategy.))) + (let* ((counted-provider (declare-source-strategy-provider (quote counted) + (lambda (_) (set! classifications (+ classifications 1)) #t) + (lambda (_) (set! compilations (+ compilations 1)) + (values (cadr recipe) engine)))) + (strategy (.o (:: self SourceStrategy.) provider: counted-provider))) + (let-values (((compiled-recipe compiled-engine) (bind-source-strategy strategy))) + (check classifications => 1) + (check compilations => 1) + (check compiled-recipe => (list (quote counted) (cadr recipe))) + (check (eq? compiled-engine engine) => #t)))))) + (test-case "Rust lowering consumes admitted products without reading declaration slots" + (let* ((strategy (.o (:: self ShellSourceStrategy.) regions: bash-word-regions + results: bash-results parts: bash-parts commands: bash-commands + scanner: bash-command-scanner)) + (reference (command-source-rust-module-source strategy)) + (compilations 0)) + (let-values (((recipe engine) (bind-source-strategy strategy))) + ;; Only the provider remains: rereading any declaration slot must fail. + (let* ((admitted-provider (declare-source-strategy-provider 'shell (lambda (_) #t) + (lambda (_) (set! compilations (+ compilations 1)) + (values (cadr recipe) engine)))) + (bound (.o (:: self SourceStrategy.) provider: admitted-provider))) + (check (command-source-rust-module-source bound) => reference) + (check compilations => 1))))) + (test-case "common contextual product rejects malformed and ambiguous envelopes" + (let-values (((recipe engine) (bind-source-strategy LineSourceStrategy.))) + (let (ir (contextual-program-ir (source-engine-program engine))) + (check (contextual-parser-ir-valid? ir) => #t) + (check (contextual-ir-ref (contextual-ir-ref ir 'recognition) 'dialect) => 'lines) + (for-each (lambda (bad) (check (contextual-parser-ir-valid? bad) => #f)) + (list '(invalid) (cons '(root-kind . SourceFile) ir) + (cons '(foreign . 1) ir) + (map (lambda (row) (if (eq? (car row) 'root-kind) + '(root-kind . Missing) row)) ir) + (map (lambda (row) (if (eq? (car row) 'recognition) + (cons 'recognition (list (cons 'dialect 'lines) + (cons 'program (lambda () #t)))) row)) ir))))) + (let-values (((recipe engine) (bind-source-strategy LineSourceStrategy.))) + (let* ((mismatched-provider (declare-source-strategy-provider 'mismatch (lambda (_) #t) + (lambda (_) (values (cons '(foreign . 1) (cadr recipe)) engine)))) + (strategy (.o (:: self SourceStrategy.) provider: mismatched-provider))) + (check (rejects? (lambda () (bind-source-strategy strategy))) => #t)))) + (test-case "source roots must belong to the compiled result catalog" + (let-values (((recipe engine) (bind-source-strategy LineSourceStrategy.))) + (let* ((bad (make-source-engine (source-engine-scanner engine) + (source-engine-factory engine) (source-engine-parse engine) + (source-engine-receipt engine) (source-engine-results engine) 'UndeclaredRoot (source-engine-program engine))) + (bad-root-provider (declare-source-strategy-provider 'bad-root (lambda (_) #t) + (lambda (_) (values (cadr recipe) bad)))) + (strategy (.o (:: self SourceStrategy.) provider: bad-root-provider))) + (check (rejects? (lambda () (bind-source-strategy strategy))) => #t)))) + (test-case "line recipe publishes lossless UTF-8 artifacts and owned workers" + (let* ((source "#α\nsecond\n") (artifact (parse-notes source)) + (worker (make-language-scan-worker notes-language 'lines source))) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source) + (check (apply string-append (map token-lexeme (source-scanner-tokens worker 'lines))) => source)) + (check (parse-artifact-success? (parse-notes "different")) => #f)) + (test-case "source descriptors retain an owned canonical product" + (let (ir (source-language-contextual-ir notes-source)) + (check (contextual-parser-ir-valid? ir) => #t) + (set-cdr! (assq 'root-kind ir) 'Foreign) + (check (contextual-ir-ref (source-language-contextual-ir notes-source) 'root-kind) => 'Notes) + (check (parse-artifact-success? (parse-notes "#kept")) => #t))) + (test-case "Scheme source authority admits the deep mixed continuation control losslessly" + (let* ((strategy (.o (:: self ShellSourceStrategy.) regions: bash-word-regions + results: bash-results parts: bash-parts commands: bash-commands + scanner: bash-command-scanner)) + (descriptor (declare-source-language "source-control" "v1" "source-control.v1" strategy)) + (source (string-append (apply string-append (make-list 512 "{ ( ")) + "echo α; " (apply string-append (make-list 512 "); } ; ")))) + (artifact (parse-source-language descriptor source))) + (check (parse-artifact-success? artifact) => #t) + (check (parse-artifact-valid? artifact) => #t) + (check (parse-artifact-roundtrip artifact) => source))) + (test-case "source identity includes the closed engine recipe" + (let (other (declare-source-language "notes" "v1" "notes.v1" + (.o (:: self LineSourceStrategy.) root-kind: 'Different required-prefix: "#"))) + (check (equal? (source-language-digest other) (source-language-digest notes-source)) => #f))) + (test-case "binding snapshots values and rejects invalid provider overrides" + (string-set! prefix 0 #\!) + (check (parse-artifact-success? (parse-notes "#kept")) => #t) + (check (rejects? (lambda () (declare-source-language "bad" "v1" "bad.v1" (.cc LineSourceStrategy. 'root-kind "wrong")))) => #t) + (check (rejects? (lambda () (declare-source-language "bad" "v1" "bad.v1" (.cc LineSourceStrategy. 'provider (lambda (_) #t))))) => #t)))) diff --git a/t/sparse-capture-test.ss b/t/sparse-capture-test.ss new file mode 100644 index 00000000..51330eec --- /dev/null +++ b/t/sparse-capture-test.ss @@ -0,0 +1,44 @@ +;;; Dense resume from the initial frontier is the independent existing control. +(import :std/test + (only-in :gerbil-parser/t/fixtures/runtime-record-assignments/languages/records/grammar records-parser) + (only-in :gerbil-parser/src/compiler/machine parser-machine-runtime) + (only-in :gerbil-parser/src/runtime/parser parse-source/checkpoints parse-source/checkpoints/resume + current-source-stream-observer) + (only-in :gerbil-parser/src/runtime/lr-parser lr-initial-checkpoint current-lr-recognition-observer)) +(export sparse-capture-test) +(def (capture-observation source spacing dense?) + (let ((root #f) (scans '())) + (parameterize ((current-lr-recognition-observer (lambda (value) (set! root value))) + (current-source-stream-observer + (lambda (token mode fragment?) (set! scans (cons (list token mode fragment?) scans))))) + (let (result + (call-with-values + (lambda () + (if dense? + (parse-source/checkpoints/resume records-parser source spacing + (lr-initial-checkpoint (parser-machine-runtime records-parser) '()) '() '() 0 0 #f) + (parse-source/checkpoints records-parser source spacing))) list)) + ;; Includes every checkpoint field and the complete grammar tree, not + ;; merely the public artifact or a snapshot count. + (list result root (reverse scans)))))) +(def sparse-capture-test + (test-suite "sparse initial LR capture equals dense initial resume" + (test-case "all checkpoint fields match at dense and sparse spacings" + (let (source (apply string-append (make-list 64 "a=1\n"))) + (for-each (lambda (spacing) + (check (capture-observation source spacing #f) + => (capture-observation source spacing #t))) '(1 2 2.0 32 1024)))) + (test-case "empty and trivia-only sources retain exact initial boundaries" + (for-each (lambda (source) + (check (capture-observation source 2 #f) + => (capture-observation source 2 #t))) '("" " \t\n" "\r\n"))) + (test-case "Unicode strings and trivia retain modes and scan callback order" + (for-each (lambda (source) + (check (capture-observation source 2 #f) + => (capture-observation source 2 #t))) + '("a = \"猫\"\n b=2\n" "a=1\r\nb=2\r\n" "a=1\n \t"))) + (test-case "rejection and incomplete EOF retain the dense diagnostics" + (for-each (lambda (source) + (check (capture-observation source 2 #f) + => (capture-observation source 2 #t))) + '("a=" "a=1\n?" "a=\"unterminated" "a=1\n猫=2\n"))))) diff --git a/t/structured-lexical-cases.ss b/t/structured-lexical-cases.ss new file mode 100644 index 00000000..6bd512d8 --- /dev/null +++ b/t/structured-lexical-cases.ss @@ -0,0 +1,10 @@ +;;; Scheme character endpoints for the three migrated TLA+ lexical rules. +(export structured-lexical-cases) +(def structured-lexical-cases + '((identifier ("123" #f) ("1_name" 6) ("α٣_字!" 4) ("²" #f) ("α²" 1) + ("_" 1) ("４2" #f) ("４a" 2) ("" #f)) + (proof-step-name ("<1>1. QED" 5) ("<+>*." 5) ("<*>-." 5) ("<1>" 3) + ("<1>name" #f) ("<1>α٣." 6) ("<١>字..." 7) + ("<+1>x." #f) ("<1" #f) ("<>." #f)) + (proof-reference ("<1>name" 7) ("<1>α٣" 5) ("<١>字" 4) + ("<1>1." #f) ("<1>" #f) ("<*>*" #f) ("<+>_" 4)))) diff --git a/t/structured-lexical-test.ss b/t/structured-lexical-test.ss new file mode 100644 index 00000000..996fd52f --- /dev/null +++ b/t/structured-lexical-test.ss @@ -0,0 +1,45 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; Real language profiles and an independently generated portable grammar. +(import :std/test + (only-in :gerbil-parser/language-support deflanguage) + (only-in :gerbil-parser/languages/tla-plus/grammar + tla-proof-start tla-proof-reference tla-identifier) + (only-in :gerbil-parser/languages/tla-plus/parser tla-plus-sany-candidate-language-grammar) + (only-in :gerbil-parser/src/language/descriptor language-grammar-ir) + (only-in :gerbil-parser/src/runtime/scan make-text-profile-scanner) + (only-in :gerbil-parser/src/grammar/lexical-algebra text-profile?) + (only-in "structured-lexical-cases.ss" structured-lexical-cases)) +(export structured-lexical-test structured-lexical-language-grammar) +(begin + (deflanguage structured-lexical + (syntax + (lexical + (root entry) + (lex (space Space (whitespace+)) + (identifier Identifier (text-profile (ref tla-identifier))) + (proof-step-name ProofStep (text-profile (ref tla-proof-start))) + (proof-reference ProofReference (text-profile (ref tla-proof-reference)))) + (extras space))) + (rules (entry (node Entry (choice identifier proof-step-name proof-reference))))) + (bind-fixture-grammar-release structured-lexical "structured-lexical" "v1" "structured-lexical.v1") ) +(def structured-lexical-test + (test-suite "portable structured lexical IR" + (test-case "real TLA+ rules use closed IR and character endpoints at arbitrary offsets" + (let (rules (cdr (assq 'lexical-rules (language-grammar-ir tla-plus-sany-candidate-language-grammar)))) + (for-each (lambda (group) + (let* ((expression (cadr (assq (car group) rules))) + (scan (make-text-profile-scanner (cadr expression)))) + (check (car expression) => 'text-profile) + (for-each (lambda (row) + (check (scan (car row) 0) => (cadr row)) + (check (scan (string-append "字" (car row)) 1) + => (and (cadr row) (+ 1 (cadr row))))) (cdr group)))) structured-lexical-cases))) + (test-case "guards reject malformed classes, empty spans and nullable children" + (for-each (lambda (value) (check (text-profile? value) => #f)) + '((ends-in (numeric) (optional (literal "x"))) + (ends-not-in (unknown) (literal "x")) + (run-containing (numeric) (unknown) 1 #f) + (run-containing (numeric) (numeric) 2 1))) + (let (scan (make-text-profile-scanner '(run-containing (numeric) (numeric) 0 #f))) + (check (scan "" 0) => #f) + (check (scan "١" 0) => 1))))) diff --git a/t/structured-profile-test.ss b/t/structured-profile-test.ss new file mode 100644 index 00000000..3fc276de --- /dev/null +++ b/t/structured-profile-test.ss @@ -0,0 +1,66 @@ +(import (only-in :gerbil-parser/t/fixtures/fixture-release bind-fixture-grammar-release)) +;;; Independent language recipes exercise the same closed structured engine. +(import :std/test + (only-in :clan/poo/object .o .cc) + :gerbil-parser/language-support/structured + (only-in :gerbil-parser/language-support deflanguage deftext-profile) + (only-in :gerbil-parser/src/runtime/scan make-text-profile-scanner) + (only-in :gerbil-parser/src/runtime/lexical-source scan-module-text call-with-lexical-source prepare-lexical-source-plan) + (only-in :gerbil-parser/src/language/entry parse-language-source) + (only-in :gerbil-parser/src/runtime/cst parse-artifact->cst) + (only-in :gerbil-parser/src/runtime/artifact parse-artifact-ref)) +(export structured-profile-test) +(def unit-expression '(module-text "~~~" "UNIT" "!!!" "(*" "*)" "\\*" "_")) +(def unit-plan (prepare-lexical-source-plan (list (list 'text unit-expression)))) +(def (unit-text source start) + (call-with-lexical-source unit-plan source (lambda () (scan-module-text source start unit-expression)))) +(deftext-profile square-proof + (seq (literal "[") (run (numeric) 1 #f) (literal "]") + (run (union (alphabetic) (numeric) (characters "_")) 0 #f) + (run (characters ".") 0 #f))) +(def square-start (make-text-profile-scanner + '(ends-not-in (union (alphabetic) (numeric) (characters "_")) + (seq (literal "[") (run (numeric) 1 #f) (literal "]") + (run (union (alphabetic) (numeric) (characters "_")) 0 #f) + (run (characters ".") 0 #f))))) +(def square-reference (make-text-profile-scanner + '(ends-in (union (alphabetic) (numeric) (characters "_")) + (seq (literal "[") (run (numeric) 1 #f) (literal "]") + (run (union (alphabetic) (numeric) (characters "_")) 0 #f) + (run (characters ".") 0 #f))))) +(def square-identifier (make-text-profile-scanner + '(run-containing (union (alphabetic) (numeric) (characters "_")) + (union (alphabetic) (characters "_")) 1 #f))) +(begin + (deflanguage checklist + (syntax + (lexical + (root checklist) + (lex (space Space (whitespace+)) + (step StepName (text-profile (ends-not-in (union (alphabetic) (numeric) (characters "_")) (ref square-proof)))) + (word Word (identifier))) + (extras space))) + (rules (checklist (node Checklist (repeat1 (field item (reference item))))) + (item (node Item (field heading step) (field command word))))) + (bind-fixture-grammar-release checklist "checklist" "v1" "checklist.v1") ) +(def checklist-policy + (bind-structured-proof-policy + (.o (:: self StructuredProofPolicy.) proof-kind: 'Checklist step-field: 'item + name-field: 'heading body-field: 'command close: #\] terminal-command: "END"))) +(def (rejects? thunk) + (with-catch (lambda (_) #t) (lambda () (thunk) #f))) +(def structured-profile-test + (test-suite "inherited structured engine recipes" + (test-case "different proof spelling and module frames use the same scanners" + (check (square-start "[1]1. END" 0) => 5) + (check (square-reference "[1]label" 0) => 8) + (check (square-identifier "123" 0) => #f) + (check (square-identifier "1_name" 0) => 6) + (check (unit-text "preamble\n~~~ UNIT Demo ~~~" 0) => 9) + (check (square-start "<1>1. END" 0) => #f)) + (test-case "renamed CST fields and terminal command reuse the proof stack" + (check (checklist-policy (parse-artifact->cst (parse-language-source checklist-language-grammar "[1]1. END"))) => #f) + (let (diagnostic (checklist-policy (parse-artifact->cst (parse-language-source checklist-language-grammar "[1]1. OPEN")))) + (check (cdr (assq 'failureKind diagnostic)) => 'proof-level-rejected))) + (test-case "malformed overrides are rejected before scanner binding" + (check (rejects? (lambda () (bind-structured-proof-policy (.cc StructuredProofPolicy. 'denied-kinds '("wrong"))))) => #t)))) diff --git a/t/support/qualification/benchmark-qos.ss b/t/support/qualification/benchmark-qos.ss new file mode 100644 index 00000000..04aaa397 --- /dev/null +++ b/t/support/qualification/benchmark-qos.ss @@ -0,0 +1,25 @@ +;;; Test-only host scheduling normalization; never a parser runtime policy. +(import :std/ffi) +(export admit-benchmark-qos!) +(extern set-benchmark-qos) +(begin-foreign + (namespace ("gerbil-parser/t/support/qualification/benchmark-qos#" set-benchmark-qos)) + (c-declare #< +#include +static int gp_benchmark_qos(void) { + qos_class_t actual; + int priority; + if (pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0) != 0) return 0; + return pthread_get_qos_class_np(pthread_self(), &actual, &priority) == 0 + && actual == QOS_CLASS_USER_INTERACTIVE && priority == 0; +} +#else +static int gp_benchmark_qos(void) { return 1; } +#endif +C + ) + (define set-benchmark-qos (c-lambda () int "gp_benchmark_qos"))) +(def (admit-benchmark-qos!) + (unless (= (set-benchmark-qos) 1) (error "benchmark host QoS admission failed"))) diff --git a/t/support/qualification/environment.ss b/t/support/qualification/environment.ss new file mode 100644 index 00000000..13210316 --- /dev/null +++ b/t/support/qualification/environment.ss @@ -0,0 +1,32 @@ +;;; Select host-native Cargo/Rust together and preserve relocated SDK mappings. +(import ./process :std/string/misc) +(export native-environment!) +(def (native-environment!) + (def (query name command (optional? #f)) + (let (result (run-observed-process command (string-append ".data/qualification/" name ".log") timeout: 5 idle-timeout: 5)) + (if (and (zero? (process-result-status result)) (not (process-result-reason result))) + (string-trim (utf8->string (process-result-output result))) + (if optional? #f (error "native tool query failed" name))))) + (let* ((cargo (or (query "cargo-path" '("which" "cargo") #t) + (query "cargo-path" [(path-expand ".cargo/bin/rustup" (getenv "HOME")) "which" "cargo"]))) + (tool-directory (path-directory cargo))) + (setenv "PATH" (string-join [tool-directory (path-expand "bin" (gerbil-home)) + "/opt/homebrew/bin" "/usr/local/bin" "/usr/bin" "/bin" "/usr/sbin" "/sbin"] ":")) + (for-each (lambda (entry) + (let (name (car entry)) + (when (or (string-prefix? "CARGO_PROFILE_" name) + (member name '("NIX_CC" "NIX_CFLAGS_COMPILE" "NIX_LDFLAGS" "CARGO_BUILD_TARGET" + "RUSTFLAGS" "CARGO_ENCODED_RUSTFLAGS" "RUSTC" "RUSTDOC" "RUSTC_WRAPPER" "RUSTC_WORKSPACE_WRAPPER"))) + (setenv name) (displayln "NATIVE-ENV-REMOVED " name) (force-output)))) (get-environment-variables)) + (cond-expand + (darwin + (setenv "CC" "/usr/bin/clang") (setenv "CXX" "/usr/bin/clang++") + (setenv "SDKROOT" (query "sdk-path" '("/usr/bin/xcrun" "--show-sdk-path"))) + (let* ((version (query "rust-host" [(path-expand "rustc" tool-directory) "-vV"])) + (host-line (find (lambda (line) (string-prefix? "host: " line)) (string-split version #\newline)))) + (unless host-line (error "rustc did not report native host")) + (let (host (substring host-line 6 (string-length host-line))) + (setenv (string-append "CARGO_TARGET_" + (string-map (lambda (char) (if (char=? char #\-) #\_ (char-upcase char))) host) "_LINKER") "/usr/bin/clang") + (displayln "NATIVE-TOOLCHAIN " host " SDK=" (getenv "SDKROOT")) (force-output)))) + (else (void))))) diff --git a/t/support/qualification/ownership.ss b/t/support/qualification/ownership.ss new file mode 100644 index 00000000..4c9f48c3 --- /dev/null +++ b/t/support/qualification/ownership.ss @@ -0,0 +1,41 @@ +;;; Ownership starts at a direct process with a private acknowledged session. +(import :std/misc/process :std/os/signal) +(export ownership-plan stop-owned-processes!) +(def (ownership-plan root rows session-owner) + (unless (= root session-owner) (error "unverified process session" root session-owner)) + (let loop ((members (list root))) + (let (next (filter (lambda (row) + (and (member (cadr row) members) (not (member (car row) members)))) rows)) + (if (pair? next) (loop (append (map car next) members)) + (list members + (cons root (filter-map (lambda (row) + (and (member (car row) members) + (member (caddr row) members) (caddr row))) rows))))))) +(def (process-topology) + (let (text (run-process ["ps" "-axo" "pid=,ppid=,pgid="])) + (filter-map (lambda (line) + (let (numbers (filter number? (map string->number (string-split line #\space)))) + (and (= (length numbers) 3) numbers))) (string-split text #\newline)))) +(def (stop-owned-processes! process witness reaped?) + (unless (or reaped? (integer? (process-status process 0 'running))) + (let* ((root (process-pid process)) + (owner (and (file-exists? witness) (call-with-input-file witness read)))) + (if (not (equal? owner root)) + ;; Before session acknowledgement, admit only the direct unreaped child. + (with-catch void (lambda () (kill root SIGKILL))) + (let* ((plan (ownership-plan root (process-topology) owner)) + (members (car plan)) + (groups (foldl (lambda (group acc) (if (member group acc) acc (cons group acc))) '() (cadr plan)))) + (for-each (lambda (group) (with-catch void (lambda () (kill (- group) SIGTERM)))) groups) + (thread-sleep! 0.25) + ;; Refresh membership before escalation. A completed/reaped root + ;; cannot qualify a recycled group; descendants must still belong + ;; to the previously admitted group in the fresh topology. + (let ((rows (process-topology)) + (root-live? (not (integer? (process-status process 0 'running))))) + (for-each + (lambda (group) + (when (ormap (lambda (row) + (and (= (caddr row) group) (member (car row) members) + (or (not (= (car row) root)) root-live?))) rows) + (with-catch void (lambda () (kill (- group) SIGKILL))))) groups))))))) diff --git a/t/support/qualification/performance.ss b/t/support/qualification/performance.ss new file mode 100644 index 00000000..9fa409a4 --- /dev/null +++ b/t/support/qualification/performance.ss @@ -0,0 +1,52 @@ +;;; Source qualification records one content identity around every real suite. +(import ./process ./products :std/crypto/digest :std/misc/ports) +(export #t) +(def performance-suites + '(("lalr" . "t/lalr-fixed-point-benchmark-test.ss") + ("incremental" . "t/incremental-suffix-benchmark-test.ss") + ("recovery" . "t/recovery-frontier-benchmark-test.ss") + ("glr" . "t/selective-glr-benchmark-test.ss") + ("hot-path" . "t/benchmark-test.ss") + ("gql" . "t/gql/runtime-benchmark-test.ss") + ("opencypher" . "t/opencypher-large-query-benchmark-test.ss") + ("languages" . "t/versioned-language-benchmark-test.ss"))) +(def (source-identity) + (let ((files '("gerbil.pkg" "justfile")) (digest (Digest::sha256))) + (def (walk path) + (when (file-exists? path) + (if (eq? (file-type path) 'directory) + (for-each (lambda (name) (unless (string-prefix? "." name) (walk (path-expand name path)))) (directory-files path)) + (set! files (cons path files))))) + (for-each walk '("src" "languages" "t" "scripts" "tools")) + (for-each (lambda (name) (when (equal? (path-extension name) ".ss") (set! files (cons name files)))) (directory-files ".")) + (for-each (lambda (path) + (let* ((bytes (call-with-input-file path read-all-as-u8vector)) + (header (string->utf8 (string-append path (string #\nul) (number->string (u8vector-length bytes)) (string #\nul))))) + (digest-update! digest header) (digest-update! digest bytes))) (list-sort string budget 0) (andmap (lambda (name) (assoc name performance-suites)) selected)) + (error "invalid source performance selection" selected budget)) + (let ((root (path-expand ".")) (identity (source-identity)) (failures '())) + (with-product-directory "parser-source-" + (lambda (directory) + (create-symbolic-link root (path-expand "gerbil-parser" directory)) + (setenv "GERBIL_LOADPATH" directory) + (setenv "GERBIL_PATH" (path-expand ".gerbil" root)) + (setenv "GAMBOPT" (string-append (getenv "GAMBOPT" "") ",max-heap=1G,debug=q")) + (displayln "SOURCE ROOT " root) (displayln "SOURCE DIGEST " identity) (force-output) + (let loop ((rest selected)) + (when (and (pair? rest) (or continue? (null? failures))) + (let* ((name (car rest)) (suite (cdr (assoc name performance-suites))) + (result (run-observed-process ["gerbil" "test" "-v" "5" suite] + (path-expand (string-append name ".log") logs) timeout: budget idle-timeout: 5)) + (text (utf8->string (process-result-output result)))) + (unless (equal? identity (source-identity)) (error "source changed during performance qualification" name)) + (unless (and (zero? (process-result-status result)) (not (process-result-reason result)) + (andmap (lambda (pattern) (receipt-present? pattern text)) '("^CASE-OK " "^MODULE-OK " "^HARNESS-OK " "^OK$"))) + (set! failures (cons name failures))) + (loop (cdr rest))))))) + (unless (null? failures) (error "source performance qualification failed" failures)) + (displayln (if (and (= (length selected) (length performance-suites)) + (andmap (lambda (row) (member (car row) selected)) performance-suites)) + "PERFORMANCE-SOURCE-OK " "PERFORMANCE-SOURCE-SUBSET-OK ") (string-join selected ",")) (force-output))) diff --git a/t/support/qualification/plans.ss b/t/support/qualification/plans.ss new file mode 100644 index 00000000..ca1933f3 --- /dev/null +++ b/t/support/qualification/plans.ss @@ -0,0 +1,117 @@ +;;; Declarative stage ownership; diagnostics compile as one union per plan. +(import ./process ./products ./environment :std/misc/process :std/string/misc) +(export #t) +(defstruct qualification-stage (name command budget idle receipts) transparent: #t) +(def (diagnostic-modules suites) + (let loop ((rest suites) (modules '())) + (if (null? rest) modules + (let (added (cond ((equal? (car rest) "gql-actors") '("reduction-counts" "execution-counts" "matched-stages" "actors")) + ((member (car rest) '("gql" "gql-profile")) '("reduction-counts" "execution-counts" "matched-stages")) + (else '()))) + (loop (cdr rest) (foldl (lambda (module acc) (if (member module acc) acc (append acc [module]))) modules added)))))) +(def (compiled-product! root module) + (let (product (path-expand (string-append "lib/gerbil-parser/" module ".o1") root)) + (unless (file-exists? product) (error "missing compiled module; build this checkout first" product)))) +(def (compiled-main module expression) + ["gxi" "-e" + (string-append "(load \"t/fixtures/tla-sany-differential/preload.ss\") (prefer-compiled-interfaces!) (preload-module \"" + module "\") (preload-module \"gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process\")") + "-e" (string-append "(import :" module " :gerbil-parser/t/fixtures/tla-sany-differential/exit-child-process) " + expression " (test-child-process-exit! 0)")]) +(def (execute-stage! stage logs) + (displayln "NATIVE-STAGE " (qualification-stage-name stage)) (force-output) + (run-qualified! (qualification-stage-command stage) + (path-expand (string-append (qualification-stage-name stage) ".log") logs) + timeout: (qualification-stage-budget stage) idle-timeout: (qualification-stage-idle stage) + require: (qualification-stage-receipts stage))) +(def (local-diagnostic-plan suites prepared?) + (unless (and (pair? suites) (andmap (lambda (suite) (member suite '("rust" "gql" "gql-profile" "gql-actors" "ffi" "aot"))) suites)) + (error "unknown local qualification suite" suites)) + (when (and prepared? (ormap (lambda (suite) (not (member suite '("gql-profile" "gql-actors")))) suites)) + (error "prepared diagnostics only admit profile and actor suites" suites)) + (let (modules (diagnostic-modules suites)) + (append + (if (and (not prepared?) (ormap (lambda (suite) (member suite '("gql" "gql-profile" "gql-actors" "ffi"))) suites)) + [(qualification-stage "test-driver-build" '("gxi" "build-test-driver.ss" "compile") 240 30 '())] '()) + (if (and (not prepared?) (pair? modules)) + [(qualification-stage "gql-support-build" + (append '("gxc" "-V" "t/benchmarks/parser-stage-cost/benchmark.ss") + (map (lambda (module) (string-append "t/benchmarks/gql/runtime/" module ".ss")) modules)) 240 30 '())] '())))) +(def (qualify-local! suites prepared? compiled-root logs jobs) + (let ((compiled-root (path-expand compiled-root)) (logs (path-expand logs))) + (def (run name command (build? #f) (receipts '()) (budget #f)) + (execute-stage! (qualification-stage name command (or budget (if build? 240 120)) (if build? 30 5) receipts) logs)) + (native-environment!) + (unless (and (integer? jobs) (> jobs 0)) (error "jobs must be positive" jobs)) + (setenv "GERBIL_PATH" compiled-root) + (setenv "GERBIL_LOADPATH" (string-append (path-expand "lib" compiled-root) ":" (path-expand ".") ":" (path-expand "."))) + (setenv "GAMBOPT" (string-append (getenv "GAMBOPT" "") ",max-heap=3G,debug=q")) + (setenv "GERBIL_PARSER_LR_TRACE" "1") (setenv "GERBIL_BUILD_VERBOSE" "1") + (setenv "RUSTC_WRAPPER") (setenv "RUSTC_WORKSPACE_WRAPPER") + (for-each (lambda (stage) (execute-stage! stage logs)) (local-diagnostic-plan suites prepared?)) + (when prepared? + (when (pair? (diagnostic-modules suites)) + (compiled-product! compiled-root "t/benchmarks/parser-stage-cost/benchmark")) + (for-each (lambda (module) (compiled-product! compiled-root (string-append "t/benchmarks/gql/runtime/" module))) + (diagnostic-modules suites))) + (for-each + (lambda (suite) + (cond + ((equal? suite "rust") + (run "rust-build" ["cargo" "test" "--workspace" "--locked" "--no-run" "--jobs" (number->string jobs) "--verbose"] #t) + (run "rust-test" '("cargo" "test" "--workspace" "--locked" "--" "--nocapture") #f '("test result: ok\\."))) + ((member suite '("gql-profile" "gql-actors")) + (run suite (compiled-main (string-append "gerbil-parser/t/benchmarks/gql/runtime/" + (if (equal? suite "gql-profile") "matched-stages" "actors")) + "(main \"40\" \"100\")") #f + (if (equal? suite "gql-profile") '("GQL-STAGES-OK" "PARSER-COMPONENT-SUMMARY" "GQL-REDUCTION-COUNTS" "GQL-LR-EXECUTION") + '("GQL-ACTORS-OK" "PARSER-COMPONENT-SUMMARY")))) + ((equal? suite "aot") + (run "aot-build" '("gxi" "build-rust-runtime-aot.ss" "compile") #t) + (with-product-directory "parser-native-aot-" + (lambda (directory) + (let (generated (path-expand "records.rs" directory)) + (run "aot-closed" ["env" "-i" (string-append "HOME=" (getenv "HOME")) "PATH=/usr/bin:/bin" + (string-append "GERBIL_PATH=" compiled-root) + (string-append "GERBIL_PARSER_RUNTIME_AOT_LIB=" (path-expand "lib" compiled-root) ":" (path-expand "lib" (gerbil-home))) + (path-expand "bin/gerbil-parser-runtime-aot" compiled-root) + "t/fixtures/runtime-record-assignments/languages/records/grammar.ss" generated]) + (run "aot-format" ["rustfmt" "--edition" "2024" generated]) + (same-product! generated "t/fixtures/runtime-record-assignments/src/generated/records.rs"))))) + (else + (when (equal? suite "ffi") + (run "ffi-header" [(getenv "CC" "cc") "-std=c11" "-Wall" "-Wextra" "-Werror" "-Wno-unused-command-line-argument" + "-fsyntax-only" "-Iinclude" "t/ffi-header-smoke.c"]) + (run "ffi-abi-build" '("gxi" "build-ffi-tests.ss" "compile") #t) + (run "ffi-abi" (compiled-main "gerbil-parser/t/fixtures/ffi/abi-probe" "(main)") #f + '("NATIVE-ABI-OK" "ABI-RUNTIME-GENERATED-MATCH calls=3")) + (run "language-abi" (compiled-main "gerbil-parser/t/fixtures/ffi/language-probe" "(main)") #f + '("LANGUAGE-ABI-OK" "LANGUAGE-ABI-100-CALLS")) + (setenv "CARGO_TARGET_DIR" (path-expand "target")) + (run "rust-native-build" '("cargo" "build" "--locked" "--manifest-path" "t/fixtures/rust-language-abi/Cargo.toml") #t '() 90) + (setenv "GERBIL_PARSER_RUST_NATIVE_ARCHIVE" (path-expand "target/debug/librust_language_abi_fixture.a")) + (run "rust-native-link" + '("gxi" "-e" "(load \"t/fixtures/tla-sany-differential/preload.ss\") (prefer-compiled-interfaces!) (preload-test-imports \"build-rust-ffi-tests.ss\")" + "-e" "(call-with-compiled-interface-trace (lambda () (load \"build-rust-ffi-tests.ss\") (eval (quote (main \"compile\")))))") #t '() 90) + (run "rust-native-ownership" (compiled-main "gerbil-parser/t/fixtures/ffi/rust-language-probe" "(main)") #f + '("RUST-NATIVE-OK" "RUST-NATIVE-OWNERSHIP-OK handles=2 results=113" "RUST-NATIVE-100-CALLS") 90) + (qualify-runtime-host!)) + (for-each (lambda (module) (compiled-product! compiled-root module)) + (if (equal? suite "gql") '("languages/gql/parser" "src/runtime/parser") '("src/ffi/rust-runtime-aot"))) + (let (files (if (equal? suite "gql") + '("t/gql/runtime-benchmark-test.ss" "t/gql/benchmark-profile-test.ss" "t/gql/actor-test.ss") + '("t/ffi-test.ss" "t/build-product-contract-test.ss" "t/language-abi-test.ss" "t/language-abi-benchmark-test.ss"))) + (run suite (append '("gxi" "t/fixtures/tla-sany-differential/test-suite.ss") files) #f + [(string-append "TEST-SUITE-OK modules=" (number->string (length files))) "^OK$"]))))) suites) + (displayln "LOCAL-NATIVE-OK " (string-join suites ",")) (force-output))) +(def (qualify-event-fold! binary directory) + (let* ((log (path-expand "compiled-test.log" directory)) + (result (run-observed-process ["env" "-u" "GERBIL_PATH" "-u" "GERBIL_LOADPATH" "GAMBOPT=max-heap=1G,debug=q" binary] + log timeout: 120 idle-timeout: 5)) + (text (utf8->string (process-result-output result)))) + (unless (and (zero? (process-result-status result)) (not (process-result-reason result)) + (= (length (filter (lambda (line) (string-prefix? "CASE-OK " line)) (string-split text #\newline))) 67) + (receipt-present? "^HARNESS-OK" text) (receipt-present? "^OK$" text) + (not (receipt-present? "ERROR|FAILED|FAILURE|FAIL:" text))) + (error "compiled event-fold qualification failed" log)) + (displayln "COMPILED-EVENT-FOLD-OK cases=67") (force-output))) diff --git a/t/support/qualification/process-child.ss b/t/support/qualification/process-child.ss new file mode 100644 index 00000000..833ddbf7 --- /dev/null +++ b/t/support/qualification/process-child.ss @@ -0,0 +1,32 @@ +;;; A fresh POSIX session contains the compiler/test child and its descendants. +(import :std/ffi) +(export main) +(C-ffi-macrology) +(C-include "" "" "") +;;; These operations have one private consumer: this native entry boundary. +(def-C-lambda blocking-standard-streams! () int + "int result = 0; + for (int fd = 0; fd < 3; ++fd) { + int flags = fcntl(fd, F_GETFL); + if (flags >= 0 && fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) < 0) result = -1; + } + ___result = result;") +(def-C-lambda start-session! () int "___result = setsid();") +;;; Stop inherited Scheme timers before replacing the image; never fork a waiter. +(def-C-lambda replace-process! (nonnull-UTF-8-string nonnull-UTF-8-string-list) + int + "struct itimerval stopped = {{0, 0}, {0, 0}}; + if (setitimer(ITIMER_REAL, &stopped, 0) < 0 || + setitimer(ITIMER_VIRTUAL, &stopped, 0) < 0 || + setitimer(ITIMER_PROF, &stopped, 0) < 0) ___result = -1; + else ___result = execvp(___arg1, ___arg2);") +(def (main witness loadpath program . arguments) + (let (owner (start-session!)) + (when (< (blocking-standard-streams!) 0) (error "cannot establish blocking compiler streams")) + (when (< owner 0) (error "cannot establish owned process session")) + (call-with-output-file witness (lambda (port) (write owner port)))) + (if (zero? (string-length loadpath)) (setenv "GERBIL_LOADPATH") (setenv "GERBIL_LOADPATH" loadpath)) + ;; Successful exec never returns. The parent observes the target's actual + ;; wait status through its existing process port, including signal exits. + (replace-process! program (cons program arguments)) + (error "cannot replace qualification process" program)) diff --git a/t/support/qualification/process.ss b/t/support/qualification/process.ss new file mode 100644 index 00000000..5fca4d99 --- /dev/null +++ b/t/support/qualification/process.ss @@ -0,0 +1,176 @@ +;;; One native process contract shared by compilation, testing and generation. +(import :std/misc/process :std/io/tempfile :std/text/pregexp ./ownership) +(export #t) +(def qualification-library-root (path-expand "lib" (gerbil-path))) +;;; Load native runtime objects, not .ssi import metadata and macro dependencies. +;;; The SDK writer quotes argv as data; shell syntax is never evaluated. +(def (process-child-arguments witness loadpath command) + ["-e" "(load-module \"gerbil-parser/t/support/qualification/process-child\")" + "-e" (object->string + `(apply gerbil-parser/t/support/qualification/process-child#main + (quote (,witness ,loadpath ,@command))))]) +(defstruct process-result (status reason output elapsed phase) transparent: #t) +(defstruct process-interruption (reason) transparent: #t) +(def (normal-exit-status raw) + (if (zero? (modulo raw 256)) (quotient raw 256) (+ 128 (modulo raw 128)))) +(def (receipt-present? pattern output) + (any (lambda (line) (pregexp-match pattern line)) (string-split output #\newline))) + +(def (run-observed-process command log timeout: (budget 60) idle-timeout: (idle #f) + error-output?: (error-output? #t) observe: (observe void)) + ;; Reader/waiter notifications belong to this invocation, not the caller's + ;; mailbox. Cancellation can leave queued notifications behind; a fresh + ;; green-thread coordinator prevents them from contaminating the next run. + (thread-join! + (spawn + (lambda () + (run-observed-process/worker command log timeout: budget idle-timeout: idle + error-output?: error-output? observe: observe))))) + +(def (run-observed-process/worker command log timeout: (budget 60) idle-timeout: (idle #f) + error-output?: (error-output? #t) observe: (observe void)) + (unless (and (pair? command) (> budget 0) (or (not idle) (> idle 0))) + (error "expected command and positive deadlines" command budget idle)) + (create-directory* (path-directory log)) + (let (directory (make-temporary-file-name "parser-process-")) + (create-directory directory) + (try + (let* ((witness (path-expand "session" directory)) + (started (##current-time-point)) (last-output started) + (coordinator (current-thread)) (capture (open-output-u8vector)) + (output (open-output-file log)) + (process (open-process [path: "gxi" + arguments: (process-child-arguments witness (getenv "GERBIL_LOADPATH" "") command) + environment: (map (lambda (entry) (string-append (car entry) "=" + (if (equal? (car entry) "GERBIL_LOADPATH") + (string-append qualification-library-root ":" (cdr entry)) (cdr entry)))) + (if (getenv "GERBIL_LOADPATH" #f) (get-environment-variables) + (cons (cons "GERBIL_LOADPATH" "") (get-environment-variables)))) + stderr-redirection: #t])) + (reader #f) (waiter #f) (monitor #f) (raw #f) (reason #f) + (line-prefix "") (eof? #f) (output-complete? #f) (exited? #f) + (session-observed? #f)) + (close-output-port process) + (def (fail! why) + (unless reason + (set! reason why) + (stop-owned-processes! process witness (integer? raw)) + (thread-interrupt! coordinator (lambda () (raise (process-interruption why)))))) + (def (observe-error! bytes) + (when error-output? + (let loop ((index 0)) + (when (< index (u8vector-length bytes)) + (let (byte (u8vector-ref bytes index)) + (cond + ((= byte 10) + (when (equal? line-prefix "ERROR") (fail! 'error-output)) + (set! line-prefix "")) + ((< (string-length line-prefix) 10) + (set! line-prefix (string-append line-prefix (string (integer->char byte)))) + (when (or (equal? line-prefix "*** ERROR") + (and (= (string-length line-prefix) 6) + (equal? (substring line-prefix 0 5) "ERROR") + (not (or (<= 48 byte 57) (<= 65 byte 90) + (<= 97 byte 122) (= byte 95))))) + (fail! 'error-output))))) + (loop (+ index 1)))))) + (try + (set! reader + (spawn + (lambda () + (with-catch + (lambda (ex) (thread-send coordinator (cons 'exception ex))) + (lambda () + (let (buffer (make-u8vector 65536)) + (let loop () + (let (count (read-subu8vector buffer 0 65536 process 1)) + (if (zero? count) (begin (set! output-complete? #t) (thread-send coordinator '(eof))) + (let (bytes (subu8vector buffer 0 count)) + (set! last-output (##current-time-point)) + (write-subu8vector bytes 0 count capture) + (write-subu8vector bytes 0 count output) (force-output output) + (thread-send coordinator (cons 'output bytes)) + (observe-error! bytes) + (loop))))))))))) + (set! waiter + (spawn + (lambda () + (with-catch + (lambda (ex) (thread-send coordinator (cons 'exception ex))) + (lambda () + ;; SDK process-status suspends this green thread until child exit. + (set! raw (process-status process)) + (thread-send coordinator (cons 'exit raw))))))) + (set! monitor + (spawn + (lambda () + (with-catch + (lambda (ex) (thread-send coordinator (cons 'exception ex))) + (lambda () + (let loop () + (let (now (##current-time-point)) + ;; Session acknowledgement is a real, one-time completed + ;; native stage, not a heartbeat. Total time still starts + ;; before GXI and neither configured deadline is widened. + (when (and (not session-observed?) + (or (not idle) (< (- now last-output) idle)) + (file-exists? witness)) + (set! session-observed? #t) + (set! last-output now)) + (cond ((>= (- now started) budget) (fail! 'total-timeout)) + ((and idle (not (and output-complete? raw)) (>= (- now last-output) idle)) + (fail! 'idle-timeout)) + (else + (when (eq? (observe) #t) (set! last-output (##current-time-point))) + (thread-sleep! 0.05) (loop)))))))))) + (with-catch + (lambda (ex) (unless (process-interruption? ex) (raise ex))) + (lambda () + (let loop () + (unless (and eof? exited?) + (let (message (thread-receive)) + (case (car message) + ((output) + (let (bytes (cdr message)) + (write-subu8vector bytes 0 (u8vector-length bytes)) (force-output))) + ((eof) (set! eof? #t)) + ((exit) (set! exited? #t)) + ((exception) (raise (cdr message)))) + (loop)))))) + (when monitor (thread-terminate! monitor) (set! monitor #f)) + (when waiter (thread-terminate! waiter) (set! waiter #f)) + (unless raw (set! raw (process-status process 2 9))) + (let* ((command-raw raw) + (bytes (get-output-u8vector capture)) + ;; Every incoming prefix has already been checked by the reader. + ;; Only bare ERROR at EOF needs admission without a delimiter. + ;; Re-decoding and regex-scanning a large completed log happens + ;; after the monitor stops and can dominate the whole command. + (failed? (and error-output? (equal? line-prefix "ERROR")))) + (process-result (normal-exit-status command-raw) (or reason (and failed? 'error-output) + (and (not (zero? (modulo command-raw 256))) 'signal-exit)) bytes + (- (##current-time-point) started) + (if (file-exists? witness) 'session-ready 'bootstrap))) + (finally + (when monitor (thread-terminate! monitor)) + (when waiter (thread-terminate! waiter)) + (when reader (thread-terminate! reader)) + (unless raw + (stop-owned-processes! process witness #f) + (process-status process 2 #f)) + (close-port process) (close-port output) (close-port capture))))))) + +(def (run-qualified! command log timeout: (budget 60) idle-timeout: (idle #f) + require: (receipts '()) observe: (observe void)) + (displayln "PROCESS-START " log " timeout=" budget) (force-output) + (let* ((result (run-observed-process command log timeout: budget idle-timeout: idle observe: observe)) + (text (and (pair? receipts) (utf8->string (process-result-output result)))) + (missing (filter (lambda (pattern) (not (receipt-present? pattern text))) receipts))) + (displayln "PROCESS-RESULT elapsed=" (process-result-elapsed result) + " exit=" (process-result-status result) " reason=" (process-result-reason result) + " phase=" (process-result-phase result) + " missing=" missing) (force-output) + (unless (and (zero? (process-result-status result)) (not (process-result-reason result)) (null? missing)) + (error "process qualification failed" log (process-result-status result) + (process-result-reason result) missing)) + result)) diff --git a/t/support/qualification/products.ss b/t/support/qualification/products.ss new file mode 100644 index 00000000..314103e1 --- /dev/null +++ b/t/support/qualification/products.ss @@ -0,0 +1,138 @@ +;;; Native products retain SDK compiler and Cargo ownership. +(import ./process ./environment :std/io/tempfile :std/misc/ports :std/text/pregexp :std/string/misc) +(export #t) +(def (with-product-directory prefix proc) + (let (directory (make-temporary-file-name prefix)) + (create-directory directory) + (try (proc directory) (finally (delete-file-or-directory directory #t))))) +(def (sdk-compiler) + (let* ((result (run-qualified! [(path-expand "bin/gambuild-C" (gerbil-home)) "C_COMPILER"] + ".data/qualification/sdk-compiler.log" timeout: 5 idle-timeout: 5)) + (compiler (string-trim (utf8->string (process-result-output result))))) + (unless (and (not (zero? (string-length compiler))) (not (pregexp-match "[[:space:]]" compiler))) + (error "SDK C_COMPILER must name one executable" compiler)) + compiler)) +(def (same-product! actual expected) + (unless (equal? (call-with-input-file actual read-all-as-u8vector) + (call-with-input-file expected read-all-as-u8vector)) + (error "generated product differs" actual expected))) +(def (native-artifact-observer directory log) + (let ((observed (make-hash-table)) (watched (make-hash-table)) (last-scan 0)) + (lambda () + (let ((now (##current-time-point)) (changed? #f)) + (def (walk path) + (when (file-exists? path) + (if (eq? (file-type path) 'directory) + (for-each (lambda (name) (walk (path-expand name path))) (directory-files path)) + (when (member (path-extension path) '(".i" ".s" ".o")) + (let* ((info (file-info path)) + (state (list (file-info-size info) (time->seconds (file-info-last-modification-time info))))) + (when (and (> (car state) 0) (not (equal? (hash-ref observed path #f) state))) + (hash-put! observed path state) (set! changed? #t) + (displayln "NATIVE-COMPILER-OUTPUT " path " bytes=" (car state)) (force-output))))))) + (when (>= (- now last-scan) 0.5) (set! last-scan now) (walk directory)) + ;; Admit static compiler products only from this invocation's -obj records. + ;; Existing files establish a baseline; only subsequent writes are progress. + (when (file-exists? log) + (let (text (call-with-input-file log read-all-as-string)) + (for-each + (lambda (line) + (when (and (string-prefix? "invoke (" line) (string-contains line " -obj ")) + (let (match (pregexp-match "[ ]([^ ()]+[.]c)[)]" line)) + (when match + (let (prefix (path-strip-extension (cadr match))) + (for-each + (lambda (suffix) + (let (path (string-append prefix suffix)) + (unless (hash-get watched path) + (hash-put! watched path #t) + (when (file-exists? path) + (let (info (file-info path)) + (hash-put! observed path (list (file-info-size info) (time->seconds (file-info-last-modification-time info))))))))) + '(".i" ".s" ".o"))))))) (string-split text #\newline)))) + (for-each walk (hash-keys watched)) + changed?)))) + +(def (build-language-library! output modules) + (unless (and (pair? modules) (andmap (lambda (module) (pregexp-match "^[A-Za-z0-9_./-]+$" module)) modules)) + (error "expected Gerbil language module identities" modules)) + (let ((output (path-expand output)) (compiler (sdk-compiler)) (root (path-expand "."))) + (create-directory* (path-directory output)) + (with-product-directory "parser-shared-library-" + (lambda (directory) + (let* ((source (string-append directory "/bundle" (path-strip-directory directory) ".ss")) + (shim (path-expand "runtime.o" directory)) (bundle (path-expand "gpembed" directory)) + (aot-log (string-append ".data/qualification/" (path-strip-directory directory) "-aot.log")) + (ld-options (cond-expand (darwin (string-append "-dynamiclib -Wl,-install_name," output)) + (else "-shared -Wl,-z,defs"))) + (preload (with-output-to-string "" + (lambda () + (write '(gx#import-module (quote :gerbil/compiler) #f #t)) + (for-each (lambda (module) + (write `(gx#import-module (quote ,(string->symbol (string-append ":" module))) #f #t))) modules) + (write '(void)))))) + (call-with-output-file source + (lambda (port) + (write `(import ,@(let loop ((rest modules) (index 0)) + (if (null? rest) '() + (cons `(prefix-in ,(string->symbol (string-append ":" (car rest))) + ,(string->symbol (string-append "language" (number->string index) "-"))) + (loop (cdr rest) (+ index 1)))))) port) (newline port) + (write '(export main) port) (newline port) + (write '(def (main . _) (void)) port) (newline port))) + (run-qualified! [compiler "-std=c11" "-D_POSIX_C_SOURCE=200809L" "-fPIC" + (string-append "-I" (path-expand "include" (gerbil-home))) + (string-append "-I" (path-expand "include" root)) + "-DGERBIL_PARSER_LINKER=___LNK_gpembed____exe__" + "-c" "include/gerbil-parser/runtime.c" "-o" shim] + ".data/qualification/shared-runtime.log" timeout: 90 idle-timeout: 30) + (setenv "GERBIL_GSC" (path-expand "bin/gsc" (gerbil-home))) + (setenv "GERBIL_GCC" compiler) + (run-qualified! ["gxi" "-e" preload "build-shared-library.ss" source bundle + (string-append "-save-temps=obj -fPIC -fwrapv -fno-strict-aliasing -foptimize-sibling-calls -I" + (path-expand "include" root)) + (string-append ld-options " " shim)] + aot-log timeout: 90 idle-timeout: 30 observe: (native-artifact-observer directory aot-log)) + (unless (file-exists? bundle) (error "SDK did not publish native language library" bundle)) + (when (file-exists? output) (delete-file output)) (copy-file bundle output)))) + (displayln "SHARED-LIBRARY-OK " output) (force-output))) +(def (qualify-runtime-host!) + (native-environment!) + (setenv "CARGO_TARGET_DIR" (path-expand "target")) + (setenv "RUSTC_WRAPPER") (setenv "RUSTC_WORKSPACE_WRAPPER") + (with-product-directory "parser-native-host-" + (lambda (directory) + (let* ((library (path-expand (cond-expand (darwin "libparsernative.dylib") (else "libparsernative.so")) directory)) + (host (path-expand "native-host" directory)) (compiler (sdk-compiler)) + (cpu-proof? (equal? (getenv "GERBIL_PARSER_RUST_CPU_PROOF" #f) "1")) + (archive (path-expand (if cpu-proof? "target/release/librust_language_abi_fixture.a" + "target/debug/librust_language_abi_fixture.a")))) + (let (prepared (getenv "GERBIL_PARSER_NATIVE_HOST_LIBRARY" #f)) + (if prepared + (begin + (unless (file-exists? prepared) (error "prepared language library missing" prepared)) + (set! library (path-expand prepared))) + (build-language-library! library '("gerbil-parser/t/fixtures/shared-scanner/records-engine-abi")))) + (run-qualified! (append ["cargo" "build" "--locked" "--manifest-path" "t/fixtures/rust-language-abi/Cargo.toml" "--features" "standalone,tokio"] + (if cpu-proof? '("--release") '())) + ".data/native-host/rust-build.log" timeout: 90 idle-timeout: 30) + (run-qualified! (append [compiler "-std=c11" "-D_POSIX_C_SOURCE=200809L" "-Iinclude" "-I." "t/runtime-host.c" + archive library "-lpthread" "-lm"] + (cond-expand (darwin '()) (else '("-ldl"))) ["-o" host]) + ".data/native-host/link.log" timeout: 90 idle-timeout: 30) + (for-each (lambda (row) + (run-qualified! (append + (if (and (member (car row) '("tokio-cpu-ascii" "tokio-cpu-unicode")) + (getenv "CPU_PROOF_PROCESSOR" #f)) + ["taskset" "-c" (getenv "CPU_PROOF_PROCESSOR")] '()) + (cons host (cdr row))) + (string-append ".data/native-host/" (car row) ".log") timeout: 90 idle-timeout: 5 + require: [(cond ((equal? (car row) "c") "NATIVE-HOST-CLEANUP-OK") + ((equal? (car row) "rust") "RUST-HOST-CLEANUP-OK") + ((equal? (car row) "tokio") "RUST-TOKIO-HOST-OK") + ((member (car row) '("tokio-cpu-ascii" "tokio-cpu-unicode")) "RUST-TOKIO-CPU-PROOF-OK") + (else "RUST-BORROWED-HOST-OK"))])) + (append '(("c") ("rust" "rust") ("borrowed" "borrowed") ("tokio" "tokio")) + (if cpu-proof? + '(("tokio-cpu-ascii" "tokio-cpu-ascii") ("tokio-cpu-unicode" "tokio-cpu-unicode")) '())))))) + (displayln "NATIVE-HOST-OK") (force-output)) diff --git a/t/test-style-contract-test.ss b/t/test-style-contract-test.ss index 0a5f87ee..1fdb4ba4 100644 --- a/t/test-style-contract-test.ss +++ b/t/test-style-contract-test.ss @@ -4,14 +4,54 @@ (import (only-in :std/test check test-case test-suite)) (export test-style-contract-tests) -(def (test-form-allowed? form forbidden) - (cond - ((not (pair? form)) #t) - ((and (symbol? (car form)) - (memq (car form) '(quote quasiquote syntax quasisyntax))) #t) - ((and (symbol? (car form)) (memq (car form) forbidden)) #f) - (else (and (test-form-allowed? (car form) forbidden) - (test-form-allowed? (cdr form) forbidden))))) +(def (test-form-allowed? form forbidden (owned-ports '())) + ;; Explicit frames carry lexical port authority. Source width/depth must not + ;; become native call-stack depth in the compiled contract checker. + (let loop ((pending (list (cons form owned-ports)))) + (if (null? pending) #t + (let* ((frame (car pending)) (form (car frame)) + (owned-ports (cdr frame)) (rest (cdr pending))) + (def (push-forms forms ports next) + (let push ((forms forms) (next next)) + (if (null? forms) next + (push (cdr forms) (cons (cons (car forms) ports) next))))) + (def (push-pair) + (loop (cons (cons (car form) owned-ports) + (cons (cons (cdr form) owned-ports) rest)))) + (cond + ((not (pair? form)) (loop rest)) + ((and (symbol? (car form)) + (memq (car form) '(quote quasiquote syntax quasisyntax))) + (loop rest)) + ((and (list? form) + (memq (car form) '(call-with-output-string call-with-output-file)) + (pair? (cdr form))) + (let (handler (last form)) + (if (and (list? handler) (>= (length handler) 3) + (eq? (car handler) 'lambda) + (list? (cadr handler)) (= (length (cadr handler)) 1) + (symbol? (caadr handler))) + (let ((args (reverse (cdr (reverse (cdr form))))) + (body-ports (cons (caadr handler) owned-ports))) + (loop (push-forms args owned-ports + (push-forms (cddr handler) body-ports rest)))) + (push-pair)))) + ((and (symbol? (car form)) (memq (car form) forbidden)) + ;; Fixture serialization owns its port; console output stays forbidden. + (and (or (eq? (car form) 'write-string) + (and (eq? (car form) 'display) + (or (equal? forbidden '(display)) + (equal? forbidden '(display displayln))))) + (list? form) (= (length form) 3) + (memq (caddr form) owned-ports) + (loop (cons (cons (cadr form) owned-ports) rest)))) + ;; A nested lambda cannot inherit authority for a shadowed argument. + ((and (eq? (car form) 'lambda) (pair? (cdr form)) + (list? (cadr form))) + (loop (cons (cons (cddr form) + (filter (lambda (port) (not (memq port (cadr form)))) + owned-ports)) rest))) + (else (push-pair))))))) (def (test-source-allowed? path forbidden) (call-with-input-file path @@ -37,9 +77,61 @@ (test-suite "parser test AST contract" (test-case "reject output calls but allow quoted negative fixtures" (check (test-form-allowed? '(display "snapshot") '(display)) => #f) + (check (test-form-allowed? + '(call-with-output-string (lambda (port) (display "input" port))) + '(display)) => #t) + (check (test-form-allowed? + '(call-with-output-string (lambda (port) (display "Rust" port))) + '(display displayln write-string string-append format)) => #f) + (check (test-form-allowed? '(display "snapshot" port) '(display)) => #f) + (check (test-form-allowed? '(display "snapshot" (current-output-port)) + '(display)) => #f) + (check (test-form-allowed? + '(call-with-output-string + (lambda (port) (display (display "snapshot") port))) + '(display)) => #f) + (check (test-form-allowed? + '(call-with-output-string + (lambda (port) (lambda (port) (display "snapshot" port)))) + '(display)) => #f) (check (test-form-allowed? '(quote (display input)) '(display)) => #t) (check (test-form-allowed? '(check value => expected) '(display)) => #t)) + (test-case "owned fixture ports retain the console and assembly restrictions" + (check (test-form-allowed? + '(call-with-output-string (lambda (port) (display "input" port))) + '(display displayln)) => #t) + (check (test-form-allowed? '(display "snapshot" (current-output-port)) + '(display)) => #f) + (check (test-form-allowed? + '(call-with-output-file path (lambda (port) (write-string ir port))) + '(display write-string string-append)) => #t) + (check (test-form-allowed? + '(call-with-output-file path + (lambda (port) (write-string (string-append "Rust" "text") port))) + '(write-string string-append)) => #f) + (check (test-form-allowed? + '(call-with-output-file quote (display "snapshot") (lambda (port) #t)) + '(display)) => #f) + (check (test-form-allowed? + '(call-with-output-string (lambda (port) quote (display "snapshot"))) + '(display)) => #f)) + (test-case "deep and wide ASTs retain rejection and lexical port authority" + (let deep ((n 20000) (form '(display "snapshot"))) + (if (zero? n) + (check (test-form-allowed? form '(display)) => #f) + (deep (- n 1) (list 'begin form)))) + (check (test-form-allowed? (cons 'begin (make-list 20000 '(check value => expected))) + '(display)) => #t) + (let deep ((n 20000) (form '(display "fixture" port))) + (if (zero? n) + (begin + (check (test-form-allowed? (list 'call-with-output-string (list 'lambda '(port) form)) + '(display)) => #t) + (check (test-form-allowed? (list 'call-with-output-string + (list 'lambda '(port) (list 'lambda '(port) form))) + '(display)) => #f)) + (deep (- n 1) (list 'begin form))))) (test-case "all parser tests avoid display assertions" (check (filter (lambda (path) (not (test-source-allowed? path @@ -61,6 +153,9 @@ "src/compiler/event-fold-aot.ss" "t/generate-event-strategy-fixture.ss" "t/generate-event-strategy-grammar.ss" - "t/generate-event-fold-ir.ss" "t/generate-owned-word-fixture.ss")) => '())))) + +;; gxtest discovers only exported names ending in -test. +(def test-style-contract-test test-style-contract-tests) +(export test-style-contract-test) diff --git a/t/text-profile-cases.ss b/t/text-profile-cases.ss new file mode 100644 index 00000000..e9f30b71 --- /dev/null +++ b/t/text-profile-cases.ss @@ -0,0 +1,13 @@ +;;; Character endpoints shared by Scheme and its generated Rust witness. +(export fhirpath-profile-cases) +(def fhirpath-profile-cases + '((identifier ("Patient٣.more" 8) ("_a9" 3) ("α" #f) ("A²" 1) ("A４" 2)) + (number ("1." 1) ("12.34rest" 5) ("١٢.٣٤" 5) ("²" #f) ("1..2" 1) (".1" #f)) + (date ("@2024" 5) ("@2024-" #f) ("@2024-2" #f) ("@2024-12" 8) + ("@2024-12-" #f) ("@2024-12-31" 11) ("@2024-02-29T" #f) ("@20245" 5)) + (datetime ("@2024T" 6) ("@2024Tx" 6) ("@2024T1" #f) ("@2024T12:3" #f) + ("@2024T12Z+01:00" 9) ("@2024T12+01:00" 14) + ("@2024T12+0:00" #f) ("@2024T12+01" #f) + ("@2024T12:00:01." 14) ("@2024T12:00:01.2" 16)) + (time ("@T12" 4) ("@T1" #f) ("@T12:" #f) ("@T12:34" 7) + ("@T12:34:5" #f) ("@T12:34:56." 10) ("@T12:34:56.１２" 13) ("@T12Z" 4)))) diff --git a/t/text-profile-test.ss b/t/text-profile-test.ss new file mode 100644 index 00000000..fbdba6ff --- /dev/null +++ b/t/text-profile-test.ss @@ -0,0 +1,95 @@ +;;; Closed profile execution consumes the real FHIRPath declaration data. +(import (only-in "text-profile-cases.ss" fhirpath-profile-cases) + :std/test + (only-in :gerbil-parser/src/grammar/lexical-algebra text-profile? lexical-expression?) + (only-in :gerbil-parser/src/runtime/scan make-text-profile-scanner) + (only-in :gerbil-parser/languages/fhirpath/grammar fhirpath-parser-ir) + (only-in :gerbil-parser/languages/hl7/grammar hl7-parser-ir)) +(export text-profile-test) + +;;; Independent declaration interpreter, never prepared engine predicates. +(def (class-oracle expression character) + (case (car expression) + ((numeric) (char-numeric? character)) + ((alphabetic) (char-alphabetic? character)) + ((ascii-letter) + (or (and (char>=? character #\A) (char<=? character #\Z)) + (and (char>=? character #\a) (char<=? character #\z)))) + ((characters) (and (memv character (string->list (cadr expression))) #t)) + ((union) (ormap (lambda (part) (class-oracle part character)) (cdr expression))))) + +(def text-profile-test + (test-suite "closed text profile IR" + (test-case "real language profiles preserve rollback, commitment and Unicode endpoints" + (let (rules (cdr (assq 'lexical-rules fhirpath-parser-ir))) + (for-each + (lambda (group) + (let* ((expression (cadr (assq (car group) rules))) + (scan (make-text-profile-scanner (cadr expression)))) + (check (car expression) => 'text-profile) + (for-each (lambda (row) + (check (scan (car row) 0) => (cadr row)) + (check (scan (string-append "α" (car row)) 1) + => (and (cadr row) (+ 1 (cadr row))))) (cdr group)))) + fhirpath-profile-cases))) + (test-case "HL7 segment ids keep maximal-run rejection and CRLF stays one terminator" + (let (rules (cdr (assq 'lexical-rules hl7-parser-ir))) + (for-each + (lambda (group) + (let* ((expression (cadr (assq (car group) rules))) + (scan (make-text-profile-scanner (cadr expression)))) + (for-each (lambda (row) (check (scan (car row) 0) => (cadr row))) (cdr group)))) + '((segment-id ("MSH|" 3) ("AB٣|" 3) ("AB４|" 3) ("MSHA|" #f) + ("MSH3|" #f) ("MS|" #f) ("msH|" #f) ("MSH²" 3)) + (segment-terminator ("\r\nx" 2) ("\rx" 1) ("\nx" 1) ("x" #f) ("" #f)))))) + (test-case "invalid, callback and nullable roots reject before scanner construction" + (for-each + (lambda (profile) + (check (text-profile? profile) => #f) + (check (lexical-expression? (list 'text-profile profile)) => #f) + (check (with-catch (lambda (_) #t) + (lambda () (make-text-profile-scanner profile) #f)) => #t)) + '(() callback (seq) (literal "") (run (numeric) -1 #f) + (run (numeric) 2 1) (run (numeric) 4294967296 #f) + (run (numeric) 1 4294967296) + (seq (run (numeric) 4294967295 #f) (literal "x")) (run (unknown) 1 #f) (run (characters "") 1 #f) + (optional (literal "x")) (not-next (numeric)) + (if-next (numeric) (literal "x")) + (seq (literal "x") (if-next (numeric) invalid))))) + (test-case "closed union classes agree over ASCII and Unicode boundaries" + (let ((observations 0) + (characters (append (map integer->char (iota 256)) + (string->list "α中😀٣４²é𝟡")))) + (for-each + (lambda (class) + (let (scan (make-text-profile-scanner (list 'run class 1 #f))) + (for-each + (lambda (character) + (let* ((source (string character)) + (expected (and (class-oracle class character) 1))) + (unless (equal? (scan source 0) expected) + (error "text class declaration changed" class character expected)) + (set! observations (+ observations 1)))) characters))) + '((union (ascii-letter) (characters "_") (numeric)) + (union (characters "ABCDEFGHIJKLMNOPQRSTUVWXYZ") (numeric)) + (union (alphabetic) (numeric) (characters "_")) + (union (characters "éα😀_") (numeric)) + (union (union (characters "_") (numeric)) (ascii-letter)) + (union (characters "aaé") (characters "é_")))) + (check observations => 1584))) + (test-case "prepared character unions isolate declaration string mutation" + (let* ((characters (string-copy "_α")) + (scan (make-text-profile-scanner + (list 'run (list 'union (list 'characters characters) '(numeric)) 1 #f)))) + (string-set! characters 0 #\x) + (string-set! characters 1 #\β) + (check (scan "_α٣" 0) => 3) + (check (scan "x" 0) => #f) + (check (scan "β" 0) => #f))) + (test-case "prepared profiles retain source and offset bounds" + (let (scan (make-text-profile-scanner '(run (numeric) 1 #f))) + (check (scan "1" -1) => #f) + (check (scan "1" 2) => #f) + (check (scan "1" 1) => #f) + (check (scan "1" 0.5) => #f) + (check (scan "١２x" 0) => 2))))) diff --git a/t/versioned-language-benchmark-test.ss b/t/versioned-language-benchmark-test.ss index bbf69e8c..1c276e2f 100644 --- a/t/versioned-language-benchmark-test.ss +++ b/t/versioned-language-benchmark-test.ss @@ -1,37 +1,34 @@ #!/usr/bin/env gxi ;;; -*- Gerbil -*- -(import :std/test +(import (only-in :gerbil-parser/languages/hcl/parser-test hcl-test-language) + (only-in :gerbil-parser/languages/cypher/parser-test opencypher-test-language) + (only-in :gerbil-parser/languages/tla-plus/parser-test tla-plus-core-test-language) + (only-in :gerbil-parser/language-support/development + language-loader-fixtures language-loader-fixture-count) + (only-in :gerbil-parser/languages/gql/parser-test gql-test-language) + :std/test :asp-gerbil-scheme/src/benchmark/framework - :gerbil-parser/languages/hcl/v2-24/fixtures - :gerbil-parser/languages/hcl/v2-24/parser + :gerbil-parser/languages/hcl/parser :gerbil-parser/language-support - :gerbil-parser/languages/gql/iso-39075-2024/fixtures - :gerbil-parser/languages/gql/iso-39075-2024/parser - :gerbil-parser/languages/cypher/opencypher-2024-1/fixtures - :gerbil-parser/languages/cypher/opencypher-2024-1/parser - :gerbil-parser/languages/tla-plus/v1/fixtures - :gerbil-parser/languages/tla-plus/v1/parser + :gerbil-parser/languages/gql/parser + :gerbil-parser/languages/cypher/parser + :gerbil-parser/languages/tla-plus/parser :gerbil-parser/src/runtime/artifact) (def benchmark-path "t/benchmarks/versioned-languages/benchmark.ss") +(def benchmark-corpus + (list (cons parse-hcl (language-loader-fixtures hcl-test-language 'accepted)) + (cons parse-gql (language-loader-fixtures gql-test-language 'accepted)) + (cons parse-opencypher (language-loader-fixtures opencypher-test-language 'accepted)) + (cons parse-tla-plus-core (language-loader-fixtures tla-plus-core-test-language 'accepted)))) (def (parse-language-batch) - (def (parse-corpus parse fixtures) - (for-each - (lambda (fixture) - (let* ((source (syntax-fixture-source fixture)) - (artifact (parse source))) - (unless (and (parse-artifact-success? artifact) - (equal? (parse-artifact-roundtrip artifact) source)) - (error "official corpus benchmark parse failed" - (syntax-fixture-id fixture))))) - fixtures)) - (parse-corpus parse-hcl-v2-24 hcl-v2-24-official-accepted-fixtures) - (parse-corpus parse-gql-iso-39075-2024 gql-iso-official-fixtures) - (parse-corpus parse-opencypher-2024-1 - opencypher-2024-1-accepted-fixtures) - (parse-corpus parse-tla-plus-v1 tla-plus-v1-accepted-fixtures)) + (for-each (lambda (group) + (for-each (lambda (fixture) + (let* ((source (syntax-fixture-source fixture)) (artifact ((car group) source))) + (unless (and (parse-artifact-success? artifact) (equal? (parse-artifact-roundtrip artifact) source)) + (error "official corpus benchmark parse failed" (syntax-fixture-id fixture))))) (cdr group))) benchmark-corpus)) (def versioned-language-benchmark-tests (test-suite "versioned language benchmark" @@ -43,6 +40,12 @@ (benchmark-contract-run benchmark-path parse-language-batch)) (write receipt) (newline) + (force-output) (check (benchmark-contract-receipt-pass? receipt) => #t))))) -(export versioned-language-benchmark-tests) +(def versioned-language-benchmark-test versioned-language-benchmark-tests) +(export versioned-language-benchmark-tests versioned-language-benchmark-test) + +;; Resource admission owns its measurement interval in the shared runtime. +(def test-worker-exclusive? #t) +(export test-worker-exclusive?) diff --git a/t/versioned-language-pack-test.ss b/t/versioned-language-pack-test.ss deleted file mode 100644 index 11e7f40e..00000000 --- a/t/versioned-language-pack-test.ss +++ /dev/null @@ -1,19 +0,0 @@ -;;; -*- Gerbil -*- -;;; Native clan/testing owner for colocated versioned language-pack suites. - -(import :std/test - "../languages/arithmetic/v1/parser-test" - "../languages/hcl/v2-24/parser-test" - "../languages/gql/iso-39075-2024/parser-test" - "../languages/cypher/opencypher-2024-1/parser-test" - "../languages/tla-plus/v1/parser-test") - -(export versioned-language-pack-test) - -(def versioned-language-pack-test - (test-suite "versioned language-pack acceptance" - arithmetic-v1-parser-test - hcl-v2-24-parser-test - gql-iso-39075-2024-parser-test - opencypher-2024-1-parser-test - tla-plus-v1-parser-test)) diff --git a/test-processes.ss b/test-processes.ss new file mode 100644 index 00000000..50720ac4 --- /dev/null +++ b/test-processes.ss @@ -0,0 +1,97 @@ +;;; Native qualification of process-owning test entries. +(displayln "PROCESS-QUALIFICATION-START") (force-output) +(load "t/fixtures/tla-sany-differential/preload.ss") +(prefer-compiled-interfaces!) +(call-with-compiled-interface-trace + (lambda () (eval '(import :gerbil-parser/t/support/qualification/process)))) + +(import :gerbil-parser/t/support/qualification/process) + +(defstruct process-case (name arguments exit marker budget idle) transparent: #t) + +(def (qualification-cases profile) + (cond + ((equal? profile "conformance") + (map (lambda (row) + (process-case (car row) + ["gxi" "scripts/tests/conformance-runner-test.ss" (car row)] + (cadr row) (caddr row) 10 5)) + '(("success" 0 "CONFORMANCE-CONTROL-OK success") + ("duplicate" 70 "duplicate conformance Suite") + ("failure" 1 "negative control") + ("timeout" 70 "CONFORMANCE-GROUP-TIMEOUT timeout")))) + ((equal? profile "workers") + (map (lambda (row) + (process-case (car row) + (append (if (equal? (car row) "source-first-runtime") + ["env" (string-append "GERBIL_LOADPATH=" (current-directory))] '()) + ["gxi" "t/fixtures/tla-sany-differential/test-suite.ss" "--workers" "2"] + (map (lambda (name) + (string-append "t/fixtures/tla-sany-differential/" name ".ss")) + (cadr row))) + (caddr row) (cadddr row) 30 #f)) + '(("shared-runtime" ("worker-left" "worker-right") 0 #f) + ("source-first-runtime" ("worker-left" "worker-right") 0 "TEST-SUITE-OK modules=2") + ("exclusive-phase" ("worker-exclusive" "worker-left" "worker-right") 0 "TEST-SUITE-EXCLUSIVE modules=1") + ("exclusive-affinity" ("affinity-exclusive" "affinity-left" "affinity-right") 0 "TEST-SUITE-EXCLUSIVE modules=1") + ("invalid-exclusivity" ("worker-exclusive-invalid") nonzero "invalid test Worker exclusivity") + ("runtime-affinity" ("affinity-left" "affinity-right") 0 #f) + ("failure-propagation" ("worker-failure") nonzero #f) + ("silence-watchdog" ("worker-timeout") 70 #f) + ("duplicate-ownership" ("worker-left" "worker-left") nonzero #f) + ("reexport-ownership" ("worker-reexport") nonzero #f) + ("suite-object-ownership" ("worker-suite-alias") 70 "duplicate test Suite object")))) + ((equal? profile "research") + (let (binary (path-expand "bin/gerbil-parser-conformance" (getenv "GERBIL_PATH"))) + (cons (process-case "semantic" [binary "semantic"] 0 + "CONFORMANCE-SEMANTIC-OK groups=" 240 5) + (map (lambda (name) (process-case name [binary name] 0 #f 10 5)) + '("concise-runtime" "history" "native-poo-runtime" "provenance" + "all-language-benchmark"))))) + (else (error "unknown process qualification profile" profile)))) + +(def (qualify-process! case directory) + (let* ((name (process-case-name case)) + (result (run-observed-process (process-case-arguments case) + (path-expand (string-append name ".log") directory) + timeout: (process-case-budget case) idle-timeout: (process-case-idle case) + error-output?: #f)) + (status (process-result-status result)) (expected (process-case-exit case))) + (unless (and (not (process-result-reason result)) + (if (eq? expected 'nonzero) (> status 0) (= status expected))) + ;; Preserve the last real child output in the job log as well as its artifact. + ;; Hidden-directory upload defaults must never hide the failing native stage. + (let* ((text (utf8->string (process-result-output result))) + (end (string-length text))) + (displayln "PROCESS-FAILURE " name " status=" status + " reason=" (process-result-reason result) + " phase=" (process-result-phase result) " elapsed=" (process-result-elapsed result)) + (displayln "PROCESS-FAILURE-TAIL " name) + (display (substring text (max 0 (- end 8192)) end)) + (newline) (force-output)) + (error "unexpected process exit" name status (process-result-reason result))) + (let (marker (process-case-marker case)) + (when (and marker (not (string-contains (utf8->string (process-result-output result)) marker))) + (error "missing process receipt" name marker))) + status)) + +(def (main profile . arguments) + (let* ((cases (qualification-cases profile)) + (directory (if (pair? arguments) (car arguments) + (string-append ".data/" profile "-controls"))) + (prefix (cond ((equal? profile "conformance") "CONFORMANCE-CONTROL-OK ") + ((equal? profile "workers") "WORKER-CONTROL-OK ") + (else "RESEARCH-CONTROL-OK ")))) + (create-directory* directory) + (call-with-output-file (path-expand "exits.tsv" directory) + (lambda (exits) + (for-each (lambda (case) + (let (status (qualify-process! case directory)) + (display (process-case-name case) exits) (display "\t" exits) + (display status exits) (newline exits) + (force-output exits) + (displayln prefix (process-case-name case)) (force-output))) cases))) + (when (equal? profile "research") + (displayln "DSL-RESEARCH-CLOSURE-OK: all child processes exited 0")) + (displayln "OK") (force-output) + (exit 0))) diff --git a/tools/ci/bootstrap.ss b/tools/ci/bootstrap.ss new file mode 100644 index 00000000..51a3ed6d --- /dev/null +++ b/tools/ci/bootstrap.ss @@ -0,0 +1,28 @@ +;;; Host archive extraction precedes Scheme; activation and release admission do not. +(import :std/encoding/json :std/string/misc :std/misc/process :std/misc/ports) +(export main admit-release! append-lines!) +(def (admit-release! metadata os arch) + (unless (and (equal? (hash-ref metadata "schema" #f) "gerbil-bazel.toolchain-release.v1") + (equal? (hash-ref metadata "upstreamRevision" #f) "2591dcd9b7c6d2c4e9dd8611a17c5b1a5d82bbdb") + (let (platform (hash-ref metadata "platform" #f)) + (and (hash-table? platform) (equal? (hash-ref platform "os" #f) os) + (equal? (hash-ref platform "arch" #f) arch)))) + (error "immutable Gerbil release identity mismatch" os arch))) +(def (append-lines! file lines) + (let (port (open-output-file [path: file append: #t])) + (try (for-each (lambda (line) (display line port) (newline port)) lines) (finally (close-port port))))) +(def (main root os arch) + (let* ((prefix (path-expand root)) (home (path-expand "current" prefix)) + (options (string-append "~~=" home ",~~bin=" (path-expand "bin" home) ",~~lib=" (path-expand "lib" home))) + (cores (string-trim (run-process (if (equal? os "darwin") '("sysctl" "-n" "hw.ncpu") '("getconf" "_NPROCESSORS_ONLN")))))) + (admit-release! (string->json (call-with-input-file (path-expand "gerbil-toolchain-release.json" prefix) read-all-as-string) (JSONReadOptions object-as-hash: #t)) os arch) + (unless (and (string->number cores) (> (string->number cores) 0)) (error "invalid native build capacity" cores)) + (setenv "GERBIL_PREFIX" prefix) (setenv "GERBIL_HOME" home) (setenv "GAMBOPT" options) + (setenv "PATH" (string-append (path-expand "bin" prefix) ":" (getenv "PATH"))) + (append-lines! (getenv "GITHUB_PATH") [(path-expand "bin" prefix)]) + (append-lines! (getenv "GITHUB_ENV") [(string-append "GERBIL_PREFIX=" prefix) (string-append "GERBIL_HOME=" home) + (string-append "GAMBOPT=" options) (string-append "GERBIL_BUILD_CORES=" cores)]) + (run-process/batch [(path-expand "bin/gerbil" prefix) "--version"]) + (run-process/batch [(path-expand "bin/gerbil" prefix) "interactive" "-e" "(displayln \"gerbil-ready\") (exit 0)"]) + (displayln "GERBIL-RELEASE-BOOTSTRAP-OK cores=" cores) (force-output)) + (exit 0)) diff --git a/tools/ci/install-gerbil-v19-release.sh b/tools/ci/install-gerbil-v19-release.sh deleted file mode 100755 index 34d001ba..00000000 --- a/tools/ci/install-gerbil-v19-release.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: 2026 tao3k team and Contributors -# SPDX-License-Identifier: Apache-2.0 AND LGPL-2.1-or-later - -set -euo pipefail - -release_root="${1:?usage: install-gerbil-v19-release.sh RELEASE_ROOT}" -revision=2591dcd9b7c6d2c4e9dd8611a17c5b1a5d82bbdb - -case "$(uname -s)-$(uname -m)" in - Linux-x86_64) - release_os=linux - release_arch=x86_64 - tag=gerbil-v0.19-2591dcd9b7c6d2c4e9dd8611a17c5b1a5d82bbdb-linux-x86_64-portable-full-single-host-unlimited-patchf5cedd8168cb - sha256=9844ba362fdf6f1c5e8a4411e462a466d52dee0d541f8e84f61091cec0e5e740 - ;; - Darwin-arm64) - release_os=darwin - release_arch=aarch64 - tag=gerbil-v0.19-2591dcd9b7c6d2c4e9dd8611a17c5b1a5d82bbdb-darwin-aarch64-gcc16-arm64-aot-tools-single-host-unlimited-patchf5cedd8168cb - sha256=8d9c88434aed6301eaebb719bf52472f05c2368eafc630e8a9f1d67cc9895a21 - ;; - *) - printf 'unsupported Gerbil release host: %s-%s\n' "$(uname -s)" "$(uname -m)" >&2 - exit 64 - ;; -esac - -archive="$tag.tar.gz" -download="${RUNNER_TEMP:?}/$archive" -mkdir -p "$release_root" -curl --fail --location --retry 3 --output "$download" \ - "https://github.com/tao3k/gerbil-bazel/releases/download/$tag/$archive" -echo "$sha256 $download" | shasum -a 256 --check -tar -xzf "$download" -C "$release_root" --strip-components=1 - -jq -e --arg revision "$revision" --arg os "$release_os" --arg arch "$release_arch" \ - '.schema == "gerbil-bazel.toolchain-release.v1" and - .upstreamRevision == $revision and - .platform.os == $os and .platform.arch == $arch' \ - "$release_root/gerbil-toolchain-release.json" >/dev/null - -source "$release_root/activate" -echo "$GERBIL_PREFIX/bin" >> "${GITHUB_PATH:?}" -{ - echo "GERBIL_PREFIX=$GERBIL_PREFIX" - echo "GERBIL_HOME=$GERBIL_HOME" - echo "GAMBOPT=$GAMBOPT" -} >> "${GITHUB_ENV:?}" - -"$GERBIL_PREFIX/bin/gerbil" --version -"$GERBIL_PREFIX/bin/gerbil" interactive -e '(displayln "gerbil-ready")' diff --git a/tools/ci/prepared-products.sh b/tools/ci/prepared-products.sh new file mode 100644 index 00000000..1ee29aba --- /dev/null +++ b/tools/ci/prepared-products.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Run-scoped compiled products retain paths, symlinks, permissions and mtimes. +set -euo pipefail +operation=${1:?expected pack or restore} +archive=${2:?expected archive path} +: "${GERBIL_PREPARED_HEAD:?}" "${GERBIL_SDK_SHA256:?}" "${RUNNER_OS:?}" "${RUNNER_ARCH:?}" +identity() { + printf '%s\n' "$GERBIL_PREPARED_HEAD" "$GERBIL_SDK_SHA256" "$RUNNER_OS" "$RUNNER_ARCH" "$PWD" +} +case "$operation" in + pack) + test "$(git rev-parse HEAD)" = "$GERBIL_PREPARED_HEAD" + mkdir -p .gerbil/ci + identity > .gerbil/ci/identity + tar -czf "$archive" .gerbil + ;; + restore) + test "$(git rev-parse HEAD)" = "$GERBIL_PREPARED_HEAD" + # Validate ownership before extracting or loading compiled Scheme products. + identity > "$RUNNER_TEMP/expected-scheme-identity" + tar -xOf "$archive" .gerbil/ci/identity > "$RUNNER_TEMP/actual-scheme-identity" + cmp "$RUNNER_TEMP/expected-scheme-identity" "$RUNNER_TEMP/actual-scheme-identity" + tar -xzf "$archive" + mkdir -p "$HOME/.gerbil/pkg" + gerbil pkg dir --add git.cons.io/mighty-gerbils/gerbil-directory + ;; + *) echo "unknown prepared-product operation: $operation" >&2; exit 2 ;; +esac +echo "PREPARED-SCHEME-OK $operation"